fluent-plugin-logentries 0.1.1 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 54dd09205e71e513063bb8346d512d2bf42477dd
4
- data.tar.gz: 9a4280574f047836fcb4bf5f6d3ff81cc458c121
3
+ metadata.gz: ede9927a84f4346ffdf4298c2c4613a66a96a855
4
+ data.tar.gz: 7b4572090c1ef0d26a02518700859a0bc508f349
5
5
  SHA512:
6
- metadata.gz: 3dd6de0d69e28416d7c92fe74e78ee454e798dfdfe3febc23320e92db1187d85f871ec979bae9aceb9c4e96cdbee4d1a3c9c2c8daf7d7b445d6bbb8e8ce70bc1
7
- data.tar.gz: 874d63485f551b2625bf0cb411c674b3f234d6dc8ef60708622fc6578eab7067f3a02fc9cee5b85a854a543307de9f17d2bfb84024d9242f4373523161f0fb84
6
+ metadata.gz: 5ad38d2ba03195a7a557075e113476314739e4625a1b140ed1bb320d3c75c5d1174ed7c4c9ea3c983f21406d3624627e0025d5aa7dbdc1b57d45180f93fb334e
7
+ data.tar.gz: ae09d8debfaabb0db41fde0bb5cdb479d2db14be08ee69ef6938009bd54155b51fb2776de050458281cdf49551eee8eae0ca11692a2050a5521cab96489bb3ac
data/README.md CHANGED
@@ -1,29 +1,69 @@
1
1
  # Fluent::Plugin::Logentries
2
+ Forward logs to Logentries, using token based input.
2
3
 
3
- TODO: Write a gem description
4
+ Looks at the tag/message to find out where the log should go.
4
5
 
5
6
  ## Installation
6
7
 
7
- Add this line to your application's Gemfile:
8
+ /opt/td-agent/embedded/bin/fluent-gem install fluent-plugin-logentries
8
9
 
9
- gem 'fluent-plugin-logentries'
10
+ ## Configruation file (YML)
10
11
 
11
- And then execute:
12
+ ```yaml
13
+ My-Awesome-App:
14
+ app: MY-LOGENTRIES-TOKEN
15
+ access: ANOTHER-LOGENTRIES-TOKEN (*)
16
+ error: ANOTHER-LOGENTRIES-TOKEN-1 (*)
17
+ Foo:
18
+ app: 2bfbea1e-10c3-4419-bdad-7e6435882e1f
19
+ access: 5deab21c-04b1-9122-abdc-09adb2eda22 (*)
20
+ error: 9acfbeba-c92c-1229-ccac-12c58d82ecc (*)
21
+ ```
22
+ (*) `access` and `error` are optional, if you don't use multiple log per host just provide an app token.
12
23
 
13
- $ bundle
24
+ This file is read everytime the buffer is flushed, it allows on fly modifications.
25
+ ## Usage
14
26
 
15
- Or install it yourself as:
27
+ ```
28
+ <match pattern>
29
+ type logentries
30
+ config_path /path/to/logentries-tokens.conf
31
+ </match>
32
+ ```
16
33
 
17
- $ gem install fluent-plugin-logentries
34
+ ## Parameters
18
35
 
19
- ## Usage
36
+ ### type (required)
37
+ The value must be `logentries`.
38
+
39
+ ### config_path (required)
40
+ Path of your configuration file, e.g. `/opt/logentries/tokens.conf`
41
+
42
+ ### use_ssl
43
+ Enable/disable SSL for data transfers between Fluentd and Logentries. The default is `true`.
20
44
 
21
- TODO: Write usage instructions here
45
+ ### no_ssl_port
46
+ Only in case you don't use SSL, the value must be `80`, `514`, or `10000`. The default is `80`
47
+
48
+ ### max_retries
49
+ Number of retries on failure.
50
+
51
+ ### tag_access_log, tag_error_log
52
+ This is use in case you tag your access/error log and want them to be push into another log.
22
53
 
23
54
  ## Contributing
24
55
 
25
- 1. Fork it ( http://github.com/<my-github-username>/fluent-plugin-logentries/fork )
56
+ 1. Fork it ( http://github.com/woorank/fluent-plugin-logentries/fork )
26
57
  2. Create your feature branch (`git checkout -b my-new-feature`)
27
58
  3. Commit your changes (`git commit -am 'Add some feature'`)
28
59
  4. Push to the branch (`git push origin my-new-feature`)
29
60
  5. Create new Pull Request
61
+
62
+ ## MIT
63
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
64
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
65
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
66
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
67
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
68
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
69
+ THE SOFTWARE.
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-logentries"
7
- spec.version = "0.1.1"
7
+ spec.version = "0.2.0"
8
8
  spec.authors = ["Woorank"]
9
9
  spec.email = ["dev@woorank.com"]
10
10
  spec.summary = "Logentries output plugin for Fluent event collector"
@@ -1,4 +1,6 @@
1
1
  require 'socket'
2
+ require 'yaml'
3
+ require 'openssl'
2
4
 
3
5
  class LogentriesOutput < Fluent::BufferedOutput
4
6
  class ConnectionFailure < StandardError; end
@@ -6,9 +8,12 @@ class LogentriesOutput < Fluent::BufferedOutput
6
8
  # and identifies the plugin in the configuration file.
7
9
  Fluent::Plugin.register_output('logentries', self)
8
10
 
9
- config_param :host, :string
10
- config_param :port, :integer, :default => 80
11
- config_param :path, :string
11
+ config_param :use_ssl, :bool, :default => true
12
+ config_param :no_ssl_port, :integer, :default => 80
13
+ config_param :config_path, :string
14
+ config_param :max_retries, :integer, :default => 3
15
+ config_param :tag_access_log, :string, :default => 'logs-access'
16
+ config_param :tag_error_log, :string, :default => 'logs-error'
12
17
 
13
18
  def configure(conf)
14
19
  super
@@ -23,7 +28,15 @@ class LogentriesOutput < Fluent::BufferedOutput
23
28
  end
24
29
 
25
30
  def client
26
- @_socket ||= TCPSocket.new @host, @port
31
+ @_socket ||= if @use_ssl
32
+ context = OpenSSL::SSL::SSLContext.new
33
+ socket = TCPSocket.new "api.logentries.com", 20000
34
+ ssl_client = OpenSSL::SSL::SSLSocket.new socket, context
35
+
36
+ ssl_client.connect
37
+ else
38
+ TCPSocket.new "data.logentries.com", @no_ssl_port
39
+ end
27
40
  end
28
41
 
29
42
  # This method is called when an event reaches Fluentd.
@@ -31,27 +44,39 @@ class LogentriesOutput < Fluent::BufferedOutput
31
44
  return [tag, record].to_msgpack
32
45
  end
33
46
 
34
- # Scan a given directory for logentries tokens
35
- def generate_token(path)
36
- tokens = {}
37
-
38
- Dir[path + "*.token"].each do |file|
39
- key = File.basename(file, ".token") #remove path/extension from filename
40
- #read the first line, remove unwanted char and close it
41
- tokens[key] = File.open(file, &:readline).gsub(/\r\n|\r|\n/, '')
47
+ # Create tokens hash
48
+ def generate_token
49
+ begin
50
+ YAML::load_file(@config_path)
51
+ rescue Exception => e
52
+ log.warn "Could not load configuration. #{e.message}"
42
53
  end
43
-
44
- tokens
45
54
  end
46
55
 
47
- # returns the correct token to use for a given tag / Records
56
+ # Returns the correct token to use for a given tag / records
48
57
  def get_token(tag, record, tokens)
49
- tag ||= ""
50
- record ||= ""
51
-
58
+ tag ||= ""
59
+ message = record["message"]
60
+
61
+ # Config Structure
62
+ # -----------------------
63
+ # app-name:
64
+ # app: TOKEN
65
+ # access: TOKEN (optional)
66
+ # error: TOKEN (optional)
52
67
  tokens.each do |key, value|
53
- if tag.index(key) != nil || record.index(key) != nil then
54
- return value
68
+ if tag.index(key) != nil || message.index(key) != nil then
69
+ default = value['app']
70
+
71
+ case tag
72
+ when @tag_access_log
73
+ return value['access'] || default
74
+ when @tag_error_log
75
+ return value['error'] || default
76
+
77
+ else
78
+ return default
79
+ end
55
80
  end
56
81
  end
57
82
 
@@ -60,17 +85,17 @@ class LogentriesOutput < Fluent::BufferedOutput
60
85
 
61
86
  # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
62
87
  def write(chunk)
63
- tokens = generate_token(@path)
88
+ tokens = generate_token()
89
+ return unless tokens.is_a? Hash
64
90
 
65
91
  chunk.msgpack_each do |tag, record|
66
92
  next unless record.is_a? Hash
93
+ next unless record.has_key? "message"
67
94
 
68
95
  token = get_token(tag, record, tokens)
69
96
  next if token.nil?
70
97
 
71
- if record.has_key?("message")
72
- send_logentries(record["message"] + ' ' + token)
73
- end
98
+ send_logentries(token + ' ' + record["message"])
74
99
  end
75
100
  end
76
101
 
@@ -79,11 +104,11 @@ class LogentriesOutput < Fluent::BufferedOutput
79
104
  begin
80
105
  client.puts data
81
106
  rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
82
- if retries < 2
107
+ if retries < @max_retries
83
108
  retries += 1
84
109
  @_socket = nil
85
110
  log.warn "Could not push logs to Logentries, resetting connection and trying again. #{e.message}"
86
- sleep 2**retries
111
+ sleep 5**retries
87
112
  retry
88
113
  end
89
114
  raise ConnectionFailure, "Could not push logs to Logentries after #{retries} retries. #{e.message}"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-logentries
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Woorank
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-12-19 00:00:00.000000000 Z
11
+ date: 2015-01-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler