fluent-plugin-logentries 0.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +50 -10
- data/fluent-plugin-logentries.gemspec +1 -1
- data/lib/fluent/plugin/out_logentries.rb +51 -26
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ede9927a84f4346ffdf4298c2c4613a66a96a855
|
4
|
+
data.tar.gz: 7b4572090c1ef0d26a02518700859a0bc508f349
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5ad38d2ba03195a7a557075e113476314739e4625a1b140ed1bb320d3c75c5d1174ed7c4c9ea3c983f21406d3624627e0025d5aa7dbdc1b57d45180f93fb334e
|
7
|
+
data.tar.gz: ae09d8debfaabb0db41fde0bb5cdb479d2db14be08ee69ef6938009bd54155b51fb2776de050458281cdf49551eee8eae0ca11692a2050a5521cab96489bb3ac
|
data/README.md
CHANGED
@@ -1,29 +1,69 @@
|
|
1
1
|
# Fluent::Plugin::Logentries
|
2
|
+
Forward logs to Logentries, using token based input.
|
2
3
|
|
3
|
-
|
4
|
+
Looks at the tag/message to find out where the log should go.
|
4
5
|
|
5
6
|
## Installation
|
6
7
|
|
7
|
-
|
8
|
+
/opt/td-agent/embedded/bin/fluent-gem install fluent-plugin-logentries
|
8
9
|
|
9
|
-
|
10
|
+
## Configruation file (YML)
|
10
11
|
|
11
|
-
|
12
|
+
```yaml
|
13
|
+
My-Awesome-App:
|
14
|
+
app: MY-LOGENTRIES-TOKEN
|
15
|
+
access: ANOTHER-LOGENTRIES-TOKEN (*)
|
16
|
+
error: ANOTHER-LOGENTRIES-TOKEN-1 (*)
|
17
|
+
Foo:
|
18
|
+
app: 2bfbea1e-10c3-4419-bdad-7e6435882e1f
|
19
|
+
access: 5deab21c-04b1-9122-abdc-09adb2eda22 (*)
|
20
|
+
error: 9acfbeba-c92c-1229-ccac-12c58d82ecc (*)
|
21
|
+
```
|
22
|
+
(*) `access` and `error` are optional, if you don't use multiple log per host just provide an app token.
|
12
23
|
|
13
|
-
|
24
|
+
This file is read everytime the buffer is flushed, it allows on fly modifications.
|
25
|
+
## Usage
|
14
26
|
|
15
|
-
|
27
|
+
```
|
28
|
+
<match pattern>
|
29
|
+
type logentries
|
30
|
+
config_path /path/to/logentries-tokens.conf
|
31
|
+
</match>
|
32
|
+
```
|
16
33
|
|
17
|
-
|
34
|
+
## Parameters
|
18
35
|
|
19
|
-
|
36
|
+
### type (required)
|
37
|
+
The value must be `logentries`.
|
38
|
+
|
39
|
+
### config_path (required)
|
40
|
+
Path of your configuration file, e.g. `/opt/logentries/tokens.conf`
|
41
|
+
|
42
|
+
### use_ssl
|
43
|
+
Enable/disable SSL for data transfers between Fluentd and Logentries. The default is `true`.
|
20
44
|
|
21
|
-
|
45
|
+
### no_ssl_port
|
46
|
+
Only in case you don't use SSL, the value must be `80`, `514`, or `10000`. The default is `80`
|
47
|
+
|
48
|
+
### max_retries
|
49
|
+
Number of retries on failure.
|
50
|
+
|
51
|
+
### tag_access_log, tag_error_log
|
52
|
+
This is use in case you tag your access/error log and want them to be push into another log.
|
22
53
|
|
23
54
|
## Contributing
|
24
55
|
|
25
|
-
1. Fork it ( http://github.com
|
56
|
+
1. Fork it ( http://github.com/woorank/fluent-plugin-logentries/fork )
|
26
57
|
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
58
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
28
59
|
4. Push to the branch (`git push origin my-new-feature`)
|
29
60
|
5. Create new Pull Request
|
61
|
+
|
62
|
+
## MIT
|
63
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
64
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
65
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
66
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
67
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
68
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
69
|
+
THE SOFTWARE.
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-logentries"
|
7
|
-
spec.version = "0.
|
7
|
+
spec.version = "0.2.0"
|
8
8
|
spec.authors = ["Woorank"]
|
9
9
|
spec.email = ["dev@woorank.com"]
|
10
10
|
spec.summary = "Logentries output plugin for Fluent event collector"
|
@@ -1,4 +1,6 @@
|
|
1
1
|
require 'socket'
|
2
|
+
require 'yaml'
|
3
|
+
require 'openssl'
|
2
4
|
|
3
5
|
class LogentriesOutput < Fluent::BufferedOutput
|
4
6
|
class ConnectionFailure < StandardError; end
|
@@ -6,9 +8,12 @@ class LogentriesOutput < Fluent::BufferedOutput
|
|
6
8
|
# and identifies the plugin in the configuration file.
|
7
9
|
Fluent::Plugin.register_output('logentries', self)
|
8
10
|
|
9
|
-
config_param :
|
10
|
-
config_param :
|
11
|
-
config_param :
|
11
|
+
config_param :use_ssl, :bool, :default => true
|
12
|
+
config_param :no_ssl_port, :integer, :default => 80
|
13
|
+
config_param :config_path, :string
|
14
|
+
config_param :max_retries, :integer, :default => 3
|
15
|
+
config_param :tag_access_log, :string, :default => 'logs-access'
|
16
|
+
config_param :tag_error_log, :string, :default => 'logs-error'
|
12
17
|
|
13
18
|
def configure(conf)
|
14
19
|
super
|
@@ -23,7 +28,15 @@ class LogentriesOutput < Fluent::BufferedOutput
|
|
23
28
|
end
|
24
29
|
|
25
30
|
def client
|
26
|
-
@_socket ||=
|
31
|
+
@_socket ||= if @use_ssl
|
32
|
+
context = OpenSSL::SSL::SSLContext.new
|
33
|
+
socket = TCPSocket.new "api.logentries.com", 20000
|
34
|
+
ssl_client = OpenSSL::SSL::SSLSocket.new socket, context
|
35
|
+
|
36
|
+
ssl_client.connect
|
37
|
+
else
|
38
|
+
TCPSocket.new "data.logentries.com", @no_ssl_port
|
39
|
+
end
|
27
40
|
end
|
28
41
|
|
29
42
|
# This method is called when an event reaches Fluentd.
|
@@ -31,27 +44,39 @@ class LogentriesOutput < Fluent::BufferedOutput
|
|
31
44
|
return [tag, record].to_msgpack
|
32
45
|
end
|
33
46
|
|
34
|
-
#
|
35
|
-
def generate_token
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
#read the first line, remove unwanted char and close it
|
41
|
-
tokens[key] = File.open(file, &:readline).gsub(/\r\n|\r|\n/, '')
|
47
|
+
# Create tokens hash
|
48
|
+
def generate_token
|
49
|
+
begin
|
50
|
+
YAML::load_file(@config_path)
|
51
|
+
rescue Exception => e
|
52
|
+
log.warn "Could not load configuration. #{e.message}"
|
42
53
|
end
|
43
|
-
|
44
|
-
tokens
|
45
54
|
end
|
46
55
|
|
47
|
-
#
|
56
|
+
# Returns the correct token to use for a given tag / records
|
48
57
|
def get_token(tag, record, tokens)
|
49
|
-
tag
|
50
|
-
|
51
|
-
|
58
|
+
tag ||= ""
|
59
|
+
message = record["message"]
|
60
|
+
|
61
|
+
# Config Structure
|
62
|
+
# -----------------------
|
63
|
+
# app-name:
|
64
|
+
# app: TOKEN
|
65
|
+
# access: TOKEN (optional)
|
66
|
+
# error: TOKEN (optional)
|
52
67
|
tokens.each do |key, value|
|
53
|
-
if tag.index(key) != nil ||
|
54
|
-
|
68
|
+
if tag.index(key) != nil || message.index(key) != nil then
|
69
|
+
default = value['app']
|
70
|
+
|
71
|
+
case tag
|
72
|
+
when @tag_access_log
|
73
|
+
return value['access'] || default
|
74
|
+
when @tag_error_log
|
75
|
+
return value['error'] || default
|
76
|
+
|
77
|
+
else
|
78
|
+
return default
|
79
|
+
end
|
55
80
|
end
|
56
81
|
end
|
57
82
|
|
@@ -60,17 +85,17 @@ class LogentriesOutput < Fluent::BufferedOutput
|
|
60
85
|
|
61
86
|
# NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
|
62
87
|
def write(chunk)
|
63
|
-
tokens = generate_token(
|
88
|
+
tokens = generate_token()
|
89
|
+
return unless tokens.is_a? Hash
|
64
90
|
|
65
91
|
chunk.msgpack_each do |tag, record|
|
66
92
|
next unless record.is_a? Hash
|
93
|
+
next unless record.has_key? "message"
|
67
94
|
|
68
95
|
token = get_token(tag, record, tokens)
|
69
96
|
next if token.nil?
|
70
97
|
|
71
|
-
|
72
|
-
send_logentries(record["message"] + ' ' + token)
|
73
|
-
end
|
98
|
+
send_logentries(token + ' ' + record["message"])
|
74
99
|
end
|
75
100
|
end
|
76
101
|
|
@@ -79,11 +104,11 @@ class LogentriesOutput < Fluent::BufferedOutput
|
|
79
104
|
begin
|
80
105
|
client.puts data
|
81
106
|
rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
|
82
|
-
if retries <
|
107
|
+
if retries < @max_retries
|
83
108
|
retries += 1
|
84
109
|
@_socket = nil
|
85
110
|
log.warn "Could not push logs to Logentries, resetting connection and trying again. #{e.message}"
|
86
|
-
sleep
|
111
|
+
sleep 5**retries
|
87
112
|
retry
|
88
113
|
end
|
89
114
|
raise ConnectionFailure, "Could not push logs to Logentries after #{retries} retries. #{e.message}"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-logentries
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Woorank
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-01-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|