fluent-plugin-logentries_ssl 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +28 -1
- data/fluent-plugin-logentries_ssl.gemspec +1 -1
- data/lib/fluent/plugin/out_logentries_ssl.rb +88 -77
- data/test/plugin/test_out_logentries_ssl.rb +59 -24
- metadata +3 -4
- data/Gemfile.lock +0 -52
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c447ff5f9abb188ea46d1a4e0dc80263e555791b
|
4
|
+
data.tar.gz: 82657485f8ec359ae29b6cecffdae7339a2fcbaa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1daba351d5d12445afae24f6c0d377240e2eddd5e9fb86ac67ab3078405bcb22870f385850731a1a046b86976363d50b1fd2947060b0ce4523d304421a1e041c
|
7
|
+
data.tar.gz: 3656af5f5ab6d40fd344ce6f9ebfb632821d6c35b0fa8dc0e7e49aa6ae9589f8ec1f46b991939bf9de93a200fbeb6e1a9626a62237dc7a6004afdb373737e912
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -19,6 +19,14 @@ gem install fluent-plugin-logentries_ssl
|
|
19
19
|
|
20
20
|
## Configuration
|
21
21
|
|
22
|
+
One of parameters *token\_path* or *default\_token* must be given. if token path is not defined, the Logentries token *default\_token* is used for all fluent events. When using *default\_token* with tokens from *token\_path*, it will be used as a fallback after trying to match a tag to a token in *token\_path*.
|
23
|
+
|
24
|
+
| *parameter* | *description | *default value* |
|
25
|
+
|---|---|---|
|
26
|
+
| *token_path* | Path to YAML formatted file containing 'tag: logentries-token' pairs | nil |
|
27
|
+
| *default_token* | A token string to be used either for all tags, or as fallback after token_path| nil |
|
28
|
+
|
29
|
+
|
22
30
|
```
|
23
31
|
<match pattern>
|
24
32
|
@type logentries_ssl
|
@@ -26,6 +34,16 @@ gem install fluent-plugin-logentries_ssl
|
|
26
34
|
</match>
|
27
35
|
```
|
28
36
|
|
37
|
+
or with *default\_token*:
|
38
|
+
|
39
|
+
```
|
40
|
+
<match pattern>
|
41
|
+
@type logentries_ssl
|
42
|
+
token_path /path/to/tokens.yml
|
43
|
+
default_token 'aaa-bbb-ccc'
|
44
|
+
</match>
|
45
|
+
```
|
46
|
+
|
29
47
|
```
|
30
48
|
<match pattern>
|
31
49
|
@type logentries_ssl
|
@@ -38,16 +56,19 @@ gem install fluent-plugin-logentries_ssl
|
|
38
56
|
</buffer>
|
39
57
|
</match>
|
40
58
|
|
59
|
+
````
|
41
60
|
with tokens.yml
|
61
|
+
|
42
62
|
```
|
43
63
|
tag-to-send: [logentries tcp token]
|
44
64
|
other-tag: [other token]
|
45
65
|
```
|
66
|
+
|
46
67
|
Event tag must match key in tokens file.
|
47
68
|
|
48
69
|
other configuration keys:
|
49
70
|
|
50
|
-
| *parameter* | *
|
71
|
+
| *parameter* | *description | *default value* |
|
51
72
|
|---|---|---|
|
52
73
|
| *le_host* | Logentries hostname to use | data.logentries.com |
|
53
74
|
| *le_port* | Logentries port to use | 443 |
|
@@ -55,6 +76,12 @@ other configuration keys:
|
|
55
76
|
| *json* | Send record as json | true |
|
56
77
|
| *verify_fqdn* | Verify FQDN for SSL | true |
|
57
78
|
|
79
|
+
|
80
|
+
## Alternatives
|
81
|
+
|
82
|
+
* [fluent-plugin-logentries](https://github.com/Woorank/fluent-plugin-logentries)
|
83
|
+
* [fluent-plugin-simple-logentries](https://github.com/sowawa/fluent-plugin-simple-logentries)
|
84
|
+
|
58
85
|
## Copyright
|
59
86
|
|
60
87
|
* Copyright(c) 2017- larte
|
@@ -17,100 +17,111 @@ require 'fluent/plugin/output'
|
|
17
17
|
require 'yaml'
|
18
18
|
require_relative 'logentries_ssl/message_helper.rb'
|
19
19
|
|
20
|
-
module Fluent
|
21
|
-
module
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
Fluent::Plugin
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
20
|
+
module Fluent
|
21
|
+
module Plugin
|
22
|
+
module LogentriesSSL
|
23
|
+
##
|
24
|
+
# The plugin implementation.
|
25
|
+
class Output < Fluent::Plugin::Output
|
26
|
+
include Fluent::PluginHelper::Socket
|
27
|
+
|
28
|
+
Fluent::Plugin.register_output('logentries_ssl', self)
|
29
|
+
|
30
|
+
config_param :max_retries, :integer, default: 3
|
31
|
+
config_param :le_host, :string, default: 'data.logentries.com'
|
32
|
+
config_param :le_port, :integer, default: 443
|
33
|
+
config_param :token_path, :string, default: nil
|
34
|
+
config_param :default_token, :string, default: nil
|
35
|
+
config_param :json, :bool, default: true
|
36
|
+
config_param :verify_fqdn, :bool, default: true
|
37
|
+
|
38
|
+
def configure(conf)
|
39
|
+
super
|
40
|
+
if @default_token.nil? && @token_path.nil?
|
41
|
+
raise Fluent::ConfigError, 'Define :token_path or :default_token'
|
42
|
+
end
|
43
|
+
@apptokens = @token_path.nil? ? {} : load_tokens
|
40
44
|
end
|
41
|
-
end
|
42
45
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
46
|
+
def start
|
47
|
+
super
|
48
|
+
log.trace "Creating connection to #{@le_host}"
|
49
|
+
@_client = create_client
|
50
|
+
end
|
48
51
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
52
|
+
# apparently needed for msgpack_each in :write fluent Issue-1342
|
53
|
+
def formatted_to_msgpack_binary
|
54
|
+
true
|
55
|
+
end
|
53
56
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
+
def load_tokens
|
58
|
+
begin
|
59
|
+
tokens = YAML.load_file(@token_path)
|
60
|
+
rescue StandardError => e
|
61
|
+
raise Fluent::ConfigError,
|
62
|
+
"Could not load #{@token_path}: #{e.message}"
|
63
|
+
end
|
64
|
+
tokens
|
65
|
+
end
|
57
66
|
|
58
|
-
|
59
|
-
|
60
|
-
return token if tag.casecmp(name).zero?
|
67
|
+
def format(tag, _time, record)
|
68
|
+
[tag, record].to_msgpack
|
61
69
|
end
|
62
|
-
nil
|
63
|
-
end
|
64
70
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
71
|
+
def tag_token(tag)
|
72
|
+
@apptokens.each do |name, token|
|
73
|
+
return token if tag.casecmp(name).zero?
|
74
|
+
end
|
75
|
+
@default_token.nil? ? nil : @default_token
|
76
|
+
end
|
77
|
+
|
78
|
+
def write(chunk)
|
79
|
+
chunk.msgpack_each do |tag, record|
|
80
|
+
token = tag_token(tag)
|
81
|
+
next unless token
|
82
|
+
data = @json ? record.to_json : record
|
83
|
+
MessageHelper.split_record(token, "#{token} #{data} \n")
|
84
|
+
.each do |payload|
|
85
|
+
with_retries { client.write(payload) }
|
76
86
|
end
|
77
87
|
end
|
78
88
|
end
|
79
|
-
end
|
80
89
|
|
81
|
-
|
90
|
+
private
|
82
91
|
|
83
|
-
|
84
|
-
|
85
|
-
|
92
|
+
def create_client
|
93
|
+
socket_create(:tls, @le_host, @le_port, verify_fqdn: @verify_fqdn)
|
94
|
+
end
|
86
95
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
96
|
+
def close_client
|
97
|
+
@_client.close if @_client
|
98
|
+
@_client = nil
|
99
|
+
end
|
91
100
|
|
92
|
-
|
93
|
-
|
94
|
-
|
101
|
+
def client
|
102
|
+
@_client ||= create_client
|
103
|
+
end
|
95
104
|
|
96
|
-
|
97
|
-
|
98
|
-
|
105
|
+
def retry?(n)
|
106
|
+
n < @max_retries
|
107
|
+
end
|
99
108
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
109
|
+
# rubocop:disable Metrics/MethodLength
|
110
|
+
def with_retries
|
111
|
+
tries = 0
|
112
|
+
begin
|
113
|
+
yield
|
114
|
+
rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED,
|
115
|
+
Errno::ENETUNREACH, Errno::ETIMEDOUT, Errno::EPIPE => e
|
116
|
+
if retry?(tries += 1)
|
117
|
+
log.warn 'Clould not push to logentries, reset and retry'\
|
118
|
+
"in #{2**tries} seconds. #{e.message}"
|
119
|
+
sleep(2**tries)
|
120
|
+
close_client
|
121
|
+
retry
|
122
|
+
end
|
123
|
+
raise 'Could not push logs to Logentries'
|
112
124
|
end
|
113
|
-
raise 'Could not push logs to Logentries'
|
114
125
|
end
|
115
126
|
end
|
116
127
|
end
|
@@ -1,7 +1,7 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/plugin/out_logentries_ssl.rb'
|
4
3
|
|
4
|
+
# rubocop:disable Metrics/ClassLength
|
5
5
|
class LogentriesSSLOutTest < Test::Unit::TestCase
|
6
6
|
setup do
|
7
7
|
Fluent::Test.setup
|
@@ -12,20 +12,20 @@ class LogentriesSSLOutTest < Test::Unit::TestCase
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def teardown
|
15
|
-
Dir.glob('test/tmp/*').each {|f| File.unlink(f) }
|
15
|
+
Dir.glob('test/tmp/*').each { |f| File.unlink(f) }
|
16
16
|
Dir.rmdir('test/tmp')
|
17
17
|
end
|
18
18
|
|
19
|
-
CONF = %
|
19
|
+
CONF = %(
|
20
20
|
token_path test/tmp/tokens.yml
|
21
21
|
max_retries 2
|
22
22
|
verify_fqdn false
|
23
|
-
|
23
|
+
).freeze
|
24
24
|
|
25
|
-
TOKENS = {
|
25
|
+
TOKENS = { 'app' => 'token', 'app2' => 'token2' }.freeze
|
26
26
|
|
27
27
|
def write_tokens
|
28
|
-
File.open(
|
28
|
+
File.open('test/tmp/tokens.yml', 'w') do |f|
|
29
29
|
f.write TOKENS.to_yaml.to_s
|
30
30
|
end
|
31
31
|
end
|
@@ -35,12 +35,12 @@ class LogentriesSSLOutTest < Test::Unit::TestCase
|
|
35
35
|
socket.stubs(:connect).returns(socket)
|
36
36
|
socket.stubs(:sync_close=)
|
37
37
|
socket.stubs(:close)
|
38
|
-
OpenSSL::SSL::SSLSocket.expects(:new).at_least_once.with(any_parameters)
|
39
|
-
|
38
|
+
OpenSSL::SSL::SSLSocket.expects(:new).at_least_once.with(any_parameters)
|
39
|
+
.returns(socket)
|
40
|
+
socket
|
40
41
|
end
|
41
42
|
|
42
|
-
|
43
|
-
test "configuration" do
|
43
|
+
test 'configuration' do
|
44
44
|
write_tokens
|
45
45
|
d = create_driver(CONF)
|
46
46
|
assert_equal 'test/tmp/tokens.yml', d.instance.token_path
|
@@ -49,7 +49,7 @@ class LogentriesSSLOutTest < Test::Unit::TestCase
|
|
49
49
|
assert_equal 'data.logentries.com', d.instance.le_host
|
50
50
|
end
|
51
51
|
|
52
|
-
test
|
52
|
+
test 'tags-tokens' do
|
53
53
|
write_tokens
|
54
54
|
d = create_driver(CONF)
|
55
55
|
assert_equal 'token', d.instance.tag_token('app')
|
@@ -57,17 +57,49 @@ class LogentriesSSLOutTest < Test::Unit::TestCase
|
|
57
57
|
assert_equal nil, d.instance.tag_token('app3')
|
58
58
|
end
|
59
59
|
|
60
|
-
test
|
60
|
+
test 'using default token' do
|
61
|
+
conf = %(
|
62
|
+
default_token aaa-bbb-ccc
|
63
|
+
max_retries 2
|
64
|
+
verify_fqdn false
|
65
|
+
)
|
66
|
+
d = create_driver(conf)
|
67
|
+
assert_equal 'aaa-bbb-ccc', d.instance.tag_token('app1')
|
68
|
+
assert_equal 'aaa-bbb-ccc', d.instance.tag_token('app2')
|
69
|
+
end
|
70
|
+
|
71
|
+
test 'using default token with token file' do
|
72
|
+
write_tokens
|
73
|
+
conf = %(
|
74
|
+
token_path test/tmp/tokens.yml
|
75
|
+
default_token aaa-bbb-ccc
|
76
|
+
max_retries 2
|
77
|
+
verify_fqdn false
|
78
|
+
)
|
79
|
+
d = create_driver(conf)
|
80
|
+
assert_equal 'token', d.instance.tag_token('app')
|
81
|
+
assert_equal 'token2', d.instance.tag_token('app2')
|
82
|
+
assert_equal 'aaa-bbb-ccc', d.instance.tag_token('app3')
|
83
|
+
end
|
84
|
+
|
85
|
+
test 'no token info' do
|
86
|
+
assert_raise Fluent::ConfigError do
|
87
|
+
create_driver('max_retries 2')
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
test 'unreadable tokens' do
|
61
92
|
assert_raise Fluent::ConfigError do
|
62
93
|
create_driver(CONF)
|
63
94
|
end
|
64
95
|
end
|
65
96
|
|
66
|
-
test
|
97
|
+
test 'sending to logentries' do
|
67
98
|
write_tokens
|
68
99
|
socket = stub_socket
|
69
|
-
message = {
|
70
|
-
socket.expects(:write)
|
100
|
+
message = { 'message' => 'Hello' }
|
101
|
+
socket.expects(:write)
|
102
|
+
.with(regexp_matches(/^token {.*message.*Hello.*}\s+/i))
|
71
103
|
d = create_driver(CONF)
|
72
104
|
time = event_time('2017-01-01 13:37:00 UTC')
|
73
105
|
d.run(default_tag: 'app') do
|
@@ -75,11 +107,12 @@ class LogentriesSSLOutTest < Test::Unit::TestCase
|
|
75
107
|
end
|
76
108
|
end
|
77
109
|
|
78
|
-
test
|
110
|
+
test 'retries on errors' do
|
79
111
|
write_tokens
|
80
112
|
socket = stub_socket
|
81
|
-
message = {
|
82
|
-
socket.expects(:write).with(anything).twice.raises(Errno::ECONNRESET)
|
113
|
+
message = { 'message' => 'Hello' }
|
114
|
+
socket.expects(:write).with(anything).twice.raises(Errno::ECONNRESET)
|
115
|
+
.then.returns('ok')
|
83
116
|
d = create_driver(CONF)
|
84
117
|
time = event_time('2017-01-01 13:37:00 UTC')
|
85
118
|
d.run(default_tag: 'app') do
|
@@ -87,11 +120,12 @@ class LogentriesSSLOutTest < Test::Unit::TestCase
|
|
87
120
|
end
|
88
121
|
end
|
89
122
|
|
90
|
-
test
|
123
|
+
test 'sending too large events to LE' do
|
91
124
|
write_tokens
|
92
125
|
socket = stub_socket
|
93
|
-
message = {
|
94
|
-
|
126
|
+
message = { 'hello' =>
|
127
|
+
'a' *
|
128
|
+
(Fluent::Plugin::LogentriesSSL::MessageHelper::MAX_SIZE + 100) }
|
95
129
|
socket.expects(:write).with(anything).twice
|
96
130
|
d = create_driver(CONF)
|
97
131
|
time = event_time('2017-01-01 13:37:00 UTC')
|
@@ -103,6 +137,7 @@ class LogentriesSSLOutTest < Test::Unit::TestCase
|
|
103
137
|
private
|
104
138
|
|
105
139
|
def create_driver(conf)
|
106
|
-
Fluent::Test::Driver::Output
|
140
|
+
Fluent::Test::Driver::Output
|
141
|
+
.new(Fluent::Plugin::LogentriesSSL::Output).configure(conf)
|
107
142
|
end
|
108
143
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-logentries_ssl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- larte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-09-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -95,7 +95,6 @@ extra_rdoc_files: []
|
|
95
95
|
files:
|
96
96
|
- ".gitignore"
|
97
97
|
- Gemfile
|
98
|
-
- Gemfile.lock
|
99
98
|
- LICENSE
|
100
99
|
- README.md
|
101
100
|
- Rakefile
|
@@ -125,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
125
124
|
version: '0'
|
126
125
|
requirements: []
|
127
126
|
rubyforge_project:
|
128
|
-
rubygems_version: 2.6.
|
127
|
+
rubygems_version: 2.6.8
|
129
128
|
signing_key:
|
130
129
|
specification_version: 4
|
131
130
|
summary: Plugin to send records to logentries
|
data/Gemfile.lock
DELETED
@@ -1,52 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
fluent-plugin-logentries_ssl (0.1.0)
|
5
|
-
fluentd (>= 0.14.10, < 2)
|
6
|
-
|
7
|
-
GEM
|
8
|
-
remote: https://rubygems.org/
|
9
|
-
specs:
|
10
|
-
cool.io (1.4.6)
|
11
|
-
fluentd (0.14.13)
|
12
|
-
cool.io (~> 1.4.5)
|
13
|
-
http_parser.rb (>= 0.5.1, < 0.7.0)
|
14
|
-
msgpack (>= 0.7.0, < 2.0.0)
|
15
|
-
serverengine (>= 2.0.4, < 3.0.0)
|
16
|
-
sigdump (~> 0.2.2)
|
17
|
-
strptime (~> 0.1.7)
|
18
|
-
tzinfo (~> 1.0)
|
19
|
-
tzinfo-data (~> 1.0)
|
20
|
-
yajl-ruby (~> 1.0)
|
21
|
-
http_parser.rb (0.6.0)
|
22
|
-
metaclass (0.0.4)
|
23
|
-
mocha (1.2.1)
|
24
|
-
metaclass (~> 0.0.1)
|
25
|
-
msgpack (1.1.0)
|
26
|
-
power_assert (1.0.1)
|
27
|
-
rake (12.0.0)
|
28
|
-
serverengine (2.0.5)
|
29
|
-
sigdump (~> 0.2.2)
|
30
|
-
sigdump (0.2.4)
|
31
|
-
strptime (0.1.9)
|
32
|
-
test-unit (3.2.3)
|
33
|
-
power_assert
|
34
|
-
thread_safe (0.3.6)
|
35
|
-
tzinfo (1.2.2)
|
36
|
-
thread_safe (~> 0.1)
|
37
|
-
tzinfo-data (1.2017.1)
|
38
|
-
tzinfo (>= 1.0.0)
|
39
|
-
yajl-ruby (1.3.0)
|
40
|
-
|
41
|
-
PLATFORMS
|
42
|
-
ruby
|
43
|
-
|
44
|
-
DEPENDENCIES
|
45
|
-
bundler (~> 1.14)
|
46
|
-
fluent-plugin-logentries_ssl!
|
47
|
-
mocha
|
48
|
-
rake (~> 12.0)
|
49
|
-
test-unit (~> 3.0)
|
50
|
-
|
51
|
-
BUNDLED WITH
|
52
|
-
1.14.6
|