fluent-plugin-cloudwatch-logs 0.5.0.pre1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +16 -0
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +12 -1
- data/lib/fluent/plugin/out_cloudwatch_logs.rb +12 -1
- data/test/plugin/test_in_cloudwatch_logs.rb +12 -8
- data/test/plugin/test_out_cloudwatch_logs.rb +26 -18
- data/test/test_helper.rb +5 -0
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 11a5a24eb3978bb6d5b29d3d08cf5b49bfd9eed85e3a3b6df46adedf31e85458
|
4
|
+
data.tar.gz: 1193f782abef03c1f89c18b25ccb2ad2d3d774681a5e489d860e5baefb71beb4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4f5d00f59b8a9c9652ab7b364bb13d9a51b6bc04ba7bcf058d2fb0a5705637c6b0156e93f94bb090f8c5e270d821a42f7b5ba7190e4ed0321f87b8d7a661f280
|
7
|
+
data.tar.gz: 32dd243642ac3d63ed8704a63074c611343a25c56a681d7c2ac8e298c8579c56ee82b4fe5f91d0ad6b236a0c094bfc86f3ba0f1b08fd4406e25e023dffec3c04
|
data/README.md
CHANGED
@@ -82,6 +82,8 @@ Fetch sample log from CloudWatch Logs:
|
|
82
82
|
#put_log_events_retry_wait 1s
|
83
83
|
#put_log_events_retry_limit 17
|
84
84
|
#put_log_events_disable_retry_limit false
|
85
|
+
#endpoint http://localhost:5000/
|
86
|
+
#json_handler json
|
85
87
|
</match>
|
86
88
|
```
|
87
89
|
|
@@ -102,6 +104,8 @@ Fetch sample log from CloudWatch Logs:
|
|
102
104
|
* `put_log_events_retry_wait`: time before retrying PutLogEvents (retry interval increases exponentially like `put_log_events_retry_wait * (2 ^ retry_count)`)
|
103
105
|
* `put_log_events_retry_limit`: maximum count of retry (if exceeding this, the events will be discarded)
|
104
106
|
* `put_log_events_disable_retry_limit`: if true, `put_log_events_retry_limit` will be ignored
|
107
|
+
* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
|
108
|
+
* `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
|
105
109
|
|
106
110
|
### in_cloudwatch_logs
|
107
111
|
|
@@ -113,6 +117,8 @@ Fetch sample log from CloudWatch Logs:
|
|
113
117
|
log_stream_name stream
|
114
118
|
#use_log_stream_name_prefix true
|
115
119
|
state_file /var/lib/fluent/group_stream.in.state
|
120
|
+
#endpoint http://localhost:5000/
|
121
|
+
#json_handler json
|
116
122
|
</source>
|
117
123
|
```
|
118
124
|
|
@@ -121,9 +127,11 @@ Fetch sample log from CloudWatch Logs:
|
|
121
127
|
* `log_stream_name`: name of log stream to fetch logs
|
122
128
|
* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
|
123
129
|
* `state_file`: file to store current state (e.g. next\_forward\_token)
|
130
|
+
* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
|
124
131
|
* `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
|
125
132
|
* `aws_sts_role_arn`: the role ARN to assume when using cross-account sts authentication
|
126
133
|
* `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
|
134
|
+
* `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
|
127
135
|
|
128
136
|
This plugin uses [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) and you can use addtional variables such as %{hostname}, %{uuid}, etc. These variables are useful to put hostname in `log_stream_name`.
|
129
137
|
|
@@ -149,6 +157,14 @@ Or, If you do not want to use IAM roll or ENV(this is just like writing to confi
|
|
149
157
|
$ rake aws_key_id=YOUR_ACCESS_KEY aws_sec_key=YOUR_SECRET_KEY region=us-east-1 test
|
150
158
|
```
|
151
159
|
|
160
|
+
If you want to run the test suite against a mock server, set `endpoint` as below:
|
161
|
+
|
162
|
+
```
|
163
|
+
$ export endpoint='http://localhost:5000/'
|
164
|
+
$ rake test
|
165
|
+
```
|
166
|
+
|
167
|
+
|
152
168
|
## Caution
|
153
169
|
|
154
170
|
- If an event message exceeds API limit (256KB), the event will be discarded.
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'fluent/plugin/input'
|
2
2
|
require 'fluent/plugin/parser'
|
3
|
+
require 'yajl'
|
3
4
|
|
4
5
|
module Fluent::Plugin
|
5
6
|
class CloudwatchLogsInput < Input
|
@@ -13,6 +14,7 @@ module Fluent::Plugin
|
|
13
14
|
config_param :aws_sts_role_arn, :string, default: nil
|
14
15
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
15
16
|
config_param :region, :string, :default => nil
|
17
|
+
config_param :endpoint, :string, :default => nil
|
16
18
|
config_param :tag, :string
|
17
19
|
config_param :log_group_name, :string
|
18
20
|
config_param :log_stream_name, :string
|
@@ -20,6 +22,7 @@ module Fluent::Plugin
|
|
20
22
|
config_param :state_file, :string
|
21
23
|
config_param :fetch_interval, :time, default: 60
|
22
24
|
config_param :http_proxy, :string, default: nil
|
25
|
+
config_param :json_handler, :enum, list: [:yajl, :json], :default => :json
|
23
26
|
|
24
27
|
config_section :parse do
|
25
28
|
config_set_default :@type, 'none'
|
@@ -41,6 +44,7 @@ module Fluent::Plugin
|
|
41
44
|
super
|
42
45
|
options = {}
|
43
46
|
options[:region] = @region if @region
|
47
|
+
options[:endpoint] = @endpoint if @endpoint
|
44
48
|
options[:http_proxy] = @http_proxy if @http_proxy
|
45
49
|
|
46
50
|
if @aws_use_sts
|
@@ -57,6 +61,13 @@ module Fluent::Plugin
|
|
57
61
|
|
58
62
|
@finished = false
|
59
63
|
thread_create(:in_cloudwatch_logs_runner, &method(:run))
|
64
|
+
|
65
|
+
@json_handler = case @json_handler
|
66
|
+
when :yajl
|
67
|
+
Yajl
|
68
|
+
when :json
|
69
|
+
JSON
|
70
|
+
end
|
60
71
|
end
|
61
72
|
|
62
73
|
def shutdown
|
@@ -121,7 +132,7 @@ module Fluent::Plugin
|
|
121
132
|
}
|
122
133
|
else
|
123
134
|
time = (event.timestamp / 1000).floor
|
124
|
-
record =
|
135
|
+
record = @json_handler.load(event.message)
|
125
136
|
router.emit(@tag, time, record)
|
126
137
|
end
|
127
138
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'fluent/plugin/output'
|
2
2
|
require 'thread'
|
3
|
+
require 'yajl'
|
3
4
|
|
4
5
|
module Fluent::Plugin
|
5
6
|
class CloudwatchLogsOutput < Output
|
@@ -15,6 +16,7 @@ module Fluent::Plugin
|
|
15
16
|
config_param :aws_sts_role_arn, :string, default: nil
|
16
17
|
config_param :aws_sts_session_name, :string, default: 'fluentd'
|
17
18
|
config_param :region, :string, :default => nil
|
19
|
+
config_param :endpoint, :string, :default => nil
|
18
20
|
config_param :log_group_name, :string, :default => nil
|
19
21
|
config_param :log_stream_name, :string, :default => nil
|
20
22
|
config_param :auto_create_stream, :bool, default: false
|
@@ -38,6 +40,7 @@ module Fluent::Plugin
|
|
38
40
|
config_param :retention_in_days, :integer, default: nil
|
39
41
|
config_param :retention_in_days_key, :string, default: nil
|
40
42
|
config_param :remove_retention_in_days, :bool, default: false
|
43
|
+
config_param :json_handler, :enum, list: [:yajl, :json], :default => :json
|
41
44
|
|
42
45
|
config_section :buffer do
|
43
46
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -79,6 +82,7 @@ module Fluent::Plugin
|
|
79
82
|
|
80
83
|
options = {}
|
81
84
|
options[:region] = @region if @region
|
85
|
+
options[:endpoint] = @endpoint if @endpoint
|
82
86
|
|
83
87
|
if @aws_use_sts
|
84
88
|
Aws.config[:region] = options[:region]
|
@@ -93,6 +97,13 @@ module Fluent::Plugin
|
|
93
97
|
@logs ||= Aws::CloudWatchLogs::Client.new(options)
|
94
98
|
@sequence_tokens = {}
|
95
99
|
@store_next_sequence_token_mutex = Mutex.new
|
100
|
+
|
101
|
+
@json_handler = case @json_handler
|
102
|
+
when :yajl
|
103
|
+
Yajl
|
104
|
+
when :json
|
105
|
+
JSON
|
106
|
+
end
|
96
107
|
end
|
97
108
|
|
98
109
|
def format(tag, time, record)
|
@@ -203,7 +214,7 @@ module Fluent::Plugin
|
|
203
214
|
if @message_keys
|
204
215
|
message = @message_keys.split(',').map {|k| record[k].to_s }.join(' ')
|
205
216
|
else
|
206
|
-
message = record
|
217
|
+
message = @json_handler.dump(record)
|
207
218
|
end
|
208
219
|
|
209
220
|
if @max_message_length
|
@@ -37,6 +37,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
37
37
|
assert_equal('stream', d.instance.log_stream_name)
|
38
38
|
assert_equal(true, d.instance.use_log_stream_name_prefix)
|
39
39
|
assert_equal('/tmp/state', d.instance.state_file)
|
40
|
+
assert_equal(:json, d.instance.json_handler)
|
40
41
|
end
|
41
42
|
|
42
43
|
def test_emit
|
@@ -80,6 +81,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
80
81
|
#{aws_key_id}
|
81
82
|
#{aws_sec_key}
|
82
83
|
#{region}
|
84
|
+
#{endpoint}
|
83
85
|
EOC
|
84
86
|
|
85
87
|
d.run(expect_emits: 2, timeout: 5)
|
@@ -100,15 +102,15 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
100
102
|
|
101
103
|
time_ms = (Time.now.to_f * 1000).floor
|
102
104
|
put_log_events([
|
103
|
-
{timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
|
104
|
-
{timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
|
105
|
+
{timestamp: time_ms + 10, message: '{"cloudwatch":"logs1"}'},
|
106
|
+
{timestamp: time_ms + 20, message: '{"cloudwatch":"logs2"}'},
|
105
107
|
])
|
106
108
|
|
107
109
|
new_log_stream("testprefix")
|
108
110
|
create_log_stream
|
109
111
|
put_log_events([
|
110
|
-
{timestamp: time_ms, message: '{"cloudwatch":"logs3"}'},
|
111
|
-
{timestamp: time_ms, message: '{"cloudwatch":"logs4"}'},
|
112
|
+
{timestamp: time_ms + 30, message: '{"cloudwatch":"logs3"}'},
|
113
|
+
{timestamp: time_ms + 40, message: '{"cloudwatch":"logs4"}'},
|
112
114
|
])
|
113
115
|
|
114
116
|
sleep 5
|
@@ -123,15 +125,16 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
123
125
|
#{aws_key_id}
|
124
126
|
#{aws_sec_key}
|
125
127
|
#{region}
|
128
|
+
#{endpoint}
|
126
129
|
EOC
|
127
130
|
d.run(expect_emits: 4, timeout: 5)
|
128
131
|
|
129
132
|
emits = d.events
|
130
133
|
assert_equal(4, emits.size)
|
131
|
-
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
|
132
|
-
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
|
133
|
-
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs3'}], emits[2])
|
134
|
-
assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs4'}], emits[3])
|
134
|
+
assert_equal(['test', ((time_ms + 10) / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
|
135
|
+
assert_equal(['test', ((time_ms + 20) / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
|
136
|
+
assert_equal(['test', ((time_ms + 30) / 1000).floor, {'cloudwatch' => 'logs3'}], emits[2])
|
137
|
+
assert_equal(['test', ((time_ms + 40) / 1000).floor, {'cloudwatch' => 'logs4'}], emits[3])
|
135
138
|
end
|
136
139
|
|
137
140
|
private
|
@@ -146,6 +149,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
146
149
|
#{aws_key_id}
|
147
150
|
#{aws_sec_key}
|
148
151
|
#{region}
|
152
|
+
#{endpoint}
|
149
153
|
EOC
|
150
154
|
end
|
151
155
|
|
@@ -40,6 +40,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
40
40
|
assert_equal("tagvalue", d.instance.log_group_aws_tags.fetch("tagkey"))
|
41
41
|
assert_equal("tagvalue_2", d.instance.log_group_aws_tags.fetch("tagkey_2"))
|
42
42
|
assert_equal(5, d.instance.retention_in_days)
|
43
|
+
assert_equal(:json, d.instance.json_handler)
|
43
44
|
end
|
44
45
|
|
45
46
|
def test_write
|
@@ -356,10 +357,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
356
357
|
]
|
357
358
|
|
358
359
|
time = Time.now
|
359
|
-
|
360
|
-
|
360
|
+
d.run(default_tag: fluentd_tag) do
|
361
|
+
records.each_with_index do |record, i|
|
362
|
+
d.feed(time.to_i + i, record)
|
363
|
+
end
|
361
364
|
end
|
362
|
-
d.run
|
363
365
|
|
364
366
|
awstags = get_log_group_tags
|
365
367
|
assert_equal("value1", awstags.fetch("tag1"))
|
@@ -384,10 +386,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
384
386
|
]
|
385
387
|
|
386
388
|
time = Time.now
|
387
|
-
|
388
|
-
|
389
|
+
d.run(default_tag: fluentd_tag) do
|
390
|
+
records.each_with_index do |record, i|
|
391
|
+
d.feed(time.to_i + i, record)
|
392
|
+
end
|
389
393
|
end
|
390
|
-
d.run
|
391
394
|
|
392
395
|
retention = get_log_group_retention_days
|
393
396
|
assert_equal(d.instance.retention_in_days, retention)
|
@@ -411,10 +414,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
411
414
|
]
|
412
415
|
|
413
416
|
time = Time.now
|
414
|
-
|
415
|
-
|
417
|
+
d.run(default_tag: fluentd_tag) do
|
418
|
+
records.each_with_index do |record, i|
|
419
|
+
d.feed(time.to_i + i, record)
|
420
|
+
end
|
416
421
|
end
|
417
|
-
d.run
|
418
422
|
|
419
423
|
assert_match(/failed to set retention policy for Log group/, d.instance.log.logs[0])
|
420
424
|
end
|
@@ -437,10 +441,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
437
441
|
]
|
438
442
|
|
439
443
|
time = Time.now
|
440
|
-
|
441
|
-
|
444
|
+
d.run(default_tag: fluentd_tag) do
|
445
|
+
records.each_with_index do |record, i|
|
446
|
+
d.feed(time.to_i + i, record)
|
447
|
+
end
|
442
448
|
end
|
443
|
-
d.run
|
444
449
|
|
445
450
|
awstags = get_log_group_tags
|
446
451
|
assert_equal("value1", awstags.fetch("tag1"))
|
@@ -464,10 +469,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
464
469
|
]
|
465
470
|
|
466
471
|
time = Time.now
|
467
|
-
|
468
|
-
|
472
|
+
d.run(default_tag: fluentd_tag) do
|
473
|
+
records.each_with_index do |record, i|
|
474
|
+
d.feed(time.to_i + i, record)
|
475
|
+
end
|
469
476
|
end
|
470
|
-
d.run
|
471
477
|
|
472
478
|
awstags = get_log_group_tags
|
473
479
|
assert_equal("value1", awstags.fetch("tag1"))
|
@@ -500,10 +506,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
|
|
500
506
|
]
|
501
507
|
|
502
508
|
time = Time.now
|
503
|
-
|
504
|
-
|
509
|
+
d.run(default_tag: fluentd_tag) do
|
510
|
+
records.each_with_index do |record, i|
|
511
|
+
d.feed(time.to_i + i, record)
|
512
|
+
end
|
505
513
|
end
|
506
|
-
d.run
|
507
514
|
|
508
515
|
sleep 10
|
509
516
|
|
@@ -607,6 +614,7 @@ auto_create_stream true
|
|
607
614
|
#{aws_key_id}
|
608
615
|
#{aws_sec_key}
|
609
616
|
#{region}
|
617
|
+
#{endpoint}
|
610
618
|
EOC
|
611
619
|
end
|
612
620
|
|
data/test/test_helper.rb
CHANGED
@@ -11,6 +11,7 @@ module CloudwatchLogsTestHelper
|
|
11
11
|
options = {}
|
12
12
|
options[:credentials] = Aws::Credentials.new(ENV['aws_key_id'], ENV['aws_sec_key']) if ENV['aws_key_id'] && ENV['aws_sec_key']
|
13
13
|
options[:region] = ENV['region'] if ENV['region']
|
14
|
+
options[:endpoint] = ENV['endpoint'] if ENV['endpoint']
|
14
15
|
options[:http_proxy] = ENV['http_proxy'] if ENV['http_proxy']
|
15
16
|
@logs ||= Aws::CloudWatchLogs::Client.new(options)
|
16
17
|
end
|
@@ -31,6 +32,10 @@ module CloudwatchLogsTestHelper
|
|
31
32
|
"region #{ENV['region']}" if ENV['region']
|
32
33
|
end
|
33
34
|
|
35
|
+
def endpoint
|
36
|
+
"endpoint #{ENV['endpoint']}" if ENV['endpoint']
|
37
|
+
end
|
38
|
+
|
34
39
|
def log_stream_name(log_stream_name_prefix = nil)
|
35
40
|
if !@log_stream_name
|
36
41
|
new_log_stream(log_stream_name_prefix)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.0
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-04-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -130,12 +130,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
130
130
|
version: '0'
|
131
131
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
132
132
|
requirements:
|
133
|
-
- - "
|
133
|
+
- - ">="
|
134
134
|
- !ruby/object:Gem::Version
|
135
|
-
version:
|
135
|
+
version: '0'
|
136
136
|
requirements: []
|
137
137
|
rubyforge_project:
|
138
|
-
rubygems_version: 2.7.
|
138
|
+
rubygems_version: 2.7.6
|
139
139
|
signing_key:
|
140
140
|
specification_version: 4
|
141
141
|
summary: CloudWatch Logs Plugin for Fluentd
|