fluent-plugin-cloudwatch-logs 0.9.3 → 0.9.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/lib/fluent/plugin/cloudwatch/logs/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_logs.rb +10 -0
- data/test/plugin/test_in_cloudwatch_logs.rb +32 -0
- data/test/test_helper.rb +1 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0b2918da30191dd56e19d14043de953c649cb6207226df51a4721b8dda6af464
|
4
|
+
data.tar.gz: a6a6def2e408711e723f6cec2f5fb706981f23479dd8cb9a3a1a674d859c1cbf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3aa61613b097ff349ba56fd9a5b7e0ee4f87ed3fa23bfa816c35decc193d1161a1771b6985c745f751cc0d05e8e0be606326e145ec19a509f1d2ebd2698f7d71
|
7
|
+
data.tar.gz: ffca6e46ee0c87dc22cd60c4cd724bdf22421daaff0338f245436edce33a8f0397f3f08daa4dd0f929cfabb6d73a2718feff15d304b5e79ca6762c0808aebd93
|
data/README.md
CHANGED
@@ -193,6 +193,7 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
|
|
193
193
|
* `log_group_name`: name of log group to fetch logs
|
194
194
|
* `log_stream_name`: name of log stream to fetch logs
|
195
195
|
* `region`: AWS Region. See [Authentication](#authentication) for more information.
|
196
|
+
* `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
|
196
197
|
* `state_file`: file to store current state (e.g. next\_forward\_token)
|
197
198
|
* `tag`: fluentd tag
|
198
199
|
* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
|
@@ -30,6 +30,7 @@ module Fluent::Plugin
|
|
30
30
|
config_param :start_time, :string, default: nil
|
31
31
|
config_param :end_time, :string, default: nil
|
32
32
|
config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
|
33
|
+
config_param :throttling_retry_seconds, :time, default: nil
|
33
34
|
|
34
35
|
config_section :parse do
|
35
36
|
config_set_default :@type, 'none'
|
@@ -183,6 +184,15 @@ module Fluent::Plugin
|
|
183
184
|
end
|
184
185
|
|
185
186
|
response.events
|
187
|
+
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
|
188
|
+
if throttling_retry_seconds
|
189
|
+
log.warn "ThrottlingException in get_log_events (#{log_stream_name}). Waiting #{throttling_retry_seconds} seconds to retry."
|
190
|
+
sleep throttling_retry_seconds
|
191
|
+
|
192
|
+
get_events(log_stream_name)
|
193
|
+
else
|
194
|
+
raise err
|
195
|
+
end
|
186
196
|
end
|
187
197
|
|
188
198
|
def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
|
@@ -28,6 +28,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
28
28
|
start_time "2019-06-18 00:00:00Z"
|
29
29
|
end_time "2020-01-18 00:00:00Z"
|
30
30
|
time_range_format "%Y-%m-%d %H:%M:%S%z"
|
31
|
+
throttling_retry_seconds 30
|
31
32
|
EOC
|
32
33
|
|
33
34
|
assert_equal('test_id', d.instance.aws_key_id)
|
@@ -43,6 +44,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
43
44
|
assert_equal(1560816000000, d.instance.start_time)
|
44
45
|
assert_equal(1579305600000, d.instance.end_time)
|
45
46
|
assert_equal("%Y-%m-%d %H:%M:%S%z", d.instance.time_range_format)
|
47
|
+
assert_equal(30, d.instance.throttling_retry_seconds)
|
46
48
|
end
|
47
49
|
|
48
50
|
test 'invalid time range' do
|
@@ -608,6 +610,36 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
|
|
608
610
|
assert_equal(["test", ((time_ms + 7000) / 1000), { "cloudwatch" => "logs7" }], events[6])
|
609
611
|
assert_equal(["test", ((time_ms + 8000) / 1000), { "cloudwatch" => "logs8" }], events[7])
|
610
612
|
end
|
613
|
+
|
614
|
+
test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException" do
|
615
|
+
config = <<-CONFIG
|
616
|
+
tag test
|
617
|
+
@type cloudwatch_logs
|
618
|
+
log_group_name #{log_group_name}
|
619
|
+
state_file /tmp/state
|
620
|
+
fetch_interval 0.1
|
621
|
+
throttling_retry_seconds 0.2
|
622
|
+
CONFIG
|
623
|
+
|
624
|
+
# it will raises the error 2 times
|
625
|
+
counter = 0
|
626
|
+
times = 2
|
627
|
+
stub(@client).get_log_events(anything) {
|
628
|
+
counter += 1
|
629
|
+
counter <= times ? raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")) : OpenStruct.new(events: [], next_forward_token: nil)
|
630
|
+
}
|
631
|
+
|
632
|
+
d = create_driver(config)
|
633
|
+
|
634
|
+
# so, it is expected to valid_next_token once
|
635
|
+
mock(d.instance).valid_next_token(nil, nil).once
|
636
|
+
|
637
|
+
log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
|
638
|
+
@client.stub_responses(:describe_log_streams, { log_streams: [log_stream], next_token: nil })
|
639
|
+
|
640
|
+
d.run
|
641
|
+
assert_equal(2, d.logs.select {|l| l =~ /Waiting 0.2 seconds to retry/ }.size)
|
642
|
+
end
|
611
643
|
end
|
612
644
|
|
613
645
|
private
|
data/test/test_helper.rb
CHANGED
@@ -42,10 +42,7 @@ module CloudwatchLogsTestHelper
|
|
42
42
|
end
|
43
43
|
|
44
44
|
def log_stream_name(log_stream_name_prefix = nil)
|
45
|
-
|
46
|
-
new_log_stream(log_stream_name_prefix)
|
47
|
-
end
|
48
|
-
@log_stream_name
|
45
|
+
@log_stream_name ||= new_log_stream(log_stream_name_prefix)
|
49
46
|
end
|
50
47
|
|
51
48
|
def new_log_stream(log_stream_name_prefix = nil)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryota Arai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-04-
|
11
|
+
date: 2020-04-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|