fluent-plugin-cloudwatch-ingest 1.4.0 → 1.5.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +7 -1
- data/CHANGELOG.md +6 -0
- data/circle.yml +1 -1
- data/fluent-plugin-cloudwatch-ingest.gemspec +1 -3
- data/lib/fluent/plugin/cloudwatch/ingest/version.rb +1 -1
- data/lib/fluent/plugin/in_cloudwatch_ingest.rb +132 -83
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e03b673f371cfe94c2f16197a40f1ff0e67d8a82
|
4
|
+
data.tar.gz: 826b1d14832d530aaa0a34532383fdeb203aff7d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d6ad1bebc3432e40916b94a7d125e75af42a3942b44f16191956293682787602076502acbaa914b5fc319406a95bb393e35b490321e21f3d097fca410341e479
|
7
|
+
data.tar.gz: 306b932009d1c270b70128451e2f2cc3b5d2e0b193ce6390ada56abbe4773111715a319ce3934e08228547fc03b830710ea9f5a85f6907b807847f972614b0b2
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -57,3 +57,9 @@ Both of these changes are designed to make debugging ingestion problems from hig
|
|
57
57
|
* Refuse to emit records with a blank (or newline only) message
|
58
58
|
* Emit metric `events.emitted.blocked` to expose these alongside logging
|
59
59
|
* Add plugin skew time to telemetry optionally emitted from the parser
|
60
|
+
|
61
|
+
## 1.5.0
|
62
|
+
|
63
|
+
* Limit the number of streams to be processed with each log stream describe call
|
64
|
+
* new parameter `max_log_streams_per_group` with a default of 50 (the default value for *limit* on API calls). This can be increased or decreased to limit the throttling of calls to AWS API
|
65
|
+
* Bail out of processing if fluentd has been stopped
|
data/circle.yml
CHANGED
@@ -5,7 +5,7 @@ test:
|
|
5
5
|
- cp pkg/*.gem ${CIRCLE_ARTIFACTS}
|
6
6
|
deployment:
|
7
7
|
release:
|
8
|
-
tag: /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)
|
8
|
+
tag: /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?(\.rc[0-9]+)*$/
|
9
9
|
owner: sampointer
|
10
10
|
commands:
|
11
11
|
- bin/deploy
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
lib = File.expand_path('../lib', __FILE__)
|
4
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
3
|
require 'fluent/plugin/cloudwatch/ingest/version'
|
@@ -32,7 +30,7 @@ Gem::Specification.new do |spec|
|
|
32
30
|
spec.add_development_dependency 'rspec'
|
33
31
|
spec.add_development_dependency 'rubocop'
|
34
32
|
|
35
|
-
spec.add_dependency 'fluentd', '~>0.14.
|
33
|
+
spec.add_dependency 'fluentd', '~>0.14.20'
|
36
34
|
spec.add_dependency 'aws-sdk', '~>2.10.9'
|
37
35
|
spec.add_dependency 'multi_json', '~>1.12'
|
38
36
|
spec.add_dependency 'statsd-ruby', '~>1.4.0'
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# rubocop:disable Metrics/BlockLength
|
1
2
|
require 'aws-sdk'
|
2
3
|
require 'fluent/config/error'
|
3
4
|
require 'fluent/plugin/input'
|
@@ -27,7 +28,8 @@ module Fluent::Plugin
|
|
27
28
|
desc 'Log group regexp to exclude, despite matching'
|
28
29
|
config_param :log_group_exclude_regexp, :string, default: ''
|
29
30
|
desc 'State file name'
|
30
|
-
config_param :state_file_name, :string, default:
|
31
|
+
config_param :state_file_name, :string, default: \
|
32
|
+
'/var/spool/td-agent/cloudwatch.state'
|
31
33
|
desc 'Fetch logs every interval'
|
32
34
|
config_param :interval, :time, default: 60
|
33
35
|
desc 'Time to pause between error conditions'
|
@@ -39,6 +41,8 @@ module Fluent::Plugin
|
|
39
41
|
config_param :aws_logging_enabled, :bool, default: false
|
40
42
|
desc 'Limit the number of events fetched in any iteration'
|
41
43
|
config_param :limit_events, :integer, default: 10_000
|
44
|
+
desc 'Limit the number of log streams to be processed with each iteration'
|
45
|
+
config_param :max_log_streams_per_group, :integer, default: 50
|
42
46
|
desc 'Do not fetch events before this time'
|
43
47
|
config_param :event_start_time, :integer, default: 0
|
44
48
|
desc 'Fetch the oldest logs first'
|
@@ -102,6 +106,8 @@ module Fluent::Plugin
|
|
102
106
|
|
103
107
|
@parser = parser_create(conf: parser_config)
|
104
108
|
log.info('Configured fluentd-plugin-cloudwatch-ingest')
|
109
|
+
|
110
|
+
@log_streams_next_token = nil
|
105
111
|
end
|
106
112
|
|
107
113
|
def start
|
@@ -120,7 +126,8 @@ module Fluent::Plugin
|
|
120
126
|
role_session_name: @sts_session_name
|
121
127
|
)
|
122
128
|
|
123
|
-
log.info(
|
129
|
+
log.info('Using STS for authentication with source account ARN: '\
|
130
|
+
"#{@sts_arn}, session name: #{@sts_session_name}")
|
124
131
|
else
|
125
132
|
log.info('Using local instance IAM role for authentication')
|
126
133
|
end
|
@@ -139,7 +146,11 @@ module Fluent::Plugin
|
|
139
146
|
def emit(event, log_group_name, log_stream_name)
|
140
147
|
@parser.parse(event, log_group_name, log_stream_name) do |time, record|
|
141
148
|
if record['message'].chomp.empty? && @drop_blank_events
|
142
|
-
log.warn(
|
149
|
+
log.warn(
|
150
|
+
'Event is blank or contains only a newline, refusing to '\
|
151
|
+
"emit. group: #{log_group_name}, "\
|
152
|
+
"stream: #{log_stream_name}, event: #{event}"\
|
153
|
+
)
|
143
154
|
metric(:increment, 'events.emitted.blocked')
|
144
155
|
next
|
145
156
|
end
|
@@ -171,7 +182,10 @@ module Fluent::Plugin
|
|
171
182
|
response.log_groups.each do |group|
|
172
183
|
if !@log_group_exclude_regexp.empty?
|
173
184
|
if regex.match(group.log_group_name)
|
174
|
-
log.info(
|
185
|
+
log.info(
|
186
|
+
"Excluding log_group #{group.log_group_name} due to "\
|
187
|
+
"log_group_exclude_regexp #{@log_group_exclude_regexp}"\
|
188
|
+
)
|
175
189
|
metric(:increment, 'api.calls.describeloggroups.excluded')
|
176
190
|
else
|
177
191
|
log_groups << group.log_group_name
|
@@ -182,7 +196,7 @@ module Fluent::Plugin
|
|
182
196
|
end
|
183
197
|
break unless response.next_token
|
184
198
|
next_token = response.next_token
|
185
|
-
rescue => boom
|
199
|
+
rescue StandardError => boom
|
186
200
|
log.error("Unable to retrieve log groups: #{boom.inspect}")
|
187
201
|
metric(:increment, 'api.calls.describeloggroups.failed')
|
188
202
|
next_token = nil
|
@@ -197,35 +211,44 @@ module Fluent::Plugin
|
|
197
211
|
|
198
212
|
def log_streams(log_group_name, log_stream_name_prefix)
|
199
213
|
log_streams = []
|
200
|
-
next_token = nil
|
201
|
-
loop do
|
202
|
-
begin
|
203
|
-
metric(:increment, 'api.calls.describelogstreams.attempted')
|
204
|
-
response = if !log_stream_name_prefix.empty?
|
205
|
-
@aws.describe_log_streams(
|
206
|
-
log_group_name: log_group_name,
|
207
|
-
log_stream_name_prefix: log_stream_name_prefix,
|
208
|
-
next_token: next_token
|
209
|
-
)
|
210
|
-
else
|
211
|
-
@aws.describe_log_streams(
|
212
|
-
log_group_name: log_group_name,
|
213
|
-
next_token: next_token
|
214
|
-
)
|
215
|
-
end
|
216
214
|
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
215
|
+
begin
|
216
|
+
metric(:increment, 'api.calls.describelogstreams.attempted')
|
217
|
+
response = if !log_stream_name_prefix.empty?
|
218
|
+
@aws.describe_log_streams(
|
219
|
+
log_group_name: log_group_name,
|
220
|
+
log_stream_name_prefix: log_stream_name_prefix,
|
221
|
+
next_token: @log_streams_next_token,
|
222
|
+
limit: @max_log_streams_per_group
|
223
|
+
)
|
224
|
+
else
|
225
|
+
@aws.describe_log_streams(
|
226
|
+
log_group_name: log_group_name,
|
227
|
+
next_token: @log_streams_next_token,
|
228
|
+
limit: @max_log_streams_per_group
|
229
|
+
)
|
230
|
+
end
|
231
|
+
|
232
|
+
response.log_streams.each { |s| log_streams << s.log_stream_name }
|
233
|
+
if log_streams.size == @max_log_streams_per_group
|
234
|
+
@log_streams_next_token = response.next_token
|
227
235
|
end
|
236
|
+
rescue StandardError => boom
|
237
|
+
prefix_message = if log_stream_name_prefix.empty?
|
238
|
+
''
|
239
|
+
else
|
240
|
+
"with stream prefix #{log_stream_name_prefix}"
|
241
|
+
end
|
242
|
+
|
243
|
+
log.error('Unable to retrieve log streams '\
|
244
|
+
"for group #{log_group_name} #{prefix_message}: "\
|
245
|
+
"#{boom.inspect}")
|
246
|
+
metric(:increment, 'api.calls.describelogstreams.failed')
|
247
|
+
|
248
|
+
sleep @error_interval
|
249
|
+
retry
|
228
250
|
end
|
251
|
+
|
229
252
|
log.info("Found #{log_streams.size} streams for #{log_group_name}")
|
230
253
|
|
231
254
|
return log_streams
|
@@ -248,11 +271,13 @@ module Fluent::Plugin
|
|
248
271
|
begin
|
249
272
|
emit(e, group, stream)
|
250
273
|
event_count += 1
|
251
|
-
rescue => boom
|
274
|
+
rescue StandardError => boom
|
252
275
|
log.error("Failed to emit event #{e}: #{boom.inspect}")
|
253
276
|
end
|
254
277
|
end
|
255
278
|
|
279
|
+
log.info("#{event_count} events processed for stream #{stream}")
|
280
|
+
|
256
281
|
has_stream_timestamp = true if state.store[group][stream]['timestamp']
|
257
282
|
|
258
283
|
if !has_stream_timestamp && response.events.count.zero?
|
@@ -278,7 +303,7 @@ module Fluent::Plugin
|
|
278
303
|
until @finished
|
279
304
|
begin
|
280
305
|
state = State.new(@state_file_name, log)
|
281
|
-
rescue => boom
|
306
|
+
rescue StandardError => boom
|
282
307
|
log.info("Failed lock state. Sleeping for #{@error_interval}: "\
|
283
308
|
"#{boom.inspect}")
|
284
309
|
sleep @error_interval
|
@@ -286,48 +311,69 @@ module Fluent::Plugin
|
|
286
311
|
end
|
287
312
|
|
288
313
|
event_count = 0
|
289
|
-
|
290
|
-
# Fetch the streams for each log group
|
314
|
+
# For each log group
|
291
315
|
log_groups(@log_group_name_prefix).each do |group|
|
292
|
-
|
293
|
-
log_streams(group, @log_stream_name_prefix).each do |stream|
|
294
|
-
state.store[group][stream] = {} unless state.store[group][stream]
|
295
|
-
|
296
|
-
log.info("processing stream: #{stream}")
|
297
|
-
|
298
|
-
# See if we have some stored state for this group and stream.
|
299
|
-
# If we have then use the stored forward_token to pick up
|
300
|
-
# from that point. Otherwise start from the start.
|
301
|
-
|
316
|
+
loop do
|
302
317
|
begin
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
318
|
+
# Fetch log group streams and emit the events
|
319
|
+
log_streams(group, @log_stream_name_prefix).each do |stream|
|
320
|
+
unless state.store[group][stream]
|
321
|
+
state.store[group][stream] = {}
|
322
|
+
end
|
323
|
+
|
324
|
+
log.info("processing stream: #{stream}")
|
325
|
+
|
326
|
+
# See if we have some stored state for this group and stream.
|
327
|
+
# If we have then use the stored forward_token to pick up
|
328
|
+
# from that point. Otherwise start from the start.
|
329
|
+
|
330
|
+
begin
|
331
|
+
event_count += process_stream(
|
332
|
+
group,
|
333
|
+
stream,
|
334
|
+
state.store[group][stream]['token'],
|
335
|
+
@event_start_time,
|
336
|
+
state
|
337
|
+
)
|
338
|
+
rescue Aws::CloudWatchLogs::Errors::InvalidParameterException
|
339
|
+
metric(:increment, 'api.calls.getlogevents.invalid_token')
|
340
|
+
log.error(
|
341
|
+
'cloudwatch token is expired or broken. '\
|
342
|
+
'trying with timestamp.'
|
343
|
+
)
|
344
|
+
|
345
|
+
# try again with timestamp instead of forward token
|
346
|
+
begin
|
347
|
+
timestamp = state.store[group][stream]['timestamp']
|
348
|
+
timestamp ||= @event_start_time
|
349
|
+
|
350
|
+
event_count += process_stream(group,
|
351
|
+
stream,
|
352
|
+
nil,
|
353
|
+
timestamp,
|
354
|
+
state)
|
355
|
+
rescue StandardError => boom
|
356
|
+
log.error(
|
357
|
+
'Unable to retrieve events for stream '\
|
358
|
+
"#{stream} in group #{group}: "\
|
359
|
+
"#{boom.inspect}"\
|
360
|
+
)
|
361
|
+
metric(:increment, 'api.calls.getlogevents.failed')
|
362
|
+
sleep @error_interval
|
363
|
+
next
|
364
|
+
end
|
365
|
+
rescue StandardError => boom
|
366
|
+
log.error("Unable to retrieve events for stream #{stream} "\
|
367
|
+
"in group #{group}: #{boom.inspect}")
|
368
|
+
metric(:increment, 'api.calls.getlogevents.failed')
|
369
|
+
sleep @error_interval
|
370
|
+
next
|
371
|
+
end
|
324
372
|
end
|
325
|
-
|
326
|
-
|
327
|
-
metric(:increment, 'api.calls.getlogevents.failed')
|
328
|
-
sleep @error_interval
|
329
|
-
next
|
373
|
+
# process log streams in batches
|
374
|
+
break if @log_streams_next_token.nil? || @finished
|
330
375
|
end
|
376
|
+
log.info("#{event_count} events processed for group #{group}")
|
331
377
|
end
|
332
378
|
end
|
333
379
|
|
@@ -335,19 +381,22 @@ module Fluent::Plugin
|
|
335
381
|
begin
|
336
382
|
state.save
|
337
383
|
state.close
|
338
|
-
rescue => boom
|
384
|
+
rescue StandardError => boom
|
339
385
|
log.error("Unable to save state file: #{boom.inspect}")
|
340
386
|
end
|
341
387
|
|
342
|
-
if
|
343
|
-
sleep_interval =
|
388
|
+
if !@finished # no need to sleep if it's finished
|
389
|
+
sleep_interval = if event_count > 0
|
390
|
+
@interval
|
391
|
+
else
|
392
|
+
@error_interval # slow down when no events
|
393
|
+
end
|
394
|
+
|
395
|
+
log.info("Pausing for #{sleep_interval}")
|
396
|
+
sleep sleep_interval
|
344
397
|
else
|
345
|
-
|
398
|
+
log.info('Stopping fluentd-plugin-cloudwatch-ingest')
|
346
399
|
end
|
347
|
-
|
348
|
-
log.info("#{event_count} events processed.")
|
349
|
-
log.info("Pausing for #{sleep_interval}")
|
350
|
-
sleep sleep_interval
|
351
400
|
end
|
352
401
|
end
|
353
402
|
|
@@ -368,7 +417,7 @@ module Fluent::Plugin
|
|
368
417
|
begin
|
369
418
|
self.statefile = Pathname.new(@filepath).open('w+')
|
370
419
|
save
|
371
|
-
rescue => boom
|
420
|
+
rescue StandardError => boom
|
372
421
|
@log.error("Unable to create new file #{statefile.path}: "\
|
373
422
|
"#{boom.inspect}")
|
374
423
|
end
|
@@ -378,13 +427,13 @@ module Fluent::Plugin
|
|
378
427
|
# exception if we can't
|
379
428
|
@log.info("Obtaining exclusive lock on state file #{statefile.path}")
|
380
429
|
lockstatus = statefile.flock(File::LOCK_EX | File::LOCK_NB)
|
381
|
-
raise CloudwatchIngestInput::State::LockFailed
|
430
|
+
raise CloudwatchIngestInput::State::LockFailed unless lockstatus
|
382
431
|
|
383
432
|
begin
|
384
433
|
@store.merge!(Psych.safe_load(statefile.read))
|
385
434
|
|
386
435
|
# Migrate old state file
|
387
|
-
@store.
|
436
|
+
@store.each_value do |streams|
|
388
437
|
streams.update(streams) do |_name, stream|
|
389
438
|
if stream.is_a? String
|
390
439
|
return { 'token' => stream, 'timestamp' => Time.now.to_i }
|
@@ -394,7 +443,7 @@ module Fluent::Plugin
|
|
394
443
|
end
|
395
444
|
|
396
445
|
@log.info("Loaded #{@store.keys.size} groups from #{statefile.path}")
|
397
|
-
rescue
|
446
|
+
rescue StandardError
|
398
447
|
statefile.close
|
399
448
|
raise
|
400
449
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-ingest
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sam Pointer
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-09-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -72,14 +72,14 @@ dependencies:
|
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: 0.14.
|
75
|
+
version: 0.14.20
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: 0.14.
|
82
|
+
version: 0.14.20
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: aws-sdk
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -163,9 +163,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
163
163
|
version: '0'
|
164
164
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
165
165
|
requirements:
|
166
|
-
- - "
|
166
|
+
- - ">"
|
167
167
|
- !ruby/object:Gem::Version
|
168
|
-
version:
|
168
|
+
version: 1.3.1
|
169
169
|
requirements: []
|
170
170
|
rubyforge_project:
|
171
171
|
rubygems_version: 2.6.12
|