fluent-plugin-cloudwatch-ingest 1.4.0 → 1.5.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 852a0773b4fd72b1ca24faebd4624b5732d3d916
4
- data.tar.gz: 9815dd011589740f0af019e450ff1def0682f11a
3
+ metadata.gz: e03b673f371cfe94c2f16197a40f1ff0e67d8a82
4
+ data.tar.gz: 826b1d14832d530aaa0a34532383fdeb203aff7d
5
5
  SHA512:
6
- metadata.gz: 10ab86bf0fde33eb65869c2db408a3a9bc1658807d4d3e6e8e5b7bd35e9f68f81c4e5a2270dc636e7dbadb3bd306dd9f39c683eba7eb05d9b2b8149df3415845
7
- data.tar.gz: 2998fe65f16a65b8582fb6b2ec0ca4b0e3ad20262e229d8f0140cd5bfb7d68bbd93b4c72ad00712ffc8f075db7cae32bc968e82dac2d910fee9f72375015574d
6
+ metadata.gz: d6ad1bebc3432e40916b94a7d125e75af42a3942b44f16191956293682787602076502acbaa914b5fc319406a95bb393e35b490321e21f3d097fca410341e479
7
+ data.tar.gz: 306b932009d1c270b70128451e2f2cc3b5d2e0b193ce6390ada56abbe4773111715a319ce3934e08228547fc03b830710ea9f5a85f6907b807847f972614b0b2
data/.rubocop.yml CHANGED
@@ -1,5 +1,11 @@
1
+ AllCops:
2
+ DisplayCopNames: true
3
+ DisplayStyleGuide: true
4
+ TargetRubyVersion: 2.1
1
5
  Metrics/MethodLength:
2
- Max: 200
6
+ Enabled: false
7
+ Metrics/BlockLength:
8
+ Enabled: false
3
9
  Style/Documentation:
4
10
  Enabled: false
5
11
  Style/ClassAndModuleChildren:
data/CHANGELOG.md CHANGED
@@ -57,3 +57,9 @@ Both of these changes are designed to make debugging ingestion problems from hig
57
57
  * Refuse to emit records with a blank (or newline only) message
58
58
  * Emit metric `events.emitted.blocked` to expose these alongside logging
59
59
  * Add plugin skew time to telemetry optionally emitted from the parser
60
+
61
+ ## 1.5.0
62
+
63
+ * Limit the number of streams to be processed with each log stream describe call
64
+ * new parameter `max_log_streams_per_group` with a default of 50 (the default value for *limit* on API calls). This can be increased or decreased to limit the throttling of calls to AWS API
65
+ * Bail out of processing if fluentd has been stopped
data/circle.yml CHANGED
@@ -5,7 +5,7 @@ test:
5
5
  - cp pkg/*.gem ${CIRCLE_ARTIFACTS}
6
6
  deployment:
7
7
  release:
8
- tag: /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$/
8
+ tag: /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?(\.rc[0-9]+)*$/
9
9
  owner: sampointer
10
10
  commands:
11
11
  - bin/deploy
@@ -1,5 +1,3 @@
1
- # coding: utf-8
2
-
3
1
  lib = File.expand_path('../lib', __FILE__)
4
2
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
3
  require 'fluent/plugin/cloudwatch/ingest/version'
@@ -32,7 +30,7 @@ Gem::Specification.new do |spec|
32
30
  spec.add_development_dependency 'rspec'
33
31
  spec.add_development_dependency 'rubocop'
34
32
 
35
- spec.add_dependency 'fluentd', '~>0.14.13'
33
+ spec.add_dependency 'fluentd', '~>0.14.20'
36
34
  spec.add_dependency 'aws-sdk', '~>2.10.9'
37
35
  spec.add_dependency 'multi_json', '~>1.12'
38
36
  spec.add_dependency 'statsd-ruby', '~>1.4.0'
@@ -2,7 +2,7 @@ module Fluent
2
2
  module Plugin
3
3
  module Cloudwatch
4
4
  module Ingest
5
- VERSION = '1.4.0'.freeze
5
+ VERSION = '1.5.0.rc1'.freeze
6
6
  end
7
7
  end
8
8
  end
@@ -1,3 +1,4 @@
1
+ # rubocop:disable Metrics/BlockLength
1
2
  require 'aws-sdk'
2
3
  require 'fluent/config/error'
3
4
  require 'fluent/plugin/input'
@@ -27,7 +28,8 @@ module Fluent::Plugin
27
28
  desc 'Log group regexp to exclude, despite matching'
28
29
  config_param :log_group_exclude_regexp, :string, default: ''
29
30
  desc 'State file name'
30
- config_param :state_file_name, :string, default: '/var/spool/td-agent/cloudwatch.state' # rubocop:disable LineLength
31
+ config_param :state_file_name, :string, default: \
32
+ '/var/spool/td-agent/cloudwatch.state'
31
33
  desc 'Fetch logs every interval'
32
34
  config_param :interval, :time, default: 60
33
35
  desc 'Time to pause between error conditions'
@@ -39,6 +41,8 @@ module Fluent::Plugin
39
41
  config_param :aws_logging_enabled, :bool, default: false
40
42
  desc 'Limit the number of events fetched in any iteration'
41
43
  config_param :limit_events, :integer, default: 10_000
44
+ desc 'Limit the number of log streams to be processed with each iteration'
45
+ config_param :max_log_streams_per_group, :integer, default: 50
42
46
  desc 'Do not fetch events before this time'
43
47
  config_param :event_start_time, :integer, default: 0
44
48
  desc 'Fetch the oldest logs first'
@@ -102,6 +106,8 @@ module Fluent::Plugin
102
106
 
103
107
  @parser = parser_create(conf: parser_config)
104
108
  log.info('Configured fluentd-plugin-cloudwatch-ingest')
109
+
110
+ @log_streams_next_token = nil
105
111
  end
106
112
 
107
113
  def start
@@ -120,7 +126,8 @@ module Fluent::Plugin
120
126
  role_session_name: @sts_session_name
121
127
  )
122
128
 
123
- log.info("Using STS for authentication with source account ARN: #{@sts_arn}, session name: #{@sts_session_name}") # rubocop:disable LineLength
129
+ log.info('Using STS for authentication with source account ARN: '\
130
+ "#{@sts_arn}, session name: #{@sts_session_name}")
124
131
  else
125
132
  log.info('Using local instance IAM role for authentication')
126
133
  end
@@ -139,7 +146,11 @@ module Fluent::Plugin
139
146
  def emit(event, log_group_name, log_stream_name)
140
147
  @parser.parse(event, log_group_name, log_stream_name) do |time, record|
141
148
  if record['message'].chomp.empty? && @drop_blank_events
142
- log.warn("Event is blank or contains only a newline, refusing to emit. group: #{log_group_name}, stream: #{log_stream_name}, event: #{event}") # rubocop:disable LineLength
149
+ log.warn(
150
+ 'Event is blank or contains only a newline, refusing to '\
151
+ "emit. group: #{log_group_name}, "\
152
+ "stream: #{log_stream_name}, event: #{event}"\
153
+ )
143
154
  metric(:increment, 'events.emitted.blocked')
144
155
  next
145
156
  end
@@ -171,7 +182,10 @@ module Fluent::Plugin
171
182
  response.log_groups.each do |group|
172
183
  if !@log_group_exclude_regexp.empty?
173
184
  if regex.match(group.log_group_name)
174
- log.info("Excluding log_group #{group.log_group_name} due to log_group_exclude_regexp #{@log_group_exclude_regexp}") # rubocop:disable LineLength
185
+ log.info(
186
+ "Excluding log_group #{group.log_group_name} due to "\
187
+ "log_group_exclude_regexp #{@log_group_exclude_regexp}"\
188
+ )
175
189
  metric(:increment, 'api.calls.describeloggroups.excluded')
176
190
  else
177
191
  log_groups << group.log_group_name
@@ -182,7 +196,7 @@ module Fluent::Plugin
182
196
  end
183
197
  break unless response.next_token
184
198
  next_token = response.next_token
185
- rescue => boom
199
+ rescue StandardError => boom
186
200
  log.error("Unable to retrieve log groups: #{boom.inspect}")
187
201
  metric(:increment, 'api.calls.describeloggroups.failed')
188
202
  next_token = nil
@@ -197,35 +211,44 @@ module Fluent::Plugin
197
211
 
198
212
  def log_streams(log_group_name, log_stream_name_prefix)
199
213
  log_streams = []
200
- next_token = nil
201
- loop do
202
- begin
203
- metric(:increment, 'api.calls.describelogstreams.attempted')
204
- response = if !log_stream_name_prefix.empty?
205
- @aws.describe_log_streams(
206
- log_group_name: log_group_name,
207
- log_stream_name_prefix: log_stream_name_prefix,
208
- next_token: next_token
209
- )
210
- else
211
- @aws.describe_log_streams(
212
- log_group_name: log_group_name,
213
- next_token: next_token
214
- )
215
- end
216
214
 
217
- response.log_streams.each { |s| log_streams << s.log_stream_name }
218
- break unless response.next_token
219
- next_token = response.next_token
220
- rescue => boom
221
- log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom.inspect}") # rubocop:disable LineLength
222
- metric(:increment, 'api.calls.describelogstreams.failed')
223
- log_streams = []
224
- next_token = nil
225
- sleep @error_interval
226
- retry
215
+ begin
216
+ metric(:increment, 'api.calls.describelogstreams.attempted')
217
+ response = if !log_stream_name_prefix.empty?
218
+ @aws.describe_log_streams(
219
+ log_group_name: log_group_name,
220
+ log_stream_name_prefix: log_stream_name_prefix,
221
+ next_token: @log_streams_next_token,
222
+ limit: @max_log_streams_per_group
223
+ )
224
+ else
225
+ @aws.describe_log_streams(
226
+ log_group_name: log_group_name,
227
+ next_token: @log_streams_next_token,
228
+ limit: @max_log_streams_per_group
229
+ )
230
+ end
231
+
232
+ response.log_streams.each { |s| log_streams << s.log_stream_name }
233
+ if log_streams.size == @max_log_streams_per_group
234
+ @log_streams_next_token = response.next_token
227
235
  end
236
+ rescue StandardError => boom
237
+ prefix_message = if log_stream_name_prefix.empty?
238
+ ''
239
+ else
240
+ "with stream prefix #{log_stream_name_prefix}"
241
+ end
242
+
243
+ log.error('Unable to retrieve log streams '\
244
+ "for group #{log_group_name} #{prefix_message}: "\
245
+ "#{boom.inspect}")
246
+ metric(:increment, 'api.calls.describelogstreams.failed')
247
+
248
+ sleep @error_interval
249
+ retry
228
250
  end
251
+
229
252
  log.info("Found #{log_streams.size} streams for #{log_group_name}")
230
253
 
231
254
  return log_streams
@@ -248,11 +271,13 @@ module Fluent::Plugin
248
271
  begin
249
272
  emit(e, group, stream)
250
273
  event_count += 1
251
- rescue => boom
274
+ rescue StandardError => boom
252
275
  log.error("Failed to emit event #{e}: #{boom.inspect}")
253
276
  end
254
277
  end
255
278
 
279
+ log.info("#{event_count} events processed for stream #{stream}")
280
+
256
281
  has_stream_timestamp = true if state.store[group][stream]['timestamp']
257
282
 
258
283
  if !has_stream_timestamp && response.events.count.zero?
@@ -278,7 +303,7 @@ module Fluent::Plugin
278
303
  until @finished
279
304
  begin
280
305
  state = State.new(@state_file_name, log)
281
- rescue => boom
306
+ rescue StandardError => boom
282
307
  log.info("Failed lock state. Sleeping for #{@error_interval}: "\
283
308
  "#{boom.inspect}")
284
309
  sleep @error_interval
@@ -286,48 +311,69 @@ module Fluent::Plugin
286
311
  end
287
312
 
288
313
  event_count = 0
289
-
290
- # Fetch the streams for each log group
314
+ # For each log group
291
315
  log_groups(@log_group_name_prefix).each do |group|
292
- # For each log stream get and emit the events
293
- log_streams(group, @log_stream_name_prefix).each do |stream|
294
- state.store[group][stream] = {} unless state.store[group][stream]
295
-
296
- log.info("processing stream: #{stream}")
297
-
298
- # See if we have some stored state for this group and stream.
299
- # If we have then use the stored forward_token to pick up
300
- # from that point. Otherwise start from the start.
301
-
316
+ loop do
302
317
  begin
303
- event_count += process_stream(group, stream,
304
- state.store[group][stream]['token'],
305
- @event_start_time, state)
306
- rescue Aws::CloudWatchLogs::Errors::InvalidParameterException
307
- metric(:increment, 'api.calls.getlogevents.invalid_token')
308
- log.error('cloudwatch token is expired or broken. '\
309
- 'trying with timestamp.')
310
-
311
- # try again with timestamp instead of forward token
312
- begin
313
- timestamp = state.store[group][stream]['timestamp']
314
- timestamp = @event_start_time unless timestamp
315
-
316
- event_count += process_stream(group, stream,
317
- nil, timestamp, state)
318
- rescue => boom
319
- log.error("Unable to retrieve events for stream #{stream} "\
320
- "in group #{group}: #{boom.inspect}") # rubocop:disable all
321
- metric(:increment, 'api.calls.getlogevents.failed')
322
- sleep @error_interval
323
- next
318
+ # Fetch log group streams and emit the events
319
+ log_streams(group, @log_stream_name_prefix).each do |stream|
320
+ unless state.store[group][stream]
321
+ state.store[group][stream] = {}
322
+ end
323
+
324
+ log.info("processing stream: #{stream}")
325
+
326
+ # See if we have some stored state for this group and stream.
327
+ # If we have then use the stored forward_token to pick up
328
+ # from that point. Otherwise start from the start.
329
+
330
+ begin
331
+ event_count += process_stream(
332
+ group,
333
+ stream,
334
+ state.store[group][stream]['token'],
335
+ @event_start_time,
336
+ state
337
+ )
338
+ rescue Aws::CloudWatchLogs::Errors::InvalidParameterException
339
+ metric(:increment, 'api.calls.getlogevents.invalid_token')
340
+ log.error(
341
+ 'cloudwatch token is expired or broken. '\
342
+ 'trying with timestamp.'
343
+ )
344
+
345
+ # try again with timestamp instead of forward token
346
+ begin
347
+ timestamp = state.store[group][stream]['timestamp']
348
+ timestamp ||= @event_start_time
349
+
350
+ event_count += process_stream(group,
351
+ stream,
352
+ nil,
353
+ timestamp,
354
+ state)
355
+ rescue StandardError => boom
356
+ log.error(
357
+ 'Unable to retrieve events for stream '\
358
+ "#{stream} in group #{group}: "\
359
+ "#{boom.inspect}"\
360
+ )
361
+ metric(:increment, 'api.calls.getlogevents.failed')
362
+ sleep @error_interval
363
+ next
364
+ end
365
+ rescue StandardError => boom
366
+ log.error("Unable to retrieve events for stream #{stream} "\
367
+ "in group #{group}: #{boom.inspect}")
368
+ metric(:increment, 'api.calls.getlogevents.failed')
369
+ sleep @error_interval
370
+ next
371
+ end
324
372
  end
325
- rescue => boom
326
- log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom.inspect}") # rubocop:disable LineLength
327
- metric(:increment, 'api.calls.getlogevents.failed')
328
- sleep @error_interval
329
- next
373
+ # process log streams in batches
374
+ break if @log_streams_next_token.nil? || @finished
330
375
  end
376
+ log.info("#{event_count} events processed for group #{group}")
331
377
  end
332
378
  end
333
379
 
@@ -335,19 +381,22 @@ module Fluent::Plugin
335
381
  begin
336
382
  state.save
337
383
  state.close
338
- rescue => boom
384
+ rescue StandardError => boom
339
385
  log.error("Unable to save state file: #{boom.inspect}")
340
386
  end
341
387
 
342
- if event_count > 0
343
- sleep_interval = @interval
388
+ if !@finished # no need to sleep if it's finished
389
+ sleep_interval = if event_count > 0
390
+ @interval
391
+ else
392
+ @error_interval # slow down when no events
393
+ end
394
+
395
+ log.info("Pausing for #{sleep_interval}")
396
+ sleep sleep_interval
344
397
  else
345
- sleep_interval = @error_interval # when there is no events, slow down
398
+ log.info('Stopping fluentd-plugin-cloudwatch-ingest')
346
399
  end
347
-
348
- log.info("#{event_count} events processed.")
349
- log.info("Pausing for #{sleep_interval}")
350
- sleep sleep_interval
351
400
  end
352
401
  end
353
402
 
@@ -368,7 +417,7 @@ module Fluent::Plugin
368
417
  begin
369
418
  self.statefile = Pathname.new(@filepath).open('w+')
370
419
  save
371
- rescue => boom
420
+ rescue StandardError => boom
372
421
  @log.error("Unable to create new file #{statefile.path}: "\
373
422
  "#{boom.inspect}")
374
423
  end
@@ -378,13 +427,13 @@ module Fluent::Plugin
378
427
  # exception if we can't
379
428
  @log.info("Obtaining exclusive lock on state file #{statefile.path}")
380
429
  lockstatus = statefile.flock(File::LOCK_EX | File::LOCK_NB)
381
- raise CloudwatchIngestInput::State::LockFailed if lockstatus == false
430
+ raise CloudwatchIngestInput::State::LockFailed unless lockstatus
382
431
 
383
432
  begin
384
433
  @store.merge!(Psych.safe_load(statefile.read))
385
434
 
386
435
  # Migrate old state file
387
- @store.each do |_group, streams|
436
+ @store.each_value do |streams|
388
437
  streams.update(streams) do |_name, stream|
389
438
  if stream.is_a? String
390
439
  return { 'token' => stream, 'timestamp' => Time.now.to_i }
@@ -394,7 +443,7 @@ module Fluent::Plugin
394
443
  end
395
444
 
396
445
  @log.info("Loaded #{@store.keys.size} groups from #{statefile.path}")
397
- rescue
446
+ rescue StandardError
398
447
  statefile.close
399
448
  raise
400
449
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch-ingest
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.0
4
+ version: 1.5.0.rc1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sam Pointer
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-08-21 00:00:00.000000000 Z
11
+ date: 2017-09-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -72,14 +72,14 @@ dependencies:
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: 0.14.13
75
+ version: 0.14.20
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: 0.14.13
82
+ version: 0.14.20
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: aws-sdk
85
85
  requirement: !ruby/object:Gem::Requirement
@@ -163,9 +163,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
163
163
  version: '0'
164
164
  required_rubygems_version: !ruby/object:Gem::Requirement
165
165
  requirements:
166
- - - ">="
166
+ - - ">"
167
167
  - !ruby/object:Gem::Version
168
- version: '0'
168
+ version: 1.3.1
169
169
  requirements: []
170
170
  rubyforge_project:
171
171
  rubygems_version: 2.6.12