fluent-plugin-cloudwatch-ingest-chaeyk 0.3.2 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/circle.yml +1 -1
- data/{fluent-plugin-cloudwatch-ingest.gemspec → fluent-plugin-cloudwatch-ingest-chaeyk.gemspec} +5 -5
- data/lib/fluent/plugin/cloudwatch/ingest/version.rb +3 -1
- data/lib/fluent/plugin/{in_cloudwatch_ingest.rb → in_cloudwatch_ingest_chaeyk.rb} +89 -21
- data/lib/fluent/plugin/{parser_cloudwatch_ingest.rb → parser_cloudwatch_ingest_chaeyk.rb} +1 -1
- metadata +10 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e539d42376be0deb2bca31af801a1d18d0c2a54
|
4
|
+
data.tar.gz: 0939b1482b07c776aa3b62228f021e6a2e201bd3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 56593d2fb6a811c6199feb4ad82158a33f28d4503be0b313bbebb2e93be6ef5c33107a17c39c08d5b8d0336376df70db1b566a1cf36a8e0bac75c4e9f66cddd0
|
7
|
+
data.tar.gz: eaebcb0661295d542a1d76ac8991b8c68fd0fbe88486feca4a089d25da21755b067ded2ac36e1e2081b83768ecc479b4b8d716ce32756a5c054c6080a3f35b07
|
data/Gemfile
CHANGED
data/circle.yml
CHANGED
@@ -6,6 +6,6 @@ test:
|
|
6
6
|
deployment:
|
7
7
|
release:
|
8
8
|
tag: /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$/
|
9
|
-
owner:
|
9
|
+
owner: chaeyk
|
10
10
|
commands:
|
11
11
|
- bin/deploy
|
data/{fluent-plugin-cloudwatch-ingest.gemspec → fluent-plugin-cloudwatch-ingest-chaeyk.gemspec}
RENAMED
@@ -6,13 +6,13 @@ require 'fluent/plugin/cloudwatch/ingest/version'
|
|
6
6
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
8
|
spec.name = 'fluent-plugin-cloudwatch-ingest-chaeyk'
|
9
|
-
spec.version = Fluent::Plugin::Cloudwatch::Ingest::VERSION
|
10
|
-
spec.authors = ['
|
11
|
-
spec.email = ['
|
9
|
+
spec.version = Fluent::Plugin::Cloudwatch::Ingest::Chaeyk::VERSION
|
10
|
+
spec.authors = ['chaeyk']
|
11
|
+
spec.email = ['chaeyk@gmail.com']
|
12
12
|
|
13
13
|
spec.summary = 'Fluentd plugin to ingest AWS Cloudwatch logs'
|
14
14
|
spec.description = 'Fluentd plugin to ingest AWS Cloudwatch logs'
|
15
|
-
spec.homepage = 'https://github.com/
|
15
|
+
spec.homepage = 'https://github.com/chaeyk/fluent-plugin-cloudwatch-ingest-chaeyk'
|
16
16
|
|
17
17
|
# Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
|
18
18
|
# delete this section to allow pushing this gem to any host.
|
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
|
|
28
28
|
spec.require_paths = ['lib']
|
29
29
|
|
30
30
|
spec.add_development_dependency 'bundler', '~> 1.10'
|
31
|
-
spec.add_development_dependency 'rake', '~> 10.
|
31
|
+
spec.add_development_dependency 'rake', '~> 10.5'
|
32
32
|
spec.add_development_dependency 'rspec'
|
33
33
|
spec.add_development_dependency 'rubocop'
|
34
34
|
|
@@ -8,7 +8,7 @@ require 'psych'
|
|
8
8
|
|
9
9
|
module Fluent::Plugin
|
10
10
|
class CloudwatchIngestInput < Fluent::Plugin::Input
|
11
|
-
Fluent::Plugin.register_input('
|
11
|
+
Fluent::Plugin.register_input('cloudwatch_ingest_chaeyk', self)
|
12
12
|
helpers :compat_parameters, :parser
|
13
13
|
|
14
14
|
desc 'The region of the source cloudwatch logs'
|
@@ -37,8 +37,10 @@ module Fluent::Plugin
|
|
37
37
|
config_param :limit_events, :integer, default: 10_000
|
38
38
|
desc 'Do not fetch events before this time'
|
39
39
|
config_param :event_start_time, :integer, default: 0
|
40
|
+
desc 'Fetch the oldest logs first'
|
41
|
+
config_param :oldest_logs_first, :bool, default: false
|
40
42
|
config_section :parse do
|
41
|
-
config_set_default :@type, '
|
43
|
+
config_set_default :@type, 'cloudwatch_ingest_chaeyk'
|
42
44
|
desc 'Regular expression with which to parse the event message'
|
43
45
|
config_param :expression, :string, default: '^(?<message>.+)$'
|
44
46
|
desc 'Take the timestamp from the event rather than the expression'
|
@@ -134,7 +136,7 @@ module Fluent::Plugin
|
|
134
136
|
break unless response.next_token
|
135
137
|
next_token = response.next_token
|
136
138
|
rescue => boom
|
137
|
-
log.error("Unable to retrieve log groups: #{boom}")
|
139
|
+
log.error("Unable to retrieve log groups: #{boom.inspect}")
|
138
140
|
next_token = nil
|
139
141
|
sleep @api_interval
|
140
142
|
retry
|
@@ -167,7 +169,7 @@ module Fluent::Plugin
|
|
167
169
|
break unless response.next_token
|
168
170
|
next_token = response.next_token
|
169
171
|
rescue => boom
|
170
|
-
log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom}") # rubocop:disable all
|
172
|
+
log.error("Unable to retrieve log streams for group #{log_group_name} with stream prefix #{log_stream_name_prefix}: #{boom.inspect}") # rubocop:disable all
|
171
173
|
log_streams = []
|
172
174
|
next_token = nil
|
173
175
|
sleep @api_interval
|
@@ -184,25 +186,36 @@ module Fluent::Plugin
|
|
184
186
|
begin
|
185
187
|
state = State.new(@state_file_name, log)
|
186
188
|
rescue => boom
|
187
|
-
log.info("Failed lock state. Sleeping for #{@interval}: #{boom}")
|
189
|
+
log.info("Failed lock state. Sleeping for #{@interval}: #{boom.inspect}")
|
188
190
|
sleep @interval
|
189
|
-
|
191
|
+
next
|
190
192
|
end
|
191
193
|
|
194
|
+
event_count = 0
|
195
|
+
|
192
196
|
# Fetch the streams for each log group
|
193
197
|
log_groups(@log_group_name_prefix).each do |group|
|
194
198
|
# For each log stream get and emit the events
|
195
199
|
log_streams(group, @log_stream_name_prefix).each do |stream|
|
200
|
+
if ! state.store[group][stream]
|
201
|
+
state.store[group][stream] = {}
|
202
|
+
end
|
203
|
+
|
196
204
|
# See if we have some stored state for this group and stream.
|
197
205
|
# If we have then use the stored forward_token to pick up
|
198
206
|
# from that point. Otherwise start from the start.
|
199
|
-
if state.store[group]
|
200
|
-
stream_token =
|
201
|
-
(state.store[group][stream] if state.store[group][stream])
|
207
|
+
if state.store[group][stream]['token']
|
208
|
+
stream_token = state.store[group][stream]['token']
|
202
209
|
else
|
203
210
|
stream_token = nil
|
204
211
|
end
|
205
212
|
|
213
|
+
if state.store[group][stream]['timestamp']
|
214
|
+
stream_timestamp = state.store[group][stream]['timestamp']
|
215
|
+
else
|
216
|
+
stream_timestamp = @event_start_time
|
217
|
+
end
|
218
|
+
|
206
219
|
begin
|
207
220
|
response = @aws.get_log_events(
|
208
221
|
log_group_name: group,
|
@@ -210,24 +223,57 @@ module Fluent::Plugin
|
|
210
223
|
next_token: stream_token,
|
211
224
|
limit: @limit_events,
|
212
225
|
start_time: @event_start_time,
|
213
|
-
start_from_head:
|
226
|
+
start_from_head: @oldest_logs_first
|
214
227
|
)
|
215
228
|
|
216
229
|
response.events.each do |e|
|
217
230
|
begin
|
218
231
|
emit(e, group, stream)
|
232
|
+
event_count = event_count + 1
|
219
233
|
rescue => boom
|
220
|
-
log.error("Failed to emit event #{e}: #{boom}")
|
234
|
+
log.error("Failed to emit event #{e}: #{boom.inspect}")
|
221
235
|
end
|
222
236
|
end
|
223
237
|
|
224
238
|
# Once all events for this stream have been processed,
|
225
239
|
# in this iteration, store the forward token
|
226
|
-
state.store[group][stream] = response.next_forward_token
|
240
|
+
state.store[group][stream]['token'] = response.next_forward_token
|
241
|
+
state.store[group][stream]['timestamp'] = response.events.last ? response.events.last.timestamp : stream_timestamp
|
242
|
+
rescue Aws::CloudWatchLogs::Errors::InvalidParameterException => boom
|
243
|
+
log.error("cloudwatch token is expired or broken. trying with timestamp.");
|
244
|
+
|
245
|
+
# try again with timestamp instead of forward token
|
246
|
+
begin
|
247
|
+
response = @aws.get_log_events(
|
248
|
+
log_group_name: group,
|
249
|
+
log_stream_name: stream,
|
250
|
+
limit: @limit_events,
|
251
|
+
start_time: stream_timestamp,
|
252
|
+
start_from_head: true
|
253
|
+
)
|
254
|
+
|
255
|
+
response.events.each do |e|
|
256
|
+
begin
|
257
|
+
emit(e, group, stream)
|
258
|
+
event_count = event_count + 1
|
259
|
+
rescue => boom
|
260
|
+
log.error("Failed to emit event #{e}: #{boom.inspect}")
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
# Once all events for this stream have been processed,
|
265
|
+
# in this iteration, store the forward token
|
266
|
+
state.store[group][stream]["token"] = response.next_forward_token
|
267
|
+
state.store[group][stream]['timestamp'] = response.events.last ? response.events.last.timestamp : steam_timestamp
|
268
|
+
rescue => boom
|
269
|
+
log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom.inspect}") # rubocop:disable all
|
270
|
+
sleep @api_interval
|
271
|
+
next
|
272
|
+
end
|
227
273
|
rescue => boom
|
228
|
-
log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom}") # rubocop:disable all
|
274
|
+
log.error("Unable to retrieve events for stream #{stream} in group #{group}: #{boom.inspect}") # rubocop:disable all
|
229
275
|
sleep @api_interval
|
230
|
-
|
276
|
+
next
|
231
277
|
end
|
232
278
|
end
|
233
279
|
end
|
@@ -238,10 +284,18 @@ module Fluent::Plugin
|
|
238
284
|
state.save
|
239
285
|
state.close
|
240
286
|
rescue
|
241
|
-
log.error("Unable to save state file: #{boom}")
|
287
|
+
log.error("Unable to save state file: #{boom.inspect}")
|
242
288
|
end
|
243
|
-
|
244
|
-
|
289
|
+
|
290
|
+
if event_count > 0
|
291
|
+
sleep_interval = @interval
|
292
|
+
else
|
293
|
+
sleep_interval = @api_interval # when there is no events, slow down
|
294
|
+
end
|
295
|
+
|
296
|
+
log.info("#{event_count} events processed.")
|
297
|
+
log.info("Pausing for #{sleep_interval}")
|
298
|
+
sleep sleep_interval
|
245
299
|
end
|
246
300
|
end
|
247
301
|
|
@@ -252,7 +306,7 @@ module Fluent::Plugin
|
|
252
306
|
def initialize(filepath, log)
|
253
307
|
@filepath = filepath
|
254
308
|
@log = log
|
255
|
-
@store = Hash.new { |h, k| h[k] = {} }
|
309
|
+
@store = Hash.new { |h, k| h[k] = Hash.new { |h1, k1| h1[k1] = {} } }
|
256
310
|
|
257
311
|
if File.exist?(filepath)
|
258
312
|
self.statefile = Pathname.new(@filepath).open('r+')
|
@@ -262,7 +316,7 @@ module Fluent::Plugin
|
|
262
316
|
self.statefile = Pathname.new(@filepath).open('w+')
|
263
317
|
save
|
264
318
|
rescue => boom
|
265
|
-
@log.error("Unable to create new file #{statefile.path}: #{boom}")
|
319
|
+
@log.error("Unable to create new file #{statefile.path}: #{boom.inspect}")
|
266
320
|
end
|
267
321
|
end
|
268
322
|
|
@@ -272,12 +326,26 @@ module Fluent::Plugin
|
|
272
326
|
lockstatus = statefile.flock(File::LOCK_EX | File::LOCK_NB)
|
273
327
|
raise CloudwatchIngestInput::State::LockFailed if lockstatus == false
|
274
328
|
|
275
|
-
|
276
|
-
|
329
|
+
begin
|
330
|
+
@store.merge!(Psych.safe_load(statefile.read))
|
331
|
+
|
332
|
+
# Migrate old state file
|
333
|
+
@store.each { |group, streams|
|
334
|
+
streams.update(streams) { |name, stream|
|
335
|
+
(stream.is_a? String) ? { 'token' => stream, 'timestamp' => Time.now.to_i } : stream
|
336
|
+
}
|
337
|
+
}
|
338
|
+
|
339
|
+
@log.info("Loaded #{@store.keys.size} groups from #{statefile.path}")
|
340
|
+
rescue
|
341
|
+
statefile.close
|
342
|
+
raise
|
343
|
+
end
|
277
344
|
end
|
278
345
|
|
279
346
|
def save
|
280
347
|
statefile.rewind
|
348
|
+
statefile.truncate(0)
|
281
349
|
statefile.write(Psych.dump(@store))
|
282
350
|
@log.info("Saved state to #{statefile.path}")
|
283
351
|
statefile.rewind
|
@@ -4,7 +4,7 @@ require 'fluent/time'
|
|
4
4
|
module Fluent
|
5
5
|
module Plugin
|
6
6
|
class CloudwatchIngestParser < RegexpParser
|
7
|
-
Plugin.register_parser('
|
7
|
+
Plugin.register_parser('cloudwatch_ingest_chaeyk', self)
|
8
8
|
|
9
9
|
config_param :expression, :string, default: '^(?<message>.+)$'
|
10
10
|
config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S.%L'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cloudwatch-ingest-chaeyk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
-
|
7
|
+
- chaeyk
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-05-
|
11
|
+
date: 2017-05-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '10.
|
33
|
+
version: '10.5'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '10.
|
40
|
+
version: '10.5'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rspec
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -96,7 +96,7 @@ dependencies:
|
|
96
96
|
version: 2.8.4
|
97
97
|
description: Fluentd plugin to ingest AWS Cloudwatch logs
|
98
98
|
email:
|
99
|
-
-
|
99
|
+
- chaeyk@gmail.com
|
100
100
|
executables: []
|
101
101
|
extensions: []
|
102
102
|
extra_rdoc_files: []
|
@@ -114,12 +114,12 @@ files:
|
|
114
114
|
- bin/deploy
|
115
115
|
- bin/setup
|
116
116
|
- circle.yml
|
117
|
-
- fluent-plugin-cloudwatch-ingest.gemspec
|
117
|
+
- fluent-plugin-cloudwatch-ingest-chaeyk.gemspec
|
118
118
|
- lib/fluent/plugin/cloudwatch/ingest.rb
|
119
119
|
- lib/fluent/plugin/cloudwatch/ingest/version.rb
|
120
|
-
- lib/fluent/plugin/
|
121
|
-
- lib/fluent/plugin/
|
122
|
-
homepage: https://github.com/
|
120
|
+
- lib/fluent/plugin/in_cloudwatch_ingest_chaeyk.rb
|
121
|
+
- lib/fluent/plugin/parser_cloudwatch_ingest_chaeyk.rb
|
122
|
+
homepage: https://github.com/chaeyk/fluent-plugin-cloudwatch-ingest-chaeyk
|
123
123
|
licenses: []
|
124
124
|
metadata:
|
125
125
|
allowed_push_host: https://rubygems.org
|