fluent-plugin-cloudwatch-logs 0.12.0 → 0.13.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ff7c8514095015603f6197f312b3e73439485ad9fd8b296522605919f5c53930
4
- data.tar.gz: b49f0d62c6033f22409c1046ebd87d884a1f693725e4f372119487225c7b0d5f
3
+ metadata.gz: c69c4ace1ac8f156cf91e67aad9587082e37939eb2d38a5948400434f2ad7550
4
+ data.tar.gz: 7d28b42e100ce69cdae91cda6771cc349803e45c1c89bca5a2dc634bef094b4f
5
5
  SHA512:
6
- metadata.gz: 2a3520d9d89be225d7fce95a4cc5acc0064b6fdf316de8c85b0a0dda8e7bcc95c3a8290e106459312e9c14aaf2639300aed693169ac200f56c795b04b38d91f6
7
- data.tar.gz: 2f3ab8a2a8eb9e98d013f184bdb9cfc7cc2607ba88a1bce37ed8ea56552d1211c1b1b565bebe7aa00791d2c5579d3348507ce612e1070d7bb8abe9dfa3b509ad
6
+ metadata.gz: cbba39e93f58da219d19726a1e8782576cb786e0cc6aac8306e03771fd7efaf967b08cae27aa9459c08b79316392f25e2c4ab3c91f65feaf13a1db89c4d97cca
7
+ data.tar.gz: b0113d75d4be4acf8b2f363d678ce6aa209be6b4c2a8c47edbf7a0c319d80656b12443c5f1948748813466d17feb871b0b92f75c3c9ca75a7d0921f4a00972d7
@@ -0,0 +1,26 @@
1
+ name: Testing on Ubuntu
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ ruby: [ '2.4', '2.5', '2.6', '2.7' ]
12
+ os:
13
+ - ubuntu-latest
14
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ - name: unit testing
21
+ env:
22
+ CI: true
23
+ run: |
24
+ gem install bundler rake
25
+ bundle install --jobs 4 --retry 3
26
+ bundle exec rake test
@@ -0,0 +1,26 @@
1
+ name: Testing on Windows
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ ruby: [ '2.4', '2.5', '2.6', '2.7' ]
12
+ os:
13
+ - windows-latest
14
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ - name: unit testing
21
+ env:
22
+ CI: true
23
+ run: |
24
+ gem install bundler rake
25
+ bundle install --jobs 4 --retry 3
26
+ bundle exec rake test
data/README.md CHANGED
@@ -222,6 +222,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
222
222
  @type cloudwatch_logs
223
223
  tag cloudwatch.in
224
224
  log_group_name group
225
+ #add_log_group_name true
226
+ #log_group_name_key group_name_key
227
+ #use_log_group_name_prefix true
225
228
  log_stream_name stream
226
229
  #use_log_stream_name_prefix true
227
230
  state_file /var/lib/fluent/group_stream.in.state
@@ -256,6 +259,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
256
259
  * `http_proxy`: use to set an optional HTTP proxy
257
260
  * `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
258
261
  * `log_group_name`: name of log group to fetch logs
262
+ * `add_log_group_name`: add record into the name of log group (default `false`)
263
+ * `log_group_name_key`: specify the key where adding record into the name of log group (default `'log_group'`)
264
+ * `use_log_group_name_prefix`: to use `log_group_name` as log group name prefix (default `false`)
259
265
  * `log_stream_name`: name of log stream to fetch logs
260
266
  * `region`: AWS Region. See [Authentication](#authentication) for more information.
261
267
  * `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
@@ -2,7 +2,7 @@ module Fluent
2
2
  module Plugin
3
3
  module Cloudwatch
4
4
  module Logs
5
- VERSION = "0.12.0"
5
+ VERSION = "0.13.4"
6
6
  end
7
7
  end
8
8
  end
@@ -22,6 +22,9 @@ module Fluent::Plugin
22
22
  config_param :endpoint, :string, default: nil
23
23
  config_param :tag, :string
24
24
  config_param :log_group_name, :string
25
+ config_param :add_log_group_name, :bool, default: false
26
+ config_param :log_group_name_key, :string, default: 'log_group'
27
+ config_param :use_log_group_name_prefix, :bool, default: false
25
28
  config_param :log_stream_name, :string, default: nil
26
29
  config_param :use_log_stream_name_prefix, :bool, default: false
27
30
  config_param :state_file, :string, default: nil,
@@ -128,6 +131,17 @@ module Fluent::Plugin
128
131
  super
129
132
  end
130
133
 
134
+ # No private for testing
135
+ def state_key_for(log_stream_name, log_group_name = nil)
136
+ if log_group_name && log_stream_name
137
+ "#{@state_file}_#{log_group_name.gsub(File::SEPARATOR, '-')}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
138
+ elsif log_stream_name
139
+ "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
140
+ else
141
+ @state_file
142
+ end
143
+ end
144
+
131
145
  private
132
146
  def configure_parser(conf)
133
147
  if conf['format']
@@ -137,28 +151,20 @@ module Fluent::Plugin
137
151
  end
138
152
  end
139
153
 
140
- def state_key_for(log_stream_name)
141
- if log_stream_name
142
- "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
143
- else
144
- @state_file
145
- end
146
- end
147
-
148
154
  def migrate_state_file_to_storage(log_stream_name)
149
155
  @next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
150
156
  File.delete(state_key_for(log_stream_name))
151
157
  end
152
158
 
153
- def next_token(log_stream_name)
159
+ def next_token(log_stream_name, log_group_name = nil)
154
160
  if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
155
161
  migrate_state_file_to_storage(log_stream_name)
156
162
  end
157
- @next_token_storage.get(:"#{state_key_for(log_stream_name)}")
163
+ @next_token_storage.get(:"#{state_key_for(log_stream_name, log_group_name)}")
158
164
  end
159
165
 
160
- def store_next_token(token, log_stream_name = nil)
161
- @next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
166
+ def store_next_token(token, log_stream_name = nil, log_group_name = nil)
167
+ @next_token_storage.put(:"#{state_key_for(log_stream_name, log_group_name)}", token)
162
168
  end
163
169
 
164
170
  def run
@@ -168,42 +174,51 @@ module Fluent::Plugin
168
174
  if Time.now > @next_fetch_time
169
175
  @next_fetch_time += @fetch_interval
170
176
 
171
- if @use_log_stream_name_prefix || @use_todays_log_stream
172
- log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
173
- begin
174
- log_streams = describe_log_streams(log_stream_name_prefix)
175
- log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
176
- log_streams.each do |log_stream|
177
- log_stream_name = log_stream.log_stream_name
178
- events = get_events(log_stream_name)
179
- metadata = if @include_metadata
180
- {
181
- "log_stream_name" => log_stream_name,
182
- "log_group_name" => @log_group_name
183
- }
184
- else
185
- {}
186
- end
187
- events.each do |event|
188
- emit(log_stream_name, event, metadata)
177
+ if @use_log_group_name_prefix
178
+ log_group_names = describe_log_groups(@log_group_name).map{|log_group|
179
+ log_group.log_group_name
180
+ }
181
+ else
182
+ log_group_names = [@log_group_name]
183
+ end
184
+ log_group_names.each do |log_group_name|
185
+ if @use_log_stream_name_prefix || @use_todays_log_stream
186
+ log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
187
+ begin
188
+ log_streams = describe_log_streams(log_stream_name_prefix, nil, nil, log_group_name)
189
+ log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
190
+ log_streams.each do |log_stream|
191
+ log_stream_name = log_stream.log_stream_name
192
+ events = get_events(log_group_name, log_stream_name)
193
+ metadata = if @include_metadata
194
+ {
195
+ "log_stream_name" => log_stream_name,
196
+ "log_group_name" => log_group_name
197
+ }
198
+ else
199
+ {}
200
+ end
201
+ events.each do |event|
202
+ emit(log_group_name, log_stream_name, event, metadata)
203
+ end
189
204
  end
205
+ rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
206
+ log.warn "'#{@log_stream_name}' prefixed log stream(s) are not found"
207
+ next
208
+ end
209
+ else
210
+ events = get_events(log_group_name, @log_stream_name)
211
+ metadata = if @include_metadata
212
+ {
213
+ "log_stream_name" => @log_stream_name,
214
+ "log_group_name" => @log_group_name
215
+ }
216
+ else
217
+ {}
218
+ end
219
+ events.each do |event|
220
+ emit(log_group_name, log_stream_name, event, metadata)
190
221
  end
191
- rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
192
- log.warn "'#{@log_stream_name}' prefixed log stream(s) are not found"
193
- next
194
- end
195
- else
196
- events = get_events(@log_stream_name)
197
- metadata = if @include_metadata
198
- {
199
- "log_stream_name" => @log_stream_name,
200
- "log_group_name" => @log_group_name
201
- }
202
- else
203
- {}
204
- end
205
- events.each do |event|
206
- emit(log_stream_name, event, metadata)
207
222
  end
208
223
  end
209
224
  end
@@ -211,12 +226,15 @@ module Fluent::Plugin
211
226
  end
212
227
  end
213
228
 
214
- def emit(stream, event, metadata)
229
+ def emit(group, stream, event, metadata)
215
230
  if @parser
216
231
  @parser.parse(event.message) {|time,record|
217
232
  if @use_aws_timestamp
218
233
  time = (event.timestamp / 1000).floor
219
234
  end
235
+ if @add_log_group_name
236
+ record[@log_group_name_key] = group
237
+ end
220
238
  unless metadata.empty?
221
239
  record.merge!("metadata" => metadata)
222
240
  end
@@ -226,6 +244,9 @@ module Fluent::Plugin
226
244
  time = (event.timestamp / 1000).floor
227
245
  begin
228
246
  record = @json_handler.load(event.message)
247
+ if @add_log_group_name
248
+ record[@log_group_name_key] = group
249
+ end
229
250
  unless metadata.empty?
230
251
  record.merge!("metadata" => metadata)
231
252
  end
@@ -237,29 +258,37 @@ module Fluent::Plugin
237
258
  end
238
259
  end
239
260
 
240
- def get_events(log_stream_name)
261
+ def get_events(log_group_name, log_stream_name)
241
262
  throttling_handler('get_log_events') do
242
263
  request = {
243
- log_group_name: @log_group_name,
264
+ log_group_name: log_group_name,
244
265
  log_stream_name: log_stream_name
245
266
  }
246
267
  request.merge!(start_time: @start_time) if @start_time
247
268
  request.merge!(end_time: @end_time) if @end_time
248
- log_next_token = next_token(log_stream_name)
269
+ if @use_log_group_name_prefix
270
+ log_next_token = next_token(log_stream_name, log_group_name)
271
+ else
272
+ log_next_token = next_token(log_stream_name)
273
+ end
249
274
  request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
250
275
  response = @logs.get_log_events(request)
251
276
  if valid_next_token(log_next_token, response.next_forward_token)
252
- store_next_token(response.next_forward_token, log_stream_name)
277
+ if @use_log_group_name_prefix
278
+ store_next_token(response.next_forward_token, log_stream_name, log_group_name)
279
+ else
280
+ store_next_token(response.next_forward_token, log_stream_name)
281
+ end
253
282
  end
254
283
 
255
284
  response.events
256
285
  end
257
286
  end
258
287
 
259
- def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
288
+ def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
260
289
  throttling_handler('describe_log_streams') do
261
290
  request = {
262
- log_group_name: @log_group_name
291
+ log_group_name: log_group_name != nil ? log_group_name : @log_group_name
263
292
  }
264
293
  request[:next_token] = next_token if next_token
265
294
  request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
@@ -270,7 +299,7 @@ module Fluent::Plugin
270
299
  log_streams = response.log_streams
271
300
  end
272
301
  if response.next_token
273
- log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
302
+ log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token, log_group_name)
274
303
  end
275
304
  log_streams
276
305
  end
@@ -289,6 +318,23 @@ module Fluent::Plugin
289
318
  end
290
319
  end
291
320
 
321
+ def describe_log_groups(log_group_name_prefix, log_groups = nil, next_token = nil)
322
+ request = {
323
+ log_group_name_prefix: log_group_name_prefix
324
+ }
325
+ request[:next_token] = next_token if next_token
326
+ response = @logs.describe_log_groups(request)
327
+ if log_groups
328
+ log_groups.concat(response.log_groups)
329
+ else
330
+ log_groups = response.log_groups
331
+ end
332
+ if response.next_token
333
+ log_groups = describe_log_groups(log_group_name_prefix, log_groups, response.next_token)
334
+ end
335
+ log_groups
336
+ end
337
+
292
338
  def valid_next_token(prev_token, next_token)
293
339
  next_token && prev_token != next_token.chomp
294
340
  end
@@ -99,6 +99,128 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
99
99
  assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
100
100
  end
101
101
 
102
+ sub_test_case "use_log_group_name_prefix true" do
103
+ test "emit" do
104
+ set_log_group_name("fluent-plugin-cloudwatch-group-prefix-test-#{Time.now.to_f}")
105
+ create_log_stream
106
+
107
+ time_ms = (Time.now.to_f * 1000).floor
108
+ put_log_events([
109
+ {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
110
+ {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
111
+ ])
112
+
113
+ sleep 5
114
+
115
+ config = <<-EOC
116
+ tag test
117
+ @type cloudwatch_logs
118
+ log_group_name fluent-plugin-cloudwatch-group-prefix-test
119
+ use_log_group_name_prefix true
120
+ log_stream_name #{log_stream_name}
121
+ state_file /tmp/state
122
+ fetch_interval 1
123
+ #{aws_key_id}
124
+ #{aws_sec_key}
125
+ #{region}
126
+ #{endpoint}
127
+ EOC
128
+
129
+ d = create_driver(config)
130
+ d.run(expect_emits: 2, timeout: 5)
131
+
132
+ emits = d.events
133
+ assert_equal(2, emits.size)
134
+ assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
135
+ assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
136
+ end
137
+
138
+ test "emit with add_log_group_name" do
139
+ set_log_group_name("fluent-plugin-cloudwatch-add-log-group-#{Time.now.to_f}")
140
+ create_log_stream
141
+
142
+ time_ms = (Time.now.to_f * 1000).floor
143
+ put_log_events([
144
+ {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
145
+ {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
146
+ ])
147
+
148
+ sleep 5
149
+
150
+ log_group_name_key = 'log_group_key'
151
+ config = <<-EOC
152
+ tag test
153
+ @type cloudwatch_logs
154
+ log_group_name fluent-plugin-cloudwatch-add-log-group
155
+ use_log_group_name_prefix true
156
+ add_log_group_name true
157
+ log_group_name_key #{log_group_name_key}
158
+ log_stream_name #{log_stream_name}
159
+ state_file /tmp/state
160
+ fetch_interval 1
161
+ #{aws_key_id}
162
+ #{aws_sec_key}
163
+ #{region}
164
+ #{endpoint}
165
+ EOC
166
+
167
+ d = create_driver(config)
168
+ d.run(expect_emits: 2, timeout: 5)
169
+
170
+ emits = d.events
171
+ assert_equal(2, emits.size)
172
+ assert_true emits[0][2].has_key?(log_group_name_key)
173
+ emits[0][2].delete(log_group_name_key)
174
+ assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
175
+ assert_true emits[1][2].has_key?(log_group_name_key)
176
+ emits[1][2].delete(log_group_name_key)
177
+ assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
178
+ end
179
+
180
+ test "emit with add_log_group_name and <parse> csv" do
181
+ cloudwatch_config = {'tag' => "test",
182
+ '@type' => 'cloudwatch_logs',
183
+ 'log_group_name' => "fluent-plugin-cloudwatch-with-csv-format",
184
+ 'log_stream_name' => "#{log_stream_name}",
185
+ 'use_log_group_name_prefix' => true,
186
+ }
187
+ cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
188
+ cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
189
+ cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
190
+ cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
191
+
192
+ csv_format_config = config_element('ROOT', '', cloudwatch_config, [
193
+ config_element('parse', '', {'@type' => 'csv',
194
+ 'keys' => 'time,message',
195
+ 'time_key' => 'time'}),
196
+ config_element('storage', '', {'@type' => 'local',
197
+ 'path' => '/tmp/state'})
198
+ ])
199
+ log_group_name = "fluent-plugin-cloudwatch-with-csv-format-#{Time.now.to_f}"
200
+ set_log_group_name(log_group_name)
201
+ create_log_stream
202
+
203
+ time_ms = (Time.now.to_f * 1000).floor
204
+ log_time_ms = time_ms - 10000
205
+ put_log_events([
206
+ {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
207
+ {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
208
+ ])
209
+
210
+ sleep 5
211
+
212
+ d = create_driver(csv_format_config)
213
+ d.run(expect_emits: 2, timeout: 5)
214
+ next_token = d.instance.instance_variable_get(:@next_token_storage)
215
+ assert_true next_token.get(d.instance.state_key_for(log_stream_name, log_group_name)).is_a?(String)
216
+
217
+ emits = d.events
218
+ assert_equal(2, emits.size)
219
+ assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
220
+ assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
221
+ end
222
+ end
223
+
102
224
  def test_emit_with_metadata
103
225
  create_log_stream
104
226
 
data/test/test_helper.rb CHANGED
@@ -17,6 +17,10 @@ module CloudwatchLogsTestHelper
17
17
  @logs ||= Aws::CloudWatchLogs::Client.new(options)
18
18
  end
19
19
 
20
+ def set_log_group_name(log_group_name)
21
+ @log_group_name = log_group_name
22
+ end
23
+
20
24
  def log_group_name
21
25
  @log_group_name ||= "fluent-plugin-cloudwatch-test-#{Time.now.to_f}"
22
26
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch-logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.12.0
4
+ version: 0.13.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryota Arai
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-27 00:00:00.000000000 Z
11
+ date: 2021-02-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -116,6 +116,8 @@ extensions: []
116
116
  extra_rdoc_files: []
117
117
  files:
118
118
  - ".github/workflows/issue-auto-closer.yml"
119
+ - ".github/workflows/linux.yml"
120
+ - ".github/workflows/windows.yml"
119
121
  - ".gitignore"
120
122
  - ".travis.yml"
121
123
  - Gemfile