fluent-plugin-cloudwatch-logs 0.9.5 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 13ef3e0d5f4fd4c12c49b16ccbd07db237b7cff26ea272e4ef6337b09b02e975
4
- data.tar.gz: 8cefeed014b1842ccce9f15479ad8cc54aa51b976281abb5d3e1c8828239aaa3
3
+ metadata.gz: 7d78abf2df32b76a85618e07a9164b6b4b59085bb56054b9cff15a947674eb26
4
+ data.tar.gz: 935d10a414f4ac4e83d619978695aa98bc2ec784c02ef5e60ec082ac7b39f667
5
5
  SHA512:
6
- metadata.gz: 41ca6fa20d0c26955fcbab7f7a5a9fb8c032a5fd09f8676ccaaf5e4dd12694ea74c7c5320590f076405c691d436bc87b820e6bc9b75dc516062b2f3e09d8a377
7
- data.tar.gz: 1f5bbd694c5962de73ee1e0ca622d8892437fb82d00b76ed87086feeba49f6d2c3a4904edad81bec88ba2d3ca47701c15cb56395470f19742c16b3168dd870e5
6
+ metadata.gz: 8f5113e44e0fb327000423a12292f1d6c25be4e5e491773e3e074b06a9c4206797ac0a3873c5b252da9dbbfa705d7989bb6b84f71d39e4080c5eb882555c914e
7
+ data.tar.gz: ee663a6f50e7788703d3bea274c194ea83d4b330d25b284112682247984cf9de344d00a4a7aba8d7f6e7589c396510e0a0970cb48bea73d73841eb1c22e3cd77
@@ -0,0 +1,12 @@
1
+ name: Autocloser
2
+ on: [issues]
3
+ jobs:
4
+ autoclose:
5
+ runs-on: ubuntu-latest
6
+ steps:
7
+ - name: Autoclose issues that did not follow issue template
8
+ uses: roots/issue-closer-action@v1.1
9
+ with:
10
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
11
+ issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
12
+ issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and CloudWatchLogs plugin versions.*)"
data/README.md CHANGED
@@ -43,6 +43,46 @@ Create IAM user with a policy like the following:
43
43
  }
44
44
  ```
45
45
 
46
+ More restricted IAM policy for `out_cloudwatch_logs` is:
47
+
48
+ ```json
49
+ {
50
+ "Version": "2012-10-17",
51
+ "Statement": [
52
+ {
53
+ "Action": [
54
+ "logs:PutLogEvents",
55
+ "logs:CreateLogGroup",
56
+ "logs:PutRetentionPolicy",
57
+ "logs:CreateLogStream",
58
+ "logs:DescribeLogGroups",
59
+ "logs:DescribeLogStreams"
60
+ ],
61
+ "Effect": "Allow",
62
+ "Resource": "*"
63
+ }
64
+ ]
65
+ }
66
+ ```
67
+
68
+ Also, more restricted IAM policy for `in_cloudwatch_logs` is:
69
+
70
+ ```json
71
+ {
72
+ "Version": "2012-10-17",
73
+ "Statement": [
74
+ {
75
+ "Action": [
76
+ "logs:GetLogEvents",
77
+ "logs:DescribeLogStreams"
78
+ ],
79
+ "Effect": "Allow",
80
+ "Resource": "*"
81
+ }
82
+ ]
83
+ }
84
+ ```
85
+
46
86
  ## Authentication
47
87
 
48
88
  There are several methods to provide authentication credentials. Be aware that there are various tradeoffs for these methods,
@@ -120,6 +160,11 @@ Fetch sample log from CloudWatch Logs:
120
160
  #endpoint http://localhost:5000/
121
161
  #json_handler json
122
162
  #log_rejected_request true
163
+ #<web_identity_credentials>
164
+ # role_arn "#{ENV['AWS_ROLE_ARN']}"
165
+ # role_session_name ROLE_SESSION_NAME
166
+ # web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
167
+ #</web_identity_credentials>
123
168
  </match>
124
169
  ```
125
170
 
@@ -154,6 +199,14 @@ Fetch sample log from CloudWatch Logs:
154
199
  * `retention_in_days_key`: use specified field of records as retention period
155
200
  * `use_tag_as_group`: to use tag as a group name
156
201
  * `use_tag_as_stream`: to use tag as a stream name
202
+ * `<web_identity_credentials>`: For EKS authentication.
203
+ * `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
204
+ * `role_session_name`: An identifier for the assumed role session. This parameter is required when using `<web_identity_credentials>`.
205
+ * `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
206
+ * `policy`: An IAM policy in JSON format. (default `nil`)
207
+ * `duration_seconds`: The duration, in seconds, of the role session. The value can range from
208
+ 900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
209
+ is set to 3600 seconds (1 hour). (default `nil`)
157
210
 
158
211
  **NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
159
212
  Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
@@ -178,6 +231,14 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
178
231
  #<parse>
179
232
  # @type none # or csv, tsv, regexp etc.
180
233
  #</parse>
234
+ #<storage>
235
+ # @type local # or redis, memcached, etc.
236
+ #</storage>
237
+ #<web_identity_credentials>
238
+ # role_arn "#{ENV['AWS_ROLE_ARN']}"
239
+ # role_session_name ROLE_SESSION_NAME
240
+ # web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
241
+ #</web_identity_credentials>
181
242
  </source>
182
243
  ```
183
244
 
@@ -194,7 +255,8 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
194
255
  * `log_stream_name`: name of log stream to fetch logs
195
256
  * `region`: AWS Region. See [Authentication](#authentication) for more information.
196
257
  * `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
197
- * `state_file`: file to store current state (e.g. next\_forward\_token)
258
+ * `include_metadata`: include metadata such as `log_group_name` and `log_stream_name`. (default: false)
259
+ * `state_file`: file to store current state (e.g. next\_forward\_token). This parameter is deprecated. Use `<storage>` instead.
198
260
  * `tag`: fluentd tag
199
261
  * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
200
262
  * `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
@@ -204,6 +266,15 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
204
266
  * `time_range_format`: specify time format for time range. (default: `%Y-%m-%d %H:%M:%S`)
205
267
  * `format`: specify CloudWatchLogs' log format. (default `nil`)
206
268
  * `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
269
+ * `<storage>`: specify storage plugin configuration. see also: https://docs.fluentd.org/v/1.0/storage#how-to-use
270
+ * `<web_identity_credentials>`: For EKS authentication.
271
+ * `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
272
+ * `role_session_name`: An identifier for the assumed role session. This parameter is required when using `<web_identity_credentials>`.
273
+ * `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
274
+ * `policy`: An IAM policy in JSON format. (default `nil`)
275
+ * `duration_seconds`: The duration, in seconds, of the role session. The value can range from
276
+ 900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
277
+ is set to 3600 seconds (1 hour). (default `nil`)
207
278
 
208
279
  ## Test
209
280
 
@@ -2,7 +2,7 @@ module Fluent
2
2
  module Plugin
3
3
  module Cloudwatch
4
4
  module Logs
5
- VERSION = "0.9.5"
5
+ VERSION = "0.11.1"
6
6
  end
7
7
  end
8
8
  end
@@ -8,20 +8,24 @@ module Fluent::Plugin
8
8
  class CloudwatchLogsInput < Input
9
9
  Fluent::Plugin.register_input('cloudwatch_logs', self)
10
10
 
11
- helpers :parser, :thread, :compat_parameters
11
+ helpers :parser, :thread, :compat_parameters, :storage
12
+
13
+ DEFAULT_STORAGE_TYPE = 'local'
12
14
 
13
15
  config_param :aws_key_id, :string, default: nil, secret: true
14
16
  config_param :aws_sec_key, :string, default: nil, secret: true
15
17
  config_param :aws_use_sts, :bool, default: false
16
18
  config_param :aws_sts_role_arn, :string, default: nil
17
19
  config_param :aws_sts_session_name, :string, default: 'fluentd'
20
+ config_param :aws_sts_endpoint_url, :string, default: nil
18
21
  config_param :region, :string, default: nil
19
22
  config_param :endpoint, :string, default: nil
20
23
  config_param :tag, :string
21
24
  config_param :log_group_name, :string
22
25
  config_param :log_stream_name, :string, default: nil
23
26
  config_param :use_log_stream_name_prefix, :bool, default: false
24
- config_param :state_file, :string
27
+ config_param :state_file, :string, default: nil,
28
+ deprecated: "Use <stroage> instead."
25
29
  config_param :fetch_interval, :time, default: 60
26
30
  config_param :http_proxy, :string, default: nil
27
31
  config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
@@ -31,11 +35,25 @@ module Fluent::Plugin
31
35
  config_param :end_time, :string, default: nil
32
36
  config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
33
37
  config_param :throttling_retry_seconds, :time, default: nil
38
+ config_param :include_metadata, :bool, default: false
39
+ config_section :web_identity_credentials, multi: false do
40
+ config_param :role_arn, :string
41
+ config_param :role_session_name, :string
42
+ config_param :web_identity_token_file, :string, default: nil #required
43
+ config_param :policy, :string, default: nil
44
+ config_param :duration_seconds, :time, default: nil
45
+ end
34
46
 
35
47
  config_section :parse do
36
48
  config_set_default :@type, 'none'
37
49
  end
38
50
 
51
+ config_section :storage do
52
+ config_set_default :usage, 'store_next_tokens'
53
+ config_set_default :@type, DEFAULT_STORAGE_TYPE
54
+ config_set_default :persistent, false
55
+ end
56
+
39
57
  def initialize
40
58
  super
41
59
 
@@ -53,6 +71,7 @@ module Fluent::Plugin
53
71
  if @start_time && @end_time && (@end_time < @start_time)
54
72
  raise Fluent::ConfigError, "end_time(#{@end_time}) should be greater than start_time(#{@start_time})."
55
73
  end
74
+ @next_token_storage = storage_create(usage: 'store_next_tokens', conf: config, default_type: DEFAULT_STORAGE_TYPE)
56
75
  end
57
76
 
58
77
  def start
@@ -64,10 +83,29 @@ module Fluent::Plugin
64
83
 
65
84
  if @aws_use_sts
66
85
  Aws.config[:region] = options[:region]
67
- options[:credentials] = Aws::AssumeRoleCredentials.new(
86
+ credentials_options = {
68
87
  role_arn: @aws_sts_role_arn,
69
88
  role_session_name: @aws_sts_session_name
70
- )
89
+ }
90
+ credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
91
+ if @region and @aws_sts_endpoint_url
92
+ credentails_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
93
+ elsif @region
94
+ credentails_options[:client] = Aws::STS::Client.new(:region => @region)
95
+ end
96
+ options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
97
+ elsif @web_identity_credentials
98
+ c = @web_identity_credentials
99
+ credentials_options = {}
100
+ credentials_options[:role_arn] = c.role_arn
101
+ credentials_options[:role_session_name] = c.role_session_name
102
+ credentials_options[:web_identity_token_file] = c.web_identity_token_file
103
+ credentials_options[:policy] = c.policy if c.policy
104
+ credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
105
+ if @region
106
+ credentials_options[:client] = Aws::STS::Client.new(:region => @region)
107
+ end
108
+ options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
71
109
  else
72
110
  options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
73
111
  end
@@ -99,20 +137,28 @@ module Fluent::Plugin
99
137
  end
100
138
  end
101
139
 
102
- def state_file_for(log_stream_name)
103
- return "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}" if log_stream_name
104
- return @state_file
140
+ def state_key_for(log_stream_name)
141
+ if log_stream_name
142
+ "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
143
+ else
144
+ @state_file
145
+ end
146
+ end
147
+
148
+ def migrate_state_file_to_storage(log_stream_name)
149
+ @next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
150
+ File.delete(state_key_for(log_stream_name))
105
151
  end
106
152
 
107
153
  def next_token(log_stream_name)
108
- return nil unless File.exist?(state_file_for(log_stream_name))
109
- File.read(state_file_for(log_stream_name)).chomp
154
+ if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
155
+ migrate_state_file_to_storage(log_stream_name)
156
+ end
157
+ @next_token_storage.get(:"#{state_key_for(log_stream_name)}")
110
158
  end
111
159
 
112
160
  def store_next_token(token, log_stream_name = nil)
113
- File.open(state_file_for(log_stream_name), 'w') do |f|
114
- f.write token
115
- end
161
+ @next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
116
162
  end
117
163
 
118
164
  def run
@@ -130,8 +176,16 @@ module Fluent::Plugin
130
176
  log_streams.each do |log_stream|
131
177
  log_stream_name = log_stream.log_stream_name
132
178
  events = get_events(log_stream_name)
179
+ metadata = if @include_metadata
180
+ {
181
+ "log_stream_name" => log_stream_name,
182
+ "log_group_name" => @log_group_name
183
+ }
184
+ else
185
+ {}
186
+ end
133
187
  events.each do |event|
134
- emit(log_stream_name, event)
188
+ emit(log_stream_name, event, metadata)
135
189
  end
136
190
  end
137
191
  rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
@@ -140,8 +194,16 @@ module Fluent::Plugin
140
194
  end
141
195
  else
142
196
  events = get_events(@log_stream_name)
197
+ metadata = if @include_metadata
198
+ {
199
+ "log_stream_name" => @log_stream_name,
200
+ "log_group_name" => @log_group_name
201
+ }
202
+ else
203
+ {}
204
+ end
143
205
  events.each do |event|
144
- emit(log_stream_name, event)
206
+ emit(log_stream_name, event, metadata)
145
207
  end
146
208
  end
147
209
  end
@@ -149,18 +211,24 @@ module Fluent::Plugin
149
211
  end
150
212
  end
151
213
 
152
- def emit(stream, event)
214
+ def emit(stream, event, metadata)
153
215
  if @parser
154
216
  @parser.parse(event.message) {|time,record|
155
217
  if @use_aws_timestamp
156
218
  time = (event.timestamp / 1000).floor
157
219
  end
220
+ unless metadata.empty?
221
+ record.merge!("metadata" => metadata)
222
+ end
158
223
  router.emit(@tag, time, record)
159
224
  }
160
225
  else
161
226
  time = (event.timestamp / 1000).floor
162
227
  begin
163
228
  record = @json_handler.load(event.message)
229
+ unless metadata.empty?
230
+ record.merge!("metadata" => metadata)
231
+ end
164
232
  router.emit(@tag, time, record)
165
233
  rescue JSON::ParserError, Yajl::ParseError => error # Catch parser errors
166
234
  log.error "Invalid JSON encountered while parsing event.message"
@@ -7,6 +7,8 @@ module Fluent::Plugin
7
7
  class CloudwatchLogsOutput < Output
8
8
  Fluent::Plugin.register_output('cloudwatch_logs', self)
9
9
 
10
+ class TooLargeEventError < Fluent::UnrecoverableError; end
11
+
10
12
  helpers :compat_parameters, :inject
11
13
 
12
14
  DEFAULT_BUFFER_TYPE = "memory"
@@ -17,6 +19,7 @@ module Fluent::Plugin
17
19
  config_param :aws_use_sts, :bool, default: false
18
20
  config_param :aws_sts_role_arn, :string, default: nil
19
21
  config_param :aws_sts_session_name, :string, default: 'fluentd'
22
+ config_param :aws_sts_endpoint_url, :string, default: nil
20
23
  config_param :region, :string, :default => nil
21
24
  config_param :endpoint, :string, :default => nil
22
25
  config_param :log_group_name, :string, :default => nil
@@ -44,6 +47,13 @@ module Fluent::Plugin
44
47
  config_param :remove_retention_in_days_key, :bool, default: false
45
48
  config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
46
49
  config_param :log_rejected_request, :bool, :default => false
50
+ config_section :web_identity_credentials, multi: false do
51
+ config_param :role_arn, :string
52
+ config_param :role_session_name, :string
53
+ config_param :web_identity_token_file, :string, default: nil #required
54
+ config_param :policy, :string, default: nil
55
+ config_param :duration_seconds, :time, default: nil
56
+ end
47
57
 
48
58
  config_section :buffer do
49
59
  config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -92,10 +102,29 @@ module Fluent::Plugin
92
102
 
93
103
  if @aws_use_sts
94
104
  Aws.config[:region] = options[:region]
95
- options[:credentials] = Aws::AssumeRoleCredentials.new(
105
+ credentials_options = {
96
106
  role_arn: @aws_sts_role_arn,
97
107
  role_session_name: @aws_sts_session_name
98
- )
108
+ }
109
+ credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
110
+ if @region and @aws_sts_endpoint_url
111
+ credentails_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
112
+ elsif @region
113
+ credentails_options[:client] = Aws::STS::Client.new(:region => @region)
114
+ end
115
+ options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
116
+ elsif @web_identity_credentials
117
+ c = @web_identity_credentials
118
+ credentials_options = {}
119
+ credentials_options[:role_arn] = c.role_arn
120
+ credentials_options[:role_session_name] = c.role_session_name
121
+ credentials_options[:web_identity_token_file] = c.web_identity_token_file
122
+ credentials_options[:policy] = c.policy if c.policy
123
+ credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
124
+ if @region
125
+ credentials_options[:client] = Aws::STS::Client.new(:region => @region)
126
+ end
127
+ options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
99
128
  else
100
129
  options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
101
130
  end
@@ -130,6 +159,9 @@ module Fluent::Plugin
130
159
  def write(chunk)
131
160
  log_group_name = extract_placeholders(@log_group_name, chunk) if @log_group_name
132
161
  log_stream_name = extract_placeholders(@log_stream_name, chunk) if @log_stream_name
162
+ aws_tags = @log_group_aws_tags.each {|k, v|
163
+ @log_group_aws_tags[extract_placeholders(k, chunk)] = extract_placeholders(v, chunk)
164
+ } if @log_group_aws_tags
133
165
 
134
166
  queue = Thread::Queue.new
135
167
 
@@ -182,7 +214,7 @@ module Fluent::Plugin
182
214
  #as we create log group only once, values from first record will persist
183
215
  record = rs[0][2]
184
216
 
185
- awstags = @log_group_aws_tags
217
+ awstags = aws_tags
186
218
  unless @log_group_aws_tags_key.nil?
187
219
  if @remove_log_group_aws_tags_key
188
220
  awstags = record.delete(@log_group_aws_tags_key)
@@ -319,8 +351,7 @@ module Fluent::Plugin
319
351
  while event = events.shift
320
352
  event_bytesize = event[:message].bytesize + EVENT_HEADER_SIZE
321
353
  if MAX_EVENT_SIZE < event_bytesize
322
- log.warn "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
323
- break
354
+ raise TooLargeEventError, "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
324
355
  end
325
356
 
326
357
  new_chunk = chunk + [event]
@@ -377,8 +408,7 @@ module Fluent::Plugin
377
408
  end
378
409
  rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException, Aws::CloudWatchLogs::Errors::DataAlreadyAcceptedException => err
379
410
  sleep 1 # to avoid too many API calls
380
- log_stream = find_log_stream(group_name, stream_name)
381
- store_next_sequence_token(group_name, stream_name, log_stream.upload_sequence_token)
411
+ store_next_sequence_token(group_name, stream_name, err.expected_sequence_token)
382
412
  log.warn "updating upload sequence token forcefully because unrecoverable error occured", {
383
413
  "error" => err,
384
414
  "log_group" => group_name,
@@ -400,7 +430,13 @@ module Fluent::Plugin
400
430
  raise err
401
431
  end
402
432
  rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
403
- if !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
433
+ if @put_log_events_retry_limit < 1
434
+ log.warn "failed to PutLogEvents and discard logs because put_log_events_retry_limit is less than 1", {
435
+ "error_class" => err.class.to_s,
436
+ "error" => err.message,
437
+ }
438
+ return
439
+ elsif !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
404
440
  log.error "failed to PutLogEvents and discard logs because retry count exceeded put_log_events_retry_limit", {
405
441
  "error_class" => err.class.to_s,
406
442
  "error" => err.message,
@@ -99,6 +99,34 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
99
99
  assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
100
100
  end
101
101
 
102
+ def test_emit_with_metadata
103
+ create_log_stream
104
+
105
+ time_ms = (Time.now.to_f * 1000).floor
106
+ put_log_events([
107
+ {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
108
+ {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
109
+ ])
110
+
111
+ sleep 5
112
+
113
+ d = create_driver(default_config + %[include_metadata true])
114
+ d.run(expect_emits: 2, timeout: 5)
115
+
116
+ emits = d.events
117
+ assert_true(emits[0][2].has_key?("metadata"))
118
+ assert_true(emits[1][2].has_key?("metadata"))
119
+ emits[0][2].delete_if {|k, v|
120
+ k == "metadata"
121
+ }
122
+ emits[1][2].delete_if {|k, v|
123
+ k == "metadata"
124
+ }
125
+ assert_equal(2, emits.size)
126
+ assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
127
+ assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
128
+ end
129
+
102
130
  def test_emit_with_aws_timestamp
103
131
  create_log_stream
104
132
 
@@ -173,7 +201,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
173
201
  '@type' => 'cloudwatch_logs',
174
202
  'log_group_name' => "#{log_group_name}",
175
203
  'log_stream_name' => "#{log_stream_name}",
176
- 'state_file' => '/tmp/state',
177
204
  }
178
205
  cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
179
206
  cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
@@ -183,7 +210,49 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
183
210
  csv_format_config = config_element('ROOT', '', cloudwatch_config, [
184
211
  config_element('parse', '', {'@type' => 'csv',
185
212
  'keys' => 'time,message',
186
- 'time_key' => 'time'})
213
+ 'time_key' => 'time'}),
214
+ config_element('storage', '', {'@type' => 'local',
215
+ 'path' => '/tmp/state'})
216
+ ])
217
+ create_log_stream
218
+
219
+ time_ms = (Time.now.to_f * 1000).floor
220
+ log_time_ms = time_ms - 10000
221
+ put_log_events([
222
+ {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
223
+ {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
224
+ ])
225
+
226
+ sleep 5
227
+
228
+ d = create_driver(csv_format_config)
229
+ d.run(expect_emits: 2, timeout: 5)
230
+
231
+ emits = d.events
232
+ assert_equal(2, emits.size)
233
+ assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
234
+ assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
235
+ end
236
+
237
+ test "emit with <parse> csv with metadata" do
238
+ cloudwatch_config = {'tag' => "test",
239
+ '@type' => 'cloudwatch_logs',
240
+ 'log_group_name' => "#{log_group_name}",
241
+ 'log_stream_name' => "#{log_stream_name}",
242
+ 'include_metadata' => true,
243
+ }
244
+ cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
245
+ cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
246
+ cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
247
+ cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
248
+
249
+ csv_format_config = config_element('ROOT', '', cloudwatch_config, [
250
+ config_element('parse', '', {'@type' => 'csv',
251
+ 'keys' => 'time,message',
252
+ 'time_key' => 'time'}),
253
+ config_element('storage', '', {'@type' => 'local',
254
+ 'path' => '/tmp/state'})
255
+
187
256
  ])
188
257
  create_log_stream
189
258
 
@@ -200,6 +269,14 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
200
269
  d.run(expect_emits: 2, timeout: 5)
201
270
 
202
271
  emits = d.events
272
+ assert_true(emits[0][2].has_key?("metadata"))
273
+ assert_true(emits[1][2].has_key?("metadata"))
274
+ emits[0][2].delete_if {|k, v|
275
+ k == "metadata"
276
+ }
277
+ emits[1][2].delete_if {|k, v|
278
+ k == "metadata"
279
+ }
203
280
  assert_equal(2, emits.size)
204
281
  assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
205
282
  assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
@@ -246,7 +323,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
246
323
  '@type' => 'cloudwatch_logs',
247
324
  'log_group_name' => "#{log_group_name}",
248
325
  'log_stream_name' => "#{log_stream_name}",
249
- 'state_file' => '/tmp/state',
250
326
  }
251
327
  cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
252
328
  cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
@@ -256,7 +332,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
256
332
  regex_format_config = config_element('ROOT', '', cloudwatch_config, [
257
333
  config_element('parse', '', {'@type' => 'regexp',
258
334
  'expression' => "/^(?<cloudwatch>[^ ]*)?/",
259
- })
335
+ }),
336
+ config_element('storage', '', {'@type' => 'local',
337
+ 'path' => '/tmp/state'})
260
338
  ])
261
339
  create_log_stream
262
340
 
@@ -552,6 +630,8 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
552
630
  end
553
631
 
554
632
  test "emit with today's log stream" do
633
+ omit "This testcase is unstable in CI." if ENV["CI"] == "true"
634
+
555
635
  config = <<-CONFIG
556
636
  tag test
557
637
  @type cloudwatch_logs
@@ -492,6 +492,47 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
492
492
  assert_equal("value2", awstags.fetch("tag2"))
493
493
  end
494
494
 
495
+ def test_log_group_aws_tags_with_placeholders
496
+ clear_log_group
497
+
498
+ config = {
499
+ "@type" => "cloudwatch_logs",
500
+ "auto_create_stream" => true,
501
+ "use_tag_as_stream" => true,
502
+ "log_group_name_key" => "group_name_key",
503
+ "log_group_aws_tags" => '{"tag1": "${tag}", "tag2": "${namespace_name}"}',
504
+ }
505
+ config.merge!(config_elementify(aws_key_id)) if aws_key_id
506
+ config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
507
+ config.merge!(config_elementify(region)) if region
508
+ config.merge!(config_elementify(endpoint)) if endpoint
509
+
510
+ d = create_driver(
511
+ Fluent::Config::Element.new('ROOT', '', config, [
512
+ Fluent::Config::Element.new('buffer', 'tag, namespace_name', {
513
+ '@type' => 'memory',
514
+ }, [])
515
+ ])
516
+ )
517
+
518
+ records = [
519
+ {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
520
+ {'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
521
+ {'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
522
+ ]
523
+
524
+ time = Time.now
525
+ d.run(default_tag: fluentd_tag) do
526
+ records.each_with_index do |record, i|
527
+ d.feed(time.to_i + i, record)
528
+ end
529
+ end
530
+
531
+ awstags = get_log_group_tags
532
+ assert_equal(fluentd_tag, awstags.fetch("tag1"))
533
+ assert_equal("fluentd", awstags.fetch("tag2"))
534
+ end
535
+
495
536
  def test_retention_in_days
496
537
  clear_log_group
497
538
 
@@ -713,6 +754,32 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
713
754
  assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
714
755
  end
715
756
 
757
+ def test_retrying_on_throttling_exception_with_put_log_events_retry_limit_as_zero
758
+ client = Aws::CloudWatchLogs::Client.new
759
+ @called = false
760
+ stub(client).put_log_events(anything) {
761
+ raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error"))
762
+ }.once.ordered
763
+
764
+ d = create_driver(<<-EOC)
765
+ #{default_config}
766
+ log_group_name #{log_group_name}
767
+ log_stream_name #{log_stream_name}
768
+ @log_level debug
769
+ put_log_events_retry_limit 0
770
+ EOC
771
+ time = event_time
772
+ d.instance.instance_variable_set(:@logs, client)
773
+ d.run(default_tag: fluentd_tag) do
774
+ d.feed(time, {'message' => 'message1'})
775
+ end
776
+
777
+ logs = d.logs
778
+ assert_equal(0, logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
779
+ assert_equal(1, logs.select {|l| l =~ /failed to PutLogEvents/ }.size)
780
+ assert_equal(0, logs.select {|l| l =~ /retry succeeded/ }.size)
781
+ end
782
+
716
783
  def test_retrying_on_throttling_exception
717
784
  resp = Object.new
718
785
  mock(resp).rejected_log_events_info {}
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch-logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.5
4
+ version: 0.11.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryota Arai
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-19 00:00:00.000000000 Z
11
+ date: 2020-10-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -108,13 +108,14 @@ dependencies:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
- description:
111
+ description:
112
112
  email:
113
113
  - ryota.arai@gmail.com
114
114
  executables: []
115
115
  extensions: []
116
116
  extra_rdoc_files: []
117
117
  files:
118
+ - ".github/workflows/issue-auto-closer.yml"
118
119
  - ".gitignore"
119
120
  - ".travis.yml"
120
121
  - Gemfile
@@ -135,7 +136,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
135
136
  licenses:
136
137
  - MIT
137
138
  metadata: {}
138
- post_install_message:
139
+ post_install_message:
139
140
  rdoc_options: []
140
141
  require_paths:
141
142
  - lib
@@ -150,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
150
151
  - !ruby/object:Gem::Version
151
152
  version: '0'
152
153
  requirements: []
153
- rubygems_version: 3.0.3
154
- signing_key:
154
+ rubygems_version: 3.1.2
155
+ signing_key:
155
156
  specification_version: 4
156
157
  summary: CloudWatch Logs Plugin for Fluentd
157
158
  test_files: