fluent-plugin-cloudwatch-logs 0.5.0.pre1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '0293939c5f3f3a26433414ba6f8f0e24e2919ffcdde9d0089c36ea812c74fa75'
4
- data.tar.gz: d2f8ccb99658afd56dd08819f4849bcb9925f590b928cd009aac380cdf130163
3
+ metadata.gz: 11a5a24eb3978bb6d5b29d3d08cf5b49bfd9eed85e3a3b6df46adedf31e85458
4
+ data.tar.gz: 1193f782abef03c1f89c18b25ccb2ad2d3d774681a5e489d860e5baefb71beb4
5
5
  SHA512:
6
- metadata.gz: 3aaef277138417fc14f8b979c05a500b86c3cce4f887993ac1079078021b6c42ccc4e77c058b3b680f23dcfba80995742513620c164b9c89a0c5050cdb4e7bc4
7
- data.tar.gz: 542aa4b296c089500eda35ea2af8f79f71a0a35f1573bbb06c1e5ae3d2ef578a1c0506142acfc6c992e51cf546ce70f17d623941d8ab283a0452808af5cf9e01
6
+ metadata.gz: 4f5d00f59b8a9c9652ab7b364bb13d9a51b6bc04ba7bcf058d2fb0a5705637c6b0156e93f94bb090f8c5e270d821a42f7b5ba7190e4ed0321f87b8d7a661f280
7
+ data.tar.gz: 32dd243642ac3d63ed8704a63074c611343a25c56a681d7c2ac8e298c8579c56ee82b4fe5f91d0ad6b236a0c094bfc86f3ba0f1b08fd4406e25e023dffec3c04
data/README.md CHANGED
@@ -82,6 +82,8 @@ Fetch sample log from CloudWatch Logs:
82
82
  #put_log_events_retry_wait 1s
83
83
  #put_log_events_retry_limit 17
84
84
  #put_log_events_disable_retry_limit false
85
+ #endpoint http://localhost:5000/
86
+ #json_handler json
85
87
  </match>
86
88
  ```
87
89
 
@@ -102,6 +104,8 @@ Fetch sample log from CloudWatch Logs:
102
104
  * `put_log_events_retry_wait`: time before retrying PutLogEvents (retry interval increases exponentially like `put_log_events_retry_wait * (2 ^ retry_count)`)
103
105
  * `put_log_events_retry_limit`: maximum count of retry (if exceeding this, the events will be discarded)
104
106
  * `put_log_events_disable_retry_limit`: if true, `put_log_events_retry_limit` will be ignored
107
+ * `endpoint`: use this parameter to connect to the local API endpoint (for testing)
108
+ * `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
105
109
 
106
110
  ### in_cloudwatch_logs
107
111
 
@@ -113,6 +117,8 @@ Fetch sample log from CloudWatch Logs:
113
117
  log_stream_name stream
114
118
  #use_log_stream_name_prefix true
115
119
  state_file /var/lib/fluent/group_stream.in.state
120
+ #endpoint http://localhost:5000/
121
+ #json_handler json
116
122
  </source>
117
123
  ```
118
124
 
@@ -121,9 +127,11 @@ Fetch sample log from CloudWatch Logs:
121
127
  * `log_stream_name`: name of log stream to fetch logs
122
128
  * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
123
129
  * `state_file`: file to store current state (e.g. next\_forward\_token)
130
+ * `endpoint`: use this parameter to connect to the local API endpoint (for testing)
124
131
  * `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
125
132
  * `aws_sts_role_arn`: the role ARN to assume when using cross-account sts authentication
126
133
  * `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
134
+ * `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
127
135
 
128
136
  This plugin uses [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) and you can use addtional variables such as %{hostname}, %{uuid}, etc. These variables are useful to put hostname in `log_stream_name`.
129
137
 
@@ -149,6 +157,14 @@ Or, If you do not want to use IAM roll or ENV(this is just like writing to confi
149
157
  $ rake aws_key_id=YOUR_ACCESS_KEY aws_sec_key=YOUR_SECRET_KEY region=us-east-1 test
150
158
  ```
151
159
 
160
+ If you want to run the test suite against a mock server, set `endpoint` as below:
161
+
162
+ ```
163
+ $ export endpoint='http://localhost:5000/'
164
+ $ rake test
165
+ ```
166
+
167
+
152
168
  ## Caution
153
169
 
154
170
  - If an event message exceeds API limit (256KB), the event will be discarded.
@@ -2,7 +2,7 @@ module Fluent
2
2
  module Plugin
3
3
  module Cloudwatch
4
4
  module Logs
5
- VERSION = "0.5.0.pre1"
5
+ VERSION = "0.5.0"
6
6
  end
7
7
  end
8
8
  end
@@ -1,5 +1,6 @@
1
1
  require 'fluent/plugin/input'
2
2
  require 'fluent/plugin/parser'
3
+ require 'yajl'
3
4
 
4
5
  module Fluent::Plugin
5
6
  class CloudwatchLogsInput < Input
@@ -13,6 +14,7 @@ module Fluent::Plugin
13
14
  config_param :aws_sts_role_arn, :string, default: nil
14
15
  config_param :aws_sts_session_name, :string, default: 'fluentd'
15
16
  config_param :region, :string, :default => nil
17
+ config_param :endpoint, :string, :default => nil
16
18
  config_param :tag, :string
17
19
  config_param :log_group_name, :string
18
20
  config_param :log_stream_name, :string
@@ -20,6 +22,7 @@ module Fluent::Plugin
20
22
  config_param :state_file, :string
21
23
  config_param :fetch_interval, :time, default: 60
22
24
  config_param :http_proxy, :string, default: nil
25
+ config_param :json_handler, :enum, list: [:yajl, :json], :default => :json
23
26
 
24
27
  config_section :parse do
25
28
  config_set_default :@type, 'none'
@@ -41,6 +44,7 @@ module Fluent::Plugin
41
44
  super
42
45
  options = {}
43
46
  options[:region] = @region if @region
47
+ options[:endpoint] = @endpoint if @endpoint
44
48
  options[:http_proxy] = @http_proxy if @http_proxy
45
49
 
46
50
  if @aws_use_sts
@@ -57,6 +61,13 @@ module Fluent::Plugin
57
61
 
58
62
  @finished = false
59
63
  thread_create(:in_cloudwatch_logs_runner, &method(:run))
64
+
65
+ @json_handler = case @json_handler
66
+ when :yajl
67
+ Yajl
68
+ when :json
69
+ JSON
70
+ end
60
71
  end
61
72
 
62
73
  def shutdown
@@ -121,7 +132,7 @@ module Fluent::Plugin
121
132
  }
122
133
  else
123
134
  time = (event.timestamp / 1000).floor
124
- record = JSON.parse(event.message)
135
+ record = @json_handler.load(event.message)
125
136
  router.emit(@tag, time, record)
126
137
  end
127
138
  end
@@ -1,5 +1,6 @@
1
1
  require 'fluent/plugin/output'
2
2
  require 'thread'
3
+ require 'yajl'
3
4
 
4
5
  module Fluent::Plugin
5
6
  class CloudwatchLogsOutput < Output
@@ -15,6 +16,7 @@ module Fluent::Plugin
15
16
  config_param :aws_sts_role_arn, :string, default: nil
16
17
  config_param :aws_sts_session_name, :string, default: 'fluentd'
17
18
  config_param :region, :string, :default => nil
19
+ config_param :endpoint, :string, :default => nil
18
20
  config_param :log_group_name, :string, :default => nil
19
21
  config_param :log_stream_name, :string, :default => nil
20
22
  config_param :auto_create_stream, :bool, default: false
@@ -38,6 +40,7 @@ module Fluent::Plugin
38
40
  config_param :retention_in_days, :integer, default: nil
39
41
  config_param :retention_in_days_key, :string, default: nil
40
42
  config_param :remove_retention_in_days, :bool, default: false
43
+ config_param :json_handler, :enum, list: [:yajl, :json], :default => :json
41
44
 
42
45
  config_section :buffer do
43
46
  config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -79,6 +82,7 @@ module Fluent::Plugin
79
82
 
80
83
  options = {}
81
84
  options[:region] = @region if @region
85
+ options[:endpoint] = @endpoint if @endpoint
82
86
 
83
87
  if @aws_use_sts
84
88
  Aws.config[:region] = options[:region]
@@ -93,6 +97,13 @@ module Fluent::Plugin
93
97
  @logs ||= Aws::CloudWatchLogs::Client.new(options)
94
98
  @sequence_tokens = {}
95
99
  @store_next_sequence_token_mutex = Mutex.new
100
+
101
+ @json_handler = case @json_handler
102
+ when :yajl
103
+ Yajl
104
+ when :json
105
+ JSON
106
+ end
96
107
  end
97
108
 
98
109
  def format(tag, time, record)
@@ -203,7 +214,7 @@ module Fluent::Plugin
203
214
  if @message_keys
204
215
  message = @message_keys.split(',').map {|k| record[k].to_s }.join(' ')
205
216
  else
206
- message = record.to_json
217
+ message = @json_handler.dump(record)
207
218
  end
208
219
 
209
220
  if @max_message_length
@@ -37,6 +37,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
37
37
  assert_equal('stream', d.instance.log_stream_name)
38
38
  assert_equal(true, d.instance.use_log_stream_name_prefix)
39
39
  assert_equal('/tmp/state', d.instance.state_file)
40
+ assert_equal(:json, d.instance.json_handler)
40
41
  end
41
42
 
42
43
  def test_emit
@@ -80,6 +81,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
80
81
  #{aws_key_id}
81
82
  #{aws_sec_key}
82
83
  #{region}
84
+ #{endpoint}
83
85
  EOC
84
86
 
85
87
  d.run(expect_emits: 2, timeout: 5)
@@ -100,15 +102,15 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
100
102
 
101
103
  time_ms = (Time.now.to_f * 1000).floor
102
104
  put_log_events([
103
- {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
104
- {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
105
+ {timestamp: time_ms + 10, message: '{"cloudwatch":"logs1"}'},
106
+ {timestamp: time_ms + 20, message: '{"cloudwatch":"logs2"}'},
105
107
  ])
106
108
 
107
109
  new_log_stream("testprefix")
108
110
  create_log_stream
109
111
  put_log_events([
110
- {timestamp: time_ms, message: '{"cloudwatch":"logs3"}'},
111
- {timestamp: time_ms, message: '{"cloudwatch":"logs4"}'},
112
+ {timestamp: time_ms + 30, message: '{"cloudwatch":"logs3"}'},
113
+ {timestamp: time_ms + 40, message: '{"cloudwatch":"logs4"}'},
112
114
  ])
113
115
 
114
116
  sleep 5
@@ -123,15 +125,16 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
123
125
  #{aws_key_id}
124
126
  #{aws_sec_key}
125
127
  #{region}
128
+ #{endpoint}
126
129
  EOC
127
130
  d.run(expect_emits: 4, timeout: 5)
128
131
 
129
132
  emits = d.events
130
133
  assert_equal(4, emits.size)
131
- assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
132
- assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
133
- assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs3'}], emits[2])
134
- assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs4'}], emits[3])
134
+ assert_equal(['test', ((time_ms + 10) / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
135
+ assert_equal(['test', ((time_ms + 20) / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
136
+ assert_equal(['test', ((time_ms + 30) / 1000).floor, {'cloudwatch' => 'logs3'}], emits[2])
137
+ assert_equal(['test', ((time_ms + 40) / 1000).floor, {'cloudwatch' => 'logs4'}], emits[3])
135
138
  end
136
139
 
137
140
  private
@@ -146,6 +149,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
146
149
  #{aws_key_id}
147
150
  #{aws_sec_key}
148
151
  #{region}
152
+ #{endpoint}
149
153
  EOC
150
154
  end
151
155
 
@@ -40,6 +40,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
40
40
  assert_equal("tagvalue", d.instance.log_group_aws_tags.fetch("tagkey"))
41
41
  assert_equal("tagvalue_2", d.instance.log_group_aws_tags.fetch("tagkey_2"))
42
42
  assert_equal(5, d.instance.retention_in_days)
43
+ assert_equal(:json, d.instance.json_handler)
43
44
  end
44
45
 
45
46
  def test_write
@@ -356,10 +357,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
356
357
  ]
357
358
 
358
359
  time = Time.now
359
- records.each_with_index do |record, i|
360
- d.emit(record, time.to_i + i)
360
+ d.run(default_tag: fluentd_tag) do
361
+ records.each_with_index do |record, i|
362
+ d.feed(time.to_i + i, record)
363
+ end
361
364
  end
362
- d.run
363
365
 
364
366
  awstags = get_log_group_tags
365
367
  assert_equal("value1", awstags.fetch("tag1"))
@@ -384,10 +386,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
384
386
  ]
385
387
 
386
388
  time = Time.now
387
- records.each_with_index do |record, i|
388
- d.emit(record, time.to_i + i)
389
+ d.run(default_tag: fluentd_tag) do
390
+ records.each_with_index do |record, i|
391
+ d.feed(time.to_i + i, record)
392
+ end
389
393
  end
390
- d.run
391
394
 
392
395
  retention = get_log_group_retention_days
393
396
  assert_equal(d.instance.retention_in_days, retention)
@@ -411,10 +414,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
411
414
  ]
412
415
 
413
416
  time = Time.now
414
- records.each_with_index do |record, i|
415
- d.emit(record, time.to_i + i)
417
+ d.run(default_tag: fluentd_tag) do
418
+ records.each_with_index do |record, i|
419
+ d.feed(time.to_i + i, record)
420
+ end
416
421
  end
417
- d.run
418
422
 
419
423
  assert_match(/failed to set retention policy for Log group/, d.instance.log.logs[0])
420
424
  end
@@ -437,10 +441,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
437
441
  ]
438
442
 
439
443
  time = Time.now
440
- records.each_with_index do |record, i|
441
- d.emit(record, time.to_i + i)
444
+ d.run(default_tag: fluentd_tag) do
445
+ records.each_with_index do |record, i|
446
+ d.feed(time.to_i + i, record)
447
+ end
442
448
  end
443
- d.run
444
449
 
445
450
  awstags = get_log_group_tags
446
451
  assert_equal("value1", awstags.fetch("tag1"))
@@ -464,10 +469,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
464
469
  ]
465
470
 
466
471
  time = Time.now
467
- records.each_with_index do |record, i|
468
- d.emit(record, time.to_i + i)
472
+ d.run(default_tag: fluentd_tag) do
473
+ records.each_with_index do |record, i|
474
+ d.feed(time.to_i + i, record)
475
+ end
469
476
  end
470
- d.run
471
477
 
472
478
  awstags = get_log_group_tags
473
479
  assert_equal("value1", awstags.fetch("tag1"))
@@ -500,10 +506,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
500
506
  ]
501
507
 
502
508
  time = Time.now
503
- records.each_with_index do |record, i|
504
- d.emit(record, time.to_i + i)
509
+ d.run(default_tag: fluentd_tag) do
510
+ records.each_with_index do |record, i|
511
+ d.feed(time.to_i + i, record)
512
+ end
505
513
  end
506
- d.run
507
514
 
508
515
  sleep 10
509
516
 
@@ -607,6 +614,7 @@ auto_create_stream true
607
614
  #{aws_key_id}
608
615
  #{aws_sec_key}
609
616
  #{region}
617
+ #{endpoint}
610
618
  EOC
611
619
  end
612
620
 
@@ -11,6 +11,7 @@ module CloudwatchLogsTestHelper
11
11
  options = {}
12
12
  options[:credentials] = Aws::Credentials.new(ENV['aws_key_id'], ENV['aws_sec_key']) if ENV['aws_key_id'] && ENV['aws_sec_key']
13
13
  options[:region] = ENV['region'] if ENV['region']
14
+ options[:endpoint] = ENV['endpoint'] if ENV['endpoint']
14
15
  options[:http_proxy] = ENV['http_proxy'] if ENV['http_proxy']
15
16
  @logs ||= Aws::CloudWatchLogs::Client.new(options)
16
17
  end
@@ -31,6 +32,10 @@ module CloudwatchLogsTestHelper
31
32
  "region #{ENV['region']}" if ENV['region']
32
33
  end
33
34
 
35
+ def endpoint
36
+ "endpoint #{ENV['endpoint']}" if ENV['endpoint']
37
+ end
38
+
34
39
  def log_stream_name(log_stream_name_prefix = nil)
35
40
  if !@log_stream_name
36
41
  new_log_stream(log_stream_name_prefix)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudwatch-logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0.pre1
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryota Arai
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-03-30 00:00:00.000000000 Z
11
+ date: 2018-04-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -130,12 +130,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
130
130
  version: '0'
131
131
  required_rubygems_version: !ruby/object:Gem::Requirement
132
132
  requirements:
133
- - - ">"
133
+ - - ">="
134
134
  - !ruby/object:Gem::Version
135
- version: 1.3.1
135
+ version: '0'
136
136
  requirements: []
137
137
  rubyforge_project:
138
- rubygems_version: 2.7.3
138
+ rubygems_version: 2.7.6
139
139
  signing_key:
140
140
  specification_version: 4
141
141
  summary: CloudWatch Logs Plugin for Fluentd