fluent-plugin-cloudwatch-logs 0.5.0.pre1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0293939c5f3f3a26433414ba6f8f0e24e2919ffcdde9d0089c36ea812c74fa75'
-  data.tar.gz: d2f8ccb99658afd56dd08819f4849bcb9925f590b928cd009aac380cdf130163
+  metadata.gz: 11a5a24eb3978bb6d5b29d3d08cf5b49bfd9eed85e3a3b6df46adedf31e85458
+  data.tar.gz: 1193f782abef03c1f89c18b25ccb2ad2d3d774681a5e489d860e5baefb71beb4
 SHA512:
-  metadata.gz: 3aaef277138417fc14f8b979c05a500b86c3cce4f887993ac1079078021b6c42ccc4e77c058b3b680f23dcfba80995742513620c164b9c89a0c5050cdb4e7bc4
-  data.tar.gz: 542aa4b296c089500eda35ea2af8f79f71a0a35f1573bbb06c1e5ae3d2ef578a1c0506142acfc6c992e51cf546ce70f17d623941d8ab283a0452808af5cf9e01
+  metadata.gz: 4f5d00f59b8a9c9652ab7b364bb13d9a51b6bc04ba7bcf058d2fb0a5705637c6b0156e93f94bb090f8c5e270d821a42f7b5ba7190e4ed0321f87b8d7a661f280
+  data.tar.gz: 32dd243642ac3d63ed8704a63074c611343a25c56a681d7c2ac8e298c8579c56ee82b4fe5f91d0ad6b236a0c094bfc86f3ba0f1b08fd4406e25e023dffec3c04

data/README.md

@@ -82,6 +82,8 @@ Fetch sample log from CloudWatch Logs:
   #put_log_events_retry_wait 1s
   #put_log_events_retry_limit 17
   #put_log_events_disable_retry_limit false
+  #endpoint http://localhost:5000/
+  #json_handler json
 </match>
 ```
 
@@ -102,6 +104,8 @@ Fetch sample log from CloudWatch Logs:
 * `put_log_events_retry_wait`: time before retrying PutLogEvents (retry interval increases exponentially like `put_log_events_retry_wait * (2 ^ retry_count)`)
 * `put_log_events_retry_limit`: maximum count of retry (if exceeding this, the events will be discarded)
 * `put_log_events_disable_retry_limit`: if true, `put_log_events_retry_limit` will be ignored
+* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
+* `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
 
 ### in_cloudwatch_logs
 
@@ -113,6 +117,8 @@ Fetch sample log from CloudWatch Logs:
   log_stream_name stream
   #use_log_stream_name_prefix true
   state_file /var/lib/fluent/group_stream.in.state
+  #endpoint http://localhost:5000/
+  #json_handler json
 </source>
 ```
 
@@ -121,9 +127,11 @@ Fetch sample log from CloudWatch Logs:
 * `log_stream_name`: name of log stream to fetch logs
 * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
 * `state_file`: file to store current state (e.g. next\_forward\_token)
+* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
 * `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
 * `aws_sts_role_arn`: the role ARN to assume when using cross-account sts authentication
 * `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
+* `json_handler`:  name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
 
 This plugin uses [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) and you can use addtional variables such as %{hostname}, %{uuid}, etc. These variables are useful to put hostname in `log_stream_name`.
 
@@ -149,6 +157,14 @@ Or, If you do not want to use IAM roll or ENV(this is just like writing to confi
 $ rake aws_key_id=YOUR_ACCESS_KEY aws_sec_key=YOUR_SECRET_KEY region=us-east-1 test
 ```
 
+If you want to run the test suite against a mock server, set `endpoint` as below:
+
+```
+$ export endpoint='http://localhost:5000/'
+$ rake test
+```
+
+
 ## Caution
 
 - If an event message exceeds API limit (256KB), the event will be discarded.

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.5.0.pre1"
+        VERSION = "0.5.0"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -1,5 +1,6 @@
 require 'fluent/plugin/input'
 require 'fluent/plugin/parser'
+require 'yajl'
 
 module Fluent::Plugin
   class CloudwatchLogsInput < Input
@@ -13,6 +14,7 @@ module Fluent::Plugin
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
     config_param :region, :string, :default => nil
+    config_param :endpoint, :string, :default => nil
     config_param :tag, :string
     config_param :log_group_name, :string
     config_param :log_stream_name, :string
@@ -20,6 +22,7 @@ module Fluent::Plugin
     config_param :state_file, :string
     config_param :fetch_interval, :time, default: 60
     config_param :http_proxy, :string, default: nil
+    config_param :json_handler, :enum, list: [:yajl, :json], :default => :json
 
     config_section :parse do
       config_set_default :@type, 'none'
@@ -41,6 +44,7 @@ module Fluent::Plugin
       super
       options = {}
       options[:region] = @region if @region
+      options[:endpoint] = @endpoint if @endpoint
       options[:http_proxy] = @http_proxy if @http_proxy
 
       if @aws_use_sts
@@ -57,6 +61,13 @@ module Fluent::Plugin
 
       @finished = false
       thread_create(:in_cloudwatch_logs_runner, &method(:run))
+
+      @json_handler = case @json_handler
+                      when :yajl
+                        Yajl
+                      when :json
+                        JSON
+                      end
     end
 
     def shutdown
@@ -121,7 +132,7 @@ module Fluent::Plugin
         }
       else
         time = (event.timestamp / 1000).floor
-        record = JSON.parse(event.message)
+        record = @json_handler.load(event.message)
         router.emit(@tag, time, record)
       end
     end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -1,5 +1,6 @@
 require 'fluent/plugin/output'
 require 'thread'
+require 'yajl'
 
 module Fluent::Plugin
   class CloudwatchLogsOutput < Output
@@ -15,6 +16,7 @@ module Fluent::Plugin
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
     config_param :region, :string, :default => nil
+    config_param :endpoint, :string, :default => nil
     config_param :log_group_name, :string, :default => nil
     config_param :log_stream_name, :string, :default => nil
     config_param :auto_create_stream, :bool, default: false
@@ -38,6 +40,7 @@ module Fluent::Plugin
     config_param :retention_in_days, :integer, default: nil
     config_param :retention_in_days_key, :string, default: nil
     config_param :remove_retention_in_days, :bool, default: false
+    config_param :json_handler, :enum, list: [:yajl, :json], :default => :json
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -79,6 +82,7 @@ module Fluent::Plugin
 
       options = {}
       options[:region] = @region if @region
+      options[:endpoint] = @endpoint if @endpoint
 
       if @aws_use_sts
         Aws.config[:region] = options[:region]
@@ -93,6 +97,13 @@ module Fluent::Plugin
       @logs ||= Aws::CloudWatchLogs::Client.new(options)
       @sequence_tokens = {}
       @store_next_sequence_token_mutex = Mutex.new
+
+      @json_handler = case @json_handler
+                      when :yajl
+                        Yajl
+                      when :json
+                        JSON
+                      end
     end
 
     def format(tag, time, record)
@@ -203,7 +214,7 @@ module Fluent::Plugin
           if @message_keys
             message = @message_keys.split(',').map {|k| record[k].to_s }.join(' ')
           else
-            message = record.to_json
+            message = @json_handler.dump(record)
           end
 
           if @max_message_length

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -37,6 +37,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
     assert_equal('stream', d.instance.log_stream_name)
     assert_equal(true, d.instance.use_log_stream_name_prefix)
     assert_equal('/tmp/state', d.instance.state_file)
+    assert_equal(:json, d.instance.json_handler)
   end
 
   def test_emit
@@ -80,6 +81,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       #{aws_key_id}
       #{aws_sec_key}
       #{region}
+      #{endpoint}
     EOC
 
     d.run(expect_emits: 2, timeout: 5)
@@ -100,15 +102,15 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
 
     time_ms = (Time.now.to_f * 1000).floor
     put_log_events([
-      {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
-      {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+      {timestamp: time_ms + 10, message: '{"cloudwatch":"logs1"}'},
+      {timestamp: time_ms + 20, message: '{"cloudwatch":"logs2"}'},
     ])
 
     new_log_stream("testprefix")
     create_log_stream
     put_log_events([
-      {timestamp: time_ms, message: '{"cloudwatch":"logs3"}'},
-      {timestamp: time_ms, message: '{"cloudwatch":"logs4"}'},
+      {timestamp: time_ms + 30, message: '{"cloudwatch":"logs3"}'},
+      {timestamp: time_ms + 40, message: '{"cloudwatch":"logs4"}'},
     ])
 
     sleep 5
@@ -123,15 +125,16 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       #{aws_key_id}
       #{aws_sec_key}
       #{region}
+      #{endpoint}
     EOC
     d.run(expect_emits: 4, timeout: 5)
 
     emits = d.events
     assert_equal(4, emits.size)
-    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
-    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
-    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs3'}], emits[2])
-    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs4'}], emits[3])
+    assert_equal(['test', ((time_ms + 10) / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+    assert_equal(['test', ((time_ms + 20) / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+    assert_equal(['test', ((time_ms + 30) / 1000).floor, {'cloudwatch' => 'logs3'}], emits[2])
+    assert_equal(['test', ((time_ms + 40) / 1000).floor, {'cloudwatch' => 'logs4'}], emits[3])
   end
 
   private
@@ -146,6 +149,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       #{aws_key_id}
       #{aws_sec_key}
       #{region}
+      #{endpoint}
     EOC
   end
 

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -40,6 +40,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal("tagvalue", d.instance.log_group_aws_tags.fetch("tagkey"))
     assert_equal("tagvalue_2", d.instance.log_group_aws_tags.fetch("tagkey_2"))
     assert_equal(5, d.instance.retention_in_days)
+    assert_equal(:json, d.instance.json_handler)
   end
 
   def test_write
@@ -356,10 +357,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     ]
 
     time = Time.now
-    records.each_with_index do |record, i|
-      d.emit(record, time.to_i + i)
+    d.run(default_tag: fluentd_tag) do
+      records.each_with_index do |record, i|
+        d.feed(time.to_i + i, record)
+      end
     end
-    d.run
 
     awstags = get_log_group_tags
     assert_equal("value1", awstags.fetch("tag1"))
@@ -384,10 +386,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     ]
 
     time = Time.now
-    records.each_with_index do |record, i|
-      d.emit(record, time.to_i + i)
+    d.run(default_tag: fluentd_tag) do
+      records.each_with_index do |record, i|
+        d.feed(time.to_i + i, record)
+      end
     end
-    d.run
 
     retention = get_log_group_retention_days
     assert_equal(d.instance.retention_in_days, retention)
@@ -411,10 +414,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     ]
 
     time = Time.now
-    records.each_with_index do |record, i|
-      d.emit(record, time.to_i + i)
+    d.run(default_tag: fluentd_tag) do
+      records.each_with_index do |record, i|
+        d.feed(time.to_i + i, record)
+      end
     end
-    d.run
 
     assert_match(/failed to set retention policy for Log group/, d.instance.log.logs[0])
   end
@@ -437,10 +441,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     ]
 
     time = Time.now
-    records.each_with_index do |record, i|
-      d.emit(record, time.to_i + i)
+    d.run(default_tag: fluentd_tag) do
+      records.each_with_index do |record, i|
+        d.feed(time.to_i + i, record)
+      end
     end
-    d.run
 
     awstags = get_log_group_tags
     assert_equal("value1", awstags.fetch("tag1"))
@@ -464,10 +469,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     ]
 
     time = Time.now
-    records.each_with_index do |record, i|
-      d.emit(record, time.to_i + i)
+    d.run(default_tag: fluentd_tag) do
+      records.each_with_index do |record, i|
+        d.feed(time.to_i + i, record)
+      end
     end
-    d.run
 
     awstags = get_log_group_tags
     assert_equal("value1", awstags.fetch("tag1"))
@@ -500,10 +506,11 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     ]
 
     time = Time.now
-    records.each_with_index do |record, i|
-      d.emit(record, time.to_i + i)
+    d.run(default_tag: fluentd_tag) do
+      records.each_with_index do |record, i|
+        d.feed(time.to_i + i, record)
+      end
     end
-    d.run
 
     sleep 10
 
@@ -607,6 +614,7 @@ auto_create_stream true
 #{aws_key_id}
 #{aws_sec_key}
 #{region}
+#{endpoint}
     EOC
   end
 

data/test/test_helper.rb

@@ -11,6 +11,7 @@ module CloudwatchLogsTestHelper
     options = {}
     options[:credentials] = Aws::Credentials.new(ENV['aws_key_id'], ENV['aws_sec_key']) if ENV['aws_key_id'] && ENV['aws_sec_key']
     options[:region] = ENV['region'] if ENV['region']
+    options[:endpoint] = ENV['endpoint'] if ENV['endpoint']
     options[:http_proxy] = ENV['http_proxy'] if ENV['http_proxy']
     @logs ||= Aws::CloudWatchLogs::Client.new(options)
   end
@@ -31,6 +32,10 @@ module CloudwatchLogsTestHelper
     "region #{ENV['region']}" if ENV['region']
   end
 
+  def endpoint
+    "endpoint #{ENV['endpoint']}" if ENV['endpoint']
+  end
+
   def log_stream_name(log_stream_name_prefix = nil)
     if !@log_stream_name
       new_log_stream(log_stream_name_prefix)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.5.0.pre1
+  version: 0.5.0
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-30 00:00:00.000000000 Z
+date: 2018-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -130,12 +130,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: CloudWatch Logs Plugin for Fluentd