fluent-plugin-cloudwatch-logs 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3809f992ae8d9967a7f1edae7ac40614496d98e38baf2d6aa18bcfbae5c72ff
-  data.tar.gz: 4eb7d7741fb3ef6e1c3d698f305c36ac49ee94df4d9350f5794a2e887e29984c
+  metadata.gz: 0d9202fe11ec14b83d2eaa50fa1797564123dd3dd20363381143955dacf89729
+  data.tar.gz: c054f3d52d90d7497f818160ab8421f8b69582884b32b6547f6073d7cff6ea16
 SHA512:
-  metadata.gz: 8da9d33a1217cae533af682e950314d5a0bd5dbaac8019f0309ef541f26434027ef4b0133bbf2e3030a87c685ba00839c766a99336b0dd0c9f1304a65698a647
-  data.tar.gz: a2df61af2a98d7dab6d2fb4b5aecbb904fd582aa80e9ae3dd4b15c00f8d320dface8b3fbfb8ba3b1cdf47641a8d4edce8e2e03eaf2eafc16ba976cf68edf9514
+  metadata.gz: 3693fb0907a876f2f0fbaac363e67bfcce42cc2d5b356d0eeb5b2a41fff8e859a2ba3ee3f08977a7f39318a67a220e7489d9d5dce82e7e82bf234fad8e1b9b9f
+  data.tar.gz: 3bf4f873cd6104e26356790e5867dfa21d17b4d67e06dc7f02b07864b83c3f302c176995c3884fce587e1a201d9c373b256b874edd99414b08b597532eef2cf0

data/README.md

@@ -134,8 +134,6 @@ Fetch sample log from CloudWatch Logs:
 * `json_handler`:  name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
 * `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
 
-This plugin uses [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) and you can use addtional variables such as %{hostname}, %{uuid}, etc. These variables are useful to put hostname in `log_stream_name`.
-
 ## Test
 
 Set credentials:

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.6.1"
+        VERSION = "0.7.0"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -22,7 +22,7 @@ module Fluent::Plugin
     config_param :log_group_name, :string, :default => nil
     config_param :log_stream_name, :string, :default => nil
     config_param :auto_create_stream, :bool, default: false
-    config_param :message_keys, :string, :default => nil
+    config_param :message_keys, :array, :default => [], value_type: :string
     config_param :max_message_length, :integer, :default => nil
     config_param :max_events_per_batch, :integer, :default => 10000
     config_param :use_tag_as_group, :bool, :default => false  # TODO: Rename to use_tag_as_group_name ?
@@ -123,6 +123,9 @@ module Fluent::Plugin
     end
 
     def write(chunk)
+      log_group_name = extract_placeholders(@log_group_name, chunk) if @log_group_name
+      log_stream_name = extract_placeholders(@log_stream_name, chunk) if @log_stream_name
+
       queue = Thread::Queue.new
 
       chunk.enum_for(:msgpack_each).select {|tag, time, record|
@@ -143,7 +146,7 @@ module Fluent::Plugin
                     record[@log_group_name_key]
                   end
                 else
-                  @log_group_name
+                  log_group_name
                 end
 
         stream = case
@@ -156,7 +159,7 @@ module Fluent::Plugin
                      record[@log_stream_name_key]
                    end
                  else
-                   @log_stream_name
+                   log_stream_name
                  end
 
         [group, stream]
@@ -214,8 +217,8 @@ module Fluent::Plugin
           time_ms = (time.to_f * 1000).floor
 
           scrub_record!(record)
-          if @message_keys
-            message = @message_keys.split(',').map {|k| record[k].to_s }.join(' ')
+          unless @message_keys.empty?
+            message = @message_keys.map {|k| record[k].to_s }.join(' ')
           else
             message = @json_handler.dump(record)
           end

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -29,6 +29,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       auto_create_stream false
       log_group_aws_tags { "tagkey": "tagvalue", "tagkey_2": "tagvalue_2"}
       retention_in_days 5
+      message_keys fluentd, aws, cloudwatch
     EOC
 
     assert_equal('test_id', d.instance.aws_key_id)
@@ -41,6 +42,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal("tagvalue_2", d.instance.log_group_aws_tags.fetch("tagkey_2"))
     assert_equal(5, d.instance.retention_in_days)
     assert_equal(:yajl, d.instance.json_handler)
+    assert_equal(["fluentd","aws","cloudwatch"], d.instance.message_keys)
   end
 
   def test_write
@@ -217,6 +219,44 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal('message2 logs2', events[1].message)
   end
 
+  def test_write_use_placeholders
+    new_log_stream
+
+    config = {'@type' => 'cloudwatch_logs',
+              'auto_create_stream' => true,
+              'message_keys' => ["message","cloudwatch"],
+              'log_stream_name' => "${tag}",
+              'log_group_name' => log_group_name}
+    config.merge!(config_elementify(aws_key_id)) if aws_key_id
+    config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+    config.merge!(config_elementify(region)) if region
+    config.merge!(config_elementify(endpoint)) if endpoint
+
+    d = create_driver(
+      Fluent::Config::Element.new('ROOT', '', config,[
+                                    Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                  '@type' => 'memory',
+                                                                  'timekey' => 3600
+                                                                }, [])
+                                  ])
+    )
+
+    time = event_time
+    d.run(default_tag: fluentd_tag) do
+      d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
+      d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
+    end
+
+    sleep 10
+
+    events = get_log_events(log_group_name, fluentd_tag)
+    assert_equal(2, events.size)
+    assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+    assert_equal('message1 logs1', events[0].message)
+    assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+    assert_equal('message2 logs2', events[1].message)
+  end
+
   def test_include_time_key
     new_log_stream
 

data/test/test_helper.rb

@@ -36,6 +36,10 @@ module CloudwatchLogsTestHelper
     "endpoint #{ENV['endpoint']}" if ENV['endpoint']
   end
 
+  def config_elementify(conf)
+    conf.split(' ').each_slice(2).map{|k, v| {k => v}}.first
+  end
+
   def log_stream_name(log_stream_name_prefix = nil)
     if !@log_stream_name
       new_log_stream(log_stream_name_prefix)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-22 00:00:00.000000000 Z
+date: 2018-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd