fluent-plugin-cloudwatch-logs 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7433cbc405896d7a542689ab5e23c6f199043760
-  data.tar.gz: c8f7680af5579da676e5b0379b9e911f5afa7dbe
+  metadata.gz: f11742cedd1eb409768c54c02aca68f71bf7eaaf
+  data.tar.gz: 49ce80a5464dc1dd99c847e6f39f914b314d83a9
 SHA512:
-  metadata.gz: 0bb5a93717b9cddfc9d29fd4e4dca9c84751747b6f10b3225f5a8be0542a418223525e4f03527131daf9d3bfec543fe2d06b372f391760535deeb1cf149c555a
-  data.tar.gz: 86be56949e121311e7ad8ebaf5fb41667fdede3d0a0a75f80418659df35664f4924756614e1bf174c03105f028fca9009a2f80fddb6a4f1a74e2bf8ae79a9d83
+  metadata.gz: b43fdfa3ee97e844180a4b775e238d7c9cac3eaa786d9ed26d897956f5d0b1a3071c5f3e65ad89b99e120a5c464a5c7114c0ef8376df563815b4d2e9a1f1b604
+  data.tar.gz: 862017735509e6e2340ecd135432d85d5edf4d5e8143ed0ef3cc9a3d08386a657ddb153e04333178838402844b3b5044010f5480ddfcb76f7137dbc6ea2919c4

data/README.md

@@ -75,6 +75,10 @@ Fetch sample log from CloudWatch Logs:
   #use_tag_as_stream false
   #include_time_key true
   #localtime true
+  #log_group_name_key group_name_key
+  #log_stream_name_key stream_name_key
+  #remove_log_group_name_key true
+  #remove_log_stream_name_key true
 </match>
 ```
 
@@ -88,6 +92,10 @@ Fetch sample log from CloudWatch Logs:
 * `use_tag_as_stream`: to use tag as a stream name
 * `include_time_key`: include time key as part of the log entry (defaults to UTC)
 * `localtime`: use localtime timezone for `include_time_key` output (overrides UTC default)
+* `log_group_name_key`: use specified field of records as log group name
+* `log_stream_name_key`: use specified field of records as log stream name
+* `remove_log_group_name_key`: remove field specified by `log_group_name_key`
+* `remove_log_stream_name_key`: remove field specified by `log_stream_name_key`
 
 ### in_cloudwatch_logs
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.1.2"
+        VERSION = "0.2.0"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -13,8 +13,12 @@ module Fluent
     config_param :message_keys, :string, :default => nil
     config_param :max_message_length, :integer, :default => nil
     config_param :max_events_per_batch, :integer, :default => 10000
-    config_param :use_tag_as_group, :bool, :default => false
-    config_param :use_tag_as_stream, :bool, :default => false
+    config_param :use_tag_as_group, :bool, :default => false  # TODO: Rename to use_tag_as_group_name ?
+    config_param :use_tag_as_stream, :bool, :default => false # TODO: Rename to use_tag_as_stream_name ?
+    config_param :log_group_name_key, :string, :default => nil
+    config_param :log_stream_name_key, :string, :default => nil
+    config_param :remove_log_group_name_key, :bool, :default => false
+    config_param :remove_log_stream_name_key, :bool, :default => false
     config_param :http_proxy, :string, default: nil
 
     MAX_EVENTS_SIZE = 1_048_576
@@ -29,6 +33,18 @@ module Fluent
       require 'aws-sdk-core'
     end
 
+    def configure(conf)
+      super
+
+      unless [conf['log_group_name'], conf['use_tag_as_group'], conf['log_group_name_key']].compact.size == 1
+        raise ConfigError, "Set only one of log_group_name, use_tag_as_group and log_group_name_key"
+      end
+
+      unless [conf['log_stream_name'], conf['use_tag_as_stream'], conf['log_stream_name_key']].compact.size == 1
+        raise ConfigError, "Set only one of log_stream_name, use_tag_as_stream and log_stream_name_key"
+      end
+    end
+
     def start
       super
 
@@ -47,10 +63,35 @@ module Fluent
     def write(chunk)
       events = []
       chunk.enum_for(:msgpack_each).chunk {|tag, time, record|
-        tag
-      }.each {|tag, rs|
-        group_name = @use_tag_as_group ? tag : @log_group_name
-        stream_name = @use_tag_as_stream ? tag : @log_stream_name
+        group = case
+                when @use_tag_as_group
+                  tag
+                when @log_group_name_key
+                  if @remove_log_group_name_key
+                    record.delete(@log_group_name_key)
+                  else
+                    record[@log_group_name_key]
+                  end
+                else
+                  @log_group_name
+                end
+
+        stream = case
+                 when @use_tag_as_stream
+                   tag
+                 when @log_stream_name_key
+                   if @remove_log_stream_name_key
+                     record.delete(@log_stream_name_key)
+                   else
+                     record[@log_stream_name_key]
+                   end
+                 else
+                   @log_stream_name
+                 end
+
+        [group, stream]
+      }.each {|group_stream, rs|
+        group_name, stream_name = group_stream
 
         unless log_group_exists?(group_name)
           if @auto_create_stream

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -43,7 +43,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
     d.run
 
-    sleep 20
+    sleep 10
 
     events = get_log_events
     assert_equal(2, events.size)
@@ -61,7 +61,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'これは日本語です'.force_encoding('UTF-8')}, time.to_i)
     d.run
 
-    sleep 20
+    sleep 10
 
     events = get_log_events
     assert_equal(1, events.size)
@@ -79,7 +79,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
     d.run
 
-    sleep 20
+    sleep 10
 
     events = get_log_events
     assert_equal(3, events.size)
@@ -97,6 +97,8 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d = create_driver(<<-EOC)
     #{default_config}
     message_keys message,cloudwatch
+    log_group_name #{log_group_name}
+    log_stream_name #{log_stream_name}
     EOC
 
     time = Time.now
@@ -104,7 +106,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
     d.run
 
-    sleep 20
+    sleep 10
 
     events = get_log_events
     assert_equal(2, events.size)
@@ -121,6 +123,8 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     #{default_config}
     message_keys message,cloudwatch
     max_message_length 10
+    log_group_name #{log_group_name}
+    log_stream_name #{log_stream_name}
     EOC
 
     time = Time.now
@@ -128,7 +132,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
     d.run
 
-    sleep 20
+    sleep 10
 
     events = get_log_events
     assert_equal(2, events.size)
@@ -145,6 +149,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     #{default_config}
     message_keys message,cloudwatch
     use_tag_as_group true
+    log_stream_name #{log_stream_name}
     EOC
 
     time = Time.now
@@ -152,7 +157,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
     d.run
 
-    sleep 20
+    sleep 10
 
     events = get_log_events(fluentd_tag)
     assert_equal(2, events.size)
@@ -169,6 +174,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     #{default_config}
     message_keys message,cloudwatch
     use_tag_as_stream true
+    log_group_name #{log_group_name}
     EOC
 
     time = Time.now
@@ -176,7 +182,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'logs2', 'message' => 'message2'}, time.to_i + 1)
     d.run
 
-    sleep 20
+    sleep 10
 
     events = get_log_events(log_group_name, fluentd_tag)
     assert_equal(2, events.size)
@@ -192,6 +198,8 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d = create_driver(<<-EOC)
     #{default_config}
     include_time_key true
+    log_group_name #{log_group_name}
+    log_stream_name #{log_stream_name}
     EOC
 
     time = Time.now
@@ -199,7 +207,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
     d.run
 
-    sleep 20
+    sleep 10
 
     events = get_log_events
     assert_equal(2, events.size)
@@ -216,6 +224,8 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     #{default_config}
     include_time_key true
     localtime true
+    log_group_name #{log_group_name}
+    log_stream_name #{log_stream_name}
     EOC
 
     time = Time.now
@@ -223,7 +233,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
     d.run
 
-    sleep 20
+    sleep 10
 
     events = get_log_events
     assert_equal(2, events.size)
@@ -233,20 +243,71 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal("{\"cloudwatch\":\"logs2\",\"time\":\"#{(time+1).strftime("%Y-%m-%dT%H:%M:%S%:z")}\"}", events[1].message)
   end
 
+  def test_log_group_name_key_and_log_stream_name_key
+    new_log_stream
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    log_group_name_key group_name_key
+    log_stream_name_key stream_name_key
+    EOC
+
+    record = {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => log_stream_name}
+
+    time = Time.now
+    d.emit(record, time.to_i)
+    d.run
+
+    sleep 10
+
+    events = get_log_events(log_group_name, log_stream_name)
+    assert_equal(1, events.size)
+    assert_equal(time.to_i * 1000, events[0].timestamp)
+    assert_equal(record, JSON.parse(events[0].message))
+  end
+
+  def test_remove_log_group_name_key_and_remove_log_stream_name_key
+    new_log_stream
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    log_group_name_key group_name_key
+    log_stream_name_key stream_name_key
+    remove_log_group_name_key true
+    remove_log_stream_name_key true
+    EOC
+
+    time = Time.now
+    d.emit({'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => log_stream_name}, time.to_i)
+    d.run
+
+    sleep 10
+
+    events = get_log_events(log_group_name, log_stream_name)
+    assert_equal(1, events.size)
+    assert_equal(time.to_i * 1000, events[0].timestamp)
+    assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
+  end
+
   private
   def default_config
     <<-EOC
-    type cloudwatch_logs
-    log_group_name #{log_group_name}
-    log_stream_name #{log_stream_name}
-    auto_create_stream true
-    #{aws_key_id}
-    #{aws_sec_key}
-    #{region}
+type cloudwatch_logs
+auto_create_stream true
+#{aws_key_id}
+#{aws_sec_key}
+#{region}
     EOC
   end
 
-  def create_driver(conf = default_config)
+  def create_driver(conf = nil)
+    unless conf
+      conf = <<-EOC
+#{default_config}
+log_group_name #{log_group_name}
+log_stream_name #{log_stream_name}
+      EOC
+    end
     Fluent::Test::BufferedOutputTestDriver.new(Fluent::CloudwatchLogsOutput, fluentd_tag).configure(conf)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-23 00:00:00.000000000 Z
+date: 2016-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -121,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: CloudWatch Logs Plugin for Fluentd