fluent-plugin-cloudwatch-logs 0.4.4 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53b798f8ec98ca39cda1b3df0365abf99c396d64669f63ca95889e97ddbd2cb6
-  data.tar.gz: bd1ee98498a8b0e438f2a4b78edb0489840fc63b9455c21467aa06c9a2757da6
+  metadata.gz: 7f8fe8ec7ee187171451b47e824e14f3c475663bad90624b20c575b9ba7fa9eb
+  data.tar.gz: 9adc8de66f127eb20aa14830d88bd6cbc08a684ce8e4bec14543317dd26028df
 SHA512:
-  metadata.gz: 5538a8b50f47804f3b9792043c9b1b52128321778730fc39104046f80954510ba9459e6d54eddd9b73a8c76bc307857abd6b1d9eceba82817a2bf6f0dd5234ef
-  data.tar.gz: cafb782f9b949114813324b0ce1c6be2517242c440a20b90466a7f06c6ce1ff035d4d5973c591229090b2849a1b6ac901456102bff5f2ea1e9b5b80c269dbc10
+  metadata.gz: 8ea28c6a7ba48d50891185c767ef5a3317d64964f89d809c643f191826cd092a77d99a12f80d3aafb3031941af1e25e12ea79a0fee374195d9882f3e3fe6bea7
+  data.tar.gz: c11a426c35c25ab35ddd066141c4c18e7df7b0f88482d4efd9544877376554001a1b1440e2068c8794bdbe12c7232bb4e07e7d7956e6b047986a68da33dcca1d

data/README.md

@@ -65,7 +65,7 @@ Fetch sample log from CloudWatch Logs:
 
 ```
 <match tag>
-  type cloudwatch_logs
+  @type cloudwatch_logs
   log_group_name log-group-name
   log_stream_name log-stream-name
   auto_create_stream true
@@ -107,7 +107,7 @@ Fetch sample log from CloudWatch Logs:
 
 ```
 <source>
-  type cloudwatch_logs
+  @type cloudwatch_logs
   tag cloudwatch.in
   log_group_name group
   log_stream_name stream
@@ -235,7 +235,7 @@ In this case IAM can be used to allow the fluentd instance in one account ("A")
 ### Configuring the plugin for STS authentication
 ```
 <source>
-  type cloudwatch_logs
+  @type cloudwatch_logs
   region us-east-1      # You must supply a region
   aws_use_sts true
   aws_sts_role_arn arn:aws:iam::ACCOUNT-B:role/fluentd

data/example/fluentd.conf

@@ -1,9 +1,9 @@
 <source>
-  type forward
+  @type forward
 </source>
 
 <source>
-  type cloudwatch_logs
+  @type cloudwatch_logs
   tag test.cloudwatch_logs.in
   log_group_name fluent-plugin-cloudwatch-example
   log_stream_name fluent-plugin-cloudwatch-example
@@ -11,13 +11,13 @@
 </source>
 
 <match test.cloudwatch_logs.out>
-  type cloudwatch_logs
+  @type cloudwatch_logs
   log_group_name fluent-plugin-cloudwatch-example
   log_stream_name fluent-plugin-cloudwatch-example
   auto_create_stream true
 </match>
 
 <match test.cloudwatch_logs.in>
-  type stdout
+  @type stdout
 </match>
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.4.4"
+        VERSION = "0.4.5"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -12,6 +12,9 @@ module Fluent
 
     config_param :aws_key_id, :string, :default => nil, :secret => true
     config_param :aws_sec_key, :string, :default => nil, :secret => true
+    config_param :aws_use_sts, :bool, default: false
+    config_param :aws_sts_role_arn, :string, default: nil
+    config_param :aws_sts_session_name, :string, default: 'fluentd'
     config_param :region, :string, :default => nil
     config_param :log_group_name, :string, :default => nil
     config_param :log_stream_name, :string, :default => nil
@@ -30,6 +33,12 @@ module Fluent
     config_param :put_log_events_retry_limit, :integer, default: 17
     config_param :put_log_events_disable_retry_limit, :bool, default: false
     config_param :concurrency, :integer, default: 1
+    config_param :log_group_aws_tags, :hash, default: nil
+    config_param :log_group_aws_tags_key, :string, default: nil
+    config_param :remove_log_group_aws_tags_key, :bool, default: false
+    config_param :retention_in_days, :integer, default: nil
+    config_param :retention_in_days_key, :string, default: nil
+    config_param :remove_retention_in_days, :bool, default: false
 
     MAX_EVENTS_SIZE = 1_048_576
     MAX_EVENT_SIZE = 256 * 1024
@@ -59,14 +68,31 @@ module Fluent
       unless [conf['log_stream_name'], conf['use_tag_as_stream'], conf['log_stream_name_key']].compact.size == 1
         raise ConfigError, "Set only one of log_stream_name, use_tag_as_stream and log_stream_name_key"
       end
+
+      if [conf['log_group_aws_tags'], conf['log_group_aws_tags_key']].compact.size > 1
+        raise ConfigError, "Set only one of log_group_aws_tags, log_group_aws_tags_key"
+      end
+
+      if [conf['retention_in_days'], conf['retention_in_days_key']].compact.size > 1
+        raise ConfigError, "Set only one of retention_in_days, retention_in_days_key"
+      end
     end
 
     def start
       super
 
       options = {}
-      options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
       options[:region] = @region if @region
+
+      if @aws_use_sts
+        Aws.config[:region] = options[:region]
+        options[:credentials] = Aws::AssumeRoleCredentials.new(
+          role_arn: @aws_sts_role_arn,
+          role_session_name: @aws_sts_session_name
+        )
+      else
+        options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
+      end
       options[:http_proxy] = @http_proxy if @http_proxy
       @logs ||= Aws::CloudWatchLogs::Client.new(options)
       @sequence_tokens = {}
@@ -124,8 +150,31 @@ module Fluent
         end
 
         unless log_group_exists?(group_name)
+          #rs = [[name, timestamp, record],[name,timestamp,record]]
+          #get tags and retention from first record
+          #as we create log group only once, values from first record will persist
+          record = rs[0][2]
+
+          awstags = @log_group_aws_tags
+          unless @log_group_aws_tags_key.nil?
+            if @remove_log_group_aws_tags_key
+              awstags = record.delete(@log_group_aws_tags_key)
+            else
+              awstags = record[@log_group_aws_tags_key]
+            end
+          end
+
+          retention_in_days = @retention_in_days
+          unless @retention_in_days_key.nil?
+            if @remove_retention_in_days_key
+              retention_in_days = record.delete(@retention_in_days_key)
+            else
+              retention_in_days = record[@retention_in_days_key]
+            end
+          end
+
           if @auto_create_stream
-            create_log_group(group_name)
+            create_log_group(group_name, awstags, retention_in_days)
           else
             log.warn "Log group '#{group_name}' does not exist"
             next
@@ -200,7 +249,7 @@ module Fluent
     end
 
     def store_next_sequence_token(group_name, stream_name, token)
-      @store_next_sequence_token_mutex.synchronize do 
+      @store_next_sequence_token_mutex.synchronize do
         @sequence_tokens[group_name][stream_name] = token
       end
     end
@@ -255,6 +304,7 @@ module Fluent
         begin
           t = Time.now
           response = @logs.put_log_events(args)
+          log.warn response.rejected_log_events_info if response.rejected_log_events_info != nil
           log.debug "Called PutLogEvents API", {
             "group" => group_name,
             "stream" => stream_name,
@@ -314,15 +364,29 @@ module Fluent
       store_next_sequence_token(group_name, stream_name, response.next_sequence_token)
     end
 
-    def create_log_group(group_name)
+    def create_log_group(group_name, log_group_aws_tags = nil, retention_in_days = nil)
       begin
-        @logs.create_log_group(log_group_name: group_name)
+        @logs.create_log_group(log_group_name: group_name, tags: log_group_aws_tags)
+        unless retention_in_days.nil?
+          put_retention_policy(group_name, retention_in_days)
+        end
         @sequence_tokens[group_name] = {}
       rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
         log.debug "Log group '#{group_name}' already exists"
       end
     end
 
+    def put_retention_policy(group_name, retention_in_days)
+      begin
+        @logs.put_retention_policy({
+          log_group_name: group_name,
+          retention_in_days: retention_in_days
+        })
+      rescue Aws::CloudWatchLogs::Errors::InvalidParameterException => error
+        log.warn "failed to set retention policy for Log group '#{group_name}' with error #{error.backtrace}"
+      end
+    end
+
     def create_log_stream(group_name, stream_name)
       begin
         @logs.create_log_stream(log_group_name: group_name, log_stream_name: stream_name)

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -15,7 +15,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver(<<-EOC)
-      type cloudwatch_logs
+      @type cloudwatch_logs
       aws_key_id test_id
       aws_sec_key test_key
       region us-east-1
@@ -71,7 +71,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
 
     d = create_driver(<<-EOC)
       tag test
-      type cloudwatch_logs
+      @type cloudwatch_logs
       log_group_name #{log_group_name}
       log_stream_name #{log_stream_name}
       state_file /tmp/state
@@ -116,7 +116,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
 
     d = create_driver(<<-EOC)
       tag test
-      type cloudwatch_logs
+      @type cloudwatch_logs
       log_group_name #{log_group_name}
       log_stream_name testprefix
       use_log_stream_name_prefix true
@@ -141,7 +141,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
   def default_config
     <<-EOC
       tag test
-      type cloudwatch_logs
+      @type cloudwatch_logs
       log_group_name #{log_group_name}
       log_stream_name #{log_stream_name}
       state_file /tmp/state

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -17,13 +17,15 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver(<<-EOC)
-      type cloudwatch_logs
+      @type cloudwatch_logs
       aws_key_id test_id
       aws_sec_key test_key
       region us-east-1
       log_group_name test_group
       log_stream_name test_stream
       auto_create_stream false
+      log_group_aws_tags { "tagkey": "tagvalue", "tagkey_2": "tagvalue_2"}
+      retention_in_days 5
     EOC
 
     assert_equal('test_id', d.instance.aws_key_id)
@@ -32,6 +34,9 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal('test_group', d.instance.log_group_name)
     assert_equal('test_stream', d.instance.log_stream_name)
     assert_equal(false, d.instance.auto_create_stream)
+    assert_equal("tagvalue", d.instance.log_group_aws_tags.fetch("tagkey"))
+    assert_equal("tagvalue_2", d.instance.log_group_aws_tags.fetch("tagkey_2"))
+    assert_equal(5, d.instance.retention_in_days)
   end
 
   def test_write
@@ -52,7 +57,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
     assert_equal('{"cloudwatch":"logs2"}', events[1].message)
 
-    assert_match(/Calling PutLogEvents API/, d.instance.log.logs[0])
+    assert_match(/Called PutLogEvents API/, d.instance.log.logs[0])
   end
 
   def test_write_utf8
@@ -270,7 +275,7 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.run
 
     # Call API once for each stream
-    assert_equal(2, d.instance.log.logs.select {|l| l =~ /Calling PutLogEvents API/ }.size)
+    assert_equal(2, d.instance.log.logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
 
     sleep 10
 
@@ -310,6 +315,188 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
   end
 
+  def test_log_group_aws_tags
+    clear_log_group
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    auto_create_stream true
+    use_tag_as_stream true
+    log_group_name_key group_name_key
+    log_group_aws_tags {"tag1": "value1", "tag2": "value2"}
+    EOC
+
+    records = [
+      {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name},
+      {'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name},
+      {'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name},
+    ]
+
+    time = Time.now
+    records.each_with_index do |record, i|
+      d.emit(record, time.to_i + i)
+    end
+    d.run
+
+    awstags = get_log_group_tags
+    assert_equal("value1", awstags.fetch("tag1"))
+    assert_equal("value2", awstags.fetch("tag2"))
+  end
+
+  def test_retention_in_days
+    clear_log_group
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    auto_create_stream true
+    use_tag_as_stream true
+    log_group_name_key group_name_key
+    retention_in_days 7
+    EOC
+
+    records = [
+      {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name},
+      {'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name},
+      {'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name},
+    ]
+
+    time = Time.now
+    records.each_with_index do |record, i|
+      d.emit(record, time.to_i + i)
+    end
+    d.run
+
+    retention = get_log_group_retention_days
+    assert_equal(d.instance.retention_in_days, retention)
+  end
+
+  def test_invalid_retention_in_days
+    clear_log_group
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    auto_create_stream true
+    use_tag_as_stream true
+    log_group_name_key group_name_key
+    retention_in_days 4
+    EOC
+
+    records = [
+      {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name},
+      {'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name},
+      {'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name},
+    ]
+
+    time = Time.now
+    records.each_with_index do |record, i|
+      d.emit(record, time.to_i + i)
+    end
+    d.run
+
+    assert_match(/failed to set retention policy for Log group/, d.instance.log.logs[0])
+  end
+
+  def test_log_group_aws_tags_key
+    clear_log_group
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    auto_create_stream true
+    use_tag_as_stream true
+    log_group_name_key group_name_key
+    log_group_aws_tags_key aws_tags
+    EOC
+
+    records = [
+      {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag1" => "value1", "tag2" => "value2"}},
+      {'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag1" => "value1", "tag2" => "value2"}},
+      {'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag1" => "value1", "tag2" => "value2"}}
+    ]
+
+    time = Time.now
+    records.each_with_index do |record, i|
+      d.emit(record, time.to_i + i)
+    end
+    d.run
+
+    awstags = get_log_group_tags
+    assert_equal("value1", awstags.fetch("tag1"))
+    assert_equal("value2", awstags.fetch("tag2"))
+  end
+
+  def test_log_group_aws_tags_key_same_group_diff_tags
+    clear_log_group
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    auto_create_stream true
+    use_tag_as_stream true
+    log_group_name_key group_name_key
+    log_group_aws_tags_key aws_tags
+    EOC
+
+    records = [
+      {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag1" => "value1", "tag2" => "value2"}},
+      {'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, 'aws_tags' => {"tag3" => "value3", "tag4" => "value4"}}
+    ]
+
+    time = Time.now
+    records.each_with_index do |record, i|
+      d.emit(record, time.to_i + i)
+    end
+    d.run
+
+    awstags = get_log_group_tags
+    assert_equal("value1", awstags.fetch("tag1"))
+    assert_equal("value2", awstags.fetch("tag2"))
+    assert_raise KeyError do
+      awstags.fetch("tag3")
+    end
+    assert_raise KeyError do
+      awstags.fetch("tag4")
+    end
+  end
+
+  def test_log_group_aws_tags_key_no_tags
+    clear_log_group
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    auto_create_stream true
+    log_group_name_key group_name_key
+    log_stream_name_key stream_name_key
+    remove_log_group_name_key true
+    remove_log_stream_name_key true
+    log_group_aws_tags_key aws_tags
+    EOC
+
+    stream = log_stream_name
+    records = [
+      {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, 'stream_name_key' => stream},
+      {'cloudwatch' => 'logs2', 'message' => 'message2', 'group_name_key' => log_group_name, 'stream_name_key' => stream}
+    ]
+
+    time = Time.now
+    records.each_with_index do |record, i|
+      d.emit(record, time.to_i + i)
+    end
+    d.run
+
+    sleep 10
+
+    awstags = get_log_group_tags
+
+    assert_raise KeyError do
+      awstags.fetch("tag1")
+    end
+
+    events = get_log_events(log_group_name, stream)
+    assert_equal(2, events.size)
+    assert_equal(time.to_i * 1000, events[0].timestamp)
+    assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
+    assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
+  end
+
   def test_retrying_on_throttling_exception
     resp = mock()
     resp.expects(:next_sequence_token)
@@ -323,10 +510,9 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     d.emit({'message' => 'message1'}, time.to_i)
     d.run
 
-    assert_match(/Calling PutLogEvents/, d.instance.log.logs[0])
-    assert_match(/failed to PutLogEvents/, d.instance.log.logs[1])
-    assert_match(/Calling PutLogEvents/, d.instance.log.logs[2])
-    assert_match(/retry succeeded/, d.instance.log.logs[3])
+    assert_match(/failed to PutLogEvents/, d.instance.log.logs[0])
+    assert_match(/Called PutLogEvents/, d.instance.log.logs[1])
+    assert_match(/retry succeeded/, d.instance.log.logs[2])
   end
 
   def test_retrying_on_throttling_exception_and_throw_away
@@ -345,12 +531,9 @@ put_log_events_retry_limit 1
     d.emit({'message' => 'message1'}, time.to_i)
     d.run
 
-    assert_match(/Calling PutLogEvents/, d.instance.log.logs[0])
+    assert_match(/failed to PutLogEvents/, d.instance.log.logs[0])
     assert_match(/failed to PutLogEvents/, d.instance.log.logs[1])
-    assert_match(/Calling PutLogEvents/, d.instance.log.logs[2])
-    assert_match(/failed to PutLogEvents/, d.instance.log.logs[3])
-    assert_match(/Calling PutLogEvents/, d.instance.log.logs[4])
-    assert_match(/failed to PutLogEvents and throwing away/, d.instance.log.logs[5])
+    assert_match(/failed to PutLogEvents and discard logs/, d.instance.log.logs[2])
   end
 
   def test_too_large_event
@@ -388,7 +571,7 @@ log_stream_name #{log_stream_name}
   private
   def default_config
     <<-EOC
-type cloudwatch_logs
+@type cloudwatch_logs
 auto_create_stream true
 #{aws_key_id}
 #{aws_sec_key}

data/test/test_helper.rb

@@ -43,6 +43,16 @@ module CloudwatchLogsTestHelper
     @log_stream_name = log_stream_name_prefix ? log_stream_name_prefix + uuid : uuid
   end
 
+  def get_log_group_tags(name = nil)
+    name ||= log_group_name
+    logs.list_tags_log_group(log_group_name: name).tags
+  end
+
+  def get_log_group_retention_days(name = nil)
+    name ||= log_group_name
+    logs.describe_log_groups(log_group_name_prefix: name, limit: 1).log_groups.first.retention_in_days
+  end
+
   def clear_log_group
     [log_group_name, fluentd_tag].each do |name|
       begin

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.5
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-22 00:00:00.000000000 Z
+date: 2018-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd