fluent-plugin-cloudwatch-logs 0.10.0 → 0.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 105e96fa516c4f972fc2b01f83c1daa30d39ff4b827b46b811b9f30514231516
-  data.tar.gz: 85741a1b0ccad0f9e207c837e7417b786ef68d1fc1552cbe80b73030e5b4191b
+  metadata.gz: e50538afdcec35527089c0d1767dff70ddc7f2b5873523d596f78a9e7d344a88
+  data.tar.gz: c8da30c9f0286bca9dca9aab693224f2165f7fdced0ceb8de07db8fb8a259b3a
 SHA512:
-  metadata.gz: 84a8458e9704102609b382b9cb8fe4c332a1c1b9c9562b89b959961bb3624958cbffc1f965eba2a502c10a0e45f1ceb427a987681d5420c35663bc6df88295ed
-  data.tar.gz: d6e22ad22c4eef5cb6c8fab7690ca75585e7ad99756926181f50c43fef70df0e52e12722e14809b793947e56ed8199fc393a318f42308f05fd6d23d6aa9a4b13
+  metadata.gz: 4b623676f4b5baec31da710b5446a189d58f063a7fe2f1f3fe721d54a32bcb619c16b45416167a24a61d68f416b2202d13451f900e7038e084600924330ef9ed
+  data.tar.gz: 1519461663014d5234ec69088a1b14a4dd345006c4ec8eebdefa87612a15563a6f65c8effacfee5edd7d0c8672765769d5e08d35d761388ac2fde0ac9669a823

data/.github/workflows/issue-auto-closer.yml

@@ -0,0 +1,12 @@
+name: Autocloser
+on: [issues]
+jobs:
+  autoclose:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Autoclose issues that did not follow issue template
+      uses: roots/issue-closer-action@v1.1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
+        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and CloudWatchLogs plugin versions.*)"

data/README.md

@@ -43,6 +43,46 @@ Create IAM user with a policy like the following:
 }
 ```
 
+More restricted IAM policy for `out_cloudwatch_logs` is:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "logs:PutLogEvents",
+                "logs:CreateLogGroup",
+                "logs:PutRetentionPolicy",
+                "logs:CreateLogStream",
+                "logs:DescribeLogGroups",
+                "logs:DescribeLogStreams"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        }
+    ]
+}
+```
+
+Also, more restricted IAM policy for `in_cloudwatch_logs` is:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "logs:GetLogEvents",
+                "logs:DescribeLogStreams"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        }
+    ]
+}
+```
+
 ## Authentication
 
 There are several methods to provide authentication credentials.  Be aware that there are various tradeoffs for these methods,
@@ -120,6 +160,11 @@ Fetch sample log from CloudWatch Logs:
   #endpoint http://localhost:5000/
   #json_handler json
   #log_rejected_request true
+  #<web_identity_credentials>
+  #  role_arn          "#{ENV['AWS_ROLE_ARN']}"
+  #  role_session_name ROLE_SESSION_NAME
+  #  web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
+  #</web_identity_credentials>
 </match>
 ```
 
@@ -154,6 +199,14 @@ Fetch sample log from CloudWatch Logs:
 * `retention_in_days_key`: use specified field of records as retention period
 * `use_tag_as_group`: to use tag as a group name
 * `use_tag_as_stream`: to use tag as a stream name
+* `<web_identity_credentials>`: For EKS authentication.
+  * `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
+  * `role_session_name`: An identifier for the assumed role session.  This parameter is required when using `<web_identity_credentials>`.
+  * `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
+  * `policy`: An IAM policy in JSON format. (default `nil`)
+  * `duration_seconds`: The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
+is set to 3600 seconds (1 hour). (default `nil`)
 
 **NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
 Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
@@ -181,6 +234,11 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   #<storage>
   # @type local # or redis, memcached, etc.
   #</storage>
+  #<web_identity_credentials>
+  #  role_arn          "#{ENV['AWS_ROLE_ARN']}"
+  #  role_session_name ROLE_SESSION_NAME
+  #  web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
+  #</web_identity_credentials>
 </source>
 ```
 
@@ -209,6 +267,14 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `format`: specify CloudWatchLogs' log format. (default `nil`)
 * `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
 * `<storage>`: specify storage plugin configuration. see also: https://docs.fluentd.org/v/1.0/storage#how-to-use
+* `<web_identity_credentials>`: For EKS authentication.
+  * `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
+  * `role_session_name`: An identifier for the assumed role session.  This parameter is required when using `<web_identity_credentials>`.
+  * `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
+  * `policy`: An IAM policy in JSON format. (default `nil`)
+  * `duration_seconds`: The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
+is set to 3600 seconds (1 hour). (default `nil`)
 
 ## Test
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.10.0"
+        VERSION = "0.11.2"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -17,6 +17,7 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, default: nil
     config_param :endpoint, :string, default: nil
     config_param :tag, :string
@@ -35,6 +36,13 @@ module Fluent::Plugin
     config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
     config_param :throttling_retry_seconds, :time, default: nil
     config_param :include_metadata, :bool, default: false
+    config_section :web_identity_credentials, multi: false do
+      config_param :role_arn, :string
+      config_param :role_session_name, :string
+      config_param :web_identity_token_file, :string, default: nil #required
+      config_param :policy, :string, default: nil
+      config_param :duration_seconds, :time, default: nil
+    end
 
     config_section :parse do
       config_set_default :@type, 'none'
@@ -75,10 +83,29 @@ module Fluent::Plugin
 
       if @aws_use_sts
         Aws.config[:region] = options[:region]
-        options[:credentials] = Aws::AssumeRoleCredentials.new(
+        credentials_options = {
           role_arn: @aws_sts_role_arn,
           role_session_name: @aws_sts_session_name
-        )
+        }
+        credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
+        if @region and @aws_sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
+        elsif @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+      elsif @web_identity_credentials
+        c = @web_identity_credentials
+        credentials_options = {}
+        credentials_options[:role_arn] = c.role_arn
+        credentials_options[:role_session_name] = c.role_session_name
+        credentials_options[:web_identity_token_file] = c.web_identity_token_file
+        credentials_options[:policy] = c.policy if c.policy
+        credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+        if @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
       else
         options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
       end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -7,6 +7,8 @@ module Fluent::Plugin
   class CloudwatchLogsOutput < Output
     Fluent::Plugin.register_output('cloudwatch_logs', self)
 
+    class TooLargeEventError < Fluent::UnrecoverableError; end
+
     helpers :compat_parameters, :inject
 
     DEFAULT_BUFFER_TYPE = "memory"
@@ -17,6 +19,7 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, :default => nil
     config_param :endpoint, :string, :default => nil
     config_param :log_group_name, :string, :default => nil
@@ -44,6 +47,13 @@ module Fluent::Plugin
     config_param :remove_retention_in_days_key, :bool, default: false
     config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
     config_param :log_rejected_request, :bool, :default => false
+    config_section :web_identity_credentials, multi: false do
+      config_param :role_arn, :string
+      config_param :role_session_name, :string
+      config_param :web_identity_token_file, :string, default: nil #required
+      config_param :policy, :string, default: nil
+      config_param :duration_seconds, :time, default: nil
+    end
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -92,10 +102,29 @@ module Fluent::Plugin
 
       if @aws_use_sts
         Aws.config[:region] = options[:region]
-        options[:credentials] = Aws::AssumeRoleCredentials.new(
+        credentials_options = {
           role_arn: @aws_sts_role_arn,
           role_session_name: @aws_sts_session_name
-        )
+        }
+        credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
+        if @region and @aws_sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
+        elsif @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+      elsif @web_identity_credentials
+        c = @web_identity_credentials
+        credentials_options = {}
+        credentials_options[:role_arn] = c.role_arn
+        credentials_options[:role_session_name] = c.role_session_name
+        credentials_options[:web_identity_token_file] = c.web_identity_token_file
+        credentials_options[:policy] = c.policy if c.policy
+        credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+        if @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
       else
         options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
       end
@@ -130,6 +159,9 @@ module Fluent::Plugin
     def write(chunk)
       log_group_name = extract_placeholders(@log_group_name, chunk) if @log_group_name
       log_stream_name = extract_placeholders(@log_stream_name, chunk) if @log_stream_name
+      aws_tags = @log_group_aws_tags.each {|k, v|
+        @log_group_aws_tags[extract_placeholders(k, chunk)] = extract_placeholders(v, chunk)
+      } if @log_group_aws_tags
 
       queue = Thread::Queue.new
 
@@ -182,7 +214,7 @@ module Fluent::Plugin
           #as we create log group only once, values from first record will persist
           record = rs[0][2]
 
-          awstags = @log_group_aws_tags
+          awstags = aws_tags
           unless @log_group_aws_tags_key.nil?
             if @remove_log_group_aws_tags_key
               awstags = record.delete(@log_group_aws_tags_key)
@@ -319,8 +351,7 @@ module Fluent::Plugin
       while event = events.shift
         event_bytesize = event[:message].bytesize + EVENT_HEADER_SIZE
         if MAX_EVENT_SIZE < event_bytesize
-          log.warn "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
-          break
+          raise TooLargeEventError, "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
         end
 
         new_chunk = chunk + [event]
@@ -399,7 +430,13 @@ module Fluent::Plugin
             raise err
           end
         rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
-          if !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
+          if @put_log_events_retry_limit < 1
+            log.warn "failed to PutLogEvents and discard logs because put_log_events_retry_limit is less than 1", {
+              "error_class" => err.class.to_s,
+              "error" => err.message,
+            }
+            return
+          elsif !@put_log_events_disable_retry_limit && @put_log_events_retry_limit < retry_count
             log.error "failed to PutLogEvents and discard logs because retry count exceeded put_log_events_retry_limit", {
               "error_class" => err.class.to_s,
               "error" => err.message,

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -630,6 +630,8 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
     end
 
     test "emit with today's log stream" do
+      omit "This testcase is unstable in CI." if ENV["CI"] == "true"
+
       config = <<-CONFIG
         tag test
         @type cloudwatch_logs

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -492,6 +492,47 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert_equal("value2", awstags.fetch("tag2"))
     end
 
+    def test_log_group_aws_tags_with_placeholders
+      clear_log_group
+
+      config = {
+        "@type" => "cloudwatch_logs",
+        "auto_create_stream" => true,
+        "use_tag_as_stream" => true,
+        "log_group_name_key" => "group_name_key",
+        "log_group_aws_tags" => '{"tag1": "${tag}", "tag2": "${namespace_name}"}',
+      }
+      config.merge!(config_elementify(aws_key_id)) if aws_key_id
+      config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+      config.merge!(config_elementify(region)) if region
+      config.merge!(config_elementify(endpoint)) if endpoint
+
+      d = create_driver(
+        Fluent::Config::Element.new('ROOT', '', config, [
+          Fluent::Config::Element.new('buffer', 'tag, namespace_name', {
+            '@type' => 'memory',
+          }, [])
+        ])
+      )
+
+      records = [
+        {'cloudwatch' => 'logs1', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
+        {'cloudwatch' => 'logs2', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
+        {'cloudwatch' => 'logs3', 'message' => 'message1', 'group_name_key' => log_group_name, "namespace_name" => "fluentd"},
+      ]
+
+      time = Time.now
+      d.run(default_tag: fluentd_tag) do
+        records.each_with_index do |record, i|
+          d.feed(time.to_i + i, record)
+        end
+      end
+
+      awstags = get_log_group_tags
+      assert_equal(fluentd_tag, awstags.fetch("tag1"))
+      assert_equal("fluentd", awstags.fetch("tag2"))
+    end
+
     def test_retention_in_days
       clear_log_group
 
@@ -713,6 +754,32 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
     end
 
+    def test_retrying_on_throttling_exception_with_put_log_events_retry_limit_as_zero
+      client = Aws::CloudWatchLogs::Client.new
+      @called = false
+      stub(client).put_log_events(anything) {
+        raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error"))
+      }.once.ordered
+
+      d = create_driver(<<-EOC)
+        #{default_config}
+        log_group_name #{log_group_name}
+        log_stream_name #{log_stream_name}
+        @log_level debug
+        put_log_events_retry_limit 0
+      EOC
+      time = event_time
+      d.instance.instance_variable_set(:@logs, client)
+      d.run(default_tag: fluentd_tag) do
+        d.feed(time, {'message' => 'message1'})
+      end
+
+      logs = d.logs
+      assert_equal(0, logs.select {|l| l =~ /Called PutLogEvents API/ }.size)
+      assert_equal(1, logs.select {|l| l =~ /failed to PutLogEvents/ }.size)
+      assert_equal(0, logs.select {|l| l =~ /retry succeeded/ }.size)
+    end
+
     def test_retrying_on_throttling_exception
       resp = Object.new
       mock(resp).rejected_log_events_info {}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.2
 platform: ruby
 authors:
 - Ryota Arai
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-23 00:00:00.000000000 Z
+date: 2020-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -108,13 +108,14 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - ryota.arai@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/issue-auto-closer.yml"
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
@@ -135,7 +136,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -150,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: CloudWatch Logs Plugin for Fluentd
 test_files: