fluent-plugin-cloudwatch-logs 0.10.2 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a66a32521964e4be0392f1d786e81750850b4f027b5ec9649a9033ab330595a
-  data.tar.gz: 6041e92eea6e3148133a78a72ad3d9702ff33f89da89216e4bff579693459725
+  metadata.gz: 17e367fb1fc1c2b58cd9a22cd9caefaf6396b3542f589b44616f160c40c3219d
+  data.tar.gz: 37eee2b6a48d17d86451a8eee88435fc1ddd572248b3ed181bf3d833d97a3d96
 SHA512:
-  metadata.gz: e8c2a9720f9e309698c1ac04e51bcb1acbe141443997dde1e9daf636cc9ce5356d3f87f7fb9ea18b7c9c760e1e9a57039e9d7aafb1bbcf98d2cbe3f657e89483
-  data.tar.gz: bf1d58dd34328aedb4d18529b9f172c05b442a3040a332ee0f5d4edfd1e76b50b2df72f2edeb195f44a77ea45628627e3f4413847203581c9610bc5cee69fb75
+  metadata.gz: 30807c60113fe1eed915092e005e83863a4cfc5911a859eb00925299a1a4e45dc0dd581006d0486d3ab176ed2babf7b1fb5e90f0b41e04838c855a578db31faf
+  data.tar.gz: a44051e6aa3f1010dd5ee17f1046dfccabd59a929ed5b8976eeda135410f681400f1f024c61a84992cc28500285148e598aad7aa2daab10a965aca0204398a28

data/README.md

@@ -160,6 +160,14 @@ Fetch sample log from CloudWatch Logs:
   #endpoint http://localhost:5000/
   #json_handler json
   #log_rejected_request true
+  #<web_identity_credentials>
+  #  role_arn          "#{ENV['AWS_ROLE_ARN']}"
+  #  role_session_name ROLE_SESSION_NAME
+  #  web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
+  #</web_identity_credentials>
+  #<format>
+  #  @type ltsv
+  #</format>
 </match>
 ```
 
@@ -194,6 +202,15 @@ Fetch sample log from CloudWatch Logs:
 * `retention_in_days_key`: use specified field of records as retention period
 * `use_tag_as_group`: to use tag as a group name
 * `use_tag_as_stream`: to use tag as a stream name
+* `<web_identity_credentials>`: For EKS authentication.
+  * `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
+  * `role_session_name`: An identifier for the assumed role session.  This parameter is required when using `<web_identity_credentials>`.
+  * `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
+  * `policy`: An IAM policy in JSON format. (default `nil`)
+  * `duration_seconds`: The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
+is set to 3600 seconds (1 hour). (default `nil`)
+* `<format>`: For specifying records format. See [formatter overview](https://docs.fluentd.org/formatter) and [formatter section overview](https://docs.fluentd.org/configuration/format-section) on the official documentation.
 
 **NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
 Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
@@ -205,6 +222,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   @type cloudwatch_logs
   tag cloudwatch.in
   log_group_name group
+  #add_log_group_name true
+  #log_group_name_key group_name_key
+  #use_log_group_name_prefix true
   log_stream_name stream
   #use_log_stream_name_prefix true
   state_file /var/lib/fluent/group_stream.in.state
@@ -221,6 +241,11 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   #<storage>
   # @type local # or redis, memcached, etc.
   #</storage>
+  #<web_identity_credentials>
+  #  role_arn          "#{ENV['AWS_ROLE_ARN']}"
+  #  role_session_name ROLE_SESSION_NAME
+  #  web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
+  #</web_identity_credentials>
 </source>
 ```
 
@@ -234,6 +259,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `http_proxy`: use to set an optional HTTP proxy
 * `json_handler`:  name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
 * `log_group_name`: name of log group to fetch logs
+* `add_log_group_name`: add record into the name of log group (default `false`)
+* `log_group_name_key`: specify the key where adding record into the name of log group (default `'log_group'`)
+* `use_log_group_name_prefix`: to use `log_group_name` as log group name prefix (default `false`)
 * `log_stream_name`: name of log stream to fetch logs
 * `region`: AWS Region.  See [Authentication](#authentication) for more information.
 * `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
@@ -249,6 +277,14 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `format`: specify CloudWatchLogs' log format. (default `nil`)
 * `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
 * `<storage>`: specify storage plugin configuration. see also: https://docs.fluentd.org/v/1.0/storage#how-to-use
+* `<web_identity_credentials>`: For EKS authentication.
+  * `role_arn`: The Amazon Resource Name (ARN) of the role to assume. This parameter is required when using `<web_identity_credentials>`.
+  * `role_session_name`: An identifier for the assumed role session.  This parameter is required when using `<web_identity_credentials>`.
+  * `web_identity_token_file`: The absolute path to the file on disk containing the OIDC token. This parameter is required when using `<web_identity_credentials>`.
+  * `policy`: An IAM policy in JSON format. (default `nil`)
+  * `duration_seconds`: The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
+is set to 3600 seconds (1 hour). (default `nil`)
 
 ## Test
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.10.2"
+        VERSION = "0.13.0"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -17,10 +17,14 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, default: nil
     config_param :endpoint, :string, default: nil
     config_param :tag, :string
     config_param :log_group_name, :string
+    config_param :add_log_group_name, :bool, default: false
+    config_param :log_group_name_key, :string, default: 'log_group'
+    config_param :use_log_group_name_prefix, :bool, default: false
     config_param :log_stream_name, :string, default: nil
     config_param :use_log_stream_name_prefix, :bool, default: false
     config_param :state_file, :string, default: nil,
@@ -35,6 +39,13 @@ module Fluent::Plugin
     config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
     config_param :throttling_retry_seconds, :time, default: nil
     config_param :include_metadata, :bool, default: false
+    config_section :web_identity_credentials, multi: false do
+      config_param :role_arn, :string
+      config_param :role_session_name, :string
+      config_param :web_identity_token_file, :string, default: nil #required
+      config_param :policy, :string, default: nil
+      config_param :duration_seconds, :time, default: nil
+    end
 
     config_section :parse do
       config_set_default :@type, 'none'
@@ -75,10 +86,29 @@ module Fluent::Plugin
 
       if @aws_use_sts
         Aws.config[:region] = options[:region]
-        options[:credentials] = Aws::AssumeRoleCredentials.new(
+        credentials_options = {
           role_arn: @aws_sts_role_arn,
           role_session_name: @aws_sts_session_name
-        )
+        }
+        credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
+        if @region and @aws_sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
+        elsif @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+      elsif @web_identity_credentials
+        c = @web_identity_credentials
+        credentials_options = {}
+        credentials_options[:role_arn] = c.role_arn
+        credentials_options[:role_session_name] = c.role_session_name
+        credentials_options[:web_identity_token_file] = c.web_identity_token_file
+        credentials_options[:policy] = c.policy if c.policy
+        credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+        if @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
       else
         options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
       end
@@ -101,6 +131,17 @@ module Fluent::Plugin
       super
     end
 
+    # No private for testing
+    def state_key_for(log_stream_name, log_group_name = nil)
+      if log_group_name && log_stream_name
+        "#{@state_file}_#{log_group_name.gsub(File::SEPARATOR, '-')}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
+      elsif log_stream_name
+        "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
+      else
+        @state_file
+      end
+    end
+
     private
     def configure_parser(conf)
       if conf['format']
@@ -110,28 +151,20 @@ module Fluent::Plugin
       end
     end
 
-    def state_key_for(log_stream_name)
-      if log_stream_name
-        "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
-      else
-        @state_file
-      end
-    end
-
     def migrate_state_file_to_storage(log_stream_name)
       @next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
       File.delete(state_key_for(log_stream_name))
     end
 
-    def next_token(log_stream_name)
+    def next_token(log_stream_name, log_group_name = nil)
       if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
         migrate_state_file_to_storage(log_stream_name)
       end
-      @next_token_storage.get(:"#{state_key_for(log_stream_name)}")
+      @next_token_storage.get(:"#{state_key_for(log_stream_name, log_group_name)}")
     end
 
-    def store_next_token(token, log_stream_name = nil)
-      @next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
+    def store_next_token(token, log_stream_name = nil, log_group_name = nil)
+      @next_token_storage.put(:"#{state_key_for(log_stream_name, log_group_name)}", token)
     end
 
     def run
@@ -141,55 +174,67 @@ module Fluent::Plugin
         if Time.now > @next_fetch_time
           @next_fetch_time += @fetch_interval
 
-          if @use_log_stream_name_prefix || @use_todays_log_stream
-            log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
-            begin
-              log_streams = describe_log_streams(log_stream_name_prefix)
-              log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
-              log_streams.each do |log_stream|
-                log_stream_name = log_stream.log_stream_name
-                events = get_events(log_stream_name)
-                metadata = if @include_metadata
-                             {
-                               "log_stream_name" => log_stream_name,
-                               "log_group_name" => @log_group_name
-                             }
-                           else
-                             {}
-                           end
-                events.each do |event|
-                  emit(log_stream_name, event, metadata)
+          if @use_log_group_name_prefix
+            log_group_names = describe_log_groups(@log_group_name).map{|log_group|
+              log_group.log_group_name
+            }
+          else
+            log_group_names = [@log_group_name]
+          end
+          log_group_names.each do |log_group_name|
+            if @use_log_stream_name_prefix || @use_todays_log_stream
+              log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
+              begin
+                log_streams = describe_log_streams(log_stream_name_prefix)
+                log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
+                log_streams.each do |log_stream|
+                  log_stream_name = log_stream.log_stream_name
+                  events = get_events(log_group_name, log_stream_name)
+                  metadata = if @include_metadata
+                               {
+                                 "log_stream_name" => log_stream_name,
+                                 "log_group_name" => @log_group_name
+                               }
+                             else
+                               {}
+                             end
+                  events.each do |event|
+                    emit(log_group_name, log_stream_name, event, metadata)
+                  end
                 end
+              rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+                log.warn "'#{@log_stream_name}' prefixed log stream(s) are not found"
+                next
+              end
+            else
+              events = get_events(log_group_name, @log_stream_name)
+              metadata = if @include_metadata
+                           {
+                             "log_stream_name" => @log_stream_name,
+                             "log_group_name" => @log_group_name
+                           }
+                         else
+                           {}
+                         end
+              events.each do |event|
+                emit(log_group_name, log_stream_name, event, metadata)
               end
-            rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
-              log.warn "'#{@log_stream_name}' prefixed log stream(s) are not found"
-              next
-            end
-          else
-            events = get_events(@log_stream_name)
-            metadata = if @include_metadata
-                          {
-                            "log_stream_name" => @log_stream_name,
-                            "log_group_name" => @log_group_name
-                          }
-                        else
-                          {}
-                        end
-            events.each do |event|
-              emit(log_stream_name, event, metadata)
             end
           end
-        end
         sleep 1
+        end
       end
     end
 
-    def emit(stream, event, metadata)
+    def emit(group, stream, event, metadata)
       if @parser
         @parser.parse(event.message) {|time,record|
           if @use_aws_timestamp
             time = (event.timestamp / 1000).floor
           end
+          if @add_log_group_name
+            record[@log_group_name_key] = group
+          end
           unless metadata.empty?
             record.merge!("metadata" => metadata)
           end
@@ -199,6 +244,9 @@ module Fluent::Plugin
         time = (event.timestamp / 1000).floor
         begin
           record = @json_handler.load(event.message)
+          if @add_log_group_name
+            record[@log_group_name_key] = group
+          end
           unless metadata.empty?
             record.merge!("metadata" => metadata)
           end
@@ -210,19 +258,23 @@ module Fluent::Plugin
       end
     end
 
-    def get_events(log_stream_name)
+    def get_events(log_group_name, log_stream_name)
       throttling_handler('get_log_events') do
         request = {
-          log_group_name: @log_group_name,
+          log_group_name: log_group_name,
           log_stream_name: log_stream_name
         }
         request.merge!(start_time: @start_time) if @start_time
         request.merge!(end_time: @end_time) if @end_time
-        log_next_token = next_token(log_stream_name)
+        log_next_token = next_token(log_group_name, log_stream_name)
         request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
         response = @logs.get_log_events(request)
         if valid_next_token(log_next_token, response.next_forward_token)
-          store_next_token(response.next_forward_token, log_stream_name)
+          if @use_log_group_name_prefix
+            store_next_token(response.next_forward_token, log_stream_name, log_group_name)
+          else
+            store_next_token(response.next_forward_token, log_stream_name)
+          end
         end
 
         response.events
@@ -262,6 +314,23 @@ module Fluent::Plugin
       end
     end
 
+    def describe_log_groups(log_group_name_prefix, log_groups = nil, next_token = nil)
+      request = {
+        log_group_name_prefix: log_group_name_prefix
+      }
+      request[:next_token] = next_token if next_token
+      response = @logs.describe_log_groups(request)
+      if log_groups
+        log_groups.concat(response.log_groups)
+      else
+        log_groups = response.log_groups
+      end
+      if response.next_token
+        log_groups = describe_log_groups(log_group_name_prefix, log_groups, response.next_token)
+      end
+      log_groups
+    end
+
     def valid_next_token(prev_token, next_token)
       next_token && prev_token != next_token.chomp
     end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -9,7 +9,7 @@ module Fluent::Plugin
 
     class TooLargeEventError < Fluent::UnrecoverableError; end
 
-    helpers :compat_parameters, :inject
+    helpers :compat_parameters, :inject, :formatter
 
     DEFAULT_BUFFER_TYPE = "memory"
 
@@ -19,6 +19,7 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, :default => nil
     config_param :endpoint, :string, :default => nil
     config_param :log_group_name, :string, :default => nil
@@ -46,10 +47,20 @@ module Fluent::Plugin
     config_param :remove_retention_in_days_key, :bool, default: false
     config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
     config_param :log_rejected_request, :bool, :default => false
+    config_section :web_identity_credentials, multi: false do
+      config_param :role_arn, :string
+      config_param :role_session_name, :string
+      config_param :web_identity_token_file, :string, default: nil #required
+      config_param :policy, :string, default: nil
+      config_param :duration_seconds, :time, default: nil
+    end
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
     end
+    config_section :format do
+      config_set_default :@type, 'json'
+    end
 
     MAX_EVENTS_SIZE = 1_048_576
     MAX_EVENT_SIZE = 256 * 1024
@@ -80,6 +91,22 @@ module Fluent::Plugin
       if [conf['retention_in_days'], conf['retention_in_days_key']].compact.size > 1
         raise ConfigError, "Set only one of retention_in_days, retention_in_days_key"
       end
+
+      formatter_conf = conf.elements('format').first
+      @formatter_proc = unless formatter_conf
+                          unless @message_keys.empty?
+                            Proc.new { |tag, time, record|
+                              @message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
+                            }
+                          else
+                            Proc.new { |tag, time, record|
+                              @json_handler.dump(record)
+                            }
+                          end
+                        else
+                          formatter = formatter_create(usage: 'cloudwatch-logs-plugin', conf: formatter_conf)
+                          formatter.method(:format)
+                        end
     end
 
     def start
@@ -94,10 +121,29 @@ module Fluent::Plugin
 
       if @aws_use_sts
         Aws.config[:region] = options[:region]
-        options[:credentials] = Aws::AssumeRoleCredentials.new(
+        credentials_options = {
           role_arn: @aws_sts_role_arn,
           role_session_name: @aws_sts_session_name
-        )
+        }
+        credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
+        if @region and @aws_sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
+        elsif @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+      elsif @web_identity_credentials
+        c = @web_identity_credentials
+        credentials_options = {}
+        credentials_options[:role_arn] = c.role_arn
+        credentials_options[:role_session_name] = c.role_session_name
+        credentials_options[:web_identity_token_file] = c.web_identity_token_file
+        credentials_options[:policy] = c.policy if c.policy
+        credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+        if @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
       else
         options[:credentials] = Aws::Credentials.new(@aws_key_id, @aws_sec_key) if @aws_key_id && @aws_sec_key
       end
@@ -237,11 +283,7 @@ module Fluent::Plugin
           time_ms = (time.to_f * 1000).floor
 
           scrub_record!(record)
-          unless @message_keys.empty?
-            message = @message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
-          else
-            message = @json_handler.dump(record)
-          end
+          message = @formatter_proc.call(t, time, record)
 
           if message.empty?
             log.warn "Within specified message_key(s): (#{@message_keys.join(',')}) do not have non-empty record. Skip."

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -99,6 +99,128 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
     end
 
+    sub_test_case "use_log_group_name_prefix true" do
+      test "emit" do
+        set_log_group_name("fluent-plugin-cloudwatch-group-prefix-test-#{Time.now.to_f}")
+        create_log_stream
+
+        time_ms = (Time.now.to_f * 1000).floor
+        put_log_events([
+          {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+          {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+        ])
+
+        sleep 5
+
+        config = <<-EOC
+          tag test
+          @type cloudwatch_logs
+          log_group_name fluent-plugin-cloudwatch-group-prefix-test
+          use_log_group_name_prefix true
+          log_stream_name #{log_stream_name}
+          state_file /tmp/state
+          fetch_interval 1
+          #{aws_key_id}
+          #{aws_sec_key}
+          #{region}
+          #{endpoint}
+        EOC
+
+        d = create_driver(config)
+        d.run(expect_emits: 2, timeout: 5)
+
+        emits = d.events
+        assert_equal(2, emits.size)
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+      end
+
+      test "emit with add_log_group_name" do
+        set_log_group_name("fluent-plugin-cloudwatch-add-log-group-#{Time.now.to_f}")
+        create_log_stream
+
+        time_ms = (Time.now.to_f * 1000).floor
+        put_log_events([
+          {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+          {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+        ])
+
+        sleep 5
+
+        log_group_name_key = 'log_group_key'
+        config = <<-EOC
+          tag test
+          @type cloudwatch_logs
+          log_group_name fluent-plugin-cloudwatch-add-log-group
+          use_log_group_name_prefix true
+          add_log_group_name true
+          log_group_name_key #{log_group_name_key}
+          log_stream_name #{log_stream_name}
+          state_file /tmp/state
+          fetch_interval 1
+          #{aws_key_id}
+          #{aws_sec_key}
+          #{region}
+          #{endpoint}
+        EOC
+
+        d = create_driver(config)
+        d.run(expect_emits: 2, timeout: 5)
+
+        emits = d.events
+        assert_equal(2, emits.size)
+        assert_true emits[0][2].has_key?(log_group_name_key)
+        emits[0][2].delete(log_group_name_key)
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+        assert_true emits[1][2].has_key?(log_group_name_key)
+        emits[1][2].delete(log_group_name_key)
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+      end
+
+      test "emit with add_log_group_name and <parse> csv" do
+        cloudwatch_config = {'tag' => "test",
+                             '@type' => 'cloudwatch_logs',
+                             'log_group_name' => "fluent-plugin-cloudwatch-with-csv-format",
+                             'log_stream_name' => "#{log_stream_name}",
+                             'use_log_group_name_prefix' => true,
+                            }
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
+
+        csv_format_config = config_element('ROOT', '', cloudwatch_config, [
+                                             config_element('parse', '', {'@type' => 'csv',
+                                                                          'keys' => 'time,message',
+                                                                          'time_key' => 'time'}),
+                                             config_element('storage', '', {'@type' => 'local',
+                                                                            'path' => '/tmp/state'})
+                                           ])
+        log_group_name = "fluent-plugin-cloudwatch-with-csv-format-#{Time.now.to_f}"
+        set_log_group_name(log_group_name)
+        create_log_stream
+
+        time_ms = (Time.now.to_f * 1000).floor
+        log_time_ms = time_ms - 10000
+        put_log_events([
+          {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
+          {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
+        ])
+
+        sleep 5
+
+        d = create_driver(csv_format_config)
+        d.run(expect_emits: 2, timeout: 5)
+        next_token = d.instance.instance_variable_get(:@next_token_storage)
+        assert_true next_token.get(d.instance.state_key_for(log_stream_name, log_group_name)).is_a?(String)
+
+        emits = d.events
+        assert_equal(2, emits.size)
+        assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+        assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+      end
+    end
+
     def test_emit_with_metadata
       create_log_stream
 

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -76,6 +76,143 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
     end
 
+    sub_test_case "formatter" do
+      test "csv" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'csv',
+                                                                      'fields' => ["message","cloudwatch"],
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('"","logs1"', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('"","logs2"', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+
+      test "ltsv" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'ltsv',
+                                                                      'fields' => ["message","cloudwatch"],
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('cloudwatch:logs1', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('cloudwatch:logs2', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+
+      test "single_value" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'single_value',
+                                                                      'message_key' => "cloudwatch",
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'Hi!'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'Hi!'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('logs1', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('logs2', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+    end
+
     def test_write_utf8
       new_log_stream
 

data/test/test_helper.rb

@@ -17,6 +17,10 @@ module CloudwatchLogsTestHelper
     @logs ||= Aws::CloudWatchLogs::Client.new(options)
   end
 
+  def set_log_group_name(log_group_name)
+    @log_group_name = log_group_name
+  end
+
   def log_group_name
     @log_group_name ||= "fluent-plugin-cloudwatch-test-#{Time.now.to_f}"
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.13.0
 platform: ruby
 authors:
 - Ryota Arai
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-21 00:00:00.000000000 Z
+date: 2020-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -108,7 +108,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - ryota.arai@gmail.com
 executables: []
@@ -136,7 +136,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -151,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: CloudWatch Logs Plugin for Fluentd
 test_files: