fluent-plugin-cloudwatch-logs 0.11.1 → 0.13.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d78abf2df32b76a85618e07a9164b6b4b59085bb56054b9cff15a947674eb26
-  data.tar.gz: 935d10a414f4ac4e83d619978695aa98bc2ec784c02ef5e60ec082ac7b39f667
+  metadata.gz: 51935a06f74f11a53f106e38e4f08132a7342d6509dc38c0d468d54e19e5fa28
+  data.tar.gz: 738e36cb222adca31a2f79e56ec71cc691f26468eb8025bb9d61773d0dd9d633
 SHA512:
-  metadata.gz: 8f5113e44e0fb327000423a12292f1d6c25be4e5e491773e3e074b06a9c4206797ac0a3873c5b252da9dbbfa705d7989bb6b84f71d39e4080c5eb882555c914e
-  data.tar.gz: ee663a6f50e7788703d3bea274c194ea83d4b330d25b284112682247984cf9de344d00a4a7aba8d7f6e7589c396510e0a0970cb48bea73d73841eb1c22e3cd77
+  metadata.gz: 06675b2f4beedcf0d39c79bf93c59586c0d11816b97a22ee3f472fa6fc892e666c96f8600cfc8f98fba3b2b82f40dd452ec9b451caf6170da199fcf71ad42bff
+  data.tar.gz: 294a5c7fe16c450a58478b982cba8d174d603885234619b5631c1da314b5b7b8a0874424d2b5c997a4790333f99739ecac3cd47a4af97836bf15c9da90ed25af

data/.github/workflows/linux.yml

@@ -0,0 +1,26 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/windows.yml

@@ -0,0 +1,26 @@
+name: Testing on Windows
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        os:
+          - windows-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/README.md

@@ -165,6 +165,9 @@ Fetch sample log from CloudWatch Logs:
   #  role_session_name ROLE_SESSION_NAME
   #  web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
   #</web_identity_credentials>
+  #<format>
+  #  @type ltsv
+  #</format>
 </match>
 ```
 
@@ -207,6 +210,7 @@ Fetch sample log from CloudWatch Logs:
   * `duration_seconds`: The duration, in seconds, of the role session. The value can range from
 900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
 is set to 3600 seconds (1 hour). (default `nil`)
+* `<format>`: For specifying records format. See [formatter overview](https://docs.fluentd.org/formatter) and [formatter section overview](https://docs.fluentd.org/configuration/format-section) on the official documentation.
 
 **NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
 Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
@@ -218,6 +222,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   @type cloudwatch_logs
   tag cloudwatch.in
   log_group_name group
+  #add_log_group_name true
+  #log_group_name_key group_name_key
+  #use_log_group_name_prefix true
   log_stream_name stream
   #use_log_stream_name_prefix true
   state_file /var/lib/fluent/group_stream.in.state
@@ -252,6 +259,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `http_proxy`: use to set an optional HTTP proxy
 * `json_handler`:  name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
 * `log_group_name`: name of log group to fetch logs
+* `add_log_group_name`: add record into the name of log group (default `false`)
+* `log_group_name_key`: specify the key where adding record into the name of log group (default `'log_group'`)
+* `use_log_group_name_prefix`: to use `log_group_name` as log group name prefix (default `false`)
 * `log_stream_name`: name of log stream to fetch logs
 * `region`: AWS Region.  See [Authentication](#authentication) for more information.
 * `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.11.1"
+        VERSION = "0.13.2"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -22,6 +22,9 @@ module Fluent::Plugin
     config_param :endpoint, :string, default: nil
     config_param :tag, :string
     config_param :log_group_name, :string
+    config_param :add_log_group_name, :bool, default: false
+    config_param :log_group_name_key, :string, default: 'log_group'
+    config_param :use_log_group_name_prefix, :bool, default: false
     config_param :log_stream_name, :string, default: nil
     config_param :use_log_stream_name_prefix, :bool, default: false
     config_param :state_file, :string, default: nil,
@@ -89,9 +92,9 @@ module Fluent::Plugin
         }
         credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
         if @region and @aws_sts_endpoint_url
-          credentails_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
         elsif @region
-          credentails_options[:client] = Aws::STS::Client.new(:region => @region)
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
         end
         options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
       elsif @web_identity_credentials
@@ -128,6 +131,17 @@ module Fluent::Plugin
       super
     end
 
+    # No private for testing
+    def state_key_for(log_stream_name, log_group_name = nil)
+      if log_group_name && log_stream_name
+        "#{@state_file}_#{log_group_name.gsub(File::SEPARATOR, '-')}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
+      elsif log_stream_name
+        "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
+      else
+        @state_file
+      end
+    end
+
     private
     def configure_parser(conf)
       if conf['format']
@@ -137,28 +151,20 @@ module Fluent::Plugin
       end
     end
 
-    def state_key_for(log_stream_name)
-      if log_stream_name
-        "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
-      else
-        @state_file
-      end
-    end
-
     def migrate_state_file_to_storage(log_stream_name)
       @next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
       File.delete(state_key_for(log_stream_name))
     end
 
-    def next_token(log_stream_name)
+    def next_token(log_stream_name, log_group_name = nil)
       if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
         migrate_state_file_to_storage(log_stream_name)
       end
-      @next_token_storage.get(:"#{state_key_for(log_stream_name)}")
+      @next_token_storage.get(:"#{state_key_for(log_stream_name, log_group_name)}")
     end
 
-    def store_next_token(token, log_stream_name = nil)
-      @next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
+    def store_next_token(token, log_stream_name = nil, log_group_name = nil)
+      @next_token_storage.put(:"#{state_key_for(log_stream_name, log_group_name)}", token)
     end
 
     def run
@@ -168,55 +174,67 @@ module Fluent::Plugin
         if Time.now > @next_fetch_time
           @next_fetch_time += @fetch_interval
 
-          if @use_log_stream_name_prefix || @use_todays_log_stream
-            log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
-            begin
-              log_streams = describe_log_streams(log_stream_name_prefix)
-              log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
-              log_streams.each do |log_stream|
-                log_stream_name = log_stream.log_stream_name
-                events = get_events(log_stream_name)
-                metadata = if @include_metadata
-                             {
-                               "log_stream_name" => log_stream_name,
-                               "log_group_name" => @log_group_name
-                             }
-                           else
-                             {}
-                           end
-                events.each do |event|
-                  emit(log_stream_name, event, metadata)
+          if @use_log_group_name_prefix
+            log_group_names = describe_log_groups(@log_group_name).map{|log_group|
+              log_group.log_group_name
+            }
+          else
+            log_group_names = [@log_group_name]
+          end
+          log_group_names.each do |log_group_name|
+            if @use_log_stream_name_prefix || @use_todays_log_stream
+              log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
+              begin
+                log_streams = describe_log_streams(log_stream_name_prefix, nil, nil, log_group_name)
+                log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
+                log_streams.each do |log_stream|
+                  log_stream_name = log_stream.log_stream_name
+                  events = get_events(log_group_name, log_stream_name)
+                  metadata = if @include_metadata
+                               {
+                                 "log_stream_name" => log_stream_name,
+                                 "log_group_name" => log_group_name
+                               }
+                             else
+                               {}
+                             end
+                  events.each do |event|
+                    emit(log_group_name, log_stream_name, event, metadata)
+                  end
                 end
+              rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+                log.warn "'#{@log_stream_name}' prefixed log stream(s) are not found"
+                next
+              end
+            else
+              events = get_events(log_group_name, @log_stream_name)
+              metadata = if @include_metadata
+                           {
+                             "log_stream_name" => @log_stream_name,
+                             "log_group_name" => @log_group_name
+                           }
+                         else
+                           {}
+                         end
+              events.each do |event|
+                emit(log_group_name, log_stream_name, event, metadata)
               end
-            rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
-              log.warn "'#{@log_stream_name}' prefixed log stream(s) are not found"
-              next
-            end
-          else
-            events = get_events(@log_stream_name)
-            metadata = if @include_metadata
-                          {
-                            "log_stream_name" => @log_stream_name,
-                            "log_group_name" => @log_group_name
-                          }
-                        else
-                          {}
-                        end
-            events.each do |event|
-              emit(log_stream_name, event, metadata)
             end
           end
-        end
         sleep 1
+        end
       end
     end
 
-    def emit(stream, event, metadata)
+    def emit(group, stream, event, metadata)
       if @parser
         @parser.parse(event.message) {|time,record|
           if @use_aws_timestamp
             time = (event.timestamp / 1000).floor
           end
+          if @add_log_group_name
+            record[@log_group_name_key] = group
+          end
           unless metadata.empty?
             record.merge!("metadata" => metadata)
           end
@@ -226,6 +244,9 @@ module Fluent::Plugin
         time = (event.timestamp / 1000).floor
         begin
           record = @json_handler.load(event.message)
+          if @add_log_group_name
+            record[@log_group_name_key] = group
+          end
           unless metadata.empty?
             record.merge!("metadata" => metadata)
           end
@@ -237,29 +258,33 @@ module Fluent::Plugin
       end
     end
 
-    def get_events(log_stream_name)
+    def get_events(log_group_name, log_stream_name)
       throttling_handler('get_log_events') do
         request = {
-          log_group_name: @log_group_name,
+          log_group_name: log_group_name,
           log_stream_name: log_stream_name
         }
         request.merge!(start_time: @start_time) if @start_time
         request.merge!(end_time: @end_time) if @end_time
-        log_next_token = next_token(log_stream_name)
+        log_next_token = next_token(log_group_name, log_stream_name)
         request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
         response = @logs.get_log_events(request)
         if valid_next_token(log_next_token, response.next_forward_token)
-          store_next_token(response.next_forward_token, log_stream_name)
+          if @use_log_group_name_prefix
+            store_next_token(response.next_forward_token, log_stream_name, log_group_name)
+          else
+            store_next_token(response.next_forward_token, log_stream_name)
+          end
         end
 
         response.events
       end
     end
 
-    def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
+    def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
       throttling_handler('describe_log_streams') do
         request = {
-          log_group_name: @log_group_name
+          log_group_name: log_group_name != nil ? log_group_name : @log_group_name
         }
         request[:next_token] = next_token if next_token
         request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
@@ -270,7 +295,7 @@ module Fluent::Plugin
           log_streams = response.log_streams
         end
         if response.next_token
-          log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
+          log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token, log_group_name)
         end
         log_streams
       end
@@ -289,6 +314,23 @@ module Fluent::Plugin
       end
     end
 
+    def describe_log_groups(log_group_name_prefix, log_groups = nil, next_token = nil)
+      request = {
+        log_group_name_prefix: log_group_name_prefix
+      }
+      request[:next_token] = next_token if next_token
+      response = @logs.describe_log_groups(request)
+      if log_groups
+        log_groups.concat(response.log_groups)
+      else
+        log_groups = response.log_groups
+      end
+      if response.next_token
+        log_groups = describe_log_groups(log_group_name_prefix, log_groups, response.next_token)
+      end
+      log_groups
+    end
+
     def valid_next_token(prev_token, next_token)
       next_token && prev_token != next_token.chomp
     end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -9,7 +9,7 @@ module Fluent::Plugin
 
     class TooLargeEventError < Fluent::UnrecoverableError; end
 
-    helpers :compat_parameters, :inject
+    helpers :compat_parameters, :inject, :formatter
 
     DEFAULT_BUFFER_TYPE = "memory"
 
@@ -58,6 +58,9 @@ module Fluent::Plugin
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
     end
+    config_section :format do
+      config_set_default :@type, 'json'
+    end
 
     MAX_EVENTS_SIZE = 1_048_576
     MAX_EVENT_SIZE = 256 * 1024
@@ -88,6 +91,22 @@ module Fluent::Plugin
       if [conf['retention_in_days'], conf['retention_in_days_key']].compact.size > 1
         raise ConfigError, "Set only one of retention_in_days, retention_in_days_key"
       end
+
+      formatter_conf = conf.elements('format').first
+      @formatter_proc = unless formatter_conf
+                          unless @message_keys.empty?
+                            Proc.new { |tag, time, record|
+                              @message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
+                            }
+                          else
+                            Proc.new { |tag, time, record|
+                              @json_handler.dump(record)
+                            }
+                          end
+                        else
+                          formatter = formatter_create(usage: 'cloudwatch-logs-plugin', conf: formatter_conf)
+                          formatter.method(:format)
+                        end
     end
 
     def start
@@ -108,9 +127,9 @@ module Fluent::Plugin
         }
         credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
         if @region and @aws_sts_endpoint_url
-          credentails_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
         elsif @region
-          credentails_options[:client] = Aws::STS::Client.new(:region => @region)
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
         end
         options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
       elsif @web_identity_credentials
@@ -264,11 +283,7 @@ module Fluent::Plugin
           time_ms = (time.to_f * 1000).floor
 
           scrub_record!(record)
-          unless @message_keys.empty?
-            message = @message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
-          else
-            message = @json_handler.dump(record)
-          end
+          message = @formatter_proc.call(t, time, record)
 
           if message.empty?
             log.warn "Within specified message_key(s): (#{@message_keys.join(',')}) do not have non-empty record. Skip."

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -99,6 +99,128 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
     end
 
+    sub_test_case "use_log_group_name_prefix true" do
+      test "emit" do
+        set_log_group_name("fluent-plugin-cloudwatch-group-prefix-test-#{Time.now.to_f}")
+        create_log_stream
+
+        time_ms = (Time.now.to_f * 1000).floor
+        put_log_events([
+          {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+          {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+        ])
+
+        sleep 5
+
+        config = <<-EOC
+          tag test
+          @type cloudwatch_logs
+          log_group_name fluent-plugin-cloudwatch-group-prefix-test
+          use_log_group_name_prefix true
+          log_stream_name #{log_stream_name}
+          state_file /tmp/state
+          fetch_interval 1
+          #{aws_key_id}
+          #{aws_sec_key}
+          #{region}
+          #{endpoint}
+        EOC
+
+        d = create_driver(config)
+        d.run(expect_emits: 2, timeout: 5)
+
+        emits = d.events
+        assert_equal(2, emits.size)
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+      end
+
+      test "emit with add_log_group_name" do
+        set_log_group_name("fluent-plugin-cloudwatch-add-log-group-#{Time.now.to_f}")
+        create_log_stream
+
+        time_ms = (Time.now.to_f * 1000).floor
+        put_log_events([
+          {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+          {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+        ])
+
+        sleep 5
+
+        log_group_name_key = 'log_group_key'
+        config = <<-EOC
+          tag test
+          @type cloudwatch_logs
+          log_group_name fluent-plugin-cloudwatch-add-log-group
+          use_log_group_name_prefix true
+          add_log_group_name true
+          log_group_name_key #{log_group_name_key}
+          log_stream_name #{log_stream_name}
+          state_file /tmp/state
+          fetch_interval 1
+          #{aws_key_id}
+          #{aws_sec_key}
+          #{region}
+          #{endpoint}
+        EOC
+
+        d = create_driver(config)
+        d.run(expect_emits: 2, timeout: 5)
+
+        emits = d.events
+        assert_equal(2, emits.size)
+        assert_true emits[0][2].has_key?(log_group_name_key)
+        emits[0][2].delete(log_group_name_key)
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+        assert_true emits[1][2].has_key?(log_group_name_key)
+        emits[1][2].delete(log_group_name_key)
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+      end
+
+      test "emit with add_log_group_name and <parse> csv" do
+        cloudwatch_config = {'tag' => "test",
+                             '@type' => 'cloudwatch_logs',
+                             'log_group_name' => "fluent-plugin-cloudwatch-with-csv-format",
+                             'log_stream_name' => "#{log_stream_name}",
+                             'use_log_group_name_prefix' => true,
+                            }
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
+
+        csv_format_config = config_element('ROOT', '', cloudwatch_config, [
+                                             config_element('parse', '', {'@type' => 'csv',
+                                                                          'keys' => 'time,message',
+                                                                          'time_key' => 'time'}),
+                                             config_element('storage', '', {'@type' => 'local',
+                                                                            'path' => '/tmp/state'})
+                                           ])
+        log_group_name = "fluent-plugin-cloudwatch-with-csv-format-#{Time.now.to_f}"
+        set_log_group_name(log_group_name)
+        create_log_stream
+
+        time_ms = (Time.now.to_f * 1000).floor
+        log_time_ms = time_ms - 10000
+        put_log_events([
+          {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
+          {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
+        ])
+
+        sleep 5
+
+        d = create_driver(csv_format_config)
+        d.run(expect_emits: 2, timeout: 5)
+        next_token = d.instance.instance_variable_get(:@next_token_storage)
+        assert_true next_token.get(d.instance.state_key_for(log_stream_name, log_group_name)).is_a?(String)
+
+        emits = d.events
+        assert_equal(2, emits.size)
+        assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+        assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+      end
+    end
+
     def test_emit_with_metadata
       create_log_stream
 

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -76,6 +76,143 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
     end
 
+    sub_test_case "formatter" do
+      test "csv" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'csv',
+                                                                      'fields' => ["message","cloudwatch"],
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('"","logs1"', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('"","logs2"', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+
+      test "ltsv" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'ltsv',
+                                                                      'fields' => ["message","cloudwatch"],
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('cloudwatch:logs1', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('cloudwatch:logs2', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+
+      test "single_value" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'single_value',
+                                                                      'message_key' => "cloudwatch",
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'Hi!'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'Hi!'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('logs1', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('logs2', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+    end
+
     def test_write_utf8
       new_log_stream
 

data/test/test_helper.rb

@@ -17,6 +17,10 @@ module CloudwatchLogsTestHelper
     @logs ||= Aws::CloudWatchLogs::Client.new(options)
   end
 
+  def set_log_group_name(log_group_name)
+    @log_group_name = log_group_name
+  end
+
   def log_group_name
     @log_group_name ||= "fluent-plugin-cloudwatch-test-#{Time.now.to_f}"
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.13.2
 platform: ruby
 authors:
 - Ryota Arai
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-20 00:00:00.000000000 Z
+date: 2021-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -116,6 +116,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/issue-auto-closer.yml"
+- ".github/workflows/linux.yml"
+- ".github/workflows/windows.yml"
 - ".gitignore"
 - ".travis.yml"
 - Gemfile