fluent-plugin-cloudwatch-logs 0.11.0 → 0.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1dc48c250b022126a1de2b125bfa8ad3320daaa5eca5613f51ba7e6571a0b9a9
-  data.tar.gz: 23993ce51cac3aacfbe6937c1f928a00a61fbd94f64fb4ccf8c38ac8e4656787
+  metadata.gz: 7bed409ac04e91c38f792645f4a3a4dda6871273d97230c1fadf733a4c1d42fa
+  data.tar.gz: d9d49dc6cac3f3cf4035d2888f1e121cec268d73ba960245df00a2039eb88d12
 SHA512:
-  metadata.gz: 84fd2ea44c0b498364a13da89d422d39b6ea18abdb38add8fbacbc9f0c7b04b6ed18498f26e85920ffe8a7c80e5c14dce8f191c6ecc1a2f1c36809ce67e6961b
-  data.tar.gz: e16ab191ba87408d82e1ffa73564aec909a0795cca65c0ab506b2bb538f4c1cd0ad61641035c6b42056ccd4459e4ba677cb35a28d8af74edc3d1d0bd04422db1
+  metadata.gz: e16807714477b6b0b4c862875c0f94b6e07b7dd6ff183279ea0f13f7cd2467f756a2c5eb6aee60234d4bc69ba3188ab5cfd13294f0d9f3ff3cdaaeee9570ce08
+  data.tar.gz: dc25160c660f4e461b89db0517952baf2601491e523c53100eff0369f06cb856173acf829c32cef981384161bf2c55cb132c9de23a864469d1496209fe7a708a

data/.github/workflows/linux.yml

@@ -0,0 +1,26 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/windows.yml

@@ -0,0 +1,26 @@
+name: Testing on Windows
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        os:
+          - windows-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/README.md

@@ -165,6 +165,9 @@ Fetch sample log from CloudWatch Logs:
   #  role_session_name ROLE_SESSION_NAME
   #  web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
   #</web_identity_credentials>
+  #<format>
+  #  @type ltsv
+  #</format>
 </match>
 ```
 
@@ -207,6 +210,7 @@ Fetch sample log from CloudWatch Logs:
   * `duration_seconds`: The duration, in seconds, of the role session. The value can range from
 900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
 is set to 3600 seconds (1 hour). (default `nil`)
+* `<format>`: For specifying records format. See [formatter overview](https://docs.fluentd.org/formatter) and [formatter section overview](https://docs.fluentd.org/configuration/format-section) on the official documentation.
 
 **NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
 Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
@@ -218,6 +222,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   @type cloudwatch_logs
   tag cloudwatch.in
   log_group_name group
+  #add_log_group_name true
+  #log_group_name_key group_name_key
+  #use_log_group_name_prefix true
   log_stream_name stream
   #use_log_stream_name_prefix true
   state_file /var/lib/fluent/group_stream.in.state
@@ -252,6 +259,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `http_proxy`: use to set an optional HTTP proxy
 * `json_handler`:  name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
 * `log_group_name`: name of log group to fetch logs
+* `add_log_group_name`: add record into the name of log group (default `false`)
+* `log_group_name_key`: specify the key where adding record into the name of log group (default `'log_group'`)
+* `use_log_group_name_prefix`: to use `log_group_name` as log group name prefix (default `false`)
 * `log_stream_name`: name of log stream to fetch logs
 * `region`: AWS Region.  See [Authentication](#authentication) for more information.
 * `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.11.0"
+        VERSION = "0.13.1"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -17,10 +17,14 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, default: nil
     config_param :endpoint, :string, default: nil
     config_param :tag, :string
     config_param :log_group_name, :string
+    config_param :add_log_group_name, :bool, default: false
+    config_param :log_group_name_key, :string, default: 'log_group'
+    config_param :use_log_group_name_prefix, :bool, default: false
     config_param :log_stream_name, :string, default: nil
     config_param :use_log_stream_name_prefix, :bool, default: false
     config_param :state_file, :string, default: nil,
@@ -82,10 +86,17 @@ module Fluent::Plugin
 
       if @aws_use_sts
         Aws.config[:region] = options[:region]
-        options[:credentials] = Aws::AssumeRoleCredentials.new(
+        credentials_options = {
           role_arn: @aws_sts_role_arn,
           role_session_name: @aws_sts_session_name
-        )
+        }
+        credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
+        if @region and @aws_sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
+        elsif @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
       elsif @web_identity_credentials
         c = @web_identity_credentials
         credentials_options = {}
@@ -120,6 +131,17 @@ module Fluent::Plugin
       super
     end
 
+    # No private for testing
+    def state_key_for(log_stream_name, log_group_name = nil)
+      if log_group_name && log_stream_name
+        "#{@state_file}_#{log_group_name.gsub(File::SEPARATOR, '-')}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
+      elsif log_stream_name
+        "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
+      else
+        @state_file
+      end
+    end
+
     private
     def configure_parser(conf)
       if conf['format']
@@ -129,28 +151,20 @@ module Fluent::Plugin
       end
     end
 
-    def state_key_for(log_stream_name)
-      if log_stream_name
-        "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
-      else
-        @state_file
-      end
-    end
-
     def migrate_state_file_to_storage(log_stream_name)
       @next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
       File.delete(state_key_for(log_stream_name))
     end
 
-    def next_token(log_stream_name)
+    def next_token(log_stream_name, log_group_name = nil)
       if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
         migrate_state_file_to_storage(log_stream_name)
       end
-      @next_token_storage.get(:"#{state_key_for(log_stream_name)}")
+      @next_token_storage.get(:"#{state_key_for(log_stream_name, log_group_name)}")
     end
 
-    def store_next_token(token, log_stream_name = nil)
-      @next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
+    def store_next_token(token, log_stream_name = nil, log_group_name = nil)
+      @next_token_storage.put(:"#{state_key_for(log_stream_name, log_group_name)}", token)
     end
 
     def run
@@ -160,55 +174,67 @@ module Fluent::Plugin
         if Time.now > @next_fetch_time
           @next_fetch_time += @fetch_interval
 
-          if @use_log_stream_name_prefix || @use_todays_log_stream
-            log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
-            begin
-              log_streams = describe_log_streams(log_stream_name_prefix)
-              log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
-              log_streams.each do |log_stream|
-                log_stream_name = log_stream.log_stream_name
-                events = get_events(log_stream_name)
-                metadata = if @include_metadata
-                             {
-                               "log_stream_name" => log_stream_name,
-                               "log_group_name" => @log_group_name
-                             }
-                           else
-                             {}
-                           end
-                events.each do |event|
-                  emit(log_stream_name, event, metadata)
+          if @use_log_group_name_prefix
+            log_group_names = describe_log_groups(@log_group_name).map{|log_group|
+              log_group.log_group_name
+            }
+          else
+            log_group_names = [@log_group_name]
+          end
+          log_group_names.each do |log_group_name|
+            if @use_log_stream_name_prefix || @use_todays_log_stream
+              log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
+              begin
+                log_streams = describe_log_streams(log_stream_name_prefix, nil, nil, log_group_name)
+                log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
+                log_streams.each do |log_stream|
+                  log_stream_name = log_stream.log_stream_name
+                  events = get_events(log_group_name, log_stream_name)
+                  metadata = if @include_metadata
+                               {
+                                 "log_stream_name" => log_stream_name,
+                                 "log_group_name" => log_group_name
+                               }
+                             else
+                               {}
+                             end
+                  events.each do |event|
+                    emit(log_group_name, log_stream_name, event, metadata)
+                  end
                 end
+              rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+                log.warn "'#{@log_stream_name}' prefixed log stream(s) are not found"
+                next
+              end
+            else
+              events = get_events(log_group_name, @log_stream_name)
+              metadata = if @include_metadata
+                           {
+                             "log_stream_name" => @log_stream_name,
+                             "log_group_name" => @log_group_name
+                           }
+                         else
+                           {}
+                         end
+              events.each do |event|
+                emit(log_group_name, log_stream_name, event, metadata)
               end
-            rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
-              log.warn "'#{@log_stream_name}' prefixed log stream(s) are not found"
-              next
-            end
-          else
-            events = get_events(@log_stream_name)
-            metadata = if @include_metadata
-                          {
-                            "log_stream_name" => @log_stream_name,
-                            "log_group_name" => @log_group_name
-                          }
-                        else
-                          {}
-                        end
-            events.each do |event|
-              emit(log_stream_name, event, metadata)
             end
           end
-        end
         sleep 1
+        end
       end
     end
 
-    def emit(stream, event, metadata)
+    def emit(group, stream, event, metadata)
       if @parser
         @parser.parse(event.message) {|time,record|
           if @use_aws_timestamp
             time = (event.timestamp / 1000).floor
           end
+          if @add_log_group_name
+            record[@log_group_name_key] = group
+          end
           unless metadata.empty?
             record.merge!("metadata" => metadata)
           end
@@ -218,6 +244,9 @@ module Fluent::Plugin
         time = (event.timestamp / 1000).floor
         begin
           record = @json_handler.load(event.message)
+          if @add_log_group_name
+            record[@log_group_name_key] = group
+          end
           unless metadata.empty?
             record.merge!("metadata" => metadata)
           end
@@ -229,29 +258,33 @@ module Fluent::Plugin
       end
     end
 
-    def get_events(log_stream_name)
+    def get_events(log_group_name, log_stream_name)
       throttling_handler('get_log_events') do
         request = {
-          log_group_name: @log_group_name,
+          log_group_name: log_group_name,
           log_stream_name: log_stream_name
         }
         request.merge!(start_time: @start_time) if @start_time
         request.merge!(end_time: @end_time) if @end_time
-        log_next_token = next_token(log_stream_name)
+        log_next_token = next_token(log_group_name, log_stream_name)
         request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
         response = @logs.get_log_events(request)
         if valid_next_token(log_next_token, response.next_forward_token)
-          store_next_token(response.next_forward_token, log_stream_name)
+          if @use_log_group_name_prefix
+            store_next_token(response.next_forward_token, log_stream_name, log_group_name)
+          else
+            store_next_token(response.next_forward_token, log_stream_name)
+          end
         end
 
         response.events
       end
     end
 
-    def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
+    def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
       throttling_handler('describe_log_streams') do
         request = {
-          log_group_name: @log_group_name
+          log_group_name: log_group_name != nil ? log_group_name : @log_group_name
         }
         request[:next_token] = next_token if next_token
         request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
@@ -281,6 +314,23 @@ module Fluent::Plugin
       end
     end
 
+    def describe_log_groups(log_group_name_prefix, log_groups = nil, next_token = nil)
+      request = {
+        log_group_name_prefix: log_group_name_prefix
+      }
+      request[:next_token] = next_token if next_token
+      response = @logs.describe_log_groups(request)
+      if log_groups
+        log_groups.concat(response.log_groups)
+      else
+        log_groups = response.log_groups
+      end
+      if response.next_token
+        log_groups = describe_log_groups(log_group_name_prefix, log_groups, response.next_token)
+      end
+      log_groups
+    end
+
     def valid_next_token(prev_token, next_token)
       next_token && prev_token != next_token.chomp
     end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -9,7 +9,7 @@ module Fluent::Plugin
 
     class TooLargeEventError < Fluent::UnrecoverableError; end
 
-    helpers :compat_parameters, :inject
+    helpers :compat_parameters, :inject, :formatter
 
     DEFAULT_BUFFER_TYPE = "memory"
 
@@ -19,6 +19,7 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, :default => nil
     config_param :endpoint, :string, :default => nil
     config_param :log_group_name, :string, :default => nil
@@ -57,6 +58,9 @@ module Fluent::Plugin
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
     end
+    config_section :format do
+      config_set_default :@type, 'json'
+    end
 
     MAX_EVENTS_SIZE = 1_048_576
     MAX_EVENT_SIZE = 256 * 1024
@@ -87,6 +91,22 @@ module Fluent::Plugin
       if [conf['retention_in_days'], conf['retention_in_days_key']].compact.size > 1
         raise ConfigError, "Set only one of retention_in_days, retention_in_days_key"
       end
+
+      formatter_conf = conf.elements('format').first
+      @formatter_proc = unless formatter_conf
+                          unless @message_keys.empty?
+                            Proc.new { |tag, time, record|
+                              @message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
+                            }
+                          else
+                            Proc.new { |tag, time, record|
+                              @json_handler.dump(record)
+                            }
+                          end
+                        else
+                          formatter = formatter_create(usage: 'cloudwatch-logs-plugin', conf: formatter_conf)
+                          formatter.method(:format)
+                        end
     end
 
     def start
@@ -101,10 +121,17 @@ module Fluent::Plugin
 
       if @aws_use_sts
         Aws.config[:region] = options[:region]
-        options[:credentials] = Aws::AssumeRoleCredentials.new(
+        credentials_options = {
           role_arn: @aws_sts_role_arn,
           role_session_name: @aws_sts_session_name
-        )
+        }
+        credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
+        if @region and @aws_sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region, endpoint: @aws_sts_endpoint_url)
+        elsif @region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+        end
+        options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
       elsif @web_identity_credentials
         c = @web_identity_credentials
         credentials_options = {}
@@ -256,11 +283,7 @@ module Fluent::Plugin
           time_ms = (time.to_f * 1000).floor
 
           scrub_record!(record)
-          unless @message_keys.empty?
-            message = @message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
-          else
-            message = @json_handler.dump(record)
-          end
+          message = @formatter_proc.call(t, time, record)
 
           if message.empty?
             log.warn "Within specified message_key(s): (#{@message_keys.join(',')}) do not have non-empty record. Skip."

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -99,6 +99,128 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
     end
 
+    sub_test_case "use_log_group_name_prefix true" do
+      test "emit" do
+        set_log_group_name("fluent-plugin-cloudwatch-group-prefix-test-#{Time.now.to_f}")
+        create_log_stream
+
+        time_ms = (Time.now.to_f * 1000).floor
+        put_log_events([
+          {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+          {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+        ])
+
+        sleep 5
+
+        config = <<-EOC
+          tag test
+          @type cloudwatch_logs
+          log_group_name fluent-plugin-cloudwatch-group-prefix-test
+          use_log_group_name_prefix true
+          log_stream_name #{log_stream_name}
+          state_file /tmp/state
+          fetch_interval 1
+          #{aws_key_id}
+          #{aws_sec_key}
+          #{region}
+          #{endpoint}
+        EOC
+
+        d = create_driver(config)
+        d.run(expect_emits: 2, timeout: 5)
+
+        emits = d.events
+        assert_equal(2, emits.size)
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+      end
+
+      test "emit with add_log_group_name" do
+        set_log_group_name("fluent-plugin-cloudwatch-add-log-group-#{Time.now.to_f}")
+        create_log_stream
+
+        time_ms = (Time.now.to_f * 1000).floor
+        put_log_events([
+          {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+          {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+        ])
+
+        sleep 5
+
+        log_group_name_key = 'log_group_key'
+        config = <<-EOC
+          tag test
+          @type cloudwatch_logs
+          log_group_name fluent-plugin-cloudwatch-add-log-group
+          use_log_group_name_prefix true
+          add_log_group_name true
+          log_group_name_key #{log_group_name_key}
+          log_stream_name #{log_stream_name}
+          state_file /tmp/state
+          fetch_interval 1
+          #{aws_key_id}
+          #{aws_sec_key}
+          #{region}
+          #{endpoint}
+        EOC
+
+        d = create_driver(config)
+        d.run(expect_emits: 2, timeout: 5)
+
+        emits = d.events
+        assert_equal(2, emits.size)
+        assert_true emits[0][2].has_key?(log_group_name_key)
+        emits[0][2].delete(log_group_name_key)
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+        assert_true emits[1][2].has_key?(log_group_name_key)
+        emits[1][2].delete(log_group_name_key)
+        assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+      end
+
+      test "emit with add_log_group_name and <parse> csv" do
+        cloudwatch_config = {'tag' => "test",
+                             '@type' => 'cloudwatch_logs',
+                             'log_group_name' => "fluent-plugin-cloudwatch-with-csv-format",
+                             'log_stream_name' => "#{log_stream_name}",
+                             'use_log_group_name_prefix' => true,
+                            }
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
+        cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
+
+        csv_format_config = config_element('ROOT', '', cloudwatch_config, [
+                                             config_element('parse', '', {'@type' => 'csv',
+                                                                          'keys' => 'time,message',
+                                                                          'time_key' => 'time'}),
+                                             config_element('storage', '', {'@type' => 'local',
+                                                                            'path' => '/tmp/state'})
+                                           ])
+        log_group_name = "fluent-plugin-cloudwatch-with-csv-format-#{Time.now.to_f}"
+        set_log_group_name(log_group_name)
+        create_log_stream
+
+        time_ms = (Time.now.to_f * 1000).floor
+        log_time_ms = time_ms - 10000
+        put_log_events([
+          {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
+          {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
+        ])
+
+        sleep 5
+
+        d = create_driver(csv_format_config)
+        d.run(expect_emits: 2, timeout: 5)
+        next_token = d.instance.instance_variable_get(:@next_token_storage)
+        assert_true next_token.get(d.instance.state_key_for(log_stream_name, log_group_name)).is_a?(String)
+
+        emits = d.events
+        assert_equal(2, emits.size)
+        assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+        assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+      end
+    end
+
     def test_emit_with_metadata
       create_log_stream
 

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -76,6 +76,143 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
     end
 
+    sub_test_case "formatter" do
+      test "csv" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'csv',
+                                                                      'fields' => ["message","cloudwatch"],
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('"","logs1"', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('"","logs2"', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+
+      test "ltsv" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'ltsv',
+                                                                      'fields' => ["message","cloudwatch"],
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('cloudwatch:logs1', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('cloudwatch:logs2', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+
+      test "single_value" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'single_value',
+                                                                      'message_key' => "cloudwatch",
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'Hi!'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'Hi!'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('logs1', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('logs2', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+    end
+
     def test_write_utf8
       new_log_stream
 

data/test/test_helper.rb

@@ -17,6 +17,10 @@ module CloudwatchLogsTestHelper
     @logs ||= Aws::CloudWatchLogs::Client.new(options)
   end
 
+  def set_log_group_name(log_group_name)
+    @log_group_name = log_group_name
+  end
+
   def log_group_name
     @log_group_name ||= "fluent-plugin-cloudwatch-test-#{Time.now.to_f}"
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.13.1
 platform: ruby
 authors:
 - Ryota Arai
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-19 00:00:00.000000000 Z
+date: 2021-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -116,6 +116,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/issue-auto-closer.yml"
+- ".github/workflows/linux.yml"
+- ".github/workflows/windows.yml"
 - ".gitignore"
 - ".travis.yml"
 - Gemfile