fluent-plugin-cloudwatch-logs 0.11.2 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e50538afdcec35527089c0d1767dff70ddc7f2b5873523d596f78a9e7d344a88
-  data.tar.gz: c8da30c9f0286bca9dca9aab693224f2165f7fdced0ceb8de07db8fb8a259b3a
+  metadata.gz: ff7c8514095015603f6197f312b3e73439485ad9fd8b296522605919f5c53930
+  data.tar.gz: b49f0d62c6033f22409c1046ebd87d884a1f693725e4f372119487225c7b0d5f
 SHA512:
-  metadata.gz: 4b623676f4b5baec31da710b5446a189d58f063a7fe2f1f3fe721d54a32bcb619c16b45416167a24a61d68f416b2202d13451f900e7038e084600924330ef9ed
-  data.tar.gz: 1519461663014d5234ec69088a1b14a4dd345006c4ec8eebdefa87612a15563a6f65c8effacfee5edd7d0c8672765769d5e08d35d761388ac2fde0ac9669a823
+  metadata.gz: 2a3520d9d89be225d7fce95a4cc5acc0064b6fdf316de8c85b0a0dda8e7bcc95c3a8290e106459312e9c14aaf2639300aed693169ac200f56c795b04b38d91f6
+  data.tar.gz: 2f3ab8a2a8eb9e98d013f184bdb9cfc7cc2607ba88a1bce37ed8ea56552d1211c1b1b565bebe7aa00791d2c5579d3348507ce612e1070d7bb8abe9dfa3b509ad

data/README.md

@@ -165,6 +165,9 @@ Fetch sample log from CloudWatch Logs:
   #  role_session_name ROLE_SESSION_NAME
   #  web_identity_token_file "#{ENV['AWS_WEB_IDENTITY_TOKEN_FILE']}"
   #</web_identity_credentials>
+  #<format>
+  #  @type ltsv
+  #</format>
 </match>
 ```
 
@@ -207,6 +210,7 @@ Fetch sample log from CloudWatch Logs:
   * `duration_seconds`: The duration, in seconds, of the role session. The value can range from
 900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
 is set to 3600 seconds (1 hour). (default `nil`)
+* `<format>`: For specifying records format. See [formatter overview](https://docs.fluentd.org/formatter) and [formatter section overview](https://docs.fluentd.org/configuration/format-section) on the official documentation.
 
 **NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
 Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.11.2"
+        VERSION = "0.12.0"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -9,7 +9,7 @@ module Fluent::Plugin
 
     class TooLargeEventError < Fluent::UnrecoverableError; end
 
-    helpers :compat_parameters, :inject
+    helpers :compat_parameters, :inject, :formatter
 
     DEFAULT_BUFFER_TYPE = "memory"
 
@@ -58,6 +58,9 @@ module Fluent::Plugin
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
     end
+    config_section :format do
+      config_set_default :@type, 'json'
+    end
 
     MAX_EVENTS_SIZE = 1_048_576
     MAX_EVENT_SIZE = 256 * 1024
@@ -88,6 +91,22 @@ module Fluent::Plugin
       if [conf['retention_in_days'], conf['retention_in_days_key']].compact.size > 1
         raise ConfigError, "Set only one of retention_in_days, retention_in_days_key"
       end
+
+      formatter_conf = conf.elements('format').first
+      @formatter_proc = unless formatter_conf
+                          unless @message_keys.empty?
+                            Proc.new { |tag, time, record|
+                              @message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
+                            }
+                          else
+                            Proc.new { |tag, time, record|
+                              @json_handler.dump(record)
+                            }
+                          end
+                        else
+                          formatter = formatter_create(usage: 'cloudwatch-logs-plugin', conf: formatter_conf)
+                          formatter.method(:format)
+                        end
     end
 
     def start
@@ -264,11 +283,7 @@ module Fluent::Plugin
           time_ms = (time.to_f * 1000).floor
 
           scrub_record!(record)
-          unless @message_keys.empty?
-            message = @message_keys.map{|k| record[k].to_s }.reject{|e| e.empty? }.join(' ')
-          else
-            message = @json_handler.dump(record)
-          end
+          message = @formatter_proc.call(t, time, record)
 
           if message.empty?
             log.warn "Within specified message_key(s): (#{@message_keys.join(',')}) do not have non-empty record. Skip."

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -76,6 +76,143 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
     end
 
+    sub_test_case "formatter" do
+      test "csv" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'csv',
+                                                                      'fields' => ["message","cloudwatch"],
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('"","logs1"', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('"","logs2"', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+
+      test "ltsv" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'ltsv',
+                                                                      'fields' => ["message","cloudwatch"],
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('cloudwatch:logs1', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('cloudwatch:logs2', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+
+      test "single_value" do
+        new_log_stream
+
+        config = {'@type' => 'cloudwatch_logs',
+                  'auto_create_stream' => true,
+                  'log_stream_name' => log_stream_name,
+                  'log_group_name' => log_group_name,
+                  '@log_level' => 'debug'}
+        config.merge!(config_elementify(aws_key_id)) if aws_key_id
+        config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+        config.merge!(config_elementify(region)) if region
+        config.merge!(config_elementify(endpoint)) if endpoint
+
+        d = create_driver(
+          Fluent::Config::Element.new('ROOT', '', config, [
+                                        Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                      '@type' => 'memory',
+                                                                      'timekey' => 3600
+                                                                    }, []),
+                                        Fluent::Config::Element.new('format', '', {
+                                                                      '@type' => 'single_value',
+                                                                      'message_key' => "cloudwatch",
+                                                                    }, []),
+                                      ]))
+
+        time = event_time
+        d.run(default_tag: fluentd_tag, flush: true) do
+          d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'Hi!'})
+          # Addition converts EventTime to seconds
+          d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'Hi!'})
+        end
+
+        sleep 10
+
+        logs = d.logs
+        events = get_log_events
+        assert_equal(2, events.size)
+        assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+        assert_equal('logs1', events[0].message.strip)
+        assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+        assert_equal('logs2', events[1].message.strip)
+
+        assert(logs.any?{|log| log.include?("Called PutLogEvents API") })
+      end
+    end
+
     def test_write_utf8
       new_log_stream
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.11.2
+  version: 0.12.0
 platform: ruby
 authors:
 - Ryota Arai
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-20 00:00:00.000000000 Z
+date: 2020-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd