fluent-plugin-cloudwatch-logs 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b005354b842b8dfcc7e1106d762a9b5db691f6d32c8a254910ab98c125ea62c
-  data.tar.gz: 6f7198ffd262247b2f600ae8b27cad5246954c2d050f10ce32b2e182adfd9cc6
+  metadata.gz: df810968d48ed4f1c2f11cfd66972864d9d4c0594ad03453c7c1a534cd5c367d
+  data.tar.gz: a0e21faa91fa6a5e530a7b7ed145de5a016f6e1b092282682e54f02025e729b0
 SHA512:
-  metadata.gz: 10b5035a2f3d3669c61e51cf4b3aed7fa6d001b26351228cc4eb1dff863ddd0c9846e09be519efadf2914b488fa7e6c471d9e86c199f167c1def428391e4493b
-  data.tar.gz: 3415f0bdb2bd70dcb1cb199ed1eb32d17998699ee519f0eba57ad67a379b7815d8648d7d2f31f9b84fdf10c6ac96d856661a9e4ba245dd734ba13e91cb5af933
+  metadata.gz: d20bd70811d9e52969d0e0aa90d99ed4a94fc53b2de060571a00ccad58ebe9ab9fd8532d82466f0659ac3ac8547d8662ed88ab493997346c951e6568e609e40c
+  data.tar.gz: 19385acfe587971bcfa1b8ae911c021e00d406132633f190dec8f14946aa5b048c5ba68a6233942672d9f4b4b38671b7c0c3df57819544ad605a7fd895f07be4

data/README.md

@@ -48,7 +48,7 @@ Create IAM user with a policy like the following:
 There are several methods to provide authentication credentials.  Be aware that there are various tradeoffs for these methods,
 although most of these tradeoffs are highly dependent on the specific environment.
 
-### Environment 
+### Environment
 
 Set region and credentials via the environment:
 
@@ -62,7 +62,7 @@ Note: For this to work persistently the enviornment will need to be set in the s
 
 ### AWS Configuration
 
-The plugin will look for the `$HOME/.aws/config` and `$HOME/.aws/credentials` for configuration information.  To setup, as the 
+The plugin will look for the `$HOME/.aws/config` and `$HOME/.aws/credentials` for configuration information.  To setup, as the
 fluentd user, run:
 
 ```sh
@@ -71,7 +71,7 @@ aws configure
 
 ### Configuration Parameters
 
-The authentication information can also be set 
+The authentication information can also be set
 
 ## Example
 
@@ -170,6 +170,11 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   state_file /var/lib/fluent/group_stream.in.state
   #endpoint http://localhost:5000/
   #json_handler json
+  # Users can use `format` or `<parse>` directive to parse non-JSON CloudwatchLogs' log
+  # format none # or csv, tsv, regexp etc.
+  #<parse>
+  # @type none # or csv, tsv, regexp etc.
+  #</parse>
 </source>
 ```
 
@@ -190,6 +195,8 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
 * `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
 * `use_aws_timestamp`: get timestamp from Cloudwatch event for non json logs, otherwise fluentd will parse the log to get the timestamp (default `false`)
+* `format`: specify CloudWatchLogs' log format. (default `nil`)
+* `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
 
 ## Test
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.8.0"
+        VERSION = "0.8.1"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -34,6 +34,7 @@ module Fluent::Plugin
     def initialize
       super
 
+      @parser = nil
       require 'aws-sdk-cloudwatchlogs'
     end
 
@@ -82,6 +83,8 @@ module Fluent::Plugin
     def configure_parser(conf)
       if conf['format']
         @parser = parser_create
+      elsif parser_config = conf.elements('parse').first
+        @parser = parser_create(conf: parser_config)
       end
     end
 

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -5,7 +5,6 @@ require 'yajl'
 
 module Fluent::Plugin
   class CloudwatchLogsOutput < Output
-    include Fluent::MessagePackFactory::Mixin
     Fluent::Plugin.register_output('cloudwatch_logs', self)
 
     helpers :compat_parameters, :inject

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -113,6 +113,43 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
     end
 
+    test "emit with <parse> csv" do
+      cloudwatch_config = {'tag' => "test",
+                           '@type' => 'cloudwatch_logs',
+                           'log_group_name' => "#{log_group_name}",
+                           'log_stream_name' => "#{log_stream_name}",
+                           'state_file' => '/tmp/state',
+                          }
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
+
+      csv_format_config = config_element('ROOT', '', cloudwatch_config, [
+                                           config_element('parse', '', {'@type' => 'csv',
+                                                                        'keys' => 'time,message',
+                                                                        'time_key' => 'time'})
+                                         ])
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      log_time_ms = time_ms - 10000
+      put_log_events([
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
+      ])
+
+      sleep 5
+
+      d = create_driver(csv_format_config)
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+      assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+    end
+
     def test_emit_width_format
       create_log_stream
 
@@ -149,6 +186,47 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal({'cloudwatch' => 'logs2'}, emits[1][2])
     end
 
+    test "emit with <parse> regexp" do
+      cloudwatch_config = {'tag' => "test",
+                           '@type' => 'cloudwatch_logs',
+                           'log_group_name' => "#{log_group_name}",
+                           'log_stream_name' => "#{log_stream_name}",
+                           'state_file' => '/tmp/state',
+                          }
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
+
+      regex_format_config = config_element('ROOT', '', cloudwatch_config, [
+                                           config_element('parse', '', {'@type' => 'regexp',
+                                                                        'expression' => "/^(?<cloudwatch>[^ ]*)?/",
+                                                                       })
+                                         ])
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      put_log_events([
+        {timestamp: time_ms, message: 'logs1'},
+        {timestamp: time_ms, message: 'logs2'},
+      ])
+
+      sleep 5
+
+      d = create_driver(regex_format_config)
+
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal('test', emits[0][0])
+      assert_in_delta((time_ms / 1000).floor, emits[0][1], 10)
+      assert_equal({'cloudwatch' => 'logs1'}, emits[0][2])
+      assert_equal('test', emits[1][0])
+      assert_in_delta((time_ms / 1000).floor, emits[1][1], 10)
+      assert_equal({'cloudwatch' => 'logs2'}, emits[1][2])
+    end
+
     def test_emit_with_prefix
       new_log_stream("testprefix")
       create_log_stream

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-24 00:00:00.000000000 Z
+date: 2020-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd