fluent-plugin-cloudwatch-logs 0.9.5 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13ef3e0d5f4fd4c12c49b16ccbd07db237b7cff26ea272e4ef6337b09b02e975
-  data.tar.gz: 8cefeed014b1842ccce9f15479ad8cc54aa51b976281abb5d3e1c8828239aaa3
+  metadata.gz: 105e96fa516c4f972fc2b01f83c1daa30d39ff4b827b46b811b9f30514231516
+  data.tar.gz: 85741a1b0ccad0f9e207c837e7417b786ef68d1fc1552cbe80b73030e5b4191b
 SHA512:
-  metadata.gz: 41ca6fa20d0c26955fcbab7f7a5a9fb8c032a5fd09f8676ccaaf5e4dd12694ea74c7c5320590f076405c691d436bc87b820e6bc9b75dc516062b2f3e09d8a377
-  data.tar.gz: 1f5bbd694c5962de73ee1e0ca622d8892437fb82d00b76ed87086feeba49f6d2c3a4904edad81bec88ba2d3ca47701c15cb56395470f19742c16b3168dd870e5
+  metadata.gz: 84a8458e9704102609b382b9cb8fe4c332a1c1b9c9562b89b959961bb3624958cbffc1f965eba2a502c10a0e45f1ceb427a987681d5420c35663bc6df88295ed
+  data.tar.gz: d6e22ad22c4eef5cb6c8fab7690ca75585e7ad99756926181f50c43fef70df0e52e12722e14809b793947e56ed8199fc393a318f42308f05fd6d23d6aa9a4b13

data/README.md

@@ -178,6 +178,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   #<parse>
   # @type none # or csv, tsv, regexp etc.
   #</parse>
+  #<storage>
+  # @type local # or redis, memcached, etc.
+  #</storage>
 </source>
 ```
 
@@ -194,7 +197,8 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `log_stream_name`: name of log stream to fetch logs
 * `region`: AWS Region.  See [Authentication](#authentication) for more information.
 * `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
-* `state_file`: file to store current state (e.g. next\_forward\_token)
+* `include_metadata`: include metadata such as `log_group_name` and `log_stream_name`. (default: false)
+* `state_file`: file to store current state (e.g. next\_forward\_token). This parameter is deprecated. Use `<storage>` instead.
 * `tag`: fluentd tag
 * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
 * `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
@@ -204,6 +208,7 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `time_range_format`: specify time format for time range. (default: `%Y-%m-%d %H:%M:%S`)
 * `format`: specify CloudWatchLogs' log format. (default `nil`)
 * `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
+* `<storage>`: specify storage plugin configuration. see also: https://docs.fluentd.org/v/1.0/storage#how-to-use
 
 ## Test
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.9.5"
+        VERSION = "0.10.0"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -8,7 +8,9 @@ module Fluent::Plugin
   class CloudwatchLogsInput < Input
     Fluent::Plugin.register_input('cloudwatch_logs', self)
 
-    helpers :parser, :thread, :compat_parameters
+    helpers :parser, :thread, :compat_parameters, :storage
+
+    DEFAULT_STORAGE_TYPE = 'local'
 
     config_param :aws_key_id, :string, default: nil, secret: true
     config_param :aws_sec_key, :string, default: nil, secret: true
@@ -21,7 +23,8 @@ module Fluent::Plugin
     config_param :log_group_name, :string
     config_param :log_stream_name, :string, default: nil
     config_param :use_log_stream_name_prefix, :bool, default: false
-    config_param :state_file, :string
+    config_param :state_file, :string, default: nil,
+                 deprecated: "Use <stroage> instead."
     config_param :fetch_interval, :time, default: 60
     config_param :http_proxy, :string, default: nil
     config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
@@ -31,11 +34,18 @@ module Fluent::Plugin
     config_param :end_time, :string, default: nil
     config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
     config_param :throttling_retry_seconds, :time, default: nil
+    config_param :include_metadata, :bool, default: false
 
     config_section :parse do
       config_set_default :@type, 'none'
     end
 
+    config_section :storage do
+      config_set_default :usage, 'store_next_tokens'
+      config_set_default :@type, DEFAULT_STORAGE_TYPE
+      config_set_default :persistent, false
+    end
+
     def initialize
       super
 
@@ -53,6 +63,7 @@ module Fluent::Plugin
       if @start_time && @end_time && (@end_time < @start_time)
         raise Fluent::ConfigError, "end_time(#{@end_time}) should be greater than start_time(#{@start_time})."
       end
+      @next_token_storage = storage_create(usage: 'store_next_tokens', conf: config, default_type: DEFAULT_STORAGE_TYPE)
     end
 
     def start
@@ -99,20 +110,28 @@ module Fluent::Plugin
       end
     end
 
-    def state_file_for(log_stream_name)
-      return "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}" if log_stream_name
-      return @state_file
+    def state_key_for(log_stream_name)
+      if log_stream_name
+        "#{@state_file}_#{log_stream_name.gsub(File::SEPARATOR, '-')}"
+      else
+        @state_file
+      end
+    end
+
+    def migrate_state_file_to_storage(log_stream_name)
+      @next_token_storage.put(:"#{state_key_for(log_stream_name)}", File.read(state_key_for(log_stream_name)).chomp)
+      File.delete(state_key_for(log_stream_name))
     end
 
     def next_token(log_stream_name)
-      return nil unless File.exist?(state_file_for(log_stream_name))
-      File.read(state_file_for(log_stream_name)).chomp
+      if @next_token_storage.persistent && File.exist?(state_key_for(log_stream_name))
+        migrate_state_file_to_storage(log_stream_name)
+      end
+      @next_token_storage.get(:"#{state_key_for(log_stream_name)}")
     end
 
     def store_next_token(token, log_stream_name = nil)
-      File.open(state_file_for(log_stream_name), 'w') do |f|
-        f.write token
-      end
+      @next_token_storage.put(:"#{state_key_for(log_stream_name)}", token)
     end
 
     def run
@@ -130,8 +149,16 @@ module Fluent::Plugin
               log_streams.each do |log_stream|
                 log_stream_name = log_stream.log_stream_name
                 events = get_events(log_stream_name)
+                metadata = if @include_metadata
+                             {
+                               "log_stream_name" => log_stream_name,
+                               "log_group_name" => @log_group_name
+                             }
+                           else
+                             {}
+                           end
                 events.each do |event|
-                  emit(log_stream_name, event)
+                  emit(log_stream_name, event, metadata)
                 end
               end
             rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
@@ -140,8 +167,16 @@ module Fluent::Plugin
             end
           else
             events = get_events(@log_stream_name)
+            metadata = if @include_metadata
+                          {
+                            "log_stream_name" => @log_stream_name,
+                            "log_group_name" => @log_group_name
+                          }
+                        else
+                          {}
+                        end
             events.each do |event|
-              emit(log_stream_name, event)
+              emit(log_stream_name, event, metadata)
             end
           end
         end
@@ -149,18 +184,24 @@ module Fluent::Plugin
       end
     end
 
-    def emit(stream, event)
+    def emit(stream, event, metadata)
       if @parser
         @parser.parse(event.message) {|time,record|
           if @use_aws_timestamp
             time = (event.timestamp / 1000).floor
           end
+          unless metadata.empty?
+            record.merge!("metadata" => metadata)
+          end
           router.emit(@tag, time, record)
         }
       else
         time = (event.timestamp / 1000).floor
         begin
           record = @json_handler.load(event.message)
+          unless metadata.empty?
+            record.merge!("metadata" => metadata)
+          end
           router.emit(@tag, time, record)
         rescue JSON::ParserError, Yajl::ParseError => error # Catch parser errors
           log.error "Invalid JSON encountered while parsing event.message"

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -377,8 +377,7 @@ module Fluent::Plugin
           end
         rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException, Aws::CloudWatchLogs::Errors::DataAlreadyAcceptedException => err
           sleep 1 # to avoid too many API calls
-          log_stream = find_log_stream(group_name, stream_name)
-          store_next_sequence_token(group_name, stream_name, log_stream.upload_sequence_token)
+          store_next_sequence_token(group_name, stream_name, err.expected_sequence_token)
           log.warn "updating upload sequence token forcefully because unrecoverable error occured", {
             "error" => err,
             "log_group" => group_name,

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -99,6 +99,34 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
     end
 
+    def test_emit_with_metadata
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      put_log_events([
+        {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+        {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+      ])
+
+      sleep 5
+
+      d = create_driver(default_config + %[include_metadata true])
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_true(emits[0][2].has_key?("metadata"))
+      assert_true(emits[1][2].has_key?("metadata"))
+      emits[0][2].delete_if {|k, v|
+        k == "metadata"
+      }
+      emits[1][2].delete_if {|k, v|
+        k == "metadata"
+      }
+      assert_equal(2, emits.size)
+      assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+      assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+    end
+
     def test_emit_with_aws_timestamp
       create_log_stream
 
@@ -173,7 +201,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
                            '@type' => 'cloudwatch_logs',
                            'log_group_name' => "#{log_group_name}",
                            'log_stream_name' => "#{log_stream_name}",
-                           'state_file' => '/tmp/state',
                           }
       cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
       cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
@@ -183,7 +210,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       csv_format_config = config_element('ROOT', '', cloudwatch_config, [
                                            config_element('parse', '', {'@type' => 'csv',
                                                                         'keys' => 'time,message',
-                                                                        'time_key' => 'time'})
+                                                                        'time_key' => 'time'}),
+                                           config_element('storage', '', {'@type' => 'local',
+                                                                          'path' => '/tmp/state'})
                                          ])
       create_log_stream
 
@@ -205,6 +234,54 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
     end
 
+    test "emit with <parse> csv with metadata" do
+      cloudwatch_config = {'tag' => "test",
+                           '@type' => 'cloudwatch_logs',
+                           'log_group_name' => "#{log_group_name}",
+                           'log_stream_name' => "#{log_stream_name}",
+                           'include_metadata' => true,
+                          }
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(region)) if ENV['region']
+      cloudwatch_config = cloudwatch_config.merge!(config_elementify(endpoint)) if ENV['endpoint']
+
+      csv_format_config = config_element('ROOT', '', cloudwatch_config, [
+                                           config_element('parse', '', {'@type' => 'csv',
+                                                                        'keys' => 'time,message',
+                                                                        'time_key' => 'time'}),
+                                           config_element('storage', '', {'@type' => 'local',
+                                                                          'path' => '/tmp/state'})
+
+                                         ])
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      log_time_ms = time_ms - 10000
+      put_log_events([
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
+      ])
+
+      sleep 5
+
+      d = create_driver(csv_format_config)
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_true(emits[0][2].has_key?("metadata"))
+      assert_true(emits[1][2].has_key?("metadata"))
+      emits[0][2].delete_if {|k, v|
+        k == "metadata"
+      }
+      emits[1][2].delete_if {|k, v|
+        k == "metadata"
+      }
+      assert_equal(2, emits.size)
+      assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+      assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+    end
+
     def test_emit_width_format
       create_log_stream
 
@@ -246,7 +323,6 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
                            '@type' => 'cloudwatch_logs',
                            'log_group_name' => "#{log_group_name}",
                            'log_stream_name' => "#{log_stream_name}",
-                           'state_file' => '/tmp/state',
                           }
       cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_key_id)) if ENV['aws_key_id']
       cloudwatch_config = cloudwatch_config.merge!(config_elementify(aws_sec_key)) if ENV['aws_sec_key']
@@ -256,7 +332,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       regex_format_config = config_element('ROOT', '', cloudwatch_config, [
                                            config_element('parse', '', {'@type' => 'regexp',
                                                                         'expression' => "/^(?<cloudwatch>[^ ]*)?/",
-                                                                       })
+                                                                       }),
+                                           config_element('storage', '', {'@type' => 'local',
+                                                                          'path' => '/tmp/state'})
                                          ])
       create_log_stream
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 0.10.0
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-19 00:00:00.000000000 Z
+date: 2020-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd