fluent-plugin-cloudwatch-logs 0.9.0 → 0.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4b45da3678f7685f04b1ca6053a2f396dd1847421002e4d7358d501916328f6
-  data.tar.gz: acc891b68cf5182063f4093c1b0c2db65c07f9b7e115bc05678128b97eb3d6b0
+  metadata.gz: 13ef3e0d5f4fd4c12c49b16ccbd07db237b7cff26ea272e4ef6337b09b02e975
+  data.tar.gz: 8cefeed014b1842ccce9f15479ad8cc54aa51b976281abb5d3e1c8828239aaa3
 SHA512:
-  metadata.gz: dd37edc8daf19fc3c8b2ea3df12786b906960f546dbd8bb9bcdcf047cb602138ee215edac5dc51891c3956ef95e7ec05a512219d62e18974dfdb03a16f62595a
-  data.tar.gz: 6f102f4499c68f3ee2ebeacf388ecafa9cf21d3d0c5b3091420986b5dd8b5849a75fdd1c59a274892592b2e7dab0613e71829b3986905c144dd20e4c8aec02b5
+  metadata.gz: 41ca6fa20d0c26955fcbab7f7a5a9fb8c032a5fd09f8676ccaaf5e4dd12694ea74c7c5320590f076405c691d436bc87b820e6bc9b75dc516062b2f3e09d8a377
+  data.tar.gz: 1f5bbd694c5962de73ee1e0ca622d8892437fb82d00b76ed87086feeba49f6d2c3a4904edad81bec88ba2d3ca47701c15cb56395470f19742c16b3168dd870e5

data/README.md

@@ -149,7 +149,7 @@ Fetch sample log from CloudWatch Logs:
 * `remove_log_group_aws_tags_key`: remove field specified by `log_group_aws_tags_key`
 * `remove_log_group_name_key`: remove field specified by `log_group_name_key`
 * `remove_log_stream_name_key`: remove field specified by `log_stream_name_key`
-* `remove_retention_in_days`: remove field specified by `retention_in_days`
+* `remove_retention_in_days_key`: remove field specified by `retention_in_days_key`
 * `retention_in_days`: use to set the expiry time for log group when created with `auto_create_stream`. (default to no expiry)
 * `retention_in_days_key`: use specified field of records as retention period
 * `use_tag_as_group`: to use tag as a group name
@@ -170,6 +170,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   state_file /var/lib/fluent/group_stream.in.state
   #endpoint http://localhost:5000/
   #json_handler json
+  # start_time "2020-03-01 00:00:00Z"
+  # end_time "2020-04-30 15:00:00Z"
+  # time_range_format "%Y-%m-%d %H:%M:%S%z"
   # Users can use `format` or `<parse>` directive to parse non-JSON CloudwatchLogs' log
   # format none # or csv, tsv, regexp etc.
   #<parse>
@@ -190,11 +193,15 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `log_group_name`: name of log group to fetch logs
 * `log_stream_name`: name of log stream to fetch logs
 * `region`: AWS Region.  See [Authentication](#authentication) for more information.
+* `throttling_retry_seconds`: time period in seconds to retry a request when aws CloudWatch rate limit exceeds (default: nil)
 * `state_file`: file to store current state (e.g. next\_forward\_token)
 * `tag`: fluentd tag
 * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
 * `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
 * `use_aws_timestamp`: get timestamp from Cloudwatch event for non json logs, otherwise fluentd will parse the log to get the timestamp (default `false`)
+* `start_time`: specify starting time range for obtaining logs. (default: `nil`)
+* `end_time`: specify ending time range for obtaining logs. (default: `nil`)
+* `time_range_format`: specify time format for time range. (default: `%Y-%m-%d %H:%M:%S`)
 * `format`: specify CloudWatchLogs' log format. (default `nil`)
 * `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.9.0"
+        VERSION = "0.9.5"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -1,4 +1,5 @@
 require 'date'
+require 'time'
 require 'fluent/plugin/input'
 require 'fluent/plugin/parser'
 require 'yajl'
@@ -26,6 +27,10 @@ module Fluent::Plugin
     config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
     config_param :use_todays_log_stream, :bool, default: false
     config_param :use_aws_timestamp, :bool, default: false
+    config_param :start_time, :string, default: nil
+    config_param :end_time, :string, default: nil
+    config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
+    config_param :throttling_retry_seconds, :time, default: nil
 
     config_section :parse do
       config_set_default :@type, 'none'
@@ -42,6 +47,12 @@ module Fluent::Plugin
       compat_parameters_convert(conf, :parser)
       super
       configure_parser(conf)
+
+      @start_time = (Time.strptime(@start_time, @time_range_format).to_f * 1000).floor if @start_time
+      @end_time = (Time.strptime(@end_time, @time_range_format).to_f * 1000).floor if @end_time
+      if @start_time && @end_time && (@end_time < @start_time)
+        raise Fluent::ConfigError, "end_time(#{@end_time}) should be greater than start_time(#{@start_time})."
+      end
     end
 
     def start
@@ -159,36 +170,55 @@ module Fluent::Plugin
     end
 
     def get_events(log_stream_name)
-      request = {
-        log_group_name: @log_group_name,
-        log_stream_name: log_stream_name
-      }
-      log_next_token = next_token(log_stream_name)
-      request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
-      response = @logs.get_log_events(request)
-      if valid_next_token(log_next_token, response.next_forward_token)
-        store_next_token(response.next_forward_token, log_stream_name)
-      end
+      throttling_handler('get_log_events') do
+        request = {
+          log_group_name: @log_group_name,
+          log_stream_name: log_stream_name
+        }
+        request.merge!(start_time: @start_time) if @start_time
+        request.merge!(end_time: @end_time) if @end_time
+        log_next_token = next_token(log_stream_name)
+        request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
+        response = @logs.get_log_events(request)
+        if valid_next_token(log_next_token, response.next_forward_token)
+          store_next_token(response.next_forward_token, log_stream_name)
+        end
 
-      response.events
+        response.events
+      end
     end
 
     def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
-      request = {
-        log_group_name: @log_group_name
-      }
-      request[:next_token] = next_token if next_token
-      request[:log_stream_name_prefix] = log_stream_name_prefix
-      response = @logs.describe_log_streams(request)
-      if log_streams
-        log_streams.concat(response.log_streams)
-      else
-        log_streams = response.log_streams
+      throttling_handler('describe_log_streams') do
+        request = {
+          log_group_name: @log_group_name
+        }
+        request[:next_token] = next_token if next_token
+        request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
+        response = @logs.describe_log_streams(request)
+        if log_streams
+          log_streams.concat(response.log_streams)
+        else
+          log_streams = response.log_streams
+        end
+        if response.next_token
+          log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
+        end
+        log_streams
       end
-      if response.next_token
-        log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
+    end
+
+    def throttling_handler(method_name)
+      yield
+    rescue Aws::CloudWatchLogs::Errors::ThrottlingException => err
+      if throttling_retry_seconds
+        log.warn "ThrottlingException #{method_name}. Waiting #{throttling_retry_seconds} seconds to retry."
+        sleep throttling_retry_seconds
+
+        throttling_handler(method_name) { yield }
+      else
+        raise err
       end
-      log_streams
     end
 
     def valid_next_token(prev_token, next_token)

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -41,7 +41,7 @@ module Fluent::Plugin
     config_param :remove_log_group_aws_tags_key, :bool, default: false
     config_param :retention_in_days, :integer, default: nil
     config_param :retention_in_days_key, :string, default: nil
-    config_param :remove_retention_in_days, :bool, default: false
+    config_param :remove_retention_in_days_key, :bool, default: false
     config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
     config_param :log_rejected_request, :bool, :default => false
 
@@ -219,6 +219,14 @@ module Fluent::Plugin
 
         events = []
         rs.each do |t, time, record|
+          if @log_group_aws_tags_key && @remove_log_group_aws_tags_key
+            record.delete(@log_group_aws_tags_key)
+          end
+
+          if @retention_in_days_key && @remove_retention_in_days_key
+            record.delete(@retention_in_days_key)
+          end
+
           record = drop_empty_record(record)
 
           time_ms = (time.to_f * 1000).floor

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -25,6 +25,10 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
         use_log_stream_name_prefix true
         state_file /tmp/state
         use_aws_timestamp true
+        start_time "2019-06-18 00:00:00Z"
+        end_time "2020-01-18 00:00:00Z"
+        time_range_format "%Y-%m-%d %H:%M:%S%z"
+        throttling_retry_seconds 30
       EOC
 
       assert_equal('test_id', d.instance.aws_key_id)
@@ -37,6 +41,30 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal('/tmp/state', d.instance.state_file)
       assert_equal(:yajl, d.instance.json_handler)
       assert_equal(true, d.instance.use_aws_timestamp)
+      assert_equal(1560816000000, d.instance.start_time)
+      assert_equal(1579305600000, d.instance.end_time)
+      assert_equal("%Y-%m-%d %H:%M:%S%z", d.instance.time_range_format)
+      assert_equal(30, d.instance.throttling_retry_seconds)
+    end
+
+    test 'invalid time range' do
+      assert_raise(Fluent::ConfigError) do
+        create_driver(<<-EOC)
+          @type cloudwatch_logs
+          aws_key_id test_id
+          aws_sec_key test_key
+          region us-east-1
+          tag test
+          log_group_name group
+          log_stream_name stream
+          use_log_stream_name_prefix true
+          state_file /tmp/state
+          use_aws_timestamp true
+          start_time "2019-06-18 00:00:00Z"
+          end_time "2019-01-18 00:00:00Z"
+          time_range_format "%Y-%m-%d %H:%M:%S%z"
+        EOC
+      end
     end
   end
 
@@ -92,6 +120,33 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(['test', (time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
     end
 
+    def test_emit_with_aws_timestamp_and_time_range
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      before_6h_time_ms = ((Time.now.to_f - 60*60*6) * 1000).floor
+      log_time_ms = time_ms - 10000
+      put_log_events([
+        {timestamp: before_6h_time_ms, message: Time.at((before_6h_time_ms - 10000)/1000.floor).to_s + ",Cloudwatch non json logs1"},
+        {timestamp: before_6h_time_ms, message: Time.at((before_6h_time_ms - 10000)/1000.floor).to_s + ",Cloudwatch non json logs2"},
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs3"},
+      ])
+
+      sleep 5
+
+      d = create_driver(csv_format_config_aws_timestamp + %[
+        start_time #{Time.at(Time.now.to_f - 60*60*8).to_s}
+        end_time #{Time.at(Time.now.to_f - 60*60*4).to_s}
+        time_range_format "%Y-%m-%d %H:%M:%S %z"
+      ])
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal(['test', (before_6h_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+      assert_equal(['test', (before_6h_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+    end
+
     def test_emit_with_log_timestamp
       create_log_stream
 
@@ -555,6 +610,59 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(["test", ((time_ms + 7000) / 1000), { "cloudwatch" => "logs7" }], events[6])
       assert_equal(["test", ((time_ms + 8000) / 1000), { "cloudwatch" => "logs8" }], events[7])
     end
+
+    test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException in get_log_events" do
+      config = <<-CONFIG
+        tag test
+        @type cloudwatch_logs
+        log_group_name #{log_group_name}
+        state_file /tmp/state
+        fetch_interval 0.1
+        throttling_retry_seconds 0.2
+      CONFIG
+
+      # it will raises the error 2 times
+      counter = 0
+      times = 2
+      stub(@client).get_log_events(anything) {
+        counter += 1
+        counter <= times ? raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")) : OpenStruct.new(events: [], next_forward_token: nil)
+      }
+
+      d = create_driver(config)
+
+      # so, it is expected to valid_next_token once
+      mock(d.instance).valid_next_token(nil, nil).once
+
+      d.run
+      assert_equal(2, d.logs.select {|l| l =~ /ThrottlingException get_log_events. Waiting 0.2 seconds to retry/ }.size)
+    end
+
+    test "retry on Aws::CloudWatchLogs::Errors::ThrottlingException in describe_log_streams" do
+      config = <<-CONFIG
+        tag test
+        @type cloudwatch_logs
+        log_group_name #{log_group_name}
+        use_log_stream_name_prefix true
+        state_file /tmp/state
+        fetch_interval 0.1
+        throttling_retry_seconds 0.2
+      CONFIG
+
+      # it will raises the error 2 times
+      log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
+      counter = 0
+      times = 2
+      stub(@client).describe_log_streams(anything) {
+        counter += 1
+        counter <= times ? raise(Aws::CloudWatchLogs::Errors::ThrottlingException.new(nil, "error")) : OpenStruct.new(log_streams: [log_stream], next_token: nil)
+      }
+
+      d = create_driver(config)
+
+      d.run
+      assert_equal(2, d.logs.select {|l| l =~ /ThrottlingException describe_log_streams. Waiting 0.2 seconds to retry/ }.size)
+    end
   end
 
   private

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -547,6 +547,37 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert(d.logs.any?{|log| log.include?("failed to set retention policy for Log group")})
     end
 
+    def test_remove_retention_in_days_key
+      new_log_stream
+
+      d = create_driver(<<-EOC)
+        #{default_config}
+        log_group_name #{log_group_name}
+        log_stream_name #{log_stream_name}
+        retention_in_days_key retention_in_days
+        remove_retention_in_days_key true
+      EOC
+
+      records = [
+        {'cloudwatch' => 'logs1', 'message' => 'message1', 'retention_in_days' => '7'},
+        {'cloudwatch' => 'logs2', 'message' => 'message2', 'retention_in_days' => '7'},
+      ]
+
+      time = Time.now
+      d.run(default_tag: fluentd_tag) do
+        records.each_with_index do |record, i|
+          d.feed(time.to_i + i, record)
+        end
+      end
+
+      sleep 10
+
+      events = get_log_events
+      assert_equal(2, events.size)
+      assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
+      assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
+    end
+
     def test_log_group_aws_tags_key
       clear_log_group
 
@@ -576,6 +607,37 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
       assert_equal("value2", awstags.fetch("tag2"))
     end
 
+    def test_remove_log_group_aws_tags_key
+      new_log_stream
+
+      d = create_driver(<<-EOC)
+        #{default_config}
+        log_group_name #{log_group_name}
+        log_stream_name #{log_stream_name}
+        log_group_aws_tags_key log_group_tags
+        remove_log_group_aws_tags_key true
+      EOC
+
+      records = [
+        {'cloudwatch' => 'logs1', 'message' => 'message1', 'log_group_tags' => {"tag1" => "value1", "tag2" => "value2"}},
+        {'cloudwatch' => 'logs2', 'message' => 'message2', 'log_group_tags' => {"tag1" => "value1", "tag2" => "value2"}},
+      ]
+
+      time = Time.now
+      d.run(default_tag: fluentd_tag) do
+        records.each_with_index do |record, i|
+          d.feed(time.to_i + i, record)
+        end
+      end
+
+      sleep 10
+
+      events = get_log_events
+      assert_equal(2, events.size)
+      assert_equal({'cloudwatch' => 'logs1', 'message' => 'message1'}, JSON.parse(events[0].message))
+      assert_equal({'cloudwatch' => 'logs2', 'message' => 'message2'}, JSON.parse(events[1].message))
+    end
+
     def test_log_group_aws_tags_key_same_group_diff_tags
       clear_log_group
 

data/test/test_helper.rb

@@ -42,10 +42,7 @@ module CloudwatchLogsTestHelper
   end
 
   def log_stream_name(log_stream_name_prefix = nil)
-    if !@log_stream_name
-      new_log_stream(log_stream_name_prefix)
-    end
-    @log_stream_name
+    @log_stream_name ||= new_log_stream(log_stream_name_prefix)
   end
 
   def new_log_stream(log_stream_name_prefix = nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.5
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-23 00:00:00.000000000 Z
+date: 2020-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd