fluent-plugin-cloudwatch-logs 0.9.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4b45da3678f7685f04b1ca6053a2f396dd1847421002e4d7358d501916328f6
-  data.tar.gz: acc891b68cf5182063f4093c1b0c2db65c07f9b7e115bc05678128b97eb3d6b0
+  metadata.gz: 849137579f60a74de04a845cdad60ec5c31a7903b8af5248c1122a06a4d66b3a
+  data.tar.gz: 4911ef8c0a16b88501f08cfe2dd217e458025d68496b1f279662bef606a53d2d
 SHA512:
-  metadata.gz: dd37edc8daf19fc3c8b2ea3df12786b906960f546dbd8bb9bcdcf047cb602138ee215edac5dc51891c3956ef95e7ec05a512219d62e18974dfdb03a16f62595a
-  data.tar.gz: 6f102f4499c68f3ee2ebeacf388ecafa9cf21d3d0c5b3091420986b5dd8b5849a75fdd1c59a274892592b2e7dab0613e71829b3986905c144dd20e4c8aec02b5
+  metadata.gz: 72233f6fe060a41813673688fe5ca00ec36de7e6e21bc52fcdb7751091a2e7b0198110e398ad5233021527496f74eee22a3c18490d2f35c847cea3587dcf5c40
+  data.tar.gz: 13357b5705f53a84a5598132238c00fc9caf1c0b317e943e38a1ff8b3ac96d1e427a01499ce89f911291b0571ac43876528c3faaa0bd821a600d20805fe76ec7

data/README.md

@@ -170,6 +170,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
   state_file /var/lib/fluent/group_stream.in.state
   #endpoint http://localhost:5000/
   #json_handler json
+  # start_time "2020-03-01 00:00:00Z"
+  # end_time "2020-04-30 15:00:00Z"
+  # time_range_format "%Y-%m-%d %H:%M:%S%z"
   # Users can use `format` or `<parse>` directive to parse non-JSON CloudwatchLogs' log
   # format none # or csv, tsv, regexp etc.
   #<parse>
@@ -195,6 +198,9 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
 * `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
 * `use_aws_timestamp`: get timestamp from Cloudwatch event for non json logs, otherwise fluentd will parse the log to get the timestamp (default `false`)
+* `start_time`: specify starting time range for obtaining logs. (default: `nil`)
+* `end_time`: specify ending time range for obtaining logs. (default: `nil`)
+* `time_range_format`: specify time format for time range. (default: `%Y-%m-%d %H:%M:%S`)
 * `format`: specify CloudWatchLogs' log format. (default `nil`)
 * `<parse>`: specify parser plugin configuration. see also: https://docs.fluentd.org/v/1.0/parser#how-to-use
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.9.0"
+        VERSION = "0.9.1"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -1,4 +1,5 @@
 require 'date'
+require 'time'
 require 'fluent/plugin/input'
 require 'fluent/plugin/parser'
 require 'yajl'
@@ -26,6 +27,9 @@ module Fluent::Plugin
     config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
     config_param :use_todays_log_stream, :bool, default: false
     config_param :use_aws_timestamp, :bool, default: false
+    config_param :start_time, :string, default: nil
+    config_param :end_time, :string, default: nil
+    config_param :time_range_format, :string, default: "%Y-%m-%d %H:%M:%S"
 
     config_section :parse do
       config_set_default :@type, 'none'
@@ -42,6 +46,12 @@ module Fluent::Plugin
       compat_parameters_convert(conf, :parser)
       super
       configure_parser(conf)
+
+      @start_time = (Time.strptime(@start_time, @time_range_format).to_f * 1000).floor if @start_time
+      @end_time = (Time.strptime(@end_time, @time_range_format).to_f * 1000).floor if @end_time
+      if @start_time && @end_time && (@end_time < @start_time)
+        raise Fluent::ConfigError, "end_time(#{@end_time}) should be greater than start_time(#{@start_time})."
+      end
     end
 
     def start
@@ -163,6 +173,8 @@ module Fluent::Plugin
         log_group_name: @log_group_name,
         log_stream_name: log_stream_name
       }
+      request.merge!(start_time: @start_time) if @start_time
+      request.merge!(end_time: @end_time) if @end_time
       log_next_token = next_token(log_stream_name)
       request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
       response = @logs.get_log_events(request)

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -25,6 +25,9 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
         use_log_stream_name_prefix true
         state_file /tmp/state
         use_aws_timestamp true
+        start_time "2019-06-18 00:00:00Z"
+        end_time "2020-01-18 00:00:00Z"
+        time_range_format "%Y-%m-%d %H:%M:%S%z"
       EOC
 
       assert_equal('test_id', d.instance.aws_key_id)
@@ -37,6 +40,29 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal('/tmp/state', d.instance.state_file)
       assert_equal(:yajl, d.instance.json_handler)
       assert_equal(true, d.instance.use_aws_timestamp)
+      assert_equal(1560816000000, d.instance.start_time)
+      assert_equal(1579305600000, d.instance.end_time)
+      assert_equal("%Y-%m-%d %H:%M:%S%z", d.instance.time_range_format)
+    end
+
+    test 'invalid time range' do
+      assert_raise(Fluent::ConfigError) do
+        create_driver(<<-EOC)
+          @type cloudwatch_logs
+          aws_key_id test_id
+          aws_sec_key test_key
+          region us-east-1
+          tag test
+          log_group_name group
+          log_stream_name stream
+          use_log_stream_name_prefix true
+          state_file /tmp/state
+          use_aws_timestamp true
+          start_time "2019-06-18 00:00:00Z"
+          end_time "2019-01-18 00:00:00Z"
+          time_range_format "%Y-%m-%d %H:%M:%S%z"
+        EOC
+      end
     end
   end
 
@@ -92,6 +118,33 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       assert_equal(['test', (time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
     end
 
+    def test_emit_with_aws_timestamp_and_time_range
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      before_6h_time_ms = ((Time.now.to_f - 60*60*6) * 1000).floor
+      log_time_ms = time_ms - 10000
+      put_log_events([
+        {timestamp: before_6h_time_ms, message: Time.at((before_6h_time_ms - 10000)/1000.floor).to_s + ",Cloudwatch non json logs1"},
+        {timestamp: before_6h_time_ms, message: Time.at((before_6h_time_ms - 10000)/1000.floor).to_s + ",Cloudwatch non json logs2"},
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs3"},
+      ])
+
+      sleep 5
+
+      d = create_driver(csv_format_config_aws_timestamp + %[
+        start_time #{Time.at(Time.now.to_f - 60*60*8).to_s}
+        end_time #{Time.at(Time.now.to_f - 60*60*4).to_s}
+        time_range_format "%Y-%m-%d %H:%M:%S %z"
+      ])
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal(['test', (before_6h_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+      assert_equal(['test', (before_6h_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+    end
+
     def test_emit_with_log_timestamp
       create_log_stream
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-23 00:00:00.000000000 Z
+date: 2020-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd