fluent-plugin-cloudwatch-logs 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f4792d605b22ee6912e36c02a7e20037c91d9912bb68ae14dd3fbb49b7afa0b
-  data.tar.gz: 445aa7b5656044ae51b54dbc86d6a0a6509682ae9cb273b19a1f4742d123ecb4
+  metadata.gz: f3809f992ae8d9967a7f1edae7ac40614496d98e38baf2d6aa18bcfbae5c72ff
+  data.tar.gz: 4eb7d7741fb3ef6e1c3d698f305c36ac49ee94df4d9350f5794a2e887e29984c
 SHA512:
-  metadata.gz: 50726dbbe3b574499b65a54469afc04d13a2b82011253992bd662338bae1351106ab4a0e430f4f48537c8c7a8dc9866ec0fab264cba31373d1689dae5fc65082
-  data.tar.gz: 8911874548680ac11cbf36c148c1dbe0e60b869d617d34b99711dbcbeb09e93aefac66e9e8e4ba4d3427bb45f42ac0d1fad3b103d2b3f3523172d331a648f39a
+  metadata.gz: 8da9d33a1217cae533af682e950314d5a0bd5dbaac8019f0309ef541f26434027ef4b0133bbf2e3030a87c685ba00839c766a99336b0dd0c9f1304a65698a647
+  data.tar.gz: a2df61af2a98d7dab6d2fb4b5aecbb904fd582aa80e9ae3dd4b15c00f8d320dface8b3fbfb8ba3b1cdf47641a8d4edce8e2e03eaf2eafc16ba976cf68edf9514

data/README.md

@@ -132,6 +132,7 @@ Fetch sample log from CloudWatch Logs:
 * `aws_sts_role_arn`: the role ARN to assume when using cross-account sts authentication
 * `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
 * `json_handler`:  name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
+* `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
 
 This plugin uses [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) and you can use addtional variables such as %{hostname}, %{uuid}, etc. These variables are useful to put hostname in `log_stream_name`.
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.6.0"
+        VERSION = "0.6.1"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -1,3 +1,4 @@
+require 'date'
 require 'fluent/plugin/input'
 require 'fluent/plugin/parser'
 require 'yajl'
@@ -17,12 +18,13 @@ module Fluent::Plugin
     config_param :endpoint, :string, :default => nil
     config_param :tag, :string
     config_param :log_group_name, :string
-    config_param :log_stream_name, :string
+    config_param :log_stream_name, :string, :default => nil
     config_param :use_log_stream_name_prefix, :bool, default: false
     config_param :state_file, :string
     config_param :fetch_interval, :time, default: 60
     config_param :http_proxy, :string, default: nil
     config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
+    config_param :use_todays_log_stream, :bool, default: false
 
     config_section :parse do
       config_set_default :@type, 'none'
@@ -105,8 +107,10 @@ module Fluent::Plugin
         if Time.now > @next_fetch_time
           @next_fetch_time += @fetch_interval
 
-          if @use_log_stream_name_prefix
-            log_streams = describe_log_streams
+          if @use_log_stream_name_prefix || @use_todays_log_stream
+            log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
+            log_streams = describe_log_streams(log_stream_name_prefix)
+            log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
             log_streams.each do |log_stream|
               log_stream_name = log_stream.log_stream_name
               events = get_events(log_stream_name)
@@ -142,19 +146,22 @@ module Fluent::Plugin
         log_group_name: @log_group_name,
         log_stream_name: log_stream_name
       }
-      request[:next_token] = next_token(log_stream_name) if next_token(log_stream_name)
+      log_next_token = next_token(log_stream_name)
+      request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty? 
       response = @logs.get_log_events(request)
-      store_next_token(response.next_forward_token, log_stream_name)
+      if valid_next_token(log_next_token, response.next_forward_token)
+        store_next_token(response.next_forward_token, log_stream_name)
+      end
 
       response.events
     end
 
-    def describe_log_streams(log_streams = nil, next_token = nil)
+    def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
       request = {
         log_group_name: @log_group_name
       }
       request[:next_token] = next_token if next_token
-      request[:log_stream_name_prefix] = @log_stream_name
+      request[:log_stream_name_prefix] = log_stream_name_prefix
       response = @logs.describe_log_streams(request)
       if log_streams
         log_streams.concat(response.log_streams)
@@ -162,9 +169,21 @@ module Fluent::Plugin
         log_streams = response.log_streams
       end
       if response.next_token
-        log_streams = describe_log_streams(log_streams, response.next_token)
+        log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
       end
       log_streams
     end
+
+    def valid_next_token(prev_token, next_token)
+      return prev_token != next_token.chomp && !next_token.nil?
+    end
+
+    def get_todays_date
+      return Date.today.strftime("%Y/%m/%d")
+    end
+
+    def get_yesterdays_date
+      return (Date.today - 1).strftime("%Y/%m/%d")
+    end
   end
 end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -343,6 +343,7 @@ module Fluent::Plugin
             "log_stream" => stream_name,
             "new_sequence_token" => token,
           }
+          retry_count += 1
         rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => err
           if @auto_create_stream && err.message == 'The specified log stream does not exist.'
             log.warn 'Creating log stream because "The specified log stream does not exist." error is got', {
@@ -352,6 +353,7 @@ module Fluent::Plugin
             }
             create_log_stream(group_name, stream_name)
             delete_sequence_token(group_name, stream_name)
+            retry_count += 1
           else
             raise err
           end

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -1,6 +1,7 @@
 require 'test_helper'
 require 'fluent/test/driver/input'
 require 'fluent/test/helpers'
+require 'date'
 
 class CloudwatchLogsInputTest < Test::Unit::TestCase
   include CloudwatchLogsTestHelper
@@ -137,6 +138,87 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
     assert_true(emits.include? ['test', ((time_ms + 4000) / 1000).floor, {'cloudwatch' => 'logs4'}])
   end
 
+  def test_emit_with_todays_log_stream
+    new_log_stream("testprefix")
+    create_log_stream
+
+    today = DateTime.now.strftime("%Y/%m/%d")
+    yesterday = (Date.today - 1).strftime("%Y/%m/%d")
+    tomorrow = (Date.today + 1).strftime("%Y/%m/%d")
+
+
+    time_ms = (Time.now.to_f * 1000).floor
+    put_log_events([
+      {timestamp: time_ms + 1000, message: '{"cloudwatch":"logs1"}'},
+      {timestamp: time_ms + 2000, message: '{"cloudwatch":"logs2"}'},
+    ])
+
+    new_log_stream(today)
+    create_log_stream
+    put_log_events([
+      {timestamp: time_ms + 3000, message: '{"cloudwatch":"logs3"}'},
+      {timestamp: time_ms + 4000, message: '{"cloudwatch":"logs4"}'},
+    ])
+
+    new_log_stream(yesterday)
+    create_log_stream
+    put_log_events([
+      {timestamp: time_ms + 5000, message: '{"cloudwatch":"logs5"}'},
+      {timestamp: time_ms + 6000, message: '{"cloudwatch":"logs6"}'},
+    ])
+
+    new_log_stream(tomorrow)
+    create_log_stream
+    put_log_events([
+      {timestamp: time_ms + 7000, message: '{"cloudwatch":"logs7"}'},
+      {timestamp: time_ms + 8000, message: '{"cloudwatch":"logs8"}'},
+    ])
+
+    new_log_stream(today)
+    create_log_stream
+    put_log_events([
+      {timestamp: time_ms + 9000, message: '{"cloudwatch":"logs9"}'},
+      {timestamp: time_ms + 10000, message: '{"cloudwatch":"logs10"}'},
+    ])
+
+    new_log_stream(yesterday)
+    create_log_stream
+    put_log_events([
+      {timestamp: time_ms + 11000, message: '{"cloudwatch":"logs11"}'},
+      {timestamp: time_ms + 12000, message: '{"cloudwatch":"logs12"}'},
+    ])
+
+    sleep 15
+
+    d = create_driver(<<-EOC)
+      tag test
+      @type cloudwatch_logs
+      log_group_name #{log_group_name}
+      use_todays_log_stream true
+      state_file /tmp/state
+      #{aws_key_id}
+      #{aws_sec_key}
+      #{region}
+      #{endpoint}
+    EOC
+    d.run(expect_emits: 8, timeout: 15)
+
+    emits = d.events
+    assert_equal(8, emits.size)
+    assert_false(emits.include? ['test', ((time_ms + 1000) / 1000).floor, {'cloudwatch' => 'logs1'}])
+    assert_false(emits.include? ['test', ((time_ms + 2000) / 1000).floor, {'cloudwatch' => 'logs2'}])
+    assert_true(emits.include? ['test', ((time_ms + 3000) / 1000).floor, {'cloudwatch' => 'logs3'}])
+    assert_true(emits.include? ['test', ((time_ms + 4000) / 1000).floor, {'cloudwatch' => 'logs4'}])
+    assert_true(emits.include? ['test', ((time_ms + 5000) / 1000).floor, {'cloudwatch' => 'logs5'}])
+    assert_true(emits.include? ['test', ((time_ms + 6000) / 1000).floor, {'cloudwatch' => 'logs6'}])
+    assert_false(emits.include? ['test', ((time_ms + 7000) / 1000).floor, {'cloudwatch' => 'logs7'}])
+    assert_false(emits.include? ['test', ((time_ms + 8000) / 1000).floor, {'cloudwatch' => 'logs8'}])
+    assert_true(emits.include? ['test', ((time_ms + 9000) / 1000).floor, {'cloudwatch' => 'logs9'}])
+    assert_true(emits.include? ['test', ((time_ms + 10000) / 1000).floor, {'cloudwatch' => 'logs10'}])
+    assert_true(emits.include? ['test', ((time_ms + 11000) / 1000).floor, {'cloudwatch' => 'logs11'}])
+    assert_true(emits.include? ['test', ((time_ms + 12000) / 1000).floor, {'cloudwatch' => 'logs12'}])
+  end
+
   private
   def default_config
     <<-EOC

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-11 00:00:00.000000000 Z
+date: 2018-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd