fluent-plugin-cloudwatch-logs 0.7.3 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac2327006fac3805ebf9fa2861895618806da41cfb2fc7cb58940b0c4f72049e
-  data.tar.gz: 0d7387d3d62c0c834a76a3c552b2b2e0dca81115a03ebda51074933420689ed6
+  metadata.gz: 1889a17c18ac0d47b2f8d4fb6e13cebd381743275698119fbdd2cfcd6742a752
+  data.tar.gz: cbde319d241346c6935429887547ffa1ff45aedf3df134156a19bf825bb6523b
 SHA512:
-  metadata.gz: df9deb817b1c0ff936ff777efe26df89f9e07c681951fa1c2840edb49b4b02dc1e4688e7df842f9ef7608cf8b8beeb9c976b0ee55be4a87dbe84d710eec6dbed
-  data.tar.gz: c72a272b1bd52f597f839bfa8468f8c8736e51206adc43385a7c91c031c9631e183d6061a8ecd524d0b38852f9d573d8b682fb7d9140bf5b02530e6ecc90dd6f
+  metadata.gz: 4827ac8cdfb0aab178aa9590c356e17a3476310383201ae1bc585d447b81143bfdc78907b2bd08615653af77c41bbc401bcb8a12bab3aa5f2c1415c25794afd8
+  data.tar.gz: 5a7472ba731deea05d57f7e584ee5cac64a0be2368c22d38b61be2a5b1992366182592d2c191d1fdc0c85bad58036bad51c2bb99c8d3b5c05a9362aa582b70af

data/.travis.yml

@@ -0,0 +1,8 @@
+sudo: false
+language: ruby
+
+rvm:
+  - 2.6
+  - 2.5
+  - 2.4
+  - 2.3

data/README.md

@@ -154,6 +154,7 @@ Fetch sample log from CloudWatch Logs:
 * `tag`: fluentd tag
 * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
 * `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
+* `use_aws_timestamp`: get timestamp from Cloudwatch event for non json logs, otherwise fluentd will parse the log to get the timestamp (default `false`)
 
 ## Test
 

data/Rakefile

@@ -7,3 +7,4 @@ Rake::TestTask.new(:test) do |test|
   test.test_files = FileList['test/plugin/*.rb']
 end
 
+task :default => :test

data/fluent-plugin-cloudwatch-logs.gemspec

@@ -20,8 +20,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'fluentd', '>= 0.14.15'
   spec.add_dependency 'aws-sdk-cloudwatchlogs', '~> 1.0'
 
-  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "test-unit-rr"
   spec.add_development_dependency "mocha"
 end

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.7.3"
+        VERSION = "0.7.4"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -9,22 +9,23 @@ module Fluent::Plugin
 
     helpers :parser, :thread, :compat_parameters
 
-    config_param :aws_key_id, :string, :default => nil, :secret => true
-    config_param :aws_sec_key, :string, :default => nil, :secret => true
+    config_param :aws_key_id, :string, default: nil, secret: true
+    config_param :aws_sec_key, :string, default: nil, secret: true
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
-    config_param :region, :string, :default => nil
-    config_param :endpoint, :string, :default => nil
+    config_param :region, :string, default: nil
+    config_param :endpoint, :string, default: nil
     config_param :tag, :string
     config_param :log_group_name, :string
-    config_param :log_stream_name, :string, :default => nil
+    config_param :log_stream_name, :string, default: nil
     config_param :use_log_stream_name_prefix, :bool, default: false
     config_param :state_file, :string
     config_param :fetch_interval, :time, default: 60
     config_param :http_proxy, :string, default: nil
-    config_param :json_handler, :enum, list: [:yajl, :json], :default => :yajl
+    config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
     config_param :use_todays_log_stream, :bool, default: false
+    config_param :use_aws_timestamp, :bool, default: false
 
     config_section :parse do
       config_set_default :@type, 'none'
@@ -95,7 +96,7 @@ module Fluent::Plugin
     end
 
     def store_next_token(token, log_stream_name = nil)
-      open(state_file_for(log_stream_name), 'w') do |f|
+      File.open(state_file_for(log_stream_name), 'w') do |f|
         f.write token
       end
     end
@@ -136,7 +137,10 @@ module Fluent::Plugin
 
     def emit(stream, event)
       if @parser
-        @parser.parse(event.message) {|time, record|
+        @parser.parse(event.message) {|time,record|
+          if @use_aws_timestamp
+            time = (event.timestamp / 1000).floor
+          end
           router.emit(@tag, time, record)
         }
       else
@@ -152,7 +156,7 @@ module Fluent::Plugin
         log_stream_name: log_stream_name
       }
       log_next_token = next_token(log_stream_name)
-      request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty? 
+      request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
       response = @logs.get_log_events(request)
       if valid_next_token(log_next_token, response.next_forward_token)
         store_next_token(response.next_forward_token, log_stream_name)
@@ -180,15 +184,15 @@ module Fluent::Plugin
     end
 
     def valid_next_token(prev_token, next_token)
-      return prev_token != next_token.chomp && !next_token.nil?
+      next_token && prev_token != next_token.chomp
     end
 
     def get_todays_date
-      return Date.today.strftime("%Y/%m/%d")
+      Date.today.strftime("%Y/%m/%d")
     end
 
     def get_yesterdays_date
-      return (Date.today - 1).strftime("%Y/%m/%d")
+      (Date.today - 1).strftime("%Y/%m/%d")
     end
   end
 end

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -2,6 +2,7 @@ require 'test_helper'
 require 'fluent/test/driver/input'
 require 'fluent/test/helpers'
 require 'date'
+require 'fluent/plugin/in_cloudwatch_logs'
 
 class CloudwatchLogsInputTest < Test::Unit::TestCase
   include CloudwatchLogsTestHelper
@@ -9,70 +10,121 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/in_cloudwatch_logs'
-
   end
 
-  def teardown
-    clear_log_group
+  sub_test_case "configure" do
+    def test_configure
+      d = create_driver(<<-EOC)
+        @type cloudwatch_logs
+        aws_key_id test_id
+        aws_sec_key test_key
+        region us-east-1
+        tag test
+        log_group_name group
+        log_stream_name stream
+        use_log_stream_name_prefix true
+        state_file /tmp/state
+        use_aws_timestamp true
+      EOC
+
+      assert_equal('test_id', d.instance.aws_key_id)
+      assert_equal('test_key', d.instance.aws_sec_key)
+      assert_equal('us-east-1', d.instance.region)
+      assert_equal('test', d.instance.tag)
+      assert_equal('group', d.instance.log_group_name)
+      assert_equal('stream', d.instance.log_stream_name)
+      assert_equal(true, d.instance.use_log_stream_name_prefix)
+      assert_equal('/tmp/state', d.instance.state_file)
+      assert_equal(:yajl, d.instance.json_handler)
+      assert_equal(true, d.instance.use_aws_timestamp)
+    end
   end
 
-  def test_configure
-    d = create_driver(<<-EOC)
-      @type cloudwatch_logs
-      aws_key_id test_id
-      aws_sec_key test_key
-      region us-east-1
-      tag test
-      log_group_name group
-      log_stream_name stream
-      use_log_stream_name_prefix true
-      state_file /tmp/state
-    EOC
+  sub_test_case "real world" do
+    def setup
+      omit if ENV["CI"] == "true"
+    end
 
-    assert_equal('test_id', d.instance.aws_key_id)
-    assert_equal('test_key', d.instance.aws_sec_key)
-    assert_equal('us-east-1', d.instance.region)
-    assert_equal('test', d.instance.tag)
-    assert_equal('group', d.instance.log_group_name)
-    assert_equal('stream', d.instance.log_stream_name)
-    assert_equal(true, d.instance.use_log_stream_name_prefix)
-    assert_equal('/tmp/state', d.instance.state_file)
-    assert_equal(:yajl, d.instance.json_handler)
-  end
+    def teardown
+      return if ENV["CI"] == "true"
 
-  def test_emit
-    create_log_stream
+      clear_log_group
+    end
 
-    time_ms = (Time.now.to_f * 1000).floor
-    put_log_events([
-      {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
-      {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
-    ])
+    def test_emit
+      create_log_stream
 
-    sleep 5
+      time_ms = (Time.now.to_f * 1000).floor
+      put_log_events([
+        {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+        {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+      ])
 
-    d = create_driver
-    d.run(expect_emits: 2, timeout: 5)
+      sleep 5
 
-    emits = d.events
-    assert_equal(2, emits.size)
-    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
-    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
-  end
+      d = create_driver
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+      assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+    end
+
+    def test_emit_with_aws_timestamp
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      log_time_ms = time_ms - 10000
+      put_log_events([
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
+      ])
+
+      sleep 5
+
+      d = create_driver(csv_format_config_aws_timestamp)
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal(['test', (time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+      assert_equal(['test', (time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+    end
+
+    def test_emit_with_log_timestamp
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      log_time_ms = time_ms - 10000
+      put_log_events([
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs1"},
+        {timestamp: time_ms, message: Time.at(log_time_ms/1000.floor).to_s + ",Cloudwatch non json logs2"},
+      ])
+
+      sleep 5
+
+      d = create_driver(csv_format_config)
+      d.run(expect_emits: 2, timeout: 5)
 
-  def test_emit_width_format
-    create_log_stream
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs1"}], emits[0])
+      assert_equal(['test', (log_time_ms / 1000).floor, {"message"=>"Cloudwatch non json logs2"}], emits[1])
+    end
 
-    time_ms = (Time.now.to_f * 1000).floor
-    put_log_events([
-      {timestamp: time_ms, message: 'logs1'},
-      {timestamp: time_ms, message: 'logs2'},
-    ])
+    def test_emit_width_format
+      create_log_stream
 
-    sleep 5
+      time_ms = (Time.now.to_f * 1000).floor
+      put_log_events([
+        {timestamp: time_ms, message: 'logs1'},
+        {timestamp: time_ms, message: 'logs2'},
+      ])
 
-    d = create_driver(<<-EOC)
+      sleep 5
+
+      d = create_driver(<<-EOC)
       tag test
       @type cloudwatch_logs
       log_group_name #{log_group_name}
@@ -85,38 +137,38 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       #{endpoint}
     EOC
 
-    d.run(expect_emits: 2, timeout: 5)
-
-    emits = d.events
-    assert_equal(2, emits.size)
-    assert_equal('test', emits[0][0])
-    assert_in_delta((time_ms / 1000).floor, emits[0][1], 10)
-    assert_equal({'cloudwatch' => 'logs1'}, emits[0][2])
-    assert_equal('test', emits[1][0])
-    assert_in_delta((time_ms / 1000).floor, emits[1][1], 10)
-    assert_equal({'cloudwatch' => 'logs2'}, emits[1][2])
-  end
-
-  def test_emit_with_prefix
-    new_log_stream("testprefix")
-    create_log_stream
-
-    time_ms = (Time.now.to_f * 1000).floor
-    put_log_events([
-      {timestamp: time_ms + 1000, message: '{"cloudwatch":"logs1"}'},
-      {timestamp: time_ms + 2000, message: '{"cloudwatch":"logs2"}'},
-    ])
-
-    new_log_stream("testprefix")
-    create_log_stream
-    put_log_events([
-      {timestamp: time_ms + 3000, message: '{"cloudwatch":"logs3"}'},
-      {timestamp: time_ms + 4000, message: '{"cloudwatch":"logs4"}'},
-    ])
-
-    sleep 5
-
-    d = create_driver(<<-EOC)
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal('test', emits[0][0])
+      assert_in_delta((time_ms / 1000).floor, emits[0][1], 10)
+      assert_equal({'cloudwatch' => 'logs1'}, emits[0][2])
+      assert_equal('test', emits[1][0])
+      assert_in_delta((time_ms / 1000).floor, emits[1][1], 10)
+      assert_equal({'cloudwatch' => 'logs2'}, emits[1][2])
+    end
+
+    def test_emit_with_prefix
+      new_log_stream("testprefix")
+      create_log_stream
+
+      time_ms = (Time.now.to_f * 1000).floor
+      put_log_events([
+        {timestamp: time_ms + 1000, message: '{"cloudwatch":"logs1"}'},
+        {timestamp: time_ms + 2000, message: '{"cloudwatch":"logs2"}'},
+      ])
+
+      new_log_stream("testprefix")
+      create_log_stream
+      put_log_events([
+        {timestamp: time_ms + 3000, message: '{"cloudwatch":"logs3"}'},
+        {timestamp: time_ms + 4000, message: '{"cloudwatch":"logs4"}'},
+      ])
+
+      sleep 5
+
+      d = create_driver(<<-EOC)
       tag test
       @type cloudwatch_logs
       log_group_name #{log_group_name}
@@ -128,69 +180,69 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       #{region}
       #{endpoint}
     EOC
-    d.run(expect_emits: 4, timeout: 5)
-
-    emits = d.events
-    assert_equal(4, emits.size)
-    assert_true(emits.include? ['test', ((time_ms + 1000) / 1000).floor, {'cloudwatch' => 'logs1'}])
-    assert_true(emits.include? ['test', ((time_ms + 2000) / 1000).floor, {'cloudwatch' => 'logs2'}])
-    assert_true(emits.include? ['test', ((time_ms + 3000) / 1000).floor, {'cloudwatch' => 'logs3'}])
-    assert_true(emits.include? ['test', ((time_ms + 4000) / 1000).floor, {'cloudwatch' => 'logs4'}])
-  end
-
-  def test_emit_with_todays_log_stream
-    new_log_stream("testprefix")
-    create_log_stream
-
-    today = DateTime.now.strftime("%Y/%m/%d")
-    yesterday = (Date.today - 1).strftime("%Y/%m/%d")
-    tomorrow = (Date.today + 1).strftime("%Y/%m/%d")
-
-
-    time_ms = (Time.now.to_f * 1000).floor
-    put_log_events([
-      {timestamp: time_ms + 1000, message: '{"cloudwatch":"logs1"}'},
-      {timestamp: time_ms + 2000, message: '{"cloudwatch":"logs2"}'},
-    ])
-
-    new_log_stream(today)
-    create_log_stream
-    put_log_events([
-      {timestamp: time_ms + 3000, message: '{"cloudwatch":"logs3"}'},
-      {timestamp: time_ms + 4000, message: '{"cloudwatch":"logs4"}'},
-    ])
-
-    new_log_stream(yesterday)
-    create_log_stream
-    put_log_events([
-      {timestamp: time_ms + 5000, message: '{"cloudwatch":"logs5"}'},
-      {timestamp: time_ms + 6000, message: '{"cloudwatch":"logs6"}'},
-    ])
-
-    new_log_stream(tomorrow)
-    create_log_stream
-    put_log_events([
-      {timestamp: time_ms + 7000, message: '{"cloudwatch":"logs7"}'},
-      {timestamp: time_ms + 8000, message: '{"cloudwatch":"logs8"}'},
-    ])
-
-    new_log_stream(today)
-    create_log_stream
-    put_log_events([
-      {timestamp: time_ms + 9000, message: '{"cloudwatch":"logs9"}'},
-      {timestamp: time_ms + 10000, message: '{"cloudwatch":"logs10"}'},
-    ])
-
-    new_log_stream(yesterday)
-    create_log_stream
-    put_log_events([
-      {timestamp: time_ms + 11000, message: '{"cloudwatch":"logs11"}'},
-      {timestamp: time_ms + 12000, message: '{"cloudwatch":"logs12"}'},
-    ])
-
-    sleep 15
-
-    d = create_driver(<<-EOC)
+      d.run(expect_emits: 4, timeout: 5)
+
+      emits = d.events
+      assert_equal(4, emits.size)
+      assert_true(emits.include? ['test', ((time_ms + 1000) / 1000).floor, {'cloudwatch' => 'logs1'}])
+      assert_true(emits.include? ['test', ((time_ms + 2000) / 1000).floor, {'cloudwatch' => 'logs2'}])
+      assert_true(emits.include? ['test', ((time_ms + 3000) / 1000).floor, {'cloudwatch' => 'logs3'}])
+      assert_true(emits.include? ['test', ((time_ms + 4000) / 1000).floor, {'cloudwatch' => 'logs4'}])
+    end
+
+    def test_emit_with_todays_log_stream
+      new_log_stream("testprefix")
+      create_log_stream
+
+      today = DateTime.now.strftime("%Y/%m/%d")
+      yesterday = (Date.today - 1).strftime("%Y/%m/%d")
+      tomorrow = (Date.today + 1).strftime("%Y/%m/%d")
+
+
+      time_ms = (Time.now.to_f * 1000).floor
+      put_log_events([
+        {timestamp: time_ms + 1000, message: '{"cloudwatch":"logs1"}'},
+        {timestamp: time_ms + 2000, message: '{"cloudwatch":"logs2"}'},
+      ])
+
+      new_log_stream(today)
+      create_log_stream
+      put_log_events([
+        {timestamp: time_ms + 3000, message: '{"cloudwatch":"logs3"}'},
+        {timestamp: time_ms + 4000, message: '{"cloudwatch":"logs4"}'},
+      ])
+
+      new_log_stream(yesterday)
+      create_log_stream
+      put_log_events([
+        {timestamp: time_ms + 5000, message: '{"cloudwatch":"logs5"}'},
+        {timestamp: time_ms + 6000, message: '{"cloudwatch":"logs6"}'},
+      ])
+
+      new_log_stream(tomorrow)
+      create_log_stream
+      put_log_events([
+        {timestamp: time_ms + 7000, message: '{"cloudwatch":"logs7"}'},
+        {timestamp: time_ms + 8000, message: '{"cloudwatch":"logs8"}'},
+      ])
+
+      new_log_stream(today)
+      create_log_stream
+      put_log_events([
+        {timestamp: time_ms + 9000, message: '{"cloudwatch":"logs9"}'},
+        {timestamp: time_ms + 10000, message: '{"cloudwatch":"logs10"}'},
+      ])
+
+      new_log_stream(yesterday)
+      create_log_stream
+      put_log_events([
+        {timestamp: time_ms + 11000, message: '{"cloudwatch":"logs11"}'},
+        {timestamp: time_ms + 12000, message: '{"cloudwatch":"logs12"}'},
+      ])
+
+      sleep 15
+
+      d = create_driver(<<-EOC)
       tag test
       @type cloudwatch_logs
       log_group_name #{log_group_name}
@@ -201,25 +253,234 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       #{region}
       #{endpoint}
     EOC
-    d.run(expect_emits: 8, timeout: 15)
-
-    emits = d.events
-    assert_equal(8, emits.size)
-    assert_false(emits.include? ['test', ((time_ms + 1000) / 1000).floor, {'cloudwatch' => 'logs1'}])
-    assert_false(emits.include? ['test', ((time_ms + 2000) / 1000).floor, {'cloudwatch' => 'logs2'}])
-    assert_true(emits.include? ['test', ((time_ms + 3000) / 1000).floor, {'cloudwatch' => 'logs3'}])
-    assert_true(emits.include? ['test', ((time_ms + 4000) / 1000).floor, {'cloudwatch' => 'logs4'}])
-    assert_true(emits.include? ['test', ((time_ms + 5000) / 1000).floor, {'cloudwatch' => 'logs5'}])
-    assert_true(emits.include? ['test', ((time_ms + 6000) / 1000).floor, {'cloudwatch' => 'logs6'}])
-    assert_false(emits.include? ['test', ((time_ms + 7000) / 1000).floor, {'cloudwatch' => 'logs7'}])
-    assert_false(emits.include? ['test', ((time_ms + 8000) / 1000).floor, {'cloudwatch' => 'logs8'}])
-    assert_true(emits.include? ['test', ((time_ms + 9000) / 1000).floor, {'cloudwatch' => 'logs9'}])
-    assert_true(emits.include? ['test', ((time_ms + 10000) / 1000).floor, {'cloudwatch' => 'logs10'}])
-    assert_true(emits.include? ['test', ((time_ms + 11000) / 1000).floor, {'cloudwatch' => 'logs11'}])
-    assert_true(emits.include? ['test', ((time_ms + 12000) / 1000).floor, {'cloudwatch' => 'logs12'}])
+      d.run(expect_emits: 8, timeout: 15)
+
+      emits = d.events
+      assert_equal(8, emits.size)
+      assert_false(emits.include? ['test', ((time_ms + 1000) / 1000).floor, {'cloudwatch' => 'logs1'}])
+      assert_false(emits.include? ['test', ((time_ms + 2000) / 1000).floor, {'cloudwatch' => 'logs2'}])
+      assert_true(emits.include? ['test', ((time_ms + 3000) / 1000).floor, {'cloudwatch' => 'logs3'}])
+      assert_true(emits.include? ['test', ((time_ms + 4000) / 1000).floor, {'cloudwatch' => 'logs4'}])
+      assert_true(emits.include? ['test', ((time_ms + 5000) / 1000).floor, {'cloudwatch' => 'logs5'}])
+      assert_true(emits.include? ['test', ((time_ms + 6000) / 1000).floor, {'cloudwatch' => 'logs6'}])
+      assert_false(emits.include? ['test', ((time_ms + 7000) / 1000).floor, {'cloudwatch' => 'logs7'}])
+      assert_false(emits.include? ['test', ((time_ms + 8000) / 1000).floor, {'cloudwatch' => 'logs8'}])
+      assert_true(emits.include? ['test', ((time_ms + 9000) / 1000).floor, {'cloudwatch' => 'logs9'}])
+      assert_true(emits.include? ['test', ((time_ms + 10000) / 1000).floor, {'cloudwatch' => 'logs10'}])
+      assert_true(emits.include? ['test', ((time_ms + 11000) / 1000).floor, {'cloudwatch' => 'logs11'}])
+      assert_true(emits.include? ['test', ((time_ms + 12000) / 1000).floor, {'cloudwatch' => 'logs12'}])
+    end
+  end
+
+  sub_test_case "stub responses" do
+    setup do
+      @client = Aws::CloudWatchLogs::Client.new(stub_responses: true)
+      mock(Aws::CloudWatchLogs::Client).new(anything) do
+        @client
+      end
+    end
+
+    test "emit" do
+      time_ms = (Time.now.to_f * 1000).floor
+      log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
+      @client.stub_responses(:describe_log_streams, { log_streams: [log_stream], next_token: nil })
+      cloudwatch_logs_events = [
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms, message: { cloudwatch: "logs1" }.to_json, ingestion_time: time_ms),
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms, message: { cloudwatch: "logs2" }.to_json, ingestion_time: time_ms)
+      ]
+      @client.stub_responses(:get_log_events, { events: cloudwatch_logs_events, next_forward_token: nil })
+
+      d = create_driver
+      d.run(expect_emits: 2, timeout: 5)
+
+      events = d.events
+      assert_equal(2, events.size)
+      assert_equal(["test", (time_ms / 1000), { "cloudwatch" => "logs1" }], events[0])
+      assert_equal(["test", (time_ms / 1000), { "cloudwatch" => "logs2" }], events[1])
+    end
+
+    test "emit with aws_timestamp" do
+      time_ms = (Time.now.to_f * 1000).floor
+      log_time_ms = time_ms - 10000
+      log_time_str = Time.at(log_time_ms / 1000.floor).to_s
+      log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
+      @client.stub_responses(:describe_log_streams, { log_streams: [log_stream], next_token: nil })
+      cloudwatch_logs_events = [
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms, message: "#{log_time_str},Cloudwatch non json logs1"),
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms, message: "#{log_time_str},Cloudwatch non json logs2")
+      ]
+      @client.stub_responses(:get_log_events, { events: cloudwatch_logs_events, next_forward_token: nil })
+
+      d = create_driver(csv_format_config_aws_timestamp)
+      d.run(expect_emits: 2, timeout: 5)
+
+      events = d.events
+      assert_equal(2, events.size)
+      assert_equal(["test", (time_ms / 1000).floor, { "message" => "Cloudwatch non json logs1" }], events[0])
+      assert_equal(["test", (time_ms / 1000).floor, { "message" => "Cloudwatch non json logs2" }], events[1])
+    end
+
+    test "emit with log_timestamp" do
+      time_ms = (Time.now.to_f * 1000).floor
+      log_time_ms = time_ms - 10000
+      log_time_str = Time.at(log_time_ms / 1000.floor).to_s
+      log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
+      @client.stub_responses(:describe_log_streams, { log_streams: [log_stream], next_token: nil })
+      cloudwatch_logs_events = [
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms, message: "#{log_time_str},Cloudwatch non json logs1"),
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms, message: "#{log_time_str},Cloudwatch non json logs2")
+      ]
+      @client.stub_responses(:get_log_events, { events: cloudwatch_logs_events, next_forward_token: nil })
+
+      d = create_driver(csv_format_config)
+      d.run(expect_emits: 2, timeout: 5)
+
+      emits = d.events
+      assert_equal(2, emits.size)
+      assert_equal(["test", (log_time_ms / 1000).floor, { "message" => "Cloudwatch non json logs1" }], emits[0])
+      assert_equal(["test", (log_time_ms / 1000).floor, { "message" => "Cloudwatch non json logs2" }], emits[1])
+    end
+
+    test "emit with format" do
+      config = <<-CONFIG
+        tag test
+        @type cloudwatch_logs
+        log_group_name #{log_group_name}
+        log_stream_name #{log_stream_name}
+        state_file /tmp/state
+        format /^(?<cloudwatch>[^ ]*)?/
+        #{aws_key_id}
+        #{aws_sec_key}
+        #{region}
+        #{endpoint}
+      CONFIG
+      time_ms = (Time.now.to_f * 1000).floor
+
+      log_stream = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
+      @client.stub_responses(:describe_log_streams, { log_streams: [log_stream], next_token: nil })
+      cloudwatch_logs_events = [
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms, message: "logs1", ingestion_time: time_ms),
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms, message: "logs2", ingestion_time: time_ms)
+      ]
+      @client.stub_responses(:get_log_events, { events: cloudwatch_logs_events, next_forward_token: nil })
+
+      d = create_driver(config)
+      d.run(expect_emits: 2, timeout: 5)
+
+      events = d.events
+      assert_equal(2, events.size)
+      assert_equal("test", events[0][0])
+      assert_in_delta(time_ms / 1000.0, events[0][1], 1.0)
+      assert_equal({ "cloudwatch" => "logs1" }, events[0][2])
+      assert_equal("test", events[1][0])
+      assert_in_delta(time_ms / 1000.0, events[1][1], 1.0)
+      assert_equal({ "cloudwatch" => "logs2" }, events[1][2])
+    end
+
+    test "emit with prefix" do
+      config = <<-CONFIG
+        tag test
+        @type cloudwatch_logs
+        log_group_name #{log_group_name}
+        log_stream_name testprefix
+        use_log_stream_name_prefix true
+        state_file /tmp/state
+        #{aws_key_id}
+        #{aws_sec_key}
+        #{region}
+        #{endpoint}
+      CONFIG
+      time_ms = (Time.now.to_f * 1000).floor
+      log_stream1 = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
+      log_stream2 = Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "stream_name")
+      @client.stub_responses(:describe_log_streams, { log_streams: [log_stream1, log_stream2], next_token: nil })
+      cloudwatch_logs_events1 = [
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms + 1000, message: { cloudwatch: "logs1" }.to_json, ingestion_time: time_ms),
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms + 2000, message: { cloudwatch: "logs2" }.to_json, ingestion_time: time_ms)
+      ]
+      cloudwatch_logs_events2 = [
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms + 3000, message: { cloudwatch: "logs3" }.to_json, ingestion_time: time_ms),
+        Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms + 4000, message: { cloudwatch: "logs4" }.to_json, ingestion_time: time_ms)
+      ]
+      @client.stub_responses(:get_log_events, [
+        { events: cloudwatch_logs_events1, next_forward_token: nil },
+        { events: cloudwatch_logs_events2, next_forward_token: nil },
+      ])
+
+      d = create_driver(config)
+      d.run(expect_emits: 4, timeout: 5)
+
+      events = d.events
+      assert_equal(4, events.size)
+      assert_equal(["test", (time_ms + 1000) / 1000, { "cloudwatch" => "logs1" }], events[0])
+      assert_equal(["test", (time_ms + 2000) / 1000, { "cloudwatch" => "logs2" }], events[1])
+      assert_equal(["test", (time_ms + 3000) / 1000, { "cloudwatch" => "logs3" }], events[2])
+      assert_equal(["test", (time_ms + 4000) / 1000, { "cloudwatch" => "logs4" }], events[3])
+    end
+
+    test "emit with today's log stream" do
+      config = <<-CONFIG
+        tag test
+        @type cloudwatch_logs
+        log_group_name #{log_group_name}
+        use_todays_log_stream true
+        state_file /tmp/state
+        fetch_interval 0.1
+        #{aws_key_id}
+        #{aws_sec_key}
+        #{region}
+        #{endpoint}
+      CONFIG
+
+      today = Date.today.strftime("%Y/%m/%d")
+      yesterday = (Date.today - 1).strftime("%Y/%m/%d")
+      time_ms = (Time.now.to_f * 1000).floor
+
+      log_stream = ->(name) { Aws::CloudWatchLogs::Types::LogStream.new(log_stream_name: "#{name}_#{SecureRandom.uuid}") }
+      @client.stub_responses(:describe_log_streams, ->(context) {
+        if context.params[:log_stream_name_prefix].start_with?(today)
+          { log_streams: [log_stream.call(today)], next_token: nil }
+        elsif context.params[:log_stream_name_prefix].start_with?(yesterday)
+          { log_streams: [log_stream.call(yesterday)], next_token: nil }
+        else
+          { log_streams: [], next_token: nil }
+        end
+      })
+      count = 0
+      @client.stub_responses(:get_log_events, ->(context) {
+        n = count * 2 + 1
+        cloudwatch_logs_events = [
+          Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms + n * 1000, message: { cloudwatch: "logs#{n}" }.to_json, ingestion_time: time_ms),
+          Aws::CloudWatchLogs::Types::OutputLogEvent.new(timestamp: time_ms + (n + 1) * 1000, message: { cloudwatch: "logs#{n + 1}" }.to_json, ingestion_time: time_ms)
+        ]
+        count += 1
+        if context.params[:log_stream_name].start_with?(today)
+          { events: cloudwatch_logs_events, next_forward_token: nil }
+        elsif context.params[:log_stream_name].start_with?(yesterday)
+          { events: cloudwatch_logs_events, next_forward_token: nil }
+        else
+          flunk("Failed log_stream_name: #{context.params[:log_stream_name]}")
+        end
+      })
+
+      d = create_driver(config)
+      d.run(expect_emits: 8, timeout: 15)
+
+      events = d.events
+      assert_equal(8, events.size)
+      assert_equal(["test", ((time_ms + 1000) / 1000), { "cloudwatch" => "logs1" }], events[0])
+      assert_equal(["test", ((time_ms + 2000) / 1000), { "cloudwatch" => "logs2" }], events[1])
+      assert_equal(["test", ((time_ms + 3000) / 1000), { "cloudwatch" => "logs3" }], events[2])
+      assert_equal(["test", ((time_ms + 4000) / 1000), { "cloudwatch" => "logs4" }], events[3])
+      assert_equal(["test", ((time_ms + 5000) / 1000), { "cloudwatch" => "logs5" }], events[4])
+      assert_equal(["test", ((time_ms + 6000) / 1000), { "cloudwatch" => "logs6" }], events[5])
+      assert_equal(["test", ((time_ms + 7000) / 1000), { "cloudwatch" => "logs7" }], events[6])
+      assert_equal(["test", ((time_ms + 8000) / 1000), { "cloudwatch" => "logs8" }], events[7])
+    end
   end
 
   private
+
   def default_config
     <<-EOC
       tag test
@@ -235,6 +496,28 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
     EOC
   end
 
+  def csv_format_config
+    <<-EOC
+    tag test
+    @type cloudwatch_logs
+    log_group_name #{log_group_name}
+    log_stream_name #{log_stream_name}
+    state_file /tmp/state
+    fetch_interval 1
+    #{aws_key_id}
+    #{aws_sec_key}
+    #{region}
+    #{endpoint}
+    format csv
+    keys time,message
+    time_key time
+  EOC
+  end
+
+  def csv_format_config_aws_timestamp
+    csv_format_config.concat("use_aws_timestamp true")
+  end
+
   def create_driver(conf = default_config)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::CloudwatchLogsInput).configure(conf)
   end