fluent-plugin-cloudwatch-logs 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88a161bb1764e810bbf9a56b73fc7e47b05ebeb3
-  data.tar.gz: 04838391a2d452903c11bf0f5878f06c554b92f2
+  metadata.gz: 7433cbc405896d7a542689ab5e23c6f199043760
+  data.tar.gz: c8f7680af5579da676e5b0379b9e911f5afa7dbe
 SHA512:
-  metadata.gz: 00153e08af56da4d830290799b162b3935c9f88bc415467f558e96c191dd6bdeebcd4497f1c51975deb6a597841d35c04dba1ec1f2b19cdf851772357c8d49a7
-  data.tar.gz: 2ae3cf9e79a22595d80f99c043b3808a640f39fe7c11ae299b4c92641fd9fa2bb2dfd6cc45b4dc7230136ae56af1e845b420f95d0c0c2fd93c86b28e3fe9abf6
+  metadata.gz: 0bb5a93717b9cddfc9d29fd4e4dca9c84751747b6f10b3225f5a8be0542a418223525e4f03527131daf9d3bfec543fe2d06b372f391760535deeb1cf149c555a
+  data.tar.gz: 86be56949e121311e7ad8ebaf5fb41667fdede3d0a0a75f80418659df35664f4924756614e1bf174c03105f028fca9009a2f80fddb6a4f1a74e2bf8ae79a9d83

data/README.md

@@ -73,18 +73,21 @@ Fetch sample log from CloudWatch Logs:
   #max_message_length 32768
   #use_tag_as_group false
   #use_tag_as_stream false
+  #include_time_key true
+  #localtime true
 </match>
 ```
 
 * `log_group_name`: name of log group to store logs
 * `log_stream_name`: name of log stream to store logs
-* `sequence_token_file`: file to store next sequence token
 * `auto_create_stream`: to create log group and stream automatically
 * `message_keys`: keys to send messages as events
 * `max_message_length`: maximum length of the message
 * `max_events_per_batch`: maximum number of events to send at once (default 10000)
 * `use_tag_as_group`: to use tag as a group name
 * `use_tag_as_stream`: to use tag as a stream name
+* `include_time_key`: include time key as part of the log entry (defaults to UTC)
+* `localtime`: use localtime timezone for `include_time_key` output (overrides UTC default)
 
 ### in_cloudwatch_logs
 
@@ -94,6 +97,7 @@ Fetch sample log from CloudWatch Logs:
   tag cloudwatch.in
   log_group_name group
   log_stream_name stream
+  #use_log_stream_name_prefix true
   state_file /var/lib/fluent/group_stream.in.state
 </source>
 ```
@@ -101,6 +105,7 @@ Fetch sample log from CloudWatch Logs:
 * `tag`: fluentd tag
 * `log_group_name`: name of log group to fetch logs
 * `log_stream_name`: name of log stream to fetch logs
+* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
 * `state_file`: file to store current state (e.g. next\_forward\_token)
 
 ## Test

data/example/fluentd.conf

@@ -14,7 +14,6 @@
   type cloudwatch_logs
   log_group_name fluent-plugin-cloudwatch-example
   log_stream_name fluent-plugin-cloudwatch-example
-  sequence_token_file /tmp/fluent-plugin-cloudwatch-example.seq
   auto_create_stream true
 </match>
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.1.1"
+        VERSION = "0.1.2"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -13,6 +13,7 @@ module Fluent
     config_param :tag, :string
     config_param :log_group_name, :string
     config_param :log_stream_name, :string
+    config_param :use_log_stream_name_prefix, :bool, default: false
     config_param :state_file, :string
     config_param :fetch_interval, :time, default: 60
     config_param :http_proxy, :string, default: nil
@@ -57,8 +58,10 @@ module Fluent
       File.read(@state_file).chomp
     end
 
-    def store_next_token(token)
-      open(@state_file, 'w') do |f|
+    def store_next_token(token, log_stream_name = nil)
+      state_file = @state_file
+      state_file = "#{@state_file}_#{log_stream_name}" if log_stream_name
+      open(state_file, 'w') do |f|
         f.write token
       end
     end
@@ -70,15 +73,19 @@ module Fluent
         if Time.now > @next_fetch_time
           @next_fetch_time += @fetch_interval
 
-          events = get_events
-          events.each do |event|
-            if @parser
-              record = @parser.parse(event.message)
-              router.emit(@tag, record[0], record[1])
-            else
-              time = (event.timestamp / 1000).floor
-              record = JSON.parse(event.message)
-              router.emit(@tag, time, record)
+          if @use_log_stream_name_prefix
+            log_streams = describe_log_streams
+            log_streams.each do |log_stram|
+              log_stream_name = log_stram.log_stream_name
+              events = get_events(log_stream_name)
+              events.each do |event|
+                emit(event)
+              end
+            end
+          else
+            events = get_events(@log_stream_name)
+            events.each do |event|
+              emit(event)
             end
           end
         end
@@ -86,10 +93,21 @@ module Fluent
       end
     end
 
-    def get_events
+    def emit(event)
+      if @parser
+        record = @parser.parse(event.message)
+        router.emit(@tag, record[0], record[1])
+      else
+        time = (event.timestamp / 1000).floor
+        record = JSON.parse(event.message)
+        router.emit(@tag, time, record)
+      end
+    end
+
+    def get_events(log_stream_name)
       request = {
         log_group_name: @log_group_name,
-        log_stream_name: @log_stream_name,
+        log_stream_name: log_stream_name
       }
       request[:next_token] = next_token if next_token
       response = @logs.get_log_events(request)
@@ -97,5 +115,23 @@ module Fluent
 
       response.events
     end
+
+    def describe_log_streams(log_streams = nil, next_token = nil)
+      request = {
+        log_group_name: @log_group_name
+      }
+      request[:next_token] = next_token if next_token
+      request[:log_stream_name_prefix] = @log_stream_name
+      response = @logs.describe_log_streams(request)
+      if log_streams
+        log_streams << response.log_streams
+      else
+        log_streams = response.log_streams
+      end
+      if response.next_token
+        log_streams = describe_log_streams(log_streams, response.next_token)
+      end
+      log_streams
+    end
   end
 end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -2,6 +2,8 @@ module Fluent
   class CloudwatchLogsOutput < BufferedOutput
     Plugin.register_output('cloudwatch_logs', self)
 
+    include Fluent::SetTimeKeyMixin
+
     config_param :aws_key_id, :string, :default => nil, :secret => true
     config_param :aws_sec_key, :string, :default => nil, :secret => true
     config_param :region, :string, :default => nil
@@ -54,7 +56,7 @@ module Fluent
           if @auto_create_stream
             create_log_group(group_name)
           else
-            log.warn "Log group '#{group_name}' dose not exists"
+            log.warn "Log group '#{group_name}' does not exist"
             next
           end
         end
@@ -63,7 +65,7 @@ module Fluent
           if @auto_create_stream
             create_log_stream(group_name, stream_name)
           else
-            log.warn "Log stream '#{stream_name}' dose not exists"
+            log.warn "Log stream '#{stream_name}' does not exist"
             next
           end
         end
@@ -77,7 +79,9 @@ module Fluent
             message = record.to_json
           end
 
-          message.force_encoding('ASCII-8BIT')
+          # CloudWatchLogs API only accepts valid UTF-8 strings
+          # so we should encode the message to UTF-8
+          message.encode('UTF-8', :invalid => :replace)
 
           if @max_message_length
             message = message.slice(0, @max_message_length)
@@ -109,7 +113,7 @@ module Fluent
       while event = events.shift
         new_chunk = chunk + [event]
         chunk_span_too_big = new_chunk.size > 1 && new_chunk[-1][:timestamp] - new_chunk[0][:timestamp] >= 1000 * 60 * 60 * 24
-        chunk_too_big = new_chunk.inject(0) {|sum, e| sum + e[:message].length + 26 } > MAX_EVENTS_SIZE
+        chunk_too_big = new_chunk.inject(0) {|sum, e| sum + e[:message].bytesize + 26 } > MAX_EVENTS_SIZE
         chunk_too_long = @max_events_per_batch && chunk.size >= @max_events_per_batch
         if chunk_too_big or chunk_span_too_big or chunk_too_long
           put_events(group_name, stream_name, chunk)

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -22,6 +22,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
       tag test
       log_group_name group
       log_stream_name stream
+      use_log_stream_name_prefix true
       state_file /tmp/state
     EOC
 
@@ -31,6 +32,7 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
     assert_equal('test', d.instance.tag)
     assert_equal('group', d.instance.log_group_name)
     assert_equal('stream', d.instance.log_stream_name)
+    assert_equal(true, d.instance.use_log_stream_name_prefix)
     assert_equal('/tmp/state', d.instance.state_file)
   end
 
@@ -93,6 +95,48 @@ class CloudwatchLogsInputTest < Test::Unit::TestCase
     assert_equal({'cloudwatch' => 'logs2'}, emits[1][2])
   end
 
+  def test_emit_with_prefix
+    new_log_stream("testprefix")
+    create_log_stream
+
+    time_ms = (Time.now.to_f * 1000).floor
+    put_log_events([
+      {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+      {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+    ])
+
+    new_log_stream("testprefix")
+    create_log_stream
+    put_log_events([
+      {timestamp: time_ms, message: '{"cloudwatch":"logs3"}'},
+      {timestamp: time_ms, message: '{"cloudwatch":"logs4"}'},
+    ])
+
+    sleep 5
+
+    d = create_driver(<<-EOC)
+      tag test
+      type cloudwatch_logs
+      log_group_name #{log_group_name}
+      log_stream_name testprefix
+      use_log_stream_name_prefix true
+      state_file /tmp/state
+      #{aws_key_id}
+      #{aws_sec_key}
+      #{region}
+    EOC
+    d.run do
+      sleep 5
+    end
+
+    emits = d.emits
+    assert_equal(4, emits.size)
+    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs3'}], emits[2])
+    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs4'}], emits[3])
+  end
+
   private
   def default_config
     <<-EOC

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -12,7 +12,6 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
 
   def teardown
     clear_log_group
-    FileUtils.rm_f(sequence_token_file)
   end
 
 
@@ -54,6 +53,22 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal('{"cloudwatch":"logs2"}', events[1].message)
   end
 
+  def test_write_utf8
+    new_log_stream
+
+    d = create_driver
+    time = Time.now
+    d.emit({'cloudwatch' => 'これは日本語です'.force_encoding('UTF-8')}, time.to_i)
+    d.run
+
+    sleep 20
+
+    events = get_log_events
+    assert_equal(1, events.size)
+    assert_equal(time.to_i * 1000, events[0].timestamp)
+    assert_equal('{"cloudwatch":"これは日本語です"}', events[0].message)
+  end
+
   def test_write_24h_apart
     new_log_stream
 
@@ -171,13 +186,59 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal('message2 logs2', events[1].message)
   end
 
+  def test_include_time_key
+    new_log_stream
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    include_time_key true
+    EOC
+
+    time = Time.now
+    d.emit({'cloudwatch' => 'logs1'}, time.to_i)
+    d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
+    d.run
+
+    sleep 20
+
+    events = get_log_events
+    assert_equal(2, events.size)
+    assert_equal(time.to_i * 1000, events[0].timestamp)
+    assert_equal("{\"cloudwatch\":\"logs1\",\"time\":\"#{time.utc.strftime("%Y-%m-%dT%H:%M:%SZ")}\"}", events[0].message)
+    assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+    assert_equal("{\"cloudwatch\":\"logs2\",\"time\":\"#{(time+1).utc.strftime("%Y-%m-%dT%H:%M:%SZ")}\"}", events[1].message)
+  end
+
+  def test_include_time_key_localtime
+    new_log_stream
+
+    d = create_driver(<<-EOC)
+    #{default_config}
+    include_time_key true
+    localtime true
+    EOC
+
+    time = Time.now
+    d.emit({'cloudwatch' => 'logs1'}, time.to_i)
+    d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
+    d.run
+
+    sleep 20
+
+    events = get_log_events
+    assert_equal(2, events.size)
+    assert_equal(time.to_i * 1000, events[0].timestamp)
+    assert_equal("{\"cloudwatch\":\"logs1\",\"time\":\"#{time.strftime("%Y-%m-%dT%H:%M:%S%:z")}\"}", events[0].message)
+    assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+    assert_equal("{\"cloudwatch\":\"logs2\",\"time\":\"#{(time+1).strftime("%Y-%m-%dT%H:%M:%S%:z")}\"}", events[1].message)
+  end
+
   private
   def default_config
     <<-EOC
     type cloudwatch_logs
     log_group_name #{log_group_name}
     log_stream_name #{log_stream_name}
-    sequence_token_file #{sequence_token_file}
     auto_create_stream true
     #{aws_key_id}
     #{aws_sec_key}
@@ -185,11 +246,6 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     EOC
   end
 
-  def sequence_token_file
-    File.expand_path('../../tmp/sequence_token', __FILE__)
-  end
-
-
   def create_driver(conf = default_config)
     Fluent::Test::BufferedOutputTestDriver.new(Fluent::CloudwatchLogsOutput, fluentd_tag).configure(conf)
   end

data/test/test_helper.rb

@@ -29,15 +29,15 @@ module CloudwatchLogsTestHelper
     "region #{ENV['region']}" if ENV['region']
   end
 
-  def log_stream_name
+  def log_stream_name(log_stream_name_prefix = nil)
     if !@log_stream_name
-      new_log_stream
+      new_log_stream(log_stream_name_prefix)
     end
     @log_stream_name
   end
 
-  def new_log_stream
-    @log_stream_name = Time.now.to_f.to_s
+  def new_log_stream(log_stream_name_prefix = nil)
+    @log_stream_name = log_stream_name_prefix ? log_stream_name_prefix + Time.now.to_f.to_s : Time.now.to_f.to_s
   end
 
   def clear_log_group
@@ -54,7 +54,7 @@ module CloudwatchLogsTestHelper
     @fluentd_tag ||= "fluent.plugin.cloudwatch.test.#{Time.now.to_f}"
   end
 
-  def create_log_stream
+  def create_log_stream()
     begin
       logs.create_log_group(log_group_name: log_group_name)
     rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-06 00:00:00.000000000 Z
+date: 2015-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd