fluent-plugin-cloudwatch-logs 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d63e33f244adf82bff43836bd21d935eebb5fee8
-  data.tar.gz: 6e14fdd4432c7b21c0b0cf6b9c1f6574565a34b9
+  metadata.gz: 88a161bb1764e810bbf9a56b73fc7e47b05ebeb3
+  data.tar.gz: 04838391a2d452903c11bf0f5878f06c554b92f2
 SHA512:
-  metadata.gz: 87332e167639ac1d7a43404849f05bb1820faac2190e2e30ce73672159b2e52f91aa68c7bb3433f815d3969c87fcd0d46c9408a84612bed7243c66afc0e53d60
-  data.tar.gz: 52698378e0c3d7a935aec5117eceba6ae3799dedc09e125bc99dd9f2f0608448cc640c08138a3a4d6703884371e868e1252af97773bfbb30e3c117baa02dc429
+  metadata.gz: 00153e08af56da4d830290799b162b3935c9f88bc415467f558e96c191dd6bdeebcd4497f1c51975deb6a597841d35c04dba1ec1f2b19cdf851772357c8d49a7
+  data.tar.gz: 2ae3cf9e79a22595d80f99c043b3808a640f39fe7c11ae299b4c92641fd9fa2bb2dfd6cc45b4dc7230136ae56af1e845b420f95d0c0c2fd93c86b28e3fe9abf6

data/README.md

@@ -72,6 +72,7 @@ Fetch sample log from CloudWatch Logs:
   #message_keys key1,key2,key3,...
   #max_message_length 32768
   #use_tag_as_group false
+  #use_tag_as_stream false
 </match>
 ```
 
@@ -81,7 +82,9 @@ Fetch sample log from CloudWatch Logs:
 * `auto_create_stream`: to create log group and stream automatically
 * `message_keys`: keys to send messages as events
 * `max_message_length`: maximum length of the message
+* `max_events_per_batch`: maximum number of events to send at once (default 10000)
 * `use_tag_as_group`: to use tag as a group name
+* `use_tag_as_stream`: to use tag as a stream name
 
 ### in_cloudwatch_logs
 

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.1.0"
+        VERSION = "0.1.1"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -10,6 +10,7 @@ module Fluent
     config_param :auto_create_stream, :bool, default: false
     config_param :message_keys, :string, :default => nil
     config_param :max_message_length, :integer, :default => nil
+    config_param :max_events_per_batch, :integer, :default => 10000
     config_param :use_tag_as_group, :bool, :default => false
     config_param :use_tag_as_stream, :bool, :default => false
     config_param :http_proxy, :string, default: nil
@@ -106,7 +107,11 @@ module Fluent
       # The maximum batch size is 1,048,576 bytes, and this size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.
       # http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
       while event = events.shift
-        if (chunk + [event]).inject(0) {|sum, e| sum + e[:message].length + 26 } > MAX_EVENTS_SIZE
+        new_chunk = chunk + [event]
+        chunk_span_too_big = new_chunk.size > 1 && new_chunk[-1][:timestamp] - new_chunk[0][:timestamp] >= 1000 * 60 * 60 * 24
+        chunk_too_big = new_chunk.inject(0) {|sum, e| sum + e[:message].length + 26 } > MAX_EVENTS_SIZE
+        chunk_too_long = @max_events_per_batch && chunk.size >= @max_events_per_batch
+        if chunk_too_big or chunk_span_too_big or chunk_too_long
           put_events(group_name, stream_name, chunk)
           chunk = [event]
         else

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -54,6 +54,28 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal('{"cloudwatch":"logs2"}', events[1].message)
   end
 
+  def test_write_24h_apart
+    new_log_stream
+
+    d = create_driver
+    time = Time.now
+    d.emit({'cloudwatch' => 'logs0'}, time.to_i - 60 * 60 * 25)
+    d.emit({'cloudwatch' => 'logs1'}, time.to_i)
+    d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
+    d.run
+
+    sleep 20
+
+    events = get_log_events
+    assert_equal(3, events.size)
+    assert_equal((time.to_i - 60 * 60 * 25) * 1000, events[0].timestamp)
+    assert_equal('{"cloudwatch":"logs0"}', events[0].message)
+    assert_equal((time.to_i ) * 1000, events[1].timestamp)
+    assert_equal('{"cloudwatch":"logs1"}', events[1].message)
+    assert_equal((time.to_i + 1) * 1000, events[2].timestamp)
+    assert_equal('{"cloudwatch":"logs2"}', events[2].message)
+  end
+
   def test_write_with_message_keys
     new_log_stream
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-29 00:00:00.000000000 Z
+date: 2015-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd