fluent-plugin-cloudwatch-logs 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aff209d6a27c958aa30c96374bae93a17e533296
-  data.tar.gz: 002ed627b666fa502cf81b0042e30d2bd904ba32
+  metadata.gz: 26d61c290a6fd56f27524732758ea6e4f8cacd96
+  data.tar.gz: 678b4500dc46217e141328d3cd915470a691a2ad
 SHA512:
-  metadata.gz: 5dfc3b01f9f885d6885015e878803f7426ac8fabadef0f41a23c70916d1d73b72b65139ffe3455bff2015d3a66f497d464a09d564077914f3aae0bbd92824606
-  data.tar.gz: 20635343fe05b015f03cb6b81bd06f5145fe779b1ce8eb16e2d313402ba1ecf2e68cc620d6c3ca52b5a3ae83f4d3e775e8a633f00b1eac11abc46a14672854a2
+  metadata.gz: 7b6d98c07e99f57022f041d1d4c60738bf847aea9512f8bed75c47de7f25fd5dca7ca3b867289784eb08901e6312f98fe3b929edb7022ed9b2032ca7b52796a9
+  data.tar.gz: 8e86a20a0f129546c5300af5811d8f0d0d7d50907bc5622fe44fa03e8115669d8eb5411339533be5f39f0eb4b9733df4a8ced4235e6ca7ffe29c22f6f544bc90

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.0.6"
+        VERSION = "0.0.7"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -12,6 +12,8 @@ module Fluent
     config_param :max_message_length, :integer, :default => nil
     config_param :use_tag_as_group, :bool, :default => false
 
+    MAX_EVENTS_SIZE = 30720
+
     unless method_defined?(:log)
       define_method(:log) { $log }
     end
@@ -70,8 +72,9 @@ module Fluent
             message = record.to_json
           end
 
+          message.force_encoding('ASCII-8BIT')
+
           if @max_message_length
-            message.force_encoding('ASCII-8BIT')
             message = message.slice(0, @max_message_length)
           end
 
@@ -80,7 +83,7 @@ module Fluent
         # The log events in the batch must be in chronological ordered by their timestamp.
         # http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
         events = events.sort_by {|e| e[:timestamp] }
-        put_events(group_name, events)
+        put_events_by_chunk(group_name, events)
       }
     end
 
@@ -93,6 +96,25 @@ module Fluent
       @sequence_tokens[group_name][stream_name] = token
     end
 
+    def put_events_by_chunk(group_name, events)
+      chunk = []
+
+      # The maximum batch size is 32,768 bytes, and this size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.
+      # http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
+      while event = events.shift
+        if (chunk + [event]).inject(0) {|sum, e| sum + e[:message].length } > MAX_EVENTS_SIZE
+          put_events(group_name, chunk)
+          chunk = [event]
+        else
+          chunk << event
+        end
+      end
+
+      unless chunk.empty?
+        put_events(group_name, chunk)
+      end
+    end
+
     def put_events(group_name, events)
       args = {
         log_events: events,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Ryota Arai