fluent-plugin-cloudwatch-logs 0.4.1 → 0.4.2.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9e629c8fa1e69f1fc25b63be72d3891bed73eb92
-  data.tar.gz: cf4b8011d1c17117e09e00c421540717fb02064a
+  metadata.gz: b61338555b31dcd2adc46c2942735ec5f06e3adb
+  data.tar.gz: e7fb2aa5337980e1f417577f920ee209f8692f41
 SHA512:
-  metadata.gz: 56f19ed188cfae7f5672a2b3ac425c5c03ff43a454bdfa4a920df6ca368871b79c646e55dd8429340735c91c802b1740d74982430bc0dba084bf4df90b292db6
-  data.tar.gz: 05d706f093a4d4815b510df65730e85cabb5915adc66db0b7513596d936f05ad10651d2d8c3687f7da8125ff860ac719f1cd7c7ac4d03339ad968ab7127a175d
+  metadata.gz: d762517f069ed20970b7cae2c7ed5f7969c8a917136de6a0327fa7375789886f42225e794faa01340cf99b6ec063fb9d2a8ee3793ead4ca9a32425bdb1819d65
+  data.tar.gz: 9a6373471d2d97b995de4c8911756ea7f6cc87702d4454c25de77f0c6ad75d8d31e88289867e1b7b73033f20dc546fd691b1b07bcfd634329dd702ba05097921

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.4.1"
+        VERSION = "0.4.2.pre"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -102,8 +102,8 @@ module Fluent
 
           if @use_log_stream_name_prefix
             log_streams = describe_log_streams
-            log_streams.each do |log_stram|
-              log_stream_name = log_stram.log_stream_name
+            log_streams.each do |log_stream|
+              log_stream_name = log_stream.log_stream_name
               events = get_events(log_stream_name)
               events.each do |event|
                 emit(log_stream_name, event)
@@ -151,7 +151,7 @@ module Fluent
       request[:log_stream_name_prefix] = @log_stream_name
       response = @logs.describe_log_streams(request)
       if log_streams
-        log_streams << response.log_streams
+        log_streams.concat(response.log_streams)
       else
         log_streams = response.log_streams
       end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -1,4 +1,5 @@
 require 'fluent/output'
+require 'thread'
 
 module Fluent
   require 'fluent/mixin/config_placeholders'
@@ -28,6 +29,7 @@ module Fluent
     config_param :put_log_events_retry_wait, :time, default: 1.0
     config_param :put_log_events_retry_limit, :integer, default: 17
     config_param :put_log_events_disable_retry_limit, :bool, default: false
+    config_param :concurrency, :integer, default: 1
 
     MAX_EVENTS_SIZE = 1_048_576
     MAX_EVENT_SIZE = 256 * 1024
@@ -68,6 +70,7 @@ module Fluent
       options[:http_proxy] = @http_proxy if @http_proxy
       @logs ||= Aws::CloudWatchLogs::Client.new(options)
       @sequence_tokens = {}
+      @store_next_sequence_token_mutex = Mutex.new
     end
 
     def format(tag, time, record)
@@ -75,6 +78,8 @@ module Fluent
     end
 
     def write(chunk)
+      queue = Thread::Queue.new
+
       chunk.enum_for(:msgpack_each).select {|tag, time, record|
         if record.nil?
           log.warn "record is nil (tag=#{tag})"
@@ -156,8 +161,22 @@ module Fluent
         # The log events in the batch must be in chronological ordered by their timestamp.
         # http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
         events = events.sort_by {|e| e[:timestamp] }
-        put_events_by_chunk(group_name, stream_name, events)
+
+        queue << [group_name, stream_name, events]
       }
+
+      @concurrency.times do
+        queue << nil
+      end
+      threads = @concurrency.times.map do |i|
+        Thread.start do
+          while job = queue.shift
+            group_name, stream_name, events = job
+            put_events_by_chunk(group_name, stream_name, events)
+          end
+        end
+      end
+      threads.each(&:join)
     end
 
     private
@@ -177,7 +196,9 @@ module Fluent
     end
 
     def store_next_sequence_token(group_name, stream_name, token)
-      @sequence_tokens[group_name][stream_name] = token
+      @store_next_sequence_token_mutex.synchronize do 
+        @sequence_tokens[group_name][stream_name] = token
+      end
     end
 
     def put_events_by_chunk(group_name, stream_name, events)
@@ -224,17 +245,20 @@ module Fluent
       response = nil
       retry_count = 0
       until response
-        log.debug "Calling PutLogEvents API", {
-          "group" => group_name,
-          "stream" => stream_name,
-          "events_count" => events.size,
-          "events_bytesize" => events_bytesize,
-          "sequence_token" => token,
-        }
-
         args[:sequence_token] = token if token
+
         begin
+          t = Time.now
           response = @logs.put_log_events(args)
+          log.debug "Called PutLogEvents API", {
+            "group" => group_name,
+            "stream" => stream_name,
+            "events_count" => events.size,
+            "events_bytesize" => events_bytesize,
+            "sequence_token" => token,
+            "thread" => Thread.current.object_id,
+            "request_sec" => Time.now - t,
+          }
         rescue Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException, Aws::CloudWatchLogs::Errors::DataAlreadyAcceptedException => err
           sleep 1 # to avoid too many API calls
           log_stream = find_log_stream(group_name, stream_name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2.pre
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-07 00:00:00.000000000 Z
+date: 2017-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -144,12 +144,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: CloudWatch Logs Plugin for Fluentd