fluent-plugin-kinesis 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8b59ae71478266b7d619c76b71811310f2cf90a
-  data.tar.gz: 4e3049d7c6ec83dc6da5858fee3ddfa8b3e76660
+  metadata.gz: 304cad8054202f54af88f9eabb61b3795e2a5d5d
+  data.tar.gz: a8bbcb160baaccfcbe2eaf4514a0e1bb87d950f2
 SHA512:
-  metadata.gz: 1bf9e15306dbf5f88f5dcdd731d06395821c7a29f1f0d0597075db19fc638f6b2439003ef08326bac4bc9f1bbd0a124f5ced2fef5b83b0c52b81e4f6fdcbb35a
-  data.tar.gz: 1ac91c53effd50cd9ebb947f7be53ec8162f4d736e836c6076149ed4da80ed39ada244ece88fd23923e9daedb1026e35d74f50e564ced387b338d7b0de8223ed
+  metadata.gz: 3d6ad1a860cb384eb5d14063d53fd4490c85e57c815ca7442db1998d00a5b2455168164085d5911823a4046ecc48279b1ecf2de06937541a6f9e118a06ebf506
+  data.tar.gz: ff3da727150ee12fe731b63397a12f2dd54fc79d3adaf091c2504557cb2d29a3804adcc8fffa4c1b07f710a2ec8bf89ad4a53af30fd28637c9c62961f5a62b40

data/CHANGELOG.md

@@ -1,6 +1,11 @@
 # CHANGELOG
 
+## 1.3.0
+
+- Feature - Log KPL stdout/err to logger [#129](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/129) [#130](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/130)
+
 ## 1.2.0
+
 - Feature - Add reduce_max_size_error_message configuration [#127](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/127)
 - Bug fix - Fix empty batch error [#125](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/125)
 

data/README.md

@@ -236,6 +236,9 @@ A key to extract partition key from JSON object. Default `nil`, which means part
 ### debug
 Boolean. Enable if you need to debug Kinesis Producer Library metrics. Default is `false`.
 
+### enable_kpl_logging
+Boolean, default `false`. If you want to log KPL binary's stdout/err, make this parameter `true`.
+
 ### kinesis_producer
 This section is configuration for Kinesis Producer Library. Almost all of description comes from [deault_config.propertites of KPL Java Sample Application][default_config.properties]. You should specify configurations below inside `<kinesis_producer>` section like:
 

data/lib/fluent/plugin/kinesis_helper/class_methods.rb

@@ -49,6 +49,7 @@ module Fluent
         config_param :stream_name_prefix, :string, default: nil
         config_param :region,             :string, default: nil
         config_param :partition_key,      :string, default: nil
+        config_param :enable_kpl_logging, :bool,   default: false
         config_param_for_credentials
         config_param_for_format
         config_param_for_debug

data/lib/fluent/plugin/kinesis_helper/kpl.rb

@@ -48,6 +48,7 @@ module Fluent
           credentials_refresh_delay: @kinesis_producer.credentials_refresh_delay,
           debug: @debug,
           logger: log,
+          enable_kpl_logging: @enable_kpl_logging,
         }
       end
 

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 #  permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '1.2.0'
+  VERSION = '1.3.0'
 end

data/lib/kinesis_producer/daemon.rb

@@ -29,6 +29,7 @@ module KinesisProducer
       @credentials_refresh_delay = options[:credentials_refresh_delay] || 5000
       @logger                    = options[:logger]
       @debug                     = options[:debug]
+      @enable_kpl_logging        = options[:enable_kpl_logging]
 
       @executor = Concurrent::CachedThreadPool.new
       @shutdown = Concurrent::AtomicBoolean.new(false)
@@ -99,8 +100,15 @@ module KinesisProducer
     end
 
     def start_child_daemon
+      @child_stdout, stdout = IO.pipe
+      @child_stderr, stderr = IO.pipe
       @pid = Process.fork do
         Process.setsid
+        STDOUT.sync = true
+        $stdout.reopen stdout
+        $stderr.reopen stderr
+        @child_stdout.close
+        @child_stderr.close
         configuration = make_configuration_message
         credentials = make_set_credentials_message
         command = [@binary, @out_pipe.to_path, @in_pipe.to_path, to_hex(configuration), to_hex(credentials)]
@@ -110,6 +118,8 @@ module KinesisProducer
         end
         exec(*command)
       end
+      stdout.close
+      stderr.close
       sleep 1 # TODO
     end
 
@@ -118,7 +128,13 @@ module KinesisProducer
       @out_channel = @out_pipe.open('w')
     end
 
+    def enable_kpl_logging?
+      @enable_kpl_logging
+    end
+
     def start_loops
+      start_loop_for(:log_stdout) if enable_kpl_logging?
+      start_loop_for(:log_stderr) if enable_kpl_logging?
       start_loop_for(:send_message)
       start_loop_for(:receive_message)
       start_loop_for(:return_message)
@@ -135,6 +151,22 @@ module KinesisProducer
       end
     end
 
+    def log_stdout
+      line = @child_stdout.readline.chomp
+      @logger.info(line)
+    end
+
+    def log_stderr
+      line = @child_stderr.readline.chomp
+      if line.include? ' [info] '
+        @logger.info(line)
+      elsif line.include? ' [warn] '
+        @logger.warn(line)
+      else
+        @logger.error(line)
+      end
+    end
+
     def send_message
       message = @outgoing_messages.pop
       size = [message.size].pack('N*')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-21 00:00:00.000000000 Z
+date: 2017-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd