fluent-plugin-kinesis 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d8b59ae71478266b7d619c76b71811310f2cf90a
4
- data.tar.gz: 4e3049d7c6ec83dc6da5858fee3ddfa8b3e76660
3
+ metadata.gz: 304cad8054202f54af88f9eabb61b3795e2a5d5d
4
+ data.tar.gz: a8bbcb160baaccfcbe2eaf4514a0e1bb87d950f2
5
5
  SHA512:
6
- metadata.gz: 1bf9e15306dbf5f88f5dcdd731d06395821c7a29f1f0d0597075db19fc638f6b2439003ef08326bac4bc9f1bbd0a124f5ced2fef5b83b0c52b81e4f6fdcbb35a
7
- data.tar.gz: 1ac91c53effd50cd9ebb947f7be53ec8162f4d736e836c6076149ed4da80ed39ada244ece88fd23923e9daedb1026e35d74f50e564ced387b338d7b0de8223ed
6
+ metadata.gz: 3d6ad1a860cb384eb5d14063d53fd4490c85e57c815ca7442db1998d00a5b2455168164085d5911823a4046ecc48279b1ecf2de06937541a6f9e118a06ebf506
7
+ data.tar.gz: ff3da727150ee12fe731b63397a12f2dd54fc79d3adaf091c2504557cb2d29a3804adcc8fffa4c1b07f710a2ec8bf89ad4a53af30fd28637c9c62961f5a62b40
@@ -1,6 +1,11 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 1.3.0
4
+
5
+ - Feature - Log KPL stdout/err to logger [#129](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/129) [#130](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/130)
6
+
3
7
  ## 1.2.0
8
+
4
9
  - Feature - Add reduce_max_size_error_message configuration [#127](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/127)
5
10
  - Bug fix - Fix empty batch error [#125](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/125)
6
11
 
data/README.md CHANGED
@@ -236,6 +236,9 @@ A key to extract partition key from JSON object. Default `nil`, which means part
236
236
  ### debug
237
237
  Boolean. Enable if you need to debug Kinesis Producer Library metrics. Default is `false`.
238
238
 
239
+ ### enable_kpl_logging
240
+ Boolean, default `false`. If you want to log KPL binary's stdout/err, make this parameter `true`.
241
+
239
242
  ### kinesis_producer
240
243
  This section is configuration for Kinesis Producer Library. Almost all of description comes from [deault_config.propertites of KPL Java Sample Application][default_config.properties]. You should specify configurations below inside `<kinesis_producer>` section like:
241
244
 
@@ -49,6 +49,7 @@ module Fluent
49
49
  config_param :stream_name_prefix, :string, default: nil
50
50
  config_param :region, :string, default: nil
51
51
  config_param :partition_key, :string, default: nil
52
+ config_param :enable_kpl_logging, :bool, default: false
52
53
  config_param_for_credentials
53
54
  config_param_for_format
54
55
  config_param_for_debug
@@ -48,6 +48,7 @@ module Fluent
48
48
  credentials_refresh_delay: @kinesis_producer.credentials_refresh_delay,
49
49
  debug: @debug,
50
50
  logger: log,
51
+ enable_kpl_logging: @enable_kpl_logging,
51
52
  }
52
53
  end
53
54
 
@@ -13,5 +13,5 @@
13
13
  # permissions and limitations under the License.
14
14
 
15
15
  module FluentPluginKinesis
16
- VERSION = '1.2.0'
16
+ VERSION = '1.3.0'
17
17
  end
@@ -29,6 +29,7 @@ module KinesisProducer
29
29
  @credentials_refresh_delay = options[:credentials_refresh_delay] || 5000
30
30
  @logger = options[:logger]
31
31
  @debug = options[:debug]
32
+ @enable_kpl_logging = options[:enable_kpl_logging]
32
33
 
33
34
  @executor = Concurrent::CachedThreadPool.new
34
35
  @shutdown = Concurrent::AtomicBoolean.new(false)
@@ -99,8 +100,15 @@ module KinesisProducer
99
100
  end
100
101
 
101
102
  def start_child_daemon
103
+ @child_stdout, stdout = IO.pipe
104
+ @child_stderr, stderr = IO.pipe
102
105
  @pid = Process.fork do
103
106
  Process.setsid
107
+ STDOUT.sync = true
108
+ $stdout.reopen stdout
109
+ $stderr.reopen stderr
110
+ @child_stdout.close
111
+ @child_stderr.close
104
112
  configuration = make_configuration_message
105
113
  credentials = make_set_credentials_message
106
114
  command = [@binary, @out_pipe.to_path, @in_pipe.to_path, to_hex(configuration), to_hex(credentials)]
@@ -110,6 +118,8 @@ module KinesisProducer
110
118
  end
111
119
  exec(*command)
112
120
  end
121
+ stdout.close
122
+ stderr.close
113
123
  sleep 1 # TODO
114
124
  end
115
125
 
@@ -118,7 +128,13 @@ module KinesisProducer
118
128
  @out_channel = @out_pipe.open('w')
119
129
  end
120
130
 
131
+ def enable_kpl_logging?
132
+ @enable_kpl_logging
133
+ end
134
+
121
135
  def start_loops
136
+ start_loop_for(:log_stdout) if enable_kpl_logging?
137
+ start_loop_for(:log_stderr) if enable_kpl_logging?
122
138
  start_loop_for(:send_message)
123
139
  start_loop_for(:receive_message)
124
140
  start_loop_for(:return_message)
@@ -135,6 +151,22 @@ module KinesisProducer
135
151
  end
136
152
  end
137
153
 
154
+ def log_stdout
155
+ line = @child_stdout.readline.chomp
156
+ @logger.info(line)
157
+ end
158
+
159
+ def log_stderr
160
+ line = @child_stderr.readline.chomp
161
+ if line.include? ' [info] '
162
+ @logger.info(line)
163
+ elsif line.include? ' [warn] '
164
+ @logger.warn(line)
165
+ else
166
+ @logger.error(line)
167
+ end
168
+ end
169
+
138
170
  def send_message
139
171
  message = @outgoing_messages.pop
140
172
  size = [message.size].pack('N*')
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kinesis
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.0
4
+ version: 1.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-09-21 00:00:00.000000000 Z
11
+ date: 2017-09-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd