fluent-plugin-kinesis 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 304cad8054202f54af88f9eabb61b3795e2a5d5d
|
4
|
+
data.tar.gz: a8bbcb160baaccfcbe2eaf4514a0e1bb87d950f2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3d6ad1a860cb384eb5d14063d53fd4490c85e57c815ca7442db1998d00a5b2455168164085d5911823a4046ecc48279b1ecf2de06937541a6f9e118a06ebf506
|
7
|
+
data.tar.gz: ff3da727150ee12fe731b63397a12f2dd54fc79d3adaf091c2504557cb2d29a3804adcc8fffa4c1b07f710a2ec8bf89ad4a53af30fd28637c9c62961f5a62b40
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
# CHANGELOG
|
2
2
|
|
3
|
+
## 1.3.0
|
4
|
+
|
5
|
+
- Feature - Log KPL stdout/err to logger [#129](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/129) [#130](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/130)
|
6
|
+
|
3
7
|
## 1.2.0
|
8
|
+
|
4
9
|
- Feature - Add reduce_max_size_error_message configuration [#127](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/127)
|
5
10
|
- Bug fix - Fix empty batch error [#125](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/125)
|
6
11
|
|
data/README.md
CHANGED
@@ -236,6 +236,9 @@ A key to extract partition key from JSON object. Default `nil`, which means part
|
|
236
236
|
### debug
|
237
237
|
Boolean. Enable if you need to debug Kinesis Producer Library metrics. Default is `false`.
|
238
238
|
|
239
|
+
### enable_kpl_logging
|
240
|
+
Boolean, default `false`. If you want to log KPL binary's stdout/err, make this parameter `true`.
|
241
|
+
|
239
242
|
### kinesis_producer
|
240
243
|
This section is configuration for Kinesis Producer Library. Almost all of description comes from [deault_config.propertites of KPL Java Sample Application][default_config.properties]. You should specify configurations below inside `<kinesis_producer>` section like:
|
241
244
|
|
@@ -49,6 +49,7 @@ module Fluent
|
|
49
49
|
config_param :stream_name_prefix, :string, default: nil
|
50
50
|
config_param :region, :string, default: nil
|
51
51
|
config_param :partition_key, :string, default: nil
|
52
|
+
config_param :enable_kpl_logging, :bool, default: false
|
52
53
|
config_param_for_credentials
|
53
54
|
config_param_for_format
|
54
55
|
config_param_for_debug
|
@@ -29,6 +29,7 @@ module KinesisProducer
|
|
29
29
|
@credentials_refresh_delay = options[:credentials_refresh_delay] || 5000
|
30
30
|
@logger = options[:logger]
|
31
31
|
@debug = options[:debug]
|
32
|
+
@enable_kpl_logging = options[:enable_kpl_logging]
|
32
33
|
|
33
34
|
@executor = Concurrent::CachedThreadPool.new
|
34
35
|
@shutdown = Concurrent::AtomicBoolean.new(false)
|
@@ -99,8 +100,15 @@ module KinesisProducer
|
|
99
100
|
end
|
100
101
|
|
101
102
|
def start_child_daemon
|
103
|
+
@child_stdout, stdout = IO.pipe
|
104
|
+
@child_stderr, stderr = IO.pipe
|
102
105
|
@pid = Process.fork do
|
103
106
|
Process.setsid
|
107
|
+
STDOUT.sync = true
|
108
|
+
$stdout.reopen stdout
|
109
|
+
$stderr.reopen stderr
|
110
|
+
@child_stdout.close
|
111
|
+
@child_stderr.close
|
104
112
|
configuration = make_configuration_message
|
105
113
|
credentials = make_set_credentials_message
|
106
114
|
command = [@binary, @out_pipe.to_path, @in_pipe.to_path, to_hex(configuration), to_hex(credentials)]
|
@@ -110,6 +118,8 @@ module KinesisProducer
|
|
110
118
|
end
|
111
119
|
exec(*command)
|
112
120
|
end
|
121
|
+
stdout.close
|
122
|
+
stderr.close
|
113
123
|
sleep 1 # TODO
|
114
124
|
end
|
115
125
|
|
@@ -118,7 +128,13 @@ module KinesisProducer
|
|
118
128
|
@out_channel = @out_pipe.open('w')
|
119
129
|
end
|
120
130
|
|
131
|
+
def enable_kpl_logging?
|
132
|
+
@enable_kpl_logging
|
133
|
+
end
|
134
|
+
|
121
135
|
def start_loops
|
136
|
+
start_loop_for(:log_stdout) if enable_kpl_logging?
|
137
|
+
start_loop_for(:log_stderr) if enable_kpl_logging?
|
122
138
|
start_loop_for(:send_message)
|
123
139
|
start_loop_for(:receive_message)
|
124
140
|
start_loop_for(:return_message)
|
@@ -135,6 +151,22 @@ module KinesisProducer
|
|
135
151
|
end
|
136
152
|
end
|
137
153
|
|
154
|
+
def log_stdout
|
155
|
+
line = @child_stdout.readline.chomp
|
156
|
+
@logger.info(line)
|
157
|
+
end
|
158
|
+
|
159
|
+
def log_stderr
|
160
|
+
line = @child_stderr.readline.chomp
|
161
|
+
if line.include? ' [info] '
|
162
|
+
@logger.info(line)
|
163
|
+
elsif line.include? ' [warn] '
|
164
|
+
@logger.warn(line)
|
165
|
+
else
|
166
|
+
@logger.error(line)
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
138
170
|
def send_message
|
139
171
|
message = @outgoing_messages.pop
|
140
172
|
size = [message.size].pack('N*')
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kinesis
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amazon Web Services
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-09-
|
11
|
+
date: 2017-09-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|