fluent-plugin-s3 1.1.10 → 1.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/ChangeLog +4 -0
- data/README.md +15 -3
- data/VERSION +1 -1
- data/fluent-plugin-s3.gemspec +0 -1
- data/lib/fluent/plugin/in_s3.rb +8 -1
- data/test/test_in_s3.rb +110 -0
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: e034c8d5e4ae06a88f405bfe396d2c1c960da096b5010d4b45902765b00eaa6a
|
4
|
+
data.tar.gz: 679865ce71becff176745c0fcb051a6120313c5127d4ad70a8333b263c5eb67d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b0e0bf4a2072b9589e0e27ed84806c00d1fcc1ba47e70d1a628017aa7981993372b2afd7ee04d1eee5c558c2d485c3128ec64318b34fd802b8c1759fa311d71d
|
7
|
+
data.tar.gz: 6fd057fd51ce0aeed1fd64c2a39a405ee329f0b96cb92dc09faeac37f4d679fc0d7951d827f8ea0b0eda513a99f9cd89ab171b795933b1d8a44294d025fac7c8
|
data/ChangeLog
CHANGED
data/README.md
CHANGED
@@ -202,7 +202,7 @@ For `<buffer>`, you can use any record field in `path` / `s3_object_key_format`.
|
|
202
202
|
# parameters...
|
203
203
|
</buffer>
|
204
204
|
|
205
|
-
See official article for more detail: [Buffer
|
205
|
+
See official article for more detail: [Config: Buffer Section - Fluentd](https://docs.fluentd.org/configuration/buffer-section)
|
206
206
|
|
207
207
|
Note that this configuration doesn't work with fluentd v0.12.
|
208
208
|
|
@@ -322,7 +322,7 @@ You get:
|
|
322
322
|
"log/events/ts=20130111-23/events_1.gz"
|
323
323
|
"log/events/ts=20130112-00/events_0.gz"
|
324
324
|
|
325
|
-
NOTE: ${hostname} placeholder is deprecated since v0.8. You can get same result by using [configuration's embedded ruby code feature](
|
325
|
+
NOTE: ${hostname} placeholder is deprecated since v0.8. You can get same result by using [configuration's embedded ruby code feature](https://docs.fluentd.org/configuration/config-file#embedded-ruby-code).
|
326
326
|
|
327
327
|
s3_object_key_format %{path}%{time_slice}_%{hostname}%{index}.%{file_extension}
|
328
328
|
s3_object_key_format "%{path}%{time_slice}_#{Socket.gethostname}%{index}.%{file_extension}"
|
@@ -355,7 +355,7 @@ See `Use your compression algorithm` section for adding another format.
|
|
355
355
|
**`<format>` or format**
|
356
356
|
|
357
357
|
Change one line format in the S3 object. Supported formats are "out_file",
|
358
|
-
"json", "ltsv" and "single_value". See also [official Formatter article](
|
358
|
+
"json", "ltsv" and "single_value". See also [official Formatter article](https://docs.fluentd.org/formatter).
|
359
359
|
|
360
360
|
* out_file (default).
|
361
361
|
|
@@ -586,12 +586,17 @@ Given a threshold to treat events as delay, output warning logs if delayed event
|
|
586
586
|
aws_sec_key YOUR_AWS_SECRET_KEY
|
587
587
|
s3_bucket YOUR_S3_BUCKET_NAME
|
588
588
|
s3_region ap-northeast-1
|
589
|
+
add_object_metadata true
|
589
590
|
|
590
591
|
<sqs>
|
591
592
|
queue_name YOUR_SQS_QUEUE_NAME
|
592
593
|
</sqs>
|
593
594
|
</source>
|
594
595
|
|
596
|
+
**add_object_metadata**
|
597
|
+
|
598
|
+
Whether or not object metadata should be added to the record. Defaults to `false`. See below for details.
|
599
|
+
|
595
600
|
**s3_bucket (required)**
|
596
601
|
|
597
602
|
S3 bucket name.
|
@@ -647,6 +652,13 @@ The long polling interval. Default is 20.
|
|
647
652
|
|
648
653
|
Interval to retry polling SQS if polling unsuccessful, in seconds. Default is 300.
|
649
654
|
|
655
|
+
## Object Metadata Added To Records
|
656
|
+
|
657
|
+
If the `add_object_metadata` option is set to true, then the name of the bucket
|
658
|
+
and the key for a given object will be added to each log record as `s3_bucket`
|
659
|
+
and `s3_key`, respectively. This metadata can be used by filter plugins or other
|
660
|
+
downstream processors to better identify the source of a given record.
|
661
|
+
|
650
662
|
## IAM Policy
|
651
663
|
|
652
664
|
The following is an example for a IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket-test, etc.).
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.1.
|
1
|
+
1.1.11
|
data/fluent-plugin-s3.gemspec
CHANGED
@@ -10,7 +10,6 @@ Gem::Specification.new do |gem|
|
|
10
10
|
gem.version = File.read("VERSION").strip
|
11
11
|
gem.authors = ["Sadayuki Furuhashi", "Masahiro Nakagawa"]
|
12
12
|
gem.email = "frsyuki@gmail.com"
|
13
|
-
gem.has_rdoc = false
|
14
13
|
#gem.platform = Gem::Platform::RUBY
|
15
14
|
gem.files = `git ls-files`.split("\n")
|
16
15
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
data/lib/fluent/plugin/in_s3.rb
CHANGED
@@ -24,6 +24,8 @@ module Fluent::Plugin
|
|
24
24
|
|
25
25
|
desc "Use aws-sdk-ruby bundled cert"
|
26
26
|
config_param :use_bundled_cert, :bool, default: false
|
27
|
+
desc "Add object metadata to the records parsed out of a given object"
|
28
|
+
config_param :add_object_metadata, :bool, default: false
|
27
29
|
desc "AWS access key id"
|
28
30
|
config_param :aws_key_id, :string, default: nil, secret: true
|
29
31
|
desc "AWS secret key."
|
@@ -256,13 +258,18 @@ module Fluent::Plugin
|
|
256
258
|
|
257
259
|
def process(body)
|
258
260
|
s3 = body["Records"].first["s3"]
|
259
|
-
|
261
|
+
raw_key = s3["object"]["key"]
|
262
|
+
key = CGI.unescape(raw_key)
|
260
263
|
|
261
264
|
io = @bucket.object(key).get.body
|
262
265
|
content = @extractor.extract(io)
|
263
266
|
es = Fluent::MultiEventStream.new
|
264
267
|
content.each_line do |line|
|
265
268
|
@parser.parse(line) do |time, record|
|
269
|
+
if @add_object_metadata
|
270
|
+
record['s3_bucket'] = @s3_bucket
|
271
|
+
record['s3_key'] = raw_key
|
272
|
+
end
|
266
273
|
es.add(time, record)
|
267
274
|
end
|
268
275
|
end
|
data/test/test_in_s3.rb
CHANGED
@@ -224,6 +224,41 @@ EOS
|
|
224
224
|
assert_equal({ "message" => "aaa" }, events.first[2])
|
225
225
|
end
|
226
226
|
|
227
|
+
def test_one_record_with_metadata
|
228
|
+
setup_mocks
|
229
|
+
d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nadd_object_metadata true\n")
|
230
|
+
|
231
|
+
s3_object = stub(Object.new)
|
232
|
+
s3_response = stub(Object.new)
|
233
|
+
s3_response.body { StringIO.new("aaa") }
|
234
|
+
s3_object.get { s3_response }
|
235
|
+
@s3_bucket.object(anything).at_least(1) { s3_object }
|
236
|
+
|
237
|
+
body = {
|
238
|
+
"Records" => [
|
239
|
+
{
|
240
|
+
"s3" => {
|
241
|
+
"object" => {
|
242
|
+
"key" => "test_key"
|
243
|
+
}
|
244
|
+
}
|
245
|
+
}
|
246
|
+
]
|
247
|
+
}
|
248
|
+
message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
|
249
|
+
@sqs_poller.get_messages(anything, anything) do |config, stats|
|
250
|
+
config.before_request.call(stats) if config.before_request
|
251
|
+
stats.request_count += 1
|
252
|
+
if stats.request_count >= 1
|
253
|
+
d.instance.instance_variable_set(:@running, false)
|
254
|
+
end
|
255
|
+
[message]
|
256
|
+
end
|
257
|
+
d.run(expect_emits: 1)
|
258
|
+
events = d.events
|
259
|
+
assert_equal({ "s3_bucket" => "test_bucket", "s3_key" => "test_key", "message" => "aaa" }, events.first[2])
|
260
|
+
end
|
261
|
+
|
227
262
|
def test_one_record_url_encoded
|
228
263
|
setup_mocks
|
229
264
|
d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\n")
|
@@ -259,6 +294,41 @@ EOS
|
|
259
294
|
assert_equal({ "message" => "aaa" }, events.first[2])
|
260
295
|
end
|
261
296
|
|
297
|
+
def test_one_record_url_encoded_with_metadata
|
298
|
+
setup_mocks
|
299
|
+
d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nadd_object_metadata true")
|
300
|
+
|
301
|
+
s3_object = stub(Object.new)
|
302
|
+
s3_response = stub(Object.new)
|
303
|
+
s3_response.body { StringIO.new("aaa") }
|
304
|
+
s3_object.get { s3_response }
|
305
|
+
@s3_bucket.object('test key').at_least(1) { s3_object }
|
306
|
+
|
307
|
+
body = {
|
308
|
+
"Records" => [
|
309
|
+
{
|
310
|
+
"s3" => {
|
311
|
+
"object" => {
|
312
|
+
"key" => "test+key"
|
313
|
+
}
|
314
|
+
}
|
315
|
+
}
|
316
|
+
]
|
317
|
+
}
|
318
|
+
message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
|
319
|
+
@sqs_poller.get_messages(anything, anything) do |config, stats|
|
320
|
+
config.before_request.call(stats) if config.before_request
|
321
|
+
stats.request_count += 1
|
322
|
+
if stats.request_count >= 1
|
323
|
+
d.instance.instance_variable_set(:@running, false)
|
324
|
+
end
|
325
|
+
[message]
|
326
|
+
end
|
327
|
+
d.run(expect_emits: 1)
|
328
|
+
events = d.events
|
329
|
+
assert_equal({ "s3_bucket" => "test_bucket", "s3_key" => "test+key", "message" => "aaa" }, events.first[2])
|
330
|
+
end
|
331
|
+
|
262
332
|
def test_one_record_multi_line
|
263
333
|
setup_mocks
|
264
334
|
d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\n")
|
@@ -299,6 +369,46 @@ EOS
|
|
299
369
|
assert_equal(expected_records, events.map {|_tag, _time, record| record })
|
300
370
|
end
|
301
371
|
|
372
|
+
def test_one_record_multi_line_with_metadata
|
373
|
+
setup_mocks
|
374
|
+
d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nadd_object_metadata true")
|
375
|
+
|
376
|
+
s3_object = stub(Object.new)
|
377
|
+
s3_response = stub(Object.new)
|
378
|
+
s3_response.body { StringIO.new("aaa\nbbb\nccc\n") }
|
379
|
+
s3_object.get { s3_response }
|
380
|
+
@s3_bucket.object(anything).at_least(1) { s3_object }
|
381
|
+
|
382
|
+
body = {
|
383
|
+
"Records" => [
|
384
|
+
{
|
385
|
+
"s3" => {
|
386
|
+
"object" => {
|
387
|
+
"key" => "test_key"
|
388
|
+
}
|
389
|
+
}
|
390
|
+
}
|
391
|
+
]
|
392
|
+
}
|
393
|
+
message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
|
394
|
+
@sqs_poller.get_messages(anything, anything) do |config, stats|
|
395
|
+
config.before_request.call(stats) if config.before_request
|
396
|
+
stats.request_count += 1
|
397
|
+
if stats.request_count >= 1
|
398
|
+
d.instance.instance_variable_set(:@running, false)
|
399
|
+
end
|
400
|
+
[message]
|
401
|
+
end
|
402
|
+
d.run(expect_emits: 1)
|
403
|
+
events = d.events
|
404
|
+
expected_records = [
|
405
|
+
{ "s3_bucket" => "test_bucket", "s3_key" => "test_key", "message" => "aaa\n" },
|
406
|
+
{ "s3_bucket" => "test_bucket", "s3_key" => "test_key", "message" => "bbb\n" },
|
407
|
+
{ "s3_bucket" => "test_bucket", "s3_key" => "test_key", "message" => "ccc\n" }
|
408
|
+
]
|
409
|
+
assert_equal(expected_records, events.map {|_tag, _time, record| record })
|
410
|
+
end
|
411
|
+
|
302
412
|
def test_gzip_single_stream
|
303
413
|
setup_mocks
|
304
414
|
d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as gzip\nformat none\n")
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-
|
12
|
+
date: 2019-06-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -162,8 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
162
162
|
- !ruby/object:Gem::Version
|
163
163
|
version: '0'
|
164
164
|
requirements: []
|
165
|
-
|
166
|
-
rubygems_version: 2.6.14.1
|
165
|
+
rubygems_version: 3.0.3
|
167
166
|
signing_key:
|
168
167
|
specification_version: 4
|
169
168
|
summary: Amazon S3 output plugin for Fluentd event collector
|