fluent-plugin-s3-input 0.0.15 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +6 -6
- data/README.md +2 -6
- data/fluent-plugin-s3-input.gemspec +1 -1
- data/lib/fluent/plugin/out_s3_input.rb +12 -11
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2a82d84e5b945d5da217daa9eb23daf300d31129
|
|
4
|
+
data.tar.gz: a582602f70e4781516acc9a3b0c57cb217e58624
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4c809268131e4c0b146dc3e8655886b9c86cf7da4c99475b46868f67093dd88ab95c2fdee6bf7e02f7b456ba2f103989241e5e99eb1e169f409b47cffb9dc4b1
|
|
7
|
+
data.tar.gz: 306cea910ecf4e069a241ff4cd4b2a0d6d9a28d114b693894e315433c433d5cf49ae129b686e54beca486a0c70d7b79acfbe740cde2fcf60b755fd78c498cd38
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-s3-input (0.0.
|
|
4
|
+
fluent-plugin-s3-input (0.0.16)
|
|
5
5
|
aws-sdk
|
|
6
6
|
fluentd
|
|
7
7
|
oj
|
|
@@ -10,13 +10,13 @@ PATH
|
|
|
10
10
|
GEM
|
|
11
11
|
remote: https://rubygems.org/
|
|
12
12
|
specs:
|
|
13
|
-
aws-sdk (2.9.
|
|
14
|
-
aws-sdk-resources (= 2.9.
|
|
15
|
-
aws-sdk-core (2.9.
|
|
13
|
+
aws-sdk (2.9.27)
|
|
14
|
+
aws-sdk-resources (= 2.9.27)
|
|
15
|
+
aws-sdk-core (2.9.27)
|
|
16
16
|
aws-sigv4 (~> 1.0)
|
|
17
17
|
jmespath (~> 1.0)
|
|
18
|
-
aws-sdk-resources (2.9.
|
|
19
|
-
aws-sdk-core (= 2.9.
|
|
18
|
+
aws-sdk-resources (2.9.27)
|
|
19
|
+
aws-sdk-core (= 2.9.27)
|
|
20
20
|
aws-sigv4 (1.0.0)
|
|
21
21
|
cool.io (1.5.0)
|
|
22
22
|
fluentd (0.14.16)
|
data/README.md
CHANGED
|
@@ -43,6 +43,8 @@ S3 Event Example Intake
|
|
|
43
43
|
|
|
44
44
|
# read and emit the json object
|
|
45
45
|
# this plugin!
|
|
46
|
+
# if the document is an object, it will be emitted.
|
|
47
|
+
# if the document is an array, then each element of the array will be emitted
|
|
46
48
|
<match sqs.s3.event>
|
|
47
49
|
type s3_input
|
|
48
50
|
merge_record no
|
|
@@ -52,12 +54,6 @@ S3 Event Example Intake
|
|
|
52
54
|
tag s3.file.contents
|
|
53
55
|
</filter>
|
|
54
56
|
|
|
55
|
-
# Emit each record in the cloudtrail json document as a new event
|
|
56
|
-
<match s3.file.contents>
|
|
57
|
-
type record_splitter
|
|
58
|
-
split_key Records
|
|
59
|
-
tag cloudtrail
|
|
60
|
-
</match>
|
|
61
57
|
# params
|
|
62
58
|
tag my.new.tag : tag name to emit new record as
|
|
63
59
|
uncompress gzip : decompression algorithm (only gzip:-/)
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "fluent-plugin-s3-input"
|
|
7
|
-
spec.version = "0.0.
|
|
7
|
+
spec.version = "0.0.16"
|
|
8
8
|
spec.authors = ["Anthony Johnson"]
|
|
9
9
|
spec.email = ["ansoni@gmail.com"]
|
|
10
10
|
spec.description = %q{Fluentd plugin to read a file from S3 and emit it}
|
|
@@ -82,9 +82,9 @@ module Fluent
|
|
|
82
82
|
input = resp.body
|
|
83
83
|
end
|
|
84
84
|
|
|
85
|
-
|
|
85
|
+
default_record = {}
|
|
86
86
|
if @merge_record
|
|
87
|
-
|
|
87
|
+
default_record = {}.merge(record)
|
|
88
88
|
end
|
|
89
89
|
|
|
90
90
|
s3_record = {}
|
|
@@ -104,19 +104,20 @@ module Fluent
|
|
|
104
104
|
raise "Unsupported format - #{@format}"
|
|
105
105
|
end
|
|
106
106
|
|
|
107
|
-
# parse the time from the record
|
|
108
|
-
@time_keys.each do |time_key|
|
|
109
|
-
if s3_record.include? time_key
|
|
110
|
-
time=Time.strptime(new_record[time_key], @time_format).to_f
|
|
111
|
-
$log.debug "Reset time for #{time_key}, Setting time to #{time}"
|
|
112
|
-
break
|
|
113
|
-
end
|
|
114
|
-
end
|
|
115
107
|
|
|
116
108
|
s3_record.each do |a_record|
|
|
117
109
|
|
|
110
|
+
# parse the time from the record
|
|
111
|
+
@time_keys.each do |time_key|
|
|
112
|
+
if s3_record.include? time_key
|
|
113
|
+
time=Time.strptime(a_record[time_key], @time_format).to_f
|
|
114
|
+
$log.debug "Reset time for #{time_key}, Setting time to #{time}"
|
|
115
|
+
break
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
118
119
|
if @record_key == nil
|
|
119
|
-
tmp_record=a_record.merge(
|
|
120
|
+
tmp_record=a_record.merge(default_record)
|
|
120
121
|
new_record=tmp_record
|
|
121
122
|
else
|
|
122
123
|
new_record[record_key]=a_record
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-s3-input
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.16
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Anthony Johnson
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-06-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|