fluent-plugin-s3-input 0.0.14 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/fluent-plugin-s3-input.gemspec +1 -1
- data/lib/fluent/plugin/out_s3_input.rb +24 -25
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6b14ef024df9f7164aa69d827a10c909928a9579
|
|
4
|
+
data.tar.gz: f994d978313569c44fca602f8b2e4d925a2748b4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4710a0f15aec5e6514796cf4d539ecca854f7852b93279ff481e7bd34b4e71be45d372a508de51dcb9a1fd270f0e82ce7c59ce209bf2f970758958227fb0dadf
|
|
7
|
+
data.tar.gz: b1e0635afe912d93708467f111ab79b4af43902e455bc9ca6058f1abea20c072848848c6d307f15ea67f392355058db5d617747a0b46730ecc9167aeac833187
|
data/Gemfile.lock
CHANGED
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "fluent-plugin-s3-input"
|
|
7
|
-
spec.version = "0.0.
|
|
7
|
+
spec.version = "0.0.15"
|
|
8
8
|
spec.authors = ["Anthony Johnson"]
|
|
9
9
|
spec.email = ["ansoni@gmail.com"]
|
|
10
10
|
spec.description = %q{Fluentd plugin to read a file from S3 and emit it}
|
|
@@ -51,14 +51,14 @@ module Fluent
|
|
|
51
51
|
)
|
|
52
52
|
end
|
|
53
53
|
end
|
|
54
|
-
|
|
54
|
+
|
|
55
55
|
# Allow JSON data in a couple of formats
|
|
56
56
|
# {} single event
|
|
57
57
|
# [{},{}] array of events
|
|
58
58
|
# {}\n{}\n{} concatenated events (flume)
|
|
59
59
|
def normalize_json(json)
|
|
60
60
|
if json[0] != "["
|
|
61
|
-
json=json.gsub /}\n{/,"},{"
|
|
61
|
+
json=json.gsub /}\n{/,"},{"
|
|
62
62
|
json="[#{json}]"
|
|
63
63
|
end
|
|
64
64
|
json
|
|
@@ -71,14 +71,13 @@ module Fluent
|
|
|
71
71
|
s3_bucket = record[s3_bucket_key]
|
|
72
72
|
s3_key = record[s3_object_key_key]
|
|
73
73
|
s3_key_ext = s3_key.split(".")[-1]
|
|
74
|
-
resp = s3.get_object(bucket: s3_bucket, key: s3_key)
|
|
74
|
+
resp = s3.get_object(bucket: s3_bucket, key: s3_key)
|
|
75
75
|
|
|
76
76
|
if @gzip_exts.include?(s3_key_ext)
|
|
77
77
|
input = Zlib::GzipReader.new(resp.body)
|
|
78
78
|
elsif @zip_exts.include?(s3_key_ext)
|
|
79
79
|
io = Zip::InputStream.new(resp.body)
|
|
80
80
|
input = io.get_next_entry
|
|
81
|
-
#input = Zip::File.open(resp.body).entries.first.get_input_stream
|
|
82
81
|
else
|
|
83
82
|
input = resp.body
|
|
84
83
|
end
|
|
@@ -87,20 +86,19 @@ module Fluent
|
|
|
87
86
|
if @merge_record
|
|
88
87
|
new_record = {}.merge(record)
|
|
89
88
|
end
|
|
90
|
-
|
|
91
|
-
s3_record = {}
|
|
89
|
+
|
|
90
|
+
s3_record = {}
|
|
92
91
|
if @format == 'json'
|
|
93
92
|
json_data=normalize_json input.read
|
|
94
93
|
begin
|
|
95
94
|
s3_record = Oj.load(json_data)
|
|
96
95
|
rescue Oj::ParseError=>e
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
puts "Error: #{e.to_s}"
|
|
96
|
+
$log.error e.to_s
|
|
97
|
+
$log.error json_data
|
|
100
98
|
end
|
|
101
99
|
elsif @format == 'csv'
|
|
102
100
|
data = input.read
|
|
103
|
-
File.open("/tmp/s3debug", 'w') { |file| file.write(data) }
|
|
101
|
+
File.open("/tmp/s3debug", 'w') { |file| file.write(data) }
|
|
104
102
|
s3_record=CSV.parse(data).to_json
|
|
105
103
|
else
|
|
106
104
|
raise "Unsupported format - #{@format}"
|
|
@@ -108,27 +106,28 @@ module Fluent
|
|
|
108
106
|
|
|
109
107
|
# parse the time from the record
|
|
110
108
|
@time_keys.each do |time_key|
|
|
111
|
-
puts "Look for #{time_key} in #{new_record}"
|
|
112
109
|
if s3_record.include? time_key
|
|
113
|
-
|
|
114
|
-
time
|
|
115
|
-
puts "Setting time to #{time}"
|
|
110
|
+
time=Time.strptime(new_record[time_key], @time_format).to_f
|
|
111
|
+
$log.debug "Reset time for #{time_key}, Setting time to #{time}"
|
|
116
112
|
break
|
|
117
113
|
end
|
|
118
114
|
end
|
|
119
115
|
|
|
120
|
-
|
|
121
|
-
tmp_record=s3_record.merge(new_record)
|
|
122
|
-
new_record=tmp_record
|
|
123
|
-
else
|
|
124
|
-
new_record[record_key]=s3_record
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
@remove_keys.each do |key_to_remove|
|
|
128
|
-
new_record.delete(key_to_remove)
|
|
129
|
-
end
|
|
116
|
+
s3_record.each do |a_record|
|
|
130
117
|
|
|
131
|
-
|
|
118
|
+
if @record_key == nil
|
|
119
|
+
tmp_record=a_record.merge(new_record)
|
|
120
|
+
new_record=tmp_record
|
|
121
|
+
else
|
|
122
|
+
new_record[record_key]=a_record
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
@remove_keys.each do |key_to_remove|
|
|
126
|
+
new_record.delete(key_to_remove)
|
|
127
|
+
end
|
|
128
|
+
$log.debug "Emit - #{new_record}"
|
|
129
|
+
router.emit(@tag, time, new_record)
|
|
130
|
+
end
|
|
132
131
|
}
|
|
133
132
|
chain.next
|
|
134
133
|
rescue StandardError => e
|