fluent-plugin-s3-input 0.0.14 → 0.0.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/fluent-plugin-s3-input.gemspec +1 -1
- data/lib/fluent/plugin/out_s3_input.rb +24 -25
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6b14ef024df9f7164aa69d827a10c909928a9579
|
|
4
|
+
data.tar.gz: f994d978313569c44fca602f8b2e4d925a2748b4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4710a0f15aec5e6514796cf4d539ecca854f7852b93279ff481e7bd34b4e71be45d372a508de51dcb9a1fd270f0e82ce7c59ce209bf2f970758958227fb0dadf
|
|
7
|
+
data.tar.gz: b1e0635afe912d93708467f111ab79b4af43902e455bc9ca6058f1abea20c072848848c6d307f15ea67f392355058db5d617747a0b46730ecc9167aeac833187
|
data/Gemfile.lock
CHANGED
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "fluent-plugin-s3-input"
|
|
7
|
-
spec.version = "0.0.
|
|
7
|
+
spec.version = "0.0.15"
|
|
8
8
|
spec.authors = ["Anthony Johnson"]
|
|
9
9
|
spec.email = ["ansoni@gmail.com"]
|
|
10
10
|
spec.description = %q{Fluentd plugin to read a file from S3 and emit it}
|
|
@@ -51,14 +51,14 @@ module Fluent
|
|
|
51
51
|
)
|
|
52
52
|
end
|
|
53
53
|
end
|
|
54
|
-
|
|
54
|
+
|
|
55
55
|
# Allow JSON data in a couple of formats
|
|
56
56
|
# {} single event
|
|
57
57
|
# [{},{}] array of events
|
|
58
58
|
# {}\n{}\n{} concatenated events (flume)
|
|
59
59
|
def normalize_json(json)
|
|
60
60
|
if json[0] != "["
|
|
61
|
-
json=json.gsub /}\n{/,"},{"
|
|
61
|
+
json=json.gsub /}\n{/,"},{"
|
|
62
62
|
json="[#{json}]"
|
|
63
63
|
end
|
|
64
64
|
json
|
|
@@ -71,14 +71,13 @@ module Fluent
|
|
|
71
71
|
s3_bucket = record[s3_bucket_key]
|
|
72
72
|
s3_key = record[s3_object_key_key]
|
|
73
73
|
s3_key_ext = s3_key.split(".")[-1]
|
|
74
|
-
resp = s3.get_object(bucket: s3_bucket, key: s3_key)
|
|
74
|
+
resp = s3.get_object(bucket: s3_bucket, key: s3_key)
|
|
75
75
|
|
|
76
76
|
if @gzip_exts.include?(s3_key_ext)
|
|
77
77
|
input = Zlib::GzipReader.new(resp.body)
|
|
78
78
|
elsif @zip_exts.include?(s3_key_ext)
|
|
79
79
|
io = Zip::InputStream.new(resp.body)
|
|
80
80
|
input = io.get_next_entry
|
|
81
|
-
#input = Zip::File.open(resp.body).entries.first.get_input_stream
|
|
82
81
|
else
|
|
83
82
|
input = resp.body
|
|
84
83
|
end
|
|
@@ -87,20 +86,19 @@ module Fluent
|
|
|
87
86
|
if @merge_record
|
|
88
87
|
new_record = {}.merge(record)
|
|
89
88
|
end
|
|
90
|
-
|
|
91
|
-
s3_record = {}
|
|
89
|
+
|
|
90
|
+
s3_record = {}
|
|
92
91
|
if @format == 'json'
|
|
93
92
|
json_data=normalize_json input.read
|
|
94
93
|
begin
|
|
95
94
|
s3_record = Oj.load(json_data)
|
|
96
95
|
rescue Oj::ParseError=>e
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
puts "Error: #{e.to_s}"
|
|
96
|
+
$log.error e.to_s
|
|
97
|
+
$log.error json_data
|
|
100
98
|
end
|
|
101
99
|
elsif @format == 'csv'
|
|
102
100
|
data = input.read
|
|
103
|
-
File.open("/tmp/s3debug", 'w') { |file| file.write(data) }
|
|
101
|
+
File.open("/tmp/s3debug", 'w') { |file| file.write(data) }
|
|
104
102
|
s3_record=CSV.parse(data).to_json
|
|
105
103
|
else
|
|
106
104
|
raise "Unsupported format - #{@format}"
|
|
@@ -108,27 +106,28 @@ module Fluent
|
|
|
108
106
|
|
|
109
107
|
# parse the time from the record
|
|
110
108
|
@time_keys.each do |time_key|
|
|
111
|
-
puts "Look for #{time_key} in #{new_record}"
|
|
112
109
|
if s3_record.include? time_key
|
|
113
|
-
|
|
114
|
-
time
|
|
115
|
-
puts "Setting time to #{time}"
|
|
110
|
+
time=Time.strptime(new_record[time_key], @time_format).to_f
|
|
111
|
+
$log.debug "Reset time for #{time_key}, Setting time to #{time}"
|
|
116
112
|
break
|
|
117
113
|
end
|
|
118
114
|
end
|
|
119
115
|
|
|
120
|
-
|
|
121
|
-
tmp_record=s3_record.merge(new_record)
|
|
122
|
-
new_record=tmp_record
|
|
123
|
-
else
|
|
124
|
-
new_record[record_key]=s3_record
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
@remove_keys.each do |key_to_remove|
|
|
128
|
-
new_record.delete(key_to_remove)
|
|
129
|
-
end
|
|
116
|
+
s3_record.each do |a_record|
|
|
130
117
|
|
|
131
|
-
|
|
118
|
+
if @record_key == nil
|
|
119
|
+
tmp_record=a_record.merge(new_record)
|
|
120
|
+
new_record=tmp_record
|
|
121
|
+
else
|
|
122
|
+
new_record[record_key]=a_record
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
@remove_keys.each do |key_to_remove|
|
|
126
|
+
new_record.delete(key_to_remove)
|
|
127
|
+
end
|
|
128
|
+
$log.debug "Emit - #{new_record}"
|
|
129
|
+
router.emit(@tag, time, new_record)
|
|
130
|
+
end
|
|
132
131
|
}
|
|
133
132
|
chain.next
|
|
134
133
|
rescue StandardError => e
|