logstash-input-s3-sns-sqs 1.1.9 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/inputs/s3snssqs.rb +19 -10
- data/logstash-input-s3-sns-sqs.gemspec +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8b7f924e911ea9f2379c0b8f2415c4dbd8aa6f16
|
|
4
|
+
data.tar.gz: c9c44b0742020e1e911e01a9f5bad9bca55857ee
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4b2f60bbe55ff50d580f7524faa78a30235ace54f8770a54baa97009aad0041d334edd5bc87822c84714142a734e86cbf859947aa6f7200f4a1c0992521a3cf3
|
|
7
|
+
data.tar.gz: d635defff1a48274967afdcd358fff1da8f9a91cde99729558f07c1b238cff866e179ab2a7122f24dc12a547d1d784d5c5736460b5afa9ed4cb76c4c8f4c3bcd
|
data/CHANGELOG.md
CHANGED
|
@@ -100,6 +100,20 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
|
100
100
|
attr_reader :poller
|
|
101
101
|
attr_reader :s3
|
|
102
102
|
|
|
103
|
+
def suggest_codec(content_type, key)
|
|
104
|
+
require "logstash/codecs/plain"
|
|
105
|
+
require "logstash/codecs/line"
|
|
106
|
+
require "logstash/codecs/json"
|
|
107
|
+
require "logstash/codecs/json_lines"
|
|
108
|
+
if content_type == "application/json_lines" then
|
|
109
|
+
@logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
|
|
110
|
+
@codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
|
|
111
|
+
elsif content_type == "application/json" or key.end_with?(".json") then
|
|
112
|
+
@logger.info("Automatically switching from #{@codec.class.config_name} to json codec", :plugin => self.class.config_name)
|
|
113
|
+
@codec = LogStash::Codecs::JSON.new("charset" => @codec.charset)
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
103
117
|
def register
|
|
104
118
|
require "aws-sdk"
|
|
105
119
|
require 'cgi'
|
|
@@ -150,9 +164,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
|
150
164
|
bucket = CGI.unescape(record['s3']['bucket']['name'])
|
|
151
165
|
key = CGI.unescape(record['s3']['object']['key'])
|
|
152
166
|
|
|
153
|
-
if match=/\/#{s3_key_prefix}\/(?<type_folder>.*?)\/.*/.match(key)
|
|
154
|
-
event.set(s3_event_type, match['type_folder'])
|
|
155
|
-
end
|
|
156
167
|
|
|
157
168
|
# try download and :skip_delete if it fails
|
|
158
169
|
begin
|
|
@@ -179,12 +190,8 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
|
179
190
|
end
|
|
180
191
|
body = temp
|
|
181
192
|
end
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
@codec = "json"
|
|
185
|
-
else
|
|
186
|
-
@codec = "plain"
|
|
187
|
-
end
|
|
193
|
+
# Make a suggestion for a good codec
|
|
194
|
+
suggest_codec(response.content_type,record['s3']['object']['key'])
|
|
188
195
|
# process the plain text content
|
|
189
196
|
begin
|
|
190
197
|
lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
|
|
@@ -195,7 +202,9 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
|
195
202
|
|
|
196
203
|
event.set('[@metadata][s3_bucket_name]', record['s3']['bucket']['name'])
|
|
197
204
|
event.set('[@metadata][s3_object_key]', record['s3']['object']['key'])
|
|
198
|
-
|
|
205
|
+
if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
|
|
206
|
+
event.set('[@metadata][s3_object_folder]', match['type_folder'])
|
|
207
|
+
end
|
|
199
208
|
queue << event
|
|
200
209
|
end
|
|
201
210
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-input-s3-sns-sqs'
|
|
3
|
-
s.version = '1.
|
|
3
|
+
s.version = '1.2.0'
|
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
|
5
5
|
s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
|
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|