logstash-input-s3-sns-sqs 1.1.9 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4847b8327b6f423cd1c9c547651cd90494a95ea9
4
- data.tar.gz: cdba3fa9f2923e14d92a8eab686a522f98797198
3
+ metadata.gz: 8b7f924e911ea9f2379c0b8f2415c4dbd8aa6f16
4
+ data.tar.gz: c9c44b0742020e1e911e01a9f5bad9bca55857ee
5
5
  SHA512:
6
- metadata.gz: 5635694cc34a510b8e2b4ea6f600839f227d2029b415a6e0d1d240c6a27c4e8d4b35bff8d88a78a764d87afa03c9d02e290e9ef1090d27974d38778ec5ed7a9f
7
- data.tar.gz: 2bd71a99eeeb872c4b485c1ee1ee08dcb7470b859b5880f895595b78ff9496f1c6ebef75cf3158abd12e642f2bf29cdfce5e7116036d748f1a116e29a5f2f779
6
+ metadata.gz: 4b2f60bbe55ff50d580f7524faa78a30235ace54f8770a54baa97009aad0041d334edd5bc87822c84714142a734e86cbf859947aa6f7200f4a1c0992521a3cf3
7
+ data.tar.gz: d635defff1a48274967afdcd358fff1da8f9a91cde99729558f07c1b238cff866e179ab2a7122f24dc12a547d1d784d5c5736460b5afa9ed4cb76c4c8f4c3bcd
@@ -1,3 +1,7 @@
1
+ ## 1.2.0
2
+ - Add codec suggestion by content-type
3
+ - enrich metadata
4
+ - Fix some bugs
1
5
  ## 1.1.9
2
6
  - Add config for s3 folder prefix, auto codec and auto type
3
7
  ## 1.1.8
@@ -100,6 +100,20 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
100
100
  attr_reader :poller
101
101
  attr_reader :s3
102
102
 
103
+ def suggest_codec(content_type, key)
104
+ require "logstash/codecs/plain"
105
+ require "logstash/codecs/line"
106
+ require "logstash/codecs/json"
107
+ require "logstash/codecs/json_lines"
108
+ if content_type == "application/json_lines" then
109
+ @logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
110
+ @codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
111
+ elsif content_type == "application/json" or key.end_with?(".json") then
112
+ @logger.info("Automatically switching from #{@codec.class.config_name} to json codec", :plugin => self.class.config_name)
113
+ @codec = LogStash::Codecs::JSON.new("charset" => @codec.charset)
114
+ end
115
+ end
116
+
103
117
  def register
104
118
  require "aws-sdk"
105
119
  require 'cgi'
@@ -150,9 +164,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
150
164
  bucket = CGI.unescape(record['s3']['bucket']['name'])
151
165
  key = CGI.unescape(record['s3']['object']['key'])
152
166
 
153
- if match=/\/#{s3_key_prefix}\/(?<type_folder>.*?)\/.*/.match(key)
154
- event.set(s3_event_type, match['type_folder'])
155
- end
156
167
 
157
168
  # try download and :skip_delete if it fails
158
169
  begin
@@ -179,12 +190,8 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
179
190
  end
180
191
  body = temp
181
192
  end
182
- if @codec == "auto" and (response.content_type == "application/json" or record['s3']['object']['key'].end_with?(".json")) then
183
- @logger.debug("This is an json files you should switch codec to json")
184
- @codec = "json"
185
- else
186
- @codec = "plain"
187
- end
193
+ # Make a suggestion for a good codec
194
+ suggest_codec(response.content_type,record['s3']['object']['key'])
188
195
  # process the plain text content
189
196
  begin
190
197
  lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
@@ -195,7 +202,9 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
195
202
 
196
203
  event.set('[@metadata][s3_bucket_name]', record['s3']['bucket']['name'])
197
204
  event.set('[@metadata][s3_object_key]', record['s3']['object']['key'])
198
-
205
+ if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
206
+ event.set('[@metadata][s3_object_folder]', match['type_folder'])
207
+ end
199
208
  queue << event
200
209
  end
201
210
  end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-s3-sns-sqs'
3
- s.version = '1.1.9'
3
+ s.version = '1.2.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
6
6
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-s3-sns-sqs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.9
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Christian Herweg