logstash-input-s3-sns-sqs 1.1.9 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4847b8327b6f423cd1c9c547651cd90494a95ea9
-  data.tar.gz: cdba3fa9f2923e14d92a8eab686a522f98797198
+  metadata.gz: 8b7f924e911ea9f2379c0b8f2415c4dbd8aa6f16
+  data.tar.gz: c9c44b0742020e1e911e01a9f5bad9bca55857ee
 SHA512:
-  metadata.gz: 5635694cc34a510b8e2b4ea6f600839f227d2029b415a6e0d1d240c6a27c4e8d4b35bff8d88a78a764d87afa03c9d02e290e9ef1090d27974d38778ec5ed7a9f
-  data.tar.gz: 2bd71a99eeeb872c4b485c1ee1ee08dcb7470b859b5880f895595b78ff9496f1c6ebef75cf3158abd12e642f2bf29cdfce5e7116036d748f1a116e29a5f2f779
+  metadata.gz: 4b2f60bbe55ff50d580f7524faa78a30235ace54f8770a54baa97009aad0041d334edd5bc87822c84714142a734e86cbf859947aa6f7200f4a1c0992521a3cf3
+  data.tar.gz: d635defff1a48274967afdcd358fff1da8f9a91cde99729558f07c1b238cff866e179ab2a7122f24dc12a547d1d784d5c5736460b5afa9ed4cb76c4c8f4c3bcd

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.2.0
+- Add codec suggestion by content-type
+- enrich metadata 
+- Fix some bugs
 ## 1.1.9
 - Add config for s3 folder prefix, auto codec and auto type
 ## 1.1.8

data/lib/logstash/inputs/s3snssqs.rb

@@ -100,6 +100,20 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   attr_reader :poller
   attr_reader :s3
 
+  def suggest_codec(content_type, key)
+    require "logstash/codecs/plain"
+    require "logstash/codecs/line"
+    require "logstash/codecs/json"
+    require "logstash/codecs/json_lines"
+    if content_type == "application/json_lines" then
+        @logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
+        @codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
+    elsif content_type == "application/json" or key.end_with?(".json") then
+        @logger.info("Automatically switching from #{@codec.class.config_name} to json codec", :plugin => self.class.config_name)
+        @codec = LogStash::Codecs::JSON.new("charset" => @codec.charset)
+    end
+  end
+
   def register
     require "aws-sdk"
     require 'cgi'
@@ -150,9 +164,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 	          bucket = CGI.unescape(record['s3']['bucket']['name'])
 	          key    = CGI.unescape(record['s3']['object']['key'])
 
-            if match=/\/#{s3_key_prefix}\/(?<type_folder>.*?)\/.*/.match(key)
-                 event.set(s3_event_type, match['type_folder'])
-            end
 
             # try download and :skip_delete if it fails
             begin
@@ -179,12 +190,8 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
                 end
                 body = temp
               end
-              if @codec == "auto" and (response.content_type == "application/json" or record['s3']['object']['key'].end_with?(".json")) then
-                 @logger.debug("This is an json files you should switch codec to json")
-                 @codec = "json"
-              else
-                 @codec = "plain"
-              end
+              # Make a suggestion for a good codec
+              suggest_codec(response.content_type,record['s3']['object']['key'])
               # process the plain text content
               begin
                 lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
@@ -195,7 +202,9 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
                   event.set('[@metadata][s3_bucket_name]', record['s3']['bucket']['name'])
                   event.set('[@metadata][s3_object_key]', record['s3']['object']['key'])
-
+                  if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
+                    event.set('[@metadata][s3_object_folder]', match['type_folder'])
+                  end
                   queue << event
                   end
                 end

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.1.9'
+  s.version         = '1.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.1.9
+  version: 1.2.0
 platform: ruby
 authors:
 - Christian Herweg