logstash-input-s3-sns-sqs 1.1.8 → 1.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b47498a9996e26b8c9c26fa4f0b8b4162276a15
-  data.tar.gz: 63f8aa74667466c11584aac319ef43028f386a49
+  metadata.gz: 4847b8327b6f423cd1c9c547651cd90494a95ea9
+  data.tar.gz: cdba3fa9f2923e14d92a8eab686a522f98797198
 SHA512:
-  metadata.gz: 9b4ce293106f3dc37405c52d6b571aea478bb65f4deea3642da7e72e1cc49120962fca54e3ae84b4438d39224f42c5b41dd7d9da00103e07889866152eeb4f68
-  data.tar.gz: aa5d8b70deae51d4161d182dfd425cc8d2c49d5b595c78a5d8608426cd9bf6fe284c372e4aa28528f0e01235ec80cbee3412f42623576ab448bb7004c5beb4ca
+  metadata.gz: 5635694cc34a510b8e2b4ea6f600839f227d2029b415a6e0d1d240c6a27c4e8d4b35bff8d88a78a764d87afa03c9d02e290e9ef1090d27974d38778ec5ed7a9f
+  data.tar.gz: 2bd71a99eeeb872c4b485c1ee1ee08dcb7470b859b5880f895595b78ff9496f1c6ebef75cf3158abd12e642f2bf29cdfce5e7116036d748f1a116e29a5f2f779

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+## 1.1.9
+- Add config for s3 folder prefix, auto codec and auto type
 ## 1.1.8
 - Add config switch for delivery with or without SNS
 ## 1.1.6

data/lib/logstash/inputs/s3snssqs.rb

@@ -90,6 +90,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   # Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
   config :queue, :validate => :string, :required => true
+  config :s3_key_prefix, :validate => :string, :default => ''
   config :queue_owner_aws_account_id, :validate => :string, :required => false
   # Whether to delete files from S3 after processing.
   config :delete_on_success, :validate => :boolean, :default => false
@@ -149,6 +150,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 	          bucket = CGI.unescape(record['s3']['bucket']['name'])
 	          key    = CGI.unescape(record['s3']['object']['key'])
 
+            if match=/\/#{s3_key_prefix}\/(?<type_folder>.*?)\/.*/.match(key)
+                 event.set(s3_event_type, match['type_folder'])
+            end
+
             # try download and :skip_delete if it fails
             begin
               response = @s3.get_object(
@@ -174,6 +179,12 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
                 end
                 body = temp
               end
+              if @codec == "auto" and (response.content_type == "application/json" or record['s3']['object']['key'].end_with?(".json")) then
+                 @logger.debug("This is an json files you should switch codec to json")
+                 @codec = "json"
+              else
+                 @codec = "plain"
+              end
               # process the plain text content
               begin
                 lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
@@ -208,7 +219,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
           end
         end
       end
-    end
   end
 
   def run(queue)

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.1.8'
+  s.version         = '1.1.9'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.1.8
+  version: 1.1.9
 platform: ruby
 authors:
 - Christian Herweg