logstash-input-s3-sns-sqs 1.4.8 → 1.4.9

data/lib/logstash/inputs/mime/mimemagic/version.rb

@@ -0,0 +1,5 @@
+class MimeMagic
+  # MimeMagic version string
+  # @api public
+  VERSION = '0.3.2'
+end

data/lib/logstash/inputs/mime/mimemagic.rb

@@ -0,0 +1,139 @@
+require 'logstash/inputs/mime/mimemagic/tables'
+require 'logstash/inputs/mime/mimemagic/version'
+
+require 'stringio'
+
+# Mime type detection
+class MimeMagic
+  attr_reader :type, :mediatype, :subtype
+
+  # Mime type by type string
+  def initialize(type)
+    @type = type
+    @mediatype, @subtype = type.split('/', 2)
+  end
+
+  # Add custom mime type. Arguments:
+  # * <i>type</i>: Mime type
+  # * <i>options</i>: Options hash
+  #
+  # Option keys:
+  # * <i>:extensions</i>: String list or single string of file extensions
+  # * <i>:parents</i>: String list or single string of parent mime types
+  # * <i>:magic</i>: Mime magic specification
+  # * <i>:comment</i>: Comment string
+  def self.add(type, options)
+    extensions = [options[:extensions]].flatten.compact
+    TYPES[type] = [extensions,
+                  [options[:parents]].flatten.compact,
+                  options[:comment]]
+    extensions.each {|ext| EXTENSIONS[ext] = type }
+    MAGIC.unshift [type, options[:magic]] if options[:magic]
+  end
+
+  # Removes a mime type from the dictionary.  You might want to do this if
+  # you're seeing impossible conflicts (for instance, application/x-gmc-link).
+  # * <i>type</i>: The mime type to remove.  All associated extensions and magic are removed too.
+  def self.remove(type)
+    EXTENSIONS.delete_if {|ext, t| t == type }
+    MAGIC.delete_if {|t, m| t == type }
+    TYPES.delete(type)
+  end
+
+  # Returns true if type is a text format
+  def text?; mediatype == 'text' || child_of?('text/plain'); end
+
+  # Mediatype shortcuts
+  def image?; mediatype == 'image'; end
+  def audio?; mediatype == 'audio'; end
+  def video?; mediatype == 'video'; end
+
+  # Returns true if type is child of parent type
+  def child_of?(parent)
+    MimeMagic.child?(type, parent)
+  end
+
+  # Get string list of file extensions
+  def extensions
+    TYPES.key?(type) ? TYPES[type][0] : []
+  end
+
+  # Get mime comment
+  def comment
+    (TYPES.key?(type) ? TYPES[type][2] : nil).to_s
+  end
+
+  # Lookup mime type by file extension
+  def self.by_extension(ext)
+    ext = ext.to_s.downcase
+    mime = ext[0..0] == '.' ? EXTENSIONS[ext[1..-1]] : EXTENSIONS[ext]
+    mime && new(mime)
+  end
+
+  # Lookup mime type by filename
+  def self.by_path(path)
+    by_extension(File.extname(path))
+  end
+
+  # Lookup mime type by magic content analysis.
+  # This is a slow operation.
+  def self.by_magic(io)
+    mime = magic_match(io, :find)
+    mime && new(mime[0])
+  end
+
+  # Lookup all mime types by magic content analysis.
+  # This is a slower operation.
+  def self.all_by_magic(io)
+    magic_match(io, :select).map { |mime| new(mime[0]) }
+  end
+
+  # Return type as string
+  def to_s
+    type
+  end
+
+  # Allow comparison with string
+  def eql?(other)
+    type == other.to_s
+  end
+
+  def hash
+    type.hash
+  end
+
+  alias == eql?
+
+  def self.child?(child, parent)
+    child == parent || TYPES.key?(child) && TYPES[child][1].any? {|p| child?(p, parent) }
+  end
+
+  def self.magic_match(io, method)
+    return magic_match(StringIO.new(io.to_s), method) unless io.respond_to?(:read)
+
+    io.binmode if io.respond_to?(:binmode)
+    io.set_encoding(Encoding::BINARY) if io.respond_to?(:set_encoding)
+    buffer = "".force_encoding(Encoding::BINARY)
+
+    MAGIC.send(method) { |type, matches| magic_match_io(io, matches, buffer) }
+  end
+
+  def self.magic_match_io(io, matches, buffer)
+    matches.any? do |offset, value, children|
+      match =
+        if Range === offset
+          io.read(offset.begin, buffer)
+          x = io.read(offset.end - offset.begin + value.bytesize, buffer)
+          x && x.include?(value)
+        else
+          io.read(offset, buffer)
+          io.read(value.bytesize, buffer) == value
+        end
+      io.rewind
+      match && (!children || magic_match_io(io, children, buffer))
+    end
+  end
+
+  private_class_method :magic_match, :magic_match_io
+end
+

data/lib/logstash/inputs/s3snssqs.rb

@@ -8,6 +8,7 @@ require "logstash/errors"
 require 'logstash/inputs/s3sqs/patch'
 require "aws-sdk"
 require 'cgi'
+require 'logstash/inputs/mime/mimemagic'
 
 require 'java'
 java_import java.io.InputStream
@@ -237,7 +238,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       begin
         remote_object.get(:response_target => s3file)
       rescue Aws::S3::Errors::AccessDenied => e
-        @logger.debug("Unable to download file. We´ll requeue the message", :file => remote_object.inspect)
+        @logger.error("Unable to download file. We´ll requeue the message", :file => remote_object.inspect)
         throw :skip_delete
       end
     end
@@ -264,10 +265,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
         @logger.warn("Logstash S3 input, stop reading in the middle of the file, we will read it again when logstash is started")
         return false
       end
-      #@logger.info("read line #{i}", :line => line)
       #line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
+      #@logger.debug("read line", :line => line)
       instance_codec.decode(line) do |event|
-        #@logger.info("decorate event")
+        @logger.debug("decorate event")
         # We are making an assumption concerning cloudfront
         # log format, the user will use the plain or the line codec
         # and the message key will represent the actual line content.
@@ -293,6 +294,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   private
   def local_decorate_and_queue(event, queue, key, folder, metadata, bucket)
+    @logger.debug('decorating event', :event => event.to_s)
     if event_is_metadata?(event)
       @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
       update_metadata(metadata, event)
@@ -306,6 +308,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       event.set("[@metadata][s3]", { "object_key"    => key })
       event.set("[@metadata][s3]", { "bucket_name"   => bucket })
       event.set("[@metadata][s3]", { "object_folder" => folder})
+      #@logger.debug('queuing event', :event => event.to_s)
       queue << event
     end
   end
@@ -319,6 +322,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       return ""
     end
   end
+
   private
   def read_file(filename, &block)
     if gzip?(filename)
@@ -355,10 +359,12 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   private
   def gzip?(filename)
-    filename.end_with?('.gz','.gzip')
+    return true if filename.end_with?('.gz','.gzip')
+    return true if MimeMagic.by_magic(File.open(filename)).to_s == 'application/gzip'
+  rescue Exception => e
+    @logger.debug("Problem while gzip detection", :error => e)
   end
 
-
   private
   def delete_file_from_bucket(object)
     if @delete_on_success
@@ -370,7 +376,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   private
   def get_s3client
     if s3_access_key_id and s3_secret_access_key
-      @logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id) )
+      @logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key) )
       @s3_client = Aws::S3::Client.new(aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key))
     elsif @s3_role_arn
       @s3_client = Aws::S3::Client.new(aws_options_hash.merge!({ :credentials => s3_assume_role }))
@@ -499,4 +505,4 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       result
     end
   end
-end # class
+end # class

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.4.8'
+  s.version         = '1.4.9'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -23,7 +23,6 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency "logstash-mixin-aws"
-
   s.add_development_dependency 'logstash-devutils'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.4.8
+  version: 1.4.9
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-28 00:00:00.000000000 Z
+date: 2018-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,10 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- lib/logstash/inputs/mime/mimemagic.rb
+- lib/logstash/inputs/mime/mimemagic/overlay.rb
+- lib/logstash/inputs/mime/mimemagic/tables.rb
+- lib/logstash/inputs/mime/mimemagic/version.rb
 - lib/logstash/inputs/s3snssqs.rb
 - lib/logstash/inputs/s3sqs/patch.rb
 - logstash-input-s3-sns-sqs.gemspec