logstash-input-s3-sns-sqs 1.6.1 → 2.0.0

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -0,0 +1,119 @@
+# LogProcessor:
+# reads and decodes locally available file with log lines
+# and creates LogStash events from these
+require 'logstash/inputs/mime/magic_gzip_validator'
+require 'pathname'
+
+module LogProcessor
+
+  def self.included(base)
+    base.extend(self)
+  end
+
+  def process(record, logstash_event_queue)
+    file = record[:local_file]
+    codec = @codec_factory.get_codec(record)
+    folder = record[:folder]
+    type = @type_by_folder[folder] #if @type_by_folder.key?(folder)
+    metadata = {}
+    read_file(file) do |line|
+      if stop?
+        @logger.warn("Abort reading in the middle of the file, we will read it again when logstash is started")
+        throw :skip_delete
+      end
+      line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
+      codec.decode(line) do |event|
+        decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
+        logstash_event_queue << event
+      end
+    end
+    # ensure any stateful codecs (such as multi-line ) are flushed to the queue
+    codec.flush do |event|
+      decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
+      @logger.debug("Flushing an incomplete event", :event => event.to_s)
+      logstash_event_queue << event
+    end
+    # signal completion:
+    return true
+  end
+
+  private
+
+  def decorate_event(event, metadata, type, key, bucket, folder)
+    if event_is_metadata?(event)
+      @logger.debug('Updating the current cloudfront metadata', :event => event)
+      update_metadata(metadata, event)
+    else
+      # type by folder - set before "decorate()" enforces default
+      event.set('type', type) if type && !event.include?('type')
+      decorate(event)
+
+      event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
+      event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
+
+      event.set("[@metadata][s3][object_key]", key)
+      event.set("[@metadata][s3][bucket_name]", bucket)
+      event.set("[@metadata][s3][full_folder]", folder)
+      event.set("[@metadata][s3][object_folder]", get_object_folder(folder))
+    end
+  end
+
+  def gzip?(filename)
+    return true if filename.end_with?('.gz','.gzip')
+    MagicGzipValidator.new(File.new(filename, 'rb')).valid?
+  rescue Exception => e
+    @logger.warn("Problem while gzip detection", :error => e)
+  end
+
+  def read_file(filename)
+    completed = false
+    zipped = gzip?(filename)
+    file_stream = FileInputStream.new(filename)
+    if zipped
+      gzip_stream = GZIPInputStream.new(file_stream)
+      decoder = InputStreamReader.new(gzip_stream, 'UTF-8')
+    else
+      decoder = InputStreamReader.new(file_stream, 'UTF-8')
+    end
+    buffered = BufferedReader.new(decoder)
+
+    while (line = buffered.readLine())
+      yield(line)
+    end
+    completed = true
+  rescue ZipException => e
+    @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename, :error => e)
+  ensure
+    buffered.close unless buffered.nil?
+    decoder.close unless decoder.nil?
+    gzip_stream.close unless gzip_stream.nil?
+    file_stream.close unless file_stream.nil?
+    throw :skip_delete unless completed
+  end
+
+  def event_is_metadata?(event)
+    return false unless event.get("message").class == String
+    line = event.get("message")
+    version_metadata?(line) || fields_metadata?(line)
+  end
+
+  def version_metadata?(line)
+    line.start_with?('#Version: ')
+  end
+
+  def fields_metadata?(line)
+    line.start_with?('#Fields: ')
+  end
+
+  def update_metadata(metadata, event)
+    line = event.get('message').strip
+
+    if version_metadata?(line)
+      metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
+    end
+
+    if fields_metadata?(line)
+      metadata[:cloudfront_fields] = line.split(/#Fields: (.+)/).last
+    end
+  end
+end

data/lib/logstash/inputs/sqs/poller.rb

@@ -0,0 +1,183 @@
+# MessagePoller:
+# polling loop fetches messages from source queue and invokes
+# the provided code block on them
+require 'json'
+require 'cgi'
+
+class SqsPoller
+
+  # queue poller options we want to set explicitly
+  DEFAULT_OPTIONS = {
+      # we query one message at a time, so we can ensure correct error
+      # handling if we can't download a single file correctly
+      # (we will throw :skip_delete if download size isn't correct to allow
+      # for processing the event again later, so make sure to set a reasonable
+      # "DefaultVisibilityTimeout" for your queue so that there's enough time
+      # to process the log files!)
+      max_number_of_messages: 1,
+      visibility_timeout: 600,
+      # long polling; by default we use the queue's setting.
+      # A good value is 10 seconds to to balance between a quick logstash
+      # shutdown and fewer api calls.
+      wait_time_seconds: nil,
+      skip_delete: false,
+  }
+
+  # only needed in "run_with_backoff":
+  BACKOFF_SLEEP_TIME = 1
+  BACKOFF_FACTOR = 2
+  MAX_TIME_BEFORE_GIVING_UP = 60
+  # only needed in "preprocess":
+  EVENT_SOURCE = 'aws:s3'
+  EVENT_TYPE = 'ObjectCreated'
+
+  # initialization and setup happens once, outside the threads:
+  #
+  def initialize(logger, stop_semaphore, sqs_queue, options = {}, aws_options_hash)
+    @logger = logger
+    @stopped = stop_semaphore
+    @queue = sqs_queue
+    # @stopped = false # FIXME: needed per thread?
+    @from_sns = options[:from_sns]
+    @options = DEFAULT_OPTIONS.merge(options.reject { |k| [:sqs_explicit_delete, :from_sns, :queue_owner_aws_account_id, :sqs_skip_delete].include? k })
+    @options[:skip_delete] = options[:sqs_skip_delete]
+    begin
+      @logger.info("Registering SQS input", :queue => @queue)
+      sqs_client = Aws::SQS::Client.new(aws_options_hash)
+      queue_url = sqs_client.get_queue_url({
+        queue_name: @queue,
+        queue_owner_aws_account_id: @options[:queue_owner_aws_account_id]
+      }).queue_url # is a method according to docs. Was [:queue_url].
+      @poller = Aws::SQS::QueuePoller.new(queue_url,
+        :client => sqs_client
+      )
+    rescue Aws::SQS::Errors::ServiceError => e
+      @logger.error("Cannot establish connection to Amazon SQS", :error => e)
+      raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
+    end
+  end
+
+  #
+  # this is called by every worker thread:
+  #
+  def run() # not (&block) - pass explicitly (use yield below)
+    # per-thread timer to extend visibility if necessary
+    extender = nil
+    message_backoff = (@options[:visibility_timeout] * 90).to_f / 100.0
+    new_visibility = 2 * @options[:visibility_timeout]
+
+    # "shutdown handler":
+    @poller.before_request do |_|
+      if stop?
+        # kill visibility extender thread if active?
+        extender.kill if extender
+        extender = nil
+        @logger.warn('issuing :stop_polling on "stop?" signal', :queue => @queue)
+        # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
+        throw :stop_polling
+      end
+    end
+
+    run_with_backoff do
+      @poller.poll(@options) do |message|
+        @logger.debug("Inside Poller: polled message", :message => message)
+        # auto-double the timeout if processing takes too long:
+        extender = Thread.new do
+          sleep message_backoff
+          @logger.info("Extending visibility for message", :message => message)
+          @poller.change_message_visibility_timeout(message, new_visibility)
+        end
+        failed = false
+        begin
+          preprocess(message) do |record|
+            @logger.debug("we got a record", :record => record)
+            yield(record) #unless record.nil? - unnecessary; implicit
+          end
+        rescue Exception => e
+          @logger.warn("Error in poller loop", :error => e)
+          @logger.warn("Backtrace:\n\t#{e.backtrace.join("\n\t")}")
+          failed = true
+        end
+        # at this time the extender has either fired or is obsolete
+        extender.kill
+        extender = nil
+        throw :skip_delete if failed
+      end
+    end
+  end
+
+  private
+
+  def stop?
+    @stopped.value
+  end
+
+  def preprocess(message)
+    @logger.debug("Inside Preprocess: Start", :message => message)
+    payload = JSON.parse(message.body)
+    payload = JSON.parse(payload['Message']) if @from_sns
+    @logger.debug("Payload in Preprocess: ", :payload => payload)
+    return nil unless payload['Records']
+    payload['Records'].each do |record|
+      @logger.debug("We found a record", :record => record)
+      # in case there are any events with Records that aren't s3 object-created events and can't therefore be
+      # processed by this plugin, we will skip them and remove them from queue
+      if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
+        @logger.debug("record is valid")
+        bucket  = CGI.unescape(record['s3']['bucket']['name'])
+        key     = CGI.unescape(record['s3']['object']['key'])
+        size    = record['s3']['object']['size']
+        yield({
+          bucket: bucket,
+          key: key,
+          size: size,
+          folder: get_type_folder(key)
+        })
+
+        # -v- this stuff goes into s3 and processor handling: -v-
+
+        # type_folder = get_object_folder(key)
+        # Set input codec by :set_codec_by_folder
+        # instance_codec = set_codec(type_folder) unless set_codec_by_folder["#{type_folder}"].nil?
+        # try download and :skip_delete if it fails
+        #if record['s3']['object']['size'] < 10000000 then
+        # process_log(bucket, key, type_folder, instance_codec, queue, message, size)
+        #else
+        #  @logger.info("Your file is too big")
+        #end
+      end
+    end
+  end
+
+  # Runs an AWS request inside a Ruby block with an exponential backoff in case
+  # we experience a ServiceError.
+  # @param [Integer] max_time maximum amount of time to sleep before giving up.
+  # @param [Integer] sleep_time the initial amount of time to sleep before retrying.
+  # instead of requiring
+  # @param [Block] block Ruby code block to execute
+  # and then doing a "block.call",
+  # we yield to the passed block.
+  def run_with_backoff(max_time = MAX_TIME_BEFORE_GIVING_UP, sleep_time = BACKOFF_SLEEP_TIME)
+    next_sleep = sleep_time
+    begin
+      yield
+      next_sleep = sleep_time
+    rescue Aws::SQS::Errors::ServiceError => e
+      @logger.warn("Aws::SQS::Errors::ServiceError ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time, :error => e)
+      sleep(next_sleep)
+      next_sleep = next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR
+      retry
+    end
+  end
+
+  def get_type_folder(key)
+    # TEST THIS!
+    # if match = /.*\/?(?<type_folder>)\/[^\/]*.match(key)
+    #   return match['type_folder']
+    # end
+    folder = ::File.dirname(key)
+    return '' if folder == '.'
+    return folder
+  end
+
+end

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.6.1'
-  s.licenses        = ['Apache License (2.0)']
+  s.version         = '2.0.0'
+  s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Christian Herweg"]
@@ -21,9 +21,8 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
 
-  s.add_runtime_dependency 'logstash-codec-json'
-  s.add_runtime_dependency "logstash-mixin-aws"
-  s.add_development_dependency 'logstash-devutils'
-
+  s.add_runtime_dependency 'logstash-codec-json', '~> 3.0'
+  s.add_runtime_dependency 'logstash-mixin-aws', '~> 4.3'
+  s.add_development_dependency 'logstash-codec-json_stream', '~> 1.0'
+  s.add_development_dependency 'logstash-devutils', '~> 1.3'
 end
-

data/spec/inputs/s3sqs_spec.rb

@@ -1,9 +1,60 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
-require "logstash/inputs/s3sqs"
+require "logstash/plugin"
+require "logstash/inputs/s3snssqs"
+require "fileutils"
+require "logstash/errors"
+require "logstash/event"
+require "logstash/json"
+require "logstash/codecs/base"
+require "logstash/codecs/json_stream"
+require 'rspec'
+require 'rspec/expectations'
 
-describe LogStash::Inputs::S3SQS do
 
-  true.should be_true
 
-end
+describe LogStash::Inputs::S3SNSSQS do
+  class LogStash::Inputs::S3SNSSQS
+    public :process_local_log # use method without error logging for better visibility of errors
+  end
+  let(:codec_options) { {} }
+
+  let(:input) { LogStash::Inputs::S3SNSSQS.new(config) }
+  subject { input }
+
+  context "default parser choice" do
+    it "should return true" do
+      expect(true).to be true
+    end
+  end
+
+  let(:compressed_log_file) { File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'log-stream.real-formatted') }
+  let(:key) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
+  let(:folder) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
+  let(:instance_codec) { "json" }
+  let(:queue) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
+  let(:bucket) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
+  let(:message) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
+  let(:size) { "123344" }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:config) { {"queue" => queue, "codec" => "json", "temporary_directory" => temporary_directory } }
+  context 'compressed_log_file' do
+
+    subject do
+      LogStash::Inputs::S3SNSSQS.new(config)
+    end
+    let(:queue) { [] }
+    before do
+      @codec = LogStash::Codecs::JSONStream.new
+      @codec.charset = "UTF-8"
+      expect( subject.process_local_log(compressed_log_file, key, folder, @codec.clone, queue, bucket, message, size) ).to be true
+      $stderr.puts "method #{queue.to_s}"
+    end
+
+    it '.process_local_log => process compressed log file and verfied logstash event queue with the correct number of events' do
+      expect( queue.size ).to eq(38)
+      expect( queue.clear).to be_empty
+    end
+
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-24 00:00:00.000000000 Z
+date: 2019-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -33,45 +33,59 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   name: logstash-codec-json
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.3'
   name: logstash-mixin-aws
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
+  name: logstash-codec-json_stream
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
   a stand-alone program
@@ -86,15 +100,19 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
-- lib/logstash/inputs/mime/MagicgzipValidator.rb
+- lib/logstash/inputs/codec_factory.rb
+- lib/logstash/inputs/mime/magic_gzip_validator.rb
+- lib/logstash/inputs/s3/client_factory.rb
+- lib/logstash/inputs/s3/downloader.rb
 - lib/logstash/inputs/s3snssqs.rb
+- lib/logstash/inputs/s3snssqs/log_processor.rb
 - lib/logstash/inputs/s3sqs/patch.rb
+- lib/logstash/inputs/sqs/poller.rb
 - logstash-input-s3-sns-sqs.gemspec
 - spec/inputs/s3sqs_spec.rb
-- spec/spec_helper.rb
 homepage: https://github.com/cherweg/logstash-input-s3-sns-sqs
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: input
@@ -121,4 +139,3 @@ summary: Get logs from AWS s3 buckets as issued by an object-created event via s
   -> sqs.
 test_files:
 - spec/inputs/s3sqs_spec.rb
-- spec/spec_helper.rb