logstash-input-crowdstrike_fdr 2.1.2

data/lib/logstash/inputs/mime/magic_gzip_validator.rb

@@ -0,0 +1,53 @@
+class MagicGzipValidator
+  attr_reader :file
+  attr_reader :starting_signature
+
+  VALID_STARTING_SIGNATURE = "1f8b"
+
+  def initialize(file)
+    raise "Expecting a file object as an argument" unless file.is_a?(File)
+
+    # Ensure there are sufficient number of bytes to determine the
+    # signature.
+    if file.stat.size < minimum_bytes_for_determining_signature
+      puts "File too small to calculate signature"
+      return false
+    end
+
+    @file = file
+    process_file!
+  end
+
+  def starting_signature_bytes
+    2
+  end
+
+  def valid?
+    @starting_signature == VALID_STARTING_SIGNATURE
+  end
+  private
+
+  def minimum_bytes_for_determining_signature
+    starting_signature_bytes
+  end
+
+  def process_file!
+    read_starting_signature!
+
+    # Ensure the file is closed after reading the starting signiture
+    # bytes
+    @file.close
+  end
+
+  def read_starting_signature!
+    @file.rewind
+    starting_bytes = @file.readpartial(starting_signature_bytes)
+    @starting_signature = starting_bytes.unpack("H*").first
+  end
+
+end
+
+#puts MagicGzipValidator.new(File.new('json.log', 'r')).valid?
+
+#=> true
+

data/lib/logstash/inputs/s3/client_factory.rb

@@ -0,0 +1,59 @@
+# not needed - Mutex is part of core lib:
+#require 'thread'
+
+class S3ClientFactory
+
+  def initialize(logger, options, aws_options_hash)
+    @logger = logger
+    @aws_options_hash = aws_options_hash
+    @s3_default_options = Hash[options[:s3_default_options].map { |k, v| [k.to_sym, v] }]
+    @aws_options_hash.merge!(@s3_default_options) unless @s3_default_options.empty?
+    @sts_client = Aws::STS::Client.new(region: options[:aws_region])
+    @credentials_by_bucket = options[:s3_credentials_by_bucket]
+    @region_by_bucket = options[:s3_region_by_bucket]
+    @logger.debug("Credentials by Bucket", :credentials => @credentials_by_bucket)
+    @default_session_name = options[:s3_role_session_name]
+    @clients_by_bucket = {}
+    @creation_mutex = Mutex.new
+  end
+
+  def get_s3_client(bucket_name)
+    bucket_symbol = bucket_name.to_sym
+    @creation_mutex.synchronize do
+      if @clients_by_bucket[bucket_symbol].nil?
+        options = @aws_options_hash.clone
+        unless @credentials_by_bucket[bucket_name].nil?
+          options.merge!(credentials: get_s3_auth(@credentials_by_bucket[bucket_name]))
+        end
+        unless @region_by_bucket[bucket_name].nil?
+          options.merge!(region: @region_by_bucket[bucket_name])
+        end
+        @clients_by_bucket[bucket_symbol] = Aws::S3::Client.new(options)
+        @logger.debug("Created a new S3 Client", :bucket_name => bucket_name, :client => @clients_by_bucket[bucket_symbol], :used_options => options)
+      end
+    end
+    # to be thread-safe, one uses this method like this:
+    # s3_client_factory.get_s3_client(my_s3_bucket) do
+    #   ... do stuff ...
+    # end
+    yield @clients_by_bucket[bucket_symbol]
+  end
+
+  private
+
+  def get_s3_auth(credentials)
+    # reminder: these are auto-refreshing!
+    if credentials.key?('role')
+      @logger.debug("Assume Role", :role => credentials["role"])
+      return Aws::AssumeRoleCredentials.new(
+        client: @sts_client,
+        role_arn: credentials['role'],
+        role_session_name: @default_session_name
+      )
+    elsif credentials.key?('access_key_id') && credentials.key?('secret_access_key')
+      @logger.debug("Fetch credentials", :access_key => credentials['access_key_id'])
+      return Aws::Credentials.new(credentials['access_key_id'], credentials['secret_access_key'])
+    end
+  end
+
+end # class

data/lib/logstash/inputs/s3/downloader.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require 'fileutils'
+require 'thread'
+
+class S3Downloader
+
+  def initialize(logger, stop_semaphore, options)
+    @logger = logger
+    @stopped = stop_semaphore
+    @factory = options[:s3_client_factory]
+    @delete_on_success = options[:delete_on_success]
+    @include_object_properties = options[:include_object_properties]
+  end
+
+  def copy_s3object_to_disk(record)
+    # (from docs) WARNING:
+    # yielding data to a block disables retries of networking errors!
+    begin
+      @factory.get_s3_client(record[:bucket]) do |s3|
+        response = s3.get_object(
+          bucket: record[:bucket],
+          key: record[:key],
+          response_target: record[:local_file]
+        )
+        record[:s3_data] = response.to_h.keep_if { |key| @include_object_properties.include?(key) }
+      end
+    rescue Aws::S3::Errors::ServiceError => e
+      @logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)
+      # prevent sqs message deletion
+      throw :skip_delete
+    end
+    throw :skip_delete if stop?
+    return true
+  end
+
+  def cleanup_local_object(record)
+    FileUtils.remove_entry_secure(record[:local_file], true) if ::File.exists?(record[:local_file])
+  rescue Exception => e
+    @logger.warn("Could not delete file", :file => record[:local_file], :error => e)
+  end
+
+  def cleanup_s3object(record)
+    return unless @delete_on_success
+    begin
+      @factory.get_s3_client(record[:bucket]) do |s3|
+        s3.delete_object(bucket: record[:bucket], key: record[:key])
+      end
+    rescue Exception => e
+      @logger.warn("Failed to delete s3 object", :record => record, :error => e)
+    end
+  end
+
+  def stop?
+    @stopped.value
+  end
+
+end # class

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -0,0 +1,143 @@
+# LogProcessor:
+# reads and decodes locally available file with log lines
+# and creates LogStash events from these
+require 'logstash/inputs/mime/magic_gzip_validator'
+require 'pathname'
+
+module LogProcessor
+
+  def self.included(base)
+    base.extend(self)
+  end
+
+  def process(record, logstash_event_queue)
+    file = record[:local_file]
+    codec = @codec_factory.get_codec(record)
+    folder = record[:folder]
+    type = @type_by_folder.fetch(record[:bucket],{})[folder]
+    metadata = {}
+    line_count = 0
+    event_count = 0
+    #start_time = Time.now
+    file_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
+    read_file(file) do |line|
+      line_count += 1
+      if stop?
+        @logger.warn("[#{Thread.current[:name]}] Abort reading in the middle of the file, we will read it again when logstash is started")
+        throw :skip_delete
+      end
+      begin
+        codec.decode(line) do |event|
+          event_count += 1
+          decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
+          #event_time = Time.now #PROFILING
+          #event.set("[@metadata][progress][begin]", start_time)
+          #event.set("[@metadata][progress][index_time]", event_time)
+          #event.set("[@metadata][progress][line]", line_count)
+          logstash_event_queue << event
+        end
+      rescue Exception => e
+        @logger.error("[#{Thread.current[:name]}] Unable to decode line", :line => line, :error => e)
+      end
+    end
+    file_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
+    processing_time = (file_t1 - file_t0)
+    #@logger.warn("[#{Thread.current[:name]}] Completed long running File ( took #{processing_time} ) s", file: record[:key], events: event_count, processing_time: processing_time  ) if processing_time > 600.0 #PROFILING
+    # ensure any stateful codecs (such as multi-line ) are flushed to the queue
+    codec.flush do |event|
+      event_count += 1
+      decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
+      @logger.debug("[#{Thread.current[:name]}] Flushing an incomplete event", :event => event.to_s)
+      logstash_event_queue << event
+    end
+    # signal completion:
+    return true
+  end
+
+  private
+
+  def decorate_event(event, metadata, type, key, bucket, s3_data)
+    if event_is_metadata?(event)
+      @logger.debug('Updating the current cloudfront metadata', :event => event)
+      update_metadata(metadata, event)
+    else
+      # type by folder - set before "decorate()" enforces default
+      event.set('type', type) if type and ! event.include?('type')
+      decorate(event)
+
+      event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
+      event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
+
+      event.set("[@metadata][s3]", s3_data)
+      event.set("[@metadata][s3][object_key]", key)
+      event.set("[@metadata][s3][bucket_name]", bucket)
+      event.set("[@metadata][s3][object_folder]", get_object_folder(key))
+
+    end
+  end
+
+  def gzip?(filename)
+    return true if filename.end_with?('.gz','.gzip')
+    MagicGzipValidator.new(File.new(filename, 'rb')).valid?
+  rescue Exception => e
+    @logger.warn("Problem while gzip detection", :error => e)
+  end
+
+  def read_file(filename)
+    zipped = gzip?(filename)
+    completed = false
+    file_stream = FileInputStream.new(filename)
+    if zipped
+      gzip_stream = GZIPInputStream.new(file_stream)
+      decoder = InputStreamReader.new(gzip_stream, 'UTF-8')
+    else
+      decoder = InputStreamReader.new(file_stream, 'UTF-8')
+    end
+    buffered = BufferedReader.new(decoder)
+
+    while (data = buffered.readLine())
+      line = StringBuilder.new(data).append("\n")
+      yield(line.toString())
+    end
+    completed = true
+  rescue ZipException => e
+    @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename, :error => e)
+  ensure
+    buffered.close unless buffered.nil?
+    decoder.close unless decoder.nil?
+    gzip_stream.close unless gzip_stream.nil?
+    file_stream.close unless file_stream.nil?
+
+    unless completed
+      @logger.warn("[#{Thread.current[:name]}] Incomplete message in read_file. We´ll throw skip_delete.", :filename => filename)
+      throw :skip_delete
+    end
+
+  end
+
+  def event_is_metadata?(event)
+    return false unless event.get("message").class == String
+    line = event.get("message")
+    version_metadata?(line) || fields_metadata?(line)
+  end
+
+  def version_metadata?(line)
+    line.start_with?('#Version: ')
+  end
+
+  def fields_metadata?(line)
+    line.start_with?('#Fields: ')
+  end
+
+  def update_metadata(metadata, event)
+    line = event.get('message').strip
+
+    if version_metadata?(line)
+      metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
+    end
+
+    if fields_metadata?(line)
+      metadata[:cloudfront_fields] = line.split(/#Fields: (.+)/).last
+    end
+  end
+end

data/lib/logstash/inputs/s3sqs/patch.rb

@@ -0,0 +1,22 @@
+# This patch was stolen from logstash-plugins/logstash-output-s3#102.
+#
+# This patch is a workaround for a JRuby issue which has been fixed in JRuby
+# 9000, but not in JRuby 1.7. See https://github.com/jruby/jruby/issues/3645
+# and https://github.com/jruby/jruby/issues/3920. This is necessary because the
+# `aws-sdk` is doing tricky name discovery to generate the correct error class.
+#
+# As per https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960,
+# this patch may be short-lived anyway.
+require 'aws-sdk'
+
+begin
+  old_stderr = $stderr
+  $stderr = StringIO.new
+
+  module Aws
+    const_set(:S3, Aws::S3)
+    const_set(:SQS, Aws::SQS)
+  end
+ensure
+  $stderr = old_stderr
+end

data/lib/logstash/inputs/sqs/poller.rb

@@ -0,0 +1,218 @@
+# MessagePoller:
+# polling loop fetches messages from source queue and invokes
+# the provided code block on them
+require 'json'
+require 'cgi'
+
+class SqsPoller
+
+  # queue poller options we want to set explicitly
+  DEFAULT_OPTIONS = {
+      # we query one message at a time, so we can ensure correct error
+      # handling if we can't download a single file correctly
+      # (we will throw :skip_delete if download size isn't correct to allow
+      # for processing the event again later, so make sure to set a reasonable
+      # "DefaultVisibilityTimeout" for your queue so that there's enough time
+      # to process the log files!)
+      max_number_of_messages: 1,
+      visibility_timeout: 10,
+      # long polling; by default we use the queue's setting.
+      # A good value is 10 seconds to to balance between a quick logstash
+      # shutdown and fewer api calls.
+      wait_time_seconds: nil,
+      #attribute_names: ["All"], # Receive all available built-in message attributes.
+      #message_attribute_names: ["All"], # Receive any custom message attributes.
+      skip_delete: false,
+  }
+
+  # only needed in "run_with_backoff":
+  BACKOFF_SLEEP_TIME = 1
+  BACKOFF_FACTOR = 2
+  MAX_TIME_BEFORE_GIVING_UP = 60
+  # only needed in "preprocess":
+  EVENT_SOURCE = 'aws:s3'
+  EVENT_TYPE = 'ObjectCreated'
+
+  # initialization and setup happens once, outside the threads:
+  #
+  def initialize(logger, stop_semaphore, poller_options = {}, client_options = {}, aws_options_hash)
+    @logger = logger
+    @stopped = stop_semaphore
+    @queue = client_options[:sqs_queue]
+    @from_sns = client_options[:from_sns]
+    @max_processing_time = client_options[:max_processing_time]
+    @sqs_delete_on_failure = client_options[:sqs_delete_on_failure]
+    @options = DEFAULT_OPTIONS.merge(poller_options)
+    begin
+      @logger.info("Registering SQS input", :queue => @queue)
+      sqs_client = Aws::SQS::Client.new(aws_options_hash)
+      if uri?(@queue)
+        queue_url = @queue
+      else
+        queue_url = sqs_client.get_queue_url({
+          queue_name: @queue,
+          queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
+        }).queue_url
+      end
+
+      @poller = Aws::SQS::QueuePoller.new(queue_url,
+        :client => sqs_client
+      )
+      @logger.info("[#{Thread.current[:name]}] connected to queue.", :queue_url => queue_url)
+    rescue Aws::SQS::Errors::ServiceError => e
+      @logger.error("Cannot establish connection to Amazon SQS", :error => e)
+      raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
+    end
+  end
+
+  # this is called by every worker thread:
+  def run() # not (&block) - pass explicitly (use yield below)
+    # per-thread timer to extend visibility if necessary
+    extender = nil
+    message_backoff = (@options[:visibility_timeout] * 95).to_f / 100.0
+    new_visibility = 2 * @options[:visibility_timeout]
+
+    # "shutdown handler":
+    @poller.before_request do |_|
+      if stop?
+        # kill visibility extender thread if active?
+        extender.kill if extender
+        extender = nil
+        @logger.warn('issuing :stop_polling on "stop?" signal', :queue => @queue)
+        # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
+        throw :stop_polling
+      end
+    end
+
+    run_with_backoff do
+      message_count = 0 #PROFILING
+      @poller.poll(@options) do |message|
+        message_count += 1 #PROFILING
+        message_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
+        # auto-increase the timeout if processing takes too long:
+        poller_thread = Thread.current
+        extender = Thread.new do
+          while new_visibility < @max_processing_time do
+
+            sleep message_backoff
+            begin
+              @poller.change_message_visibility_timeout(message, new_visibility)
+              @logger.warn("[#{Thread.current[:name]}] Extended visibility for a long running message", :visibility => new_visibility) if new_visibility > 600.0
+              new_visibility += message_backoff
+            rescue Aws::SQS::Errors::InvalidParameterValue => e
+              @logger.debug("Extending visibility failed for message", :error => e)
+            else
+              @logger.debug("[#{Thread.current[:name]}] Extended visibility for message", :visibility => new_visibility) #PROFILING
+            end
+          end
+          @logger.error("[#{Thread.current[:name]}] Maximum visibility reached! We will delete this message from queue!")
+          @poller.delete_message(message) if @sqs_delete_on_failure
+          poller_thread.kill
+        end
+        extender[:name] = "#{Thread.current[:name]}/extender" #PROFILING
+        failed = false
+        record_count = 0
+        begin
+          message_completed = catch(:skip_delete) do
+            preprocess(message) do |record|
+              record_count += 1
+              extender[:name] = "#{Thread.current[:name]}/extender/#{record[:key]}" #PROFILING
+              yield(record)
+            end
+          end
+        rescue Exception => e
+          @logger.warn("Error in poller loop", :error => e)
+          @logger.warn("Backtrace:\n\t#{e.backtrace.join("\n\t")}")
+          failed = true
+        end
+        message_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
+        unless message_completed
+          @logger.debug("[#{Thread.current[:name]}] uncompleted message at the end of poller loop. We´ll throw skip_delete.", :message_count => message_count)
+          extender.run if extender
+        end
+        # at this time the extender has either fired or is obsolete
+        extender.kill if extender
+        extender = nil
+        throw :skip_delete if failed or ! message_completed
+        #@logger.info("[#{Thread.current[:name]}] completed message.", :message => message_count)
+      end
+    end
+  end
+
+  private
+
+  def stop?
+    @stopped.value
+  end
+
+  # Customization to parse CrowdStrike Falcon Data Replicator queue messages
+  def preprocess(message)
+    @logger.debug("Inside FDR Queue Preprocess: Start", :event => message)
+    payload = JSON.parse(message.body)
+    payload = JSON.parse(payload['Message']) if @from_sns
+    @logger.debug("Payload in Preprocess: ", :payload => payload)
+
+    # skip files other than aidmaster data  (for now)
+    return nil unless payload['files'] and (payload['pathPrefix'].start_with?("data/") or payload['pathPrefix'].start_with?("aidmaster/"))
+	
+  	bucket = payload['bucket']
+    payload['files'].each do |file|
+      @logger.debug("We found a file", :file => file)
+      # in case there are any events with Records that aren't s3 object-created events and can't therefore be
+      # processed by this plugin, we will skip them and remove them from queue
+#      if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
+        key  = file['path']
+		    @logger.debug("We found a file", :key => key)
+        size    = file['size']
+		    @logger.debug("We found a file", :size => size)
+		
+        yield({
+          bucket: bucket,
+          key: key,
+          size: size,
+          folder: get_object_path(key)
+        })
+      #end
+    end
+  end
+
+
+
+  # Runs an AWS request inside a Ruby block with an exponential backoff in case
+  # we experience a ServiceError.
+  # @param [Integer] max_time maximum amount of time to sleep before giving up.
+  # @param [Integer] sleep_time the initial amount of time to sleep before retrying.
+  # instead of requiring
+  # @param [Block] block Ruby code block to execute
+  # and then doing a "block.call",
+  # we yield to the passed block.
+  def run_with_backoff(max_time = MAX_TIME_BEFORE_GIVING_UP, sleep_time = BACKOFF_SLEEP_TIME)
+    next_sleep = sleep_time
+    begin
+      yield
+      next_sleep = sleep_time
+    rescue Aws::SQS::Errors::ServiceError => e
+      @logger.warn("Aws::SQS::Errors::ServiceError ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time, :error => e)
+      sleep(next_sleep)
+      next_sleep = next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR
+      retry
+    end
+  end
+
+  def uri?(string)
+    uri = URI.parse(string)
+    %w( http https ).include?(uri.scheme)
+  rescue URI::BadURIError
+    false
+  rescue URI::InvalidURIError
+    false
+  end
+
+
+  def get_object_path(key)
+    folder = ::File.dirname(key)
+    return '' if folder == '.'
+    return folder
+  end
+
+end