logstash-filter-empowclassifier 0.3.15

data/lib/logstash/filters/cognito-client.rb

@@ -0,0 +1,48 @@
+require 'aws-sdk'
+ 
+module LogStash
+	module Filters
+		module Empow
+			class CognitoClient
+				include LogStash::Util::Loggable
+
+				def initialize(username, password, aws_region_name, aws_client_id)
+					@logger = self.logger
+
+					@logger.debug("aws region: #{aws_region_name}")
+					@logger.debug("aws aws_client_id: #{aws_client_id}")
+					@logger.debug("cognito username: #{username}")
+
+					@username = username
+					@password = password
+					@aws_region_name = aws_region_name
+					@aws_client_id = aws_client_id
+
+					Aws.config.update({
+						region: @aws_region_name,
+						credentials: Aws::Credentials.new('aaaa', 'aaaa')
+					})
+
+					@client = Aws::CognitoIdentityProvider::Client.new
+				end
+
+				def authenticate
+					resp = @client.initiate_auth({
+						auth_flow: "USER_PASSWORD_AUTH",
+						auth_parameters: {
+							'USERNAME': @username,
+							'PASSWORD': @password,
+						},
+						client_id: @aws_client_id,
+					})
+
+					id_token = resp.authentication_result.id_token
+					token_type = resp.authentication_result.token_type
+
+					token = token_type + " " + id_token
+					return id_token
+				end
+			end
+		end
+	end
+end

data/lib/logstash/filters/elastic-db.rb

@@ -0,0 +1,128 @@
+require 'elasticsearch'
+require 'hashie'
+
+module LogStash; module Filters; module Empow;
+	class PersistentKeyValueDB
+		#include LogStash::Util::Loggable
+
+		def initialize(hosts, username, password, index)
+			#@logger ||= self.logger
+
+			#@logger.debug("opening the local classification db")
+
+			@elastic ||= Elasticsearch::Client.new(:hosts => hosts)
+			@index = index
+
+			create_index index
+		end
+
+		def create_index(index)
+			return if @elastic.indices.exists? index: index
+
+			@elastic.indices.create index: index, body: {
+				mappings: {
+					_doc: {
+						properties: {
+							product_type: {
+								type: 'keyword'
+							},
+							product: {
+								type: 'keyword'
+							},
+							term_key: {
+								type: 'keyword'
+							},
+							classification: {
+								enabled: false
+							}
+						}
+					}
+				}
+			}
+		end
+
+		def query(product_type, product, term)
+			#@logger.debug("quering local classification db")
+
+			# fix nil product
+			if product.nil?
+				product = 'nil_safe_product_key'
+			end
+
+			response = @elastic.search index: @index, type: '_doc', body: {
+				query: {
+					bool: {
+						must: [
+							{ term: { product_type: product_type } },
+							{
+								bool: {
+									should: [
+										{
+											bool: {
+												must: [
+													{ term: { term_key: term } },
+													{ term: { product: product } }
+												]
+											}
+										},
+										{
+											bool: {
+												must: {
+													term: { term_key: term }
+												},
+												must_not: {
+													exists: { field: 'product' }
+												}
+											}
+										}
+									]
+								}
+							}
+						]
+					}
+				}
+			}
+
+			mash = Hashie::Mash.new response
+
+			return nil if mash.hits.hits.first.nil?
+
+			return mash.hits.hits.first._source.classification
+		end
+
+		def save(doc_id, product_type, product, term, classification)
+			#@logger.debug("saving key to local classification db")
+
+			@elastic.index index: @index, type: '_doc', id: doc_id, body: {
+				product_type: product_type,
+				product: product,
+				term_key: term,
+				classification: classification
+			}
+		end
+
+		def close
+			#@logger.debug("clsoing the local classification db")
+		end
+	end
+
+end; end; end
+
+=begin
+db = LogStash::Filters::Empow::PersistentKeyValueDB.new('192.168.3.24:9200', 'user', 'pass', 'key-val-8')
+
+db.save("am", "p3", "dummy signature", "v1")
+db.save("am", "p3", "dummy signature 2", "v1")
+
+db.save("am", "p1", "dummy", "v1")
+db.save("am", nil, "dummy", "v1")
+p db.query "am", "p1", "h1"
+db.save("am", "p1", "h1", "v1")
+p db.query "am", "p1", "h1"
+p db.query "am", "p1", "h2"
+p db.query "am", "no-such-product", "h1"
+p db.query "am", nil, "h1"
+p db.query "am", nil, "dummy"
+
+p db.query "am", "p3", "dummy signature 2"
+=end

data/lib/logstash/filters/empowclassifier.rb

@@ -0,0 +1,249 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "elasticsearch"
+
+require_relative "elastic-db"
+require_relative "local-classifier"
+require_relative "classifier"
+require_relative "center-client"
+require_relative "plugin-logic"
+require_relative "utils"
+
+#
+class LogStash::Filters::EmpowClassifier < LogStash::Filters::Base
+
+  config_name "empowclassifier"
+
+  # The mail address used when registering to the classification center.
+  config :username, :validate => :string, :required => true
+
+  # The password for the classification center.
+  config :password, :validate => :string, :required => true
+
+  # user pool id. this parameter needs to be set only when using an entire empow stack, open source users should leave this unchanged.
+  config :pool_id, :validate => :string, :default => '8dljcvt4jfif762le0ald6j'
+
+  # Size of the local response cache
+  config :cache_size, :validate => :number, :default => 10000
+
+  # The maximum number of events that may wait in memory for a classification result from the classification center
+  config :max_pending_requests, :validate => :number, :default => 10000
+
+  # Time to wait in seconds an event will wait for a classification before returning to the pipeline with no result
+  config :pending_request_timeout, :validate => :number, :default => 60
+
+  # Max number of concurrent threads classifying via the classification center
+  # These threads mostly wait on I/O during the web request, and aren't cpu intensive.
+  # Idle workers are closed after one minute, only one idle worker remains alive for incoming request on peace time.
+  config :max_classification_center_workers, :validate => :number, :default => 5
+
+  # Classfication center bulk request size
+  config :bulk_request_size, :validate => :number, :default => 50
+
+  # Seconds to wait for batch to fill up before querying the classification center.
+  config :bulk_request_interval, :validate => :number, :default => 2
+
+  # Max number of times each request will be query the classification center.
+  config :max_query_retries, :validate => :number, :default => 5
+
+  # Seconds to wait before reclassifying an in-progress request. In progress response will occur when the classification center is processing a new threat.
+  config :time_between_queries, :validate => :number, :default => 10
+
+  # Allows renaimg the log field containing the log's product type. Possible values are AM for Anti-Malware and IDS for Intrusion Detection systems.
+  # For example, if our log contained a 'log_type' field (instead of the expected product_type field),
+  # We would configure the plugin as follows:
+  # [source,ruby]
+  #   filter {
+  #     empowclassifier {
+  #       username => "happy"
+  #       password => "festivus"
+  #       product_type_field => "log_type"
+  #     }
+  #   }
+  config :product_type_field, :validate => :string, :default => "product_type"
+
+  # Allows renaimg the log field containing the log's product name.
+  # Assuming our log contained a 'product' field (instead of the expected product_name field),
+  # We would configure the plugin as follows:
+  # [source,ruby]
+  #   filter {
+  #     empowclassifier {
+  #       username => "happy"
+  #       password => "festivus"
+  #       product_type_field => "product"
+  #     }
+  #   }
+  config :product_name_field, :validate => :string, :default => "product_name"
+  config :threat_field, :validate => :string, :default => "threat"
+
+  # Configs the name of the field used to indicate whether the source described in the log was within the internal network.
+  # Example:
+  # [source,ruby]
+  #   filter {
+  #     empowclassifier {
+  #       ...
+  #       src_internal_field => "internal_src"
+  #     }
+  #   }
+  config :src_internal_field, :validate => :string, :default => "is_src_internal"
+
+  # Configs the name of the field used to indicate whether the destination described in the log was within the internal network.
+  # Example:
+  # [source,ruby]
+  #   filter {
+  #     empowclassifier {
+  #       ...
+  #       dst_internal_field => "internal_dst"
+  #     }
+  #   }
+  config :dst_internal_field, :validate => :string, :default => "is_dst_internal"
+
+  # changes the api root for customers of the commercial empow stack
+  config :base_url, :validate => :string, :default => ""
+
+  config :async_local_cache, :validate => :boolean, :default => true
+
+  # elastic config params
+  ########################
+
+  config :elastic_hosts, :validate => :array
+
+  # The index or alias to write to
+  config :elastic_index, :validate => :string, :default => "empow-intent-db"
+
+  config :elastic_user, :validate => :string
+  config :elastic_password, :validate => :password
+
+  # failure tags
+  ###############
+  config :tag_on_product_type_failure, :validate => :array, :default => ['_empow_no_product_type']
+  config :tag_on_signature_failure, :validate => :array, :default => ['_empow_no_signature']
+  config :tag_on_timeout, :validate => :array, :default => ['_empow_classifer_timeout']
+  config :tag_on_error, :validate => :array, :default => ['_empow_classifer_error']
+
+  CLASSIFICATION_URL = "https://s0apxz9wik.execute-api.us-east-2.amazonaws.com" #"https://intent.cloud.empow.co"
+  CACHE_TTL = (24*60*60)
+
+  public
+  def register
+    @logger.info("registering empow classifcation plugin")
+
+    validate_params()
+
+    local_db = create_local_database
+
+    local_classifier = LogStash::Filters::Empow::LocalClassifier.new(@cache_size, CACHE_TTL, @async_local_cache, local_db)
+
+    base_url = get_effective_url()
+    online_classifier = LogStash::Filters::Empow::ClassificationCenterClient.new(@username, @password, @pool_id, base_url)
+
+    classifer = LogStash::Filters::Empow::Classifier.new(online_classifier, local_classifier, @max_classification_center_workers, @bulk_request_size, @bulk_request_interval, @max_query_retries, @time_between_queries)
+
+    field_handler = LogStash::Filters::Empow::FieldHandler.new(@product_type_field, @product_name_field, @threat_field, @src_internal_field, @dst_internal_field)
+
+    @plugin_core ||= LogStash::Filters::Empow::PluginLogic.new(classifer, field_handler, @pending_request_timeout, @max_pending_requests, @tag_on_timeout, @tag_on_error)
+
+    @logger.info("empow classifcation plugin registered")
+  end # def register
+
+  private
+  def get_effective_url
+    if (@base_url.nil? or @base_url.strip == 0)
+      return CLASSIFICATION_URL
+    end
+
+    return CLASSIFICATION_URL
+  end
+
+  private
+  def validate_params
+    raise ArgumentError, 'threat field cannot be empty' if LogStash::Filters::Empow::Utils.is_blank_string(@threat_field)
+
+    raise ArgumentError, 'bulk_request_size must be an positive number between 1 and 1000' if (@bulk_request_size < 1 or @bulk_request_size > 1000)
+
+    raise ArgumentError, 'bulk_request_interval must be an greater or equal to 1' if (@bulk_request_interval < 1)
+  end
+
+  def close
+    @logger.info("closing the empow classifcation plugin")
+
+    @plugin_core.close
+
+    @logger.info("empow classifcation plugin closed")
+  end
+
+  def periodic_flush
+    true
+  end
+
+  public def flush(options = {})
+    @logger.debug("entered flush")
+
+    events_to_flush = []
+
+    begin
+      parked_events = @plugin_core.flush(options)
+
+      parked_events.each do |event|
+        # need to clone as original event was canceled
+        cloned_event = event.clone
+        events_to_flush << cloned_event
+      end
+
+    rescue StandardError => e
+      @logger.error("encountered an exception while processing flush. #{e}")
+    end
+
+    @logger.debug("flush ended", :flushed_event_count => events_to_flush.length)
+
+    return events_to_flush
+  end
+
+  public def filter(event)
+    res = event
+
+    begin
+      res = @plugin_core.classify(event)
+
+      if res.nil?
+        event.cancel # don't stream this event just yet ...
+        return
+      end
+
+      # event was classified and returned, not some overflow event
+      if res.equal? event
+        filter_matched(event) 
+        return
+      end
+
+      # got here with a parked event
+      res = res.clone
+      filter_matched(res)
+      
+      @logger.debug("filter matched for overflow event", :event => res)
+
+    rescue StandardError => e
+      @logger.error("encountered an exception while classifying", :error => e, :event => event, :backtrace => e.backtrace)
+
+      @tag_on_error.each{|tag| event.tag(tag)}
+
+      return res
+    end
+  end # def filter
+
+  private def create_local_database
+    # if no elastic host has been configured, no local db should be used
+    if @elastic_hosts.nil?
+      @logger.info("no local persisted cache is configured")
+      return nil
+    end
+
+    begin
+      return LogStash::Filters::Empow::PersistentKeyValueDB.new(:elastic_hosts, :elastic_user, :elastic_password, :elastic_index)
+    rescue StandardError => e
+      @logger.error("caught an exception while trying to configured persisted cache", e)
+    end
+
+    return nil
+  end
+end

data/lib/logstash/filters/field-handler.rb

@@ -0,0 +1,127 @@
+require_relative "classification-request"
+require_relative "utils"
+
+class LogStash::Filters::Empow::FieldHandler
+
+  IDS = "IDS"
+  AM = "AM"
+  CUSTOM = "CUSTOM"
+
+  public
+  def initialize(product_type_field, product_name_field, threat_field, src_internal_field, dst_internal_field)
+    @product_type_field = product_type_field
+    @product_name_field = product_name_field
+
+    if threat_field.nil? || threat_field.strip.length == 0
+      raise ArgumentError, 'threat field cannot be empty'
+    end
+
+    @threat_field = '[' + threat_field + ']'
+
+    @ids_signature_field = @threat_field + '[signature]'
+    @malware_name_field = @threat_field + '[malware_name]'
+
+    @src_internal_field = @threat_field + '[' + src_internal_field + ']'
+    @dst_internal_field = @threat_field + '[' + dst_internal_field + ']'
+
+    @hash_field = @threat_field + '[hash]'
+  end
+
+  public
+  def event_to_classification_request(event)
+    product_type = event.get(@product_type_field)
+    product = event.get(@product_name_field)
+    is_src_internal = event.get(@src_internal_field)
+    is_dst_internal = event.get(@dst_internal_field)
+
+    if product_type.nil?
+      LogStash::Filters::Empow::Utils.add_error(event, "missing_product_type")
+      return nil
+    end
+
+    is_src_internal = LogStash::Filters::Empow::Utils.convert_to_boolean(is_src_internal)
+
+    if is_src_internal.nil?
+      is_src_internal = true
+      LogStash::Filters::Empow::Utils.add_warn(event, 'src_internal_wrong_value')
+    end
+
+    is_dst_internal = LogStash::Filters::Empow::Utils.convert_to_boolean(is_dst_internal)
+
+    if is_dst_internal.nil?
+      is_dst_internal = true
+      LogStash::Filters::Empow::Utils.add_warn(event, 'dst_internal_wrong_value')
+    end
+
+    case product_type
+    when IDS
+      return nil if !is_valid_ids_request(product, event)
+    when AM
+      return nil if !is_valid_antimalware_request(product, event)
+    else # others are resolved in the cloud
+      return nil if !is_valid_product(product, event)
+    end
+
+    original_threat = event.get(@threat_field)
+
+    threat = copy_threat(original_threat)
+
+    if (threat.nil?)
+      LogStash::Filters::Empow::Utils.add_error(event, "missing_threat_field")
+      return nil
+    end
+
+    threat['is_src_internal'] = is_src_internal
+    threat['is_dst_internal'] = is_dst_internal
+
+    return LogStash::Filters::Empow::ClassificationRequest.new(product_type, product, threat)
+  end
+
+  private
+  def copy_threat(threat)
+    return nil if (threat.nil? or threat.size == 0)
+
+    res = Hash.new
+
+    threat.each do |k, v|
+      res[k] = v
+    end
+
+    return res
+  end
+
+  private
+  def is_valid_ids_request(product, event)
+    sid = event.get(@ids_signature_field)
+
+    if sid.nil? || sid.strip.length == 0
+      LogStash::Filters::Empow::Utils.add_error(event, "missing_ids_signature")
+      return false
+    end
+
+    return is_valid_product(product, event)
+  end
+
+  private
+  def is_valid_product(product, event)
+    if (product.nil? or product.strip.length == 0)
+      LogStash::Filters::Empow::Utils.add_error(event, "missing_product_name")
+      return false
+    end
+
+    return true
+  end
+
+  private
+  def is_valid_antimalware_request(product, event)
+    malware_name = event.get(@malware_name_field)
+    malware_hash = event.get(@hash_field)
+
+    if malware_hash.nil? and (malware_name.nil? or product.nil?)
+      LogStash::Filters::Empow::Utils.add_error(event, "anti_malware_missing_hash_or_name")
+      return false
+    end
+
+    return true
+  end
+end