logstash-filter-threats_classifier 1.0.4

data/lib/logstash/filters/cognito-client.rb

@@ -0,0 +1,48 @@
+require 'aws-sdk'
+ 
+module LogStash
+	module Filters
+		module Empow
+			class CognitoClient
+				include LogStash::Util::Loggable
+
+				def initialize(username, password, aws_region_name, aws_client_id)
+					@logger = self.logger
+
+					@logger.debug("aws region: #{aws_region_name}")
+					@logger.debug("aws aws_client_id: #{aws_client_id}")
+					@logger.debug("cognito username: #{username}")
+
+					@username = username
+					@password = password
+					@aws_region_name = aws_region_name
+					@aws_client_id = aws_client_id
+
+					Aws.config.update({
+						region: @aws_region_name,
+						credentials: Aws::Credentials.new('aaaa', 'aaaa')
+					})
+
+					@client = Aws::CognitoIdentityProvider::Client.new
+				end
+
+				def authenticate
+					resp = @client.initiate_auth({
+						auth_flow: "USER_PASSWORD_AUTH",
+						auth_parameters: {
+							'USERNAME': @username,
+							'PASSWORD': @password,
+						},
+						client_id: @aws_client_id,
+					})
+
+					id_token = resp.authentication_result.id_token
+					token_type = resp.authentication_result.token_type
+
+					token = token_type + " " + id_token
+					return id_token
+				end
+			end
+		end
+	end
+end

data/lib/logstash/filters/elastic-db.rb

@@ -0,0 +1,128 @@
+require 'elasticsearch'
+require 'hashie'
+
+module LogStash; module Filters; module Empow;
+	class PersistentKeyValueDB
+		#include LogStash::Util::Loggable
+
+		def initialize(hosts, username, password, index)
+			#@logger ||= self.logger
+
+			#@logger.debug("opening the local classification db")
+
+			@elastic ||= Elasticsearch::Client.new(:hosts => hosts)
+			@index = index
+
+			create_index index
+		end
+
+		def create_index(index)
+			return if @elastic.indices.exists? index: index
+
+			@elastic.indices.create index: index, body: {
+				mappings: {
+					_doc: {
+						properties: {
+							product_type: {
+								type: 'keyword'
+							},
+							product: {
+								type: 'keyword'
+							},
+							term_key: {
+								type: 'keyword'
+							},
+							classification: {
+								enabled: false
+							}
+						}
+					}
+				}
+			}
+		end
+
+		def query(product_type, product, term)
+			#@logger.debug("quering local classification db")
+
+			# fix nil product
+			if product.nil?
+				product = 'nil_safe_product_key'
+			end
+
+			response = @elastic.search index: @index, type: '_doc', body: {
+				query: {
+					bool: {
+						must: [
+							{ term: { product_type: product_type } },
+							{
+								bool: {
+									should: [
+										{
+											bool: {
+												must: [
+													{ term: { term_key: term } },
+													{ term: { product: product } }
+												]
+											}
+										},
+										{
+											bool: {
+												must: {
+													term: { term_key: term }
+												},
+												must_not: {
+													exists: { field: 'product' }
+												}
+											}
+										}
+									]
+								}
+							}
+						]
+					}
+				}
+			}
+
+			mash = Hashie::Mash.new response
+
+			return nil if mash.hits.hits.first.nil?
+
+			return mash.hits.hits.first._source.classification
+		end
+
+		def save(doc_id, product_type, product, term, classification)
+			#@logger.debug("saving key to local classification db")
+
+			@elastic.index index: @index, type: '_doc', id: doc_id, body: {
+				product_type: product_type,
+				product: product,
+				term_key: term,
+				classification: classification
+			}
+		end
+
+		def close
+			#@logger.debug("clsoing the local classification db")
+		end
+	end
+
+end; end; end
+
+=begin
+db = LogStash::Filters::Empow::PersistentKeyValueDB.new('192.168.3.24:9200', 'user', 'pass', 'key-val-8')
+
+db.save("am", "p3", "dummy signature", "v1")
+db.save("am", "p3", "dummy signature 2", "v1")
+
+db.save("am", "p1", "dummy", "v1")
+db.save("am", nil, "dummy", "v1")
+p db.query "am", "p1", "h1"
+db.save("am", "p1", "h1", "v1")
+p db.query "am", "p1", "h1"
+p db.query "am", "p1", "h2"
+p db.query "am", "no-such-product", "h1"
+p db.query "am", nil, "h1"
+p db.query "am", nil, "dummy"
+
+p db.query "am", "p3", "dummy signature 2"
+=end

data/lib/logstash/filters/field-handler.rb

@@ -0,0 +1,127 @@
+require_relative "classification-request"
+require_relative "utils"
+
+class LogStash::Filters::Empow::FieldHandler
+
+  IDS = "IDS"
+  AM = "AM"
+  CUSTOM = "CUSTOM"
+
+  public
+  def initialize(product_type_field, product_name_field, threat_field, src_internal_field, dst_internal_field)
+    @product_type_field = product_type_field
+    @product_name_field = product_name_field
+
+    if threat_field.nil? || threat_field.strip.length == 0
+      raise ArgumentError, 'threat field cannot be empty'
+    end
+
+    @threat_field = '[' + threat_field + ']'
+
+    @ids_signature_field = @threat_field + '[signature]'
+    @malware_name_field = @threat_field + '[malware_name]'
+
+    @src_internal_field = @threat_field + '[' + src_internal_field + ']'
+    @dst_internal_field = @threat_field + '[' + dst_internal_field + ']'
+
+    @blacklisted_fields = [src_internal_field, dst_internal_field]
+
+    @hash_field = @threat_field + '[hash]'
+  end
+
+  public
+  def event_to_classification_request(event)
+    product_type = event.get(@product_type_field)
+    product = event.get(@product_name_field)
+    is_src_internal = event.get(@src_internal_field)
+    is_dst_internal = event.get(@dst_internal_field)
+
+    if product_type.nil?
+      LogStash::Filters::Empow::Utils.add_error(event, "missing_product_type")
+      return nil
+    end
+
+    is_src_internal = LogStash::Filters::Empow::Utils.convert_to_boolean(is_src_internal)
+
+    if is_src_internal.nil?
+      is_src_internal = true
+      LogStash::Filters::Empow::Utils.add_warn(event, 'src_internal_wrong_value')
+    end
+
+    is_dst_internal = LogStash::Filters::Empow::Utils.convert_to_boolean(is_dst_internal)
+
+    if is_dst_internal.nil?
+      is_dst_internal = true
+      LogStash::Filters::Empow::Utils.add_warn(event, 'dst_internal_wrong_value')
+    end
+
+    case product_type
+    when IDS
+      return nil if !is_valid_ids_request(product, event)
+    when AM
+      return nil if !is_valid_antimalware_request(product, event)
+    else # others are resolved in the cloud
+      return nil if !is_valid_product(product, event)
+    end
+
+    original_threat = event.get(@threat_field)
+
+    threat = copy_threat(original_threat)
+
+    if (threat.nil?)
+      LogStash::Filters::Empow::Utils.add_error(event, "missing_threat_field")
+      return nil
+    end
+
+    return LogStash::Filters::Empow::ClassificationRequest.new(product_type, product, threat, is_src_internal, is_dst_internal)
+  end
+
+  private
+  def copy_threat(threat)
+    return nil if (threat.nil? or threat.size == 0)
+
+    res = Hash.new
+
+    threat.each do |k, v|
+      next if @blacklisted_fields.include?(k)
+      res[k] = v
+    end
+
+    return res
+  end
+
+  private
+  def is_valid_ids_request(product, event)
+    sid = event.get(@ids_signature_field)
+
+    if sid.nil? || sid.strip.length == 0
+      LogStash::Filters::Empow::Utils.add_error(event, "missing_ids_signature")
+      return false
+    end
+
+    return is_valid_product(product, event)
+  end
+
+  private
+  def is_valid_product(product, event)
+    if (product.nil? or product.strip.length == 0)
+      LogStash::Filters::Empow::Utils.add_error(event, "missing_product_name")
+      return false
+    end
+
+    return true
+  end
+
+  private
+  def is_valid_antimalware_request(product, event)
+    malware_name = event.get(@malware_name_field)
+    malware_hash = event.get(@hash_field)
+
+    if malware_hash.nil? and (malware_name.nil? or product.nil?)
+      LogStash::Filters::Empow::Utils.add_error(event, "anti_malware_missing_hash_or_name")
+      return false
+    end
+
+    return true
+  end
+end

data/lib/logstash/filters/local-classifier.rb

@@ -0,0 +1,94 @@
+require "concurrent"
+require_relative 'classifier-cache'
+
+module LogStash; module Filters; module Empow;
+	class LocalClassifier
+		include LogStash::Util::Loggable
+
+		def initialize(cache_size, ttl, async_local_db, local_db)
+			@logger ||= self.logger
+
+			@logger.debug("initializing in memory cache")
+			@logger.debug("cache size #{cache_size}")
+			@logger.debug("cache ttl #{ttl}")
+
+			@cache ||= LogStash::Filters::Empow::ClassifierCache.new(cache_size, ttl)
+			@ttl = ttl
+
+			@local_db ||= local_db
+
+			@local_db_workers ||= Concurrent::ThreadPoolExecutor.new(min_threads: 1, max_threads: 1)
+			@async_local_db ||= async_local_db
+		end
+
+		def close
+			@logger.debug("shutting down local classifier")
+
+			@local_db_workers.shutdown if !@local_db.nil?
+
+			@local_db_workers.wait_for_termination(1)
+			@logger.debug("local classifier shut down")
+		end
+
+
+		def classify(key)
+			if !key.nil?
+				cached_result = @cache.classify(key)
+				return cached_result if !cached_result.nil?
+			end
+
+			return classify_using_local_database(key)
+		end
+
+		def add_to_cache(key, val, expiration_time)
+			return if key.nil?
+
+			@logger.debug? and @logger.info("adding #{key} to cache")
+
+			@cache.put(key, val, Time.now+3600)
+		end
+
+		def save_to_cache_and_db(key, val, expiration_time)
+			return if key.nil?
+
+			@logger.debug? and @logger.info("adding #{key} to the local db and cache")
+
+			product_type = key[:product_type]
+			product = key[:product]
+			term = key[:term]
+
+			doc_id = "#{product_type}-#{product}-term"
+
+			@local_db.save(doc_id, product_type, product, term, val) if !@local_db.nil?
+			add_to_cache(key, val, expiration_time)
+		end
+
+		def read_from_local_database(key)
+			res = @local_db.query(key[:product_type], key[:product], key[:term])
+
+			if !res.nil?
+				@logger.debug("adding result from db to local cache")
+				add_to_cache(key, res, Time.now + @ttl)
+			end
+
+			return res
+		end
+
+		def read_from_local_database_async(key)
+			@local_db_workers.post do
+				read_from_local_database(key)
+			end
+		end
+
+		def classify_using_local_database(key)
+			return nil if @local_db.nil? # if a local db wasn't configured
+
+			if (@async_local_db)
+				read_from_local_database_async(key)
+				return nil
+			end
+
+			return read_from_local_database(key)
+		end
+	end
+end; end; end

data/lib/logstash/filters/plugin-logic.rb

@@ -0,0 +1,166 @@
+require 'time'
+require "concurrent"
+require_relative "classification-request"
+require_relative "field-handler"
+require_relative 'response'
+require_relative 'utils'
+
+module LogStash; module Filters; module Empow;
+	class PluginLogic
+		include LogStash::Util::Loggable
+
+		def initialize(classifer, field_handler, max_parking_time, max_parked_events, tag_on_timeout, tag_on_error)
+			@logger ||= self.logger
+			#@logger.info("initializing classifier")
+
+			@field_handler = field_handler
+
+			@max_parking_time = max_parking_time
+			@max_parked_events = max_parked_events
+			@tag_on_timeout = tag_on_timeout
+			@tag_on_error = tag_on_error
+
+			@classifer = classifer
+			@parked_events = Concurrent::Array.new
+		end
+
+		def close
+			@classifer.close
+		end
+
+		def classify(event)
+			request = @field_handler.event_to_classification_request(event)
+
+			if request.nil?
+				@tag_on_error.each{|tag| event.tag(tag)}
+				return event
+			end
+
+			if classify_event(request, event)
+				return event
+			else
+				park(event)
+
+				if @parked_events.length > @max_parked_events
+					tuple = @parked_events.shift
+					
+					if !tuple.nil?
+						unparked_event = tuple[:event]
+						unparked_event.uncancel
+						return unparked_event
+					end
+				end
+
+				return nil
+			end
+		end
+
+		def flush(options = {})
+			# tag flushed events, 
+			events_to_flush = []
+
+      		if options[:final] # indicating "final flush" special event, flush everything
+      			while tuple = @parked_events.shift do
+      				events_to_flush << tuple[:event]
+      			end
+      		else
+      			@parked_events.delete_if do |tuple|
+      				process_parked_event(tuple, events_to_flush)
+      			end
+      		end
+
+      		return events_to_flush
+		end
+
+		private def process_parked_event(tuple, events_to_flush)
+			event = tuple[:event]
+			request = @field_handler.event_to_classification_request(event)
+
+			begin
+				res = @classifer.classify(request)
+
+				if (parking_time_expired(tuple) or is_valid_classification(res))
+					tag_event(res, event)
+
+					# if we're releasing this event based on time expiration, tag it with timeout
+					if res.nil? or !res.is_final
+						@tag_on_timeout.each{|tag| event.tag(tag)}
+					end
+
+					events_to_flush << event
+					return true
+				end
+				
+			rescue StandardError => e
+				@logger.error("an error occured while processing event, event flushed backed to the stream", :request => request, :backtrace => e.backtrace)
+				return true # so that this event will be flushed out of the plugin
+			end
+
+			return false
+		end
+
+		private
+		def is_unauthorized(classification)
+			return (!classification.nil? and classification.kind_of?(LogStash::Filters::Empow::UnauthorizedReponse))
+		end
+
+		private
+		def classify_event(request, event)
+			res = @classifer.classify(request)
+
+			if is_valid_classification(res)
+				tag_event(res, event)
+				return true
+			end
+
+			return false
+		end
+
+  		private
+  		def is_valid_classification(classification)
+			return (!classification.nil? and classification.is_final())
+  		end
+
+		private
+		def tag_event(classification, event)
+			return if classification.nil?
+
+			responseBody = classification.response
+
+			@logger.debug("classification response", :classification => responseBody)
+
+			response = responseBody["response"]
+
+			if !response.nil? && response.size > 0
+				response.each do |k, v|
+					event.set("[empow_classification_response][#{k}]", v)
+				end
+			end
+
+			if !classification.is_successful()
+				@tag_on_error.each{|tag| event.tag(tag)}
+
+				if (!responseBody.nil?)
+					LogStash::Filters::Empow::Utils.add_error(event, responseBody)
+				end
+			end
+		end
+
+		private
+		def park(event)
+			tuple = {}
+			tuple[:event] = event
+			tuple[:time] = Time.now
+
+			@parked_events << tuple
+
+			event.cancel # don't stream this event just yet ...
+		end
+		
+		private
+		def parking_time_expired(tuple)
+			return (Time.now - tuple[:time]) > @max_parking_time
+		end
+	end
+
+end; end; end