logstash-filter-empowclassifier 0.3.15

data/lib/logstash/filters/local-classifier.rb

@@ -0,0 +1,94 @@
+require "concurrent"
+require_relative 'classifier-cache'
+
+module LogStash; module Filters; module Empow;
+	class LocalClassifier
+		include LogStash::Util::Loggable
+
+		def initialize(cache_size, ttl, async_local_db, local_db)
+			@logger ||= self.logger
+
+			@logger.debug("initializing in memory cache")
+			@logger.debug("cache size #{cache_size}")
+			@logger.debug("cache ttl #{ttl}")
+
+			@cache ||= LogStash::Filters::Empow::ClassifierCache.new(cache_size, ttl)
+			@ttl = ttl
+
+			@local_db ||= local_db
+
+			@local_db_workers ||= Concurrent::ThreadPoolExecutor.new(min_threads: 1, max_threads: 1)
+			@async_local_db ||= async_local_db
+		end
+
+		def close
+			@logger.debug("shutting down local classifier")
+
+			@local_db_workers.shutdown if !@local_db.nil?
+
+			@local_db_workers.wait_for_termination(1)
+			@logger.debug("local classifier shut down")
+		end
+
+
+		def classify(key)
+			if !key.nil?
+				cached_result = @cache.classify(key)
+				return cached_result if !cached_result.nil?
+			end
+
+			return classify_using_local_database(key)
+		end
+
+		def add_to_cache(key, val, expiration_time)
+			return if key.nil?
+
+			@logger.debug? and @logger.info("adding #{key} to cache")
+
+			@cache.put(key, val, Time.now+3600)
+		end
+
+		def save_to_cache_and_db(key, val, expiration_time)
+			return if key.nil?
+
+			@logger.debug? and @logger.info("adding #{key} to the local db and cache")
+
+			product_type = key[:product_type]
+			product = key[:product]
+			term = key[:term]
+
+			doc_id = "#{product_type}-#{product}-term"
+
+			@local_db.save(doc_id, product_type, product, term, val) if !@local_db.nil?
+			add_to_cache(key, val, expiration_time)
+		end
+
+		def read_from_local_database(key)
+			res = @local_db.query(key[:product_type], key[:product], key[:term])
+
+			if !res.nil?
+				@logger.debug("adding result from db to local cache")
+				add_to_cache(key, res, Time.now + @ttl)
+			end
+
+			return res
+		end
+
+		def read_from_local_database_async(key)
+			@local_db_workers.post do
+				read_from_local_database(key)
+			end
+		end
+
+		def classify_using_local_database(key)
+			return nil if @local_db.nil? # if a local db wasn't configured
+
+			if (@async_local_db)
+				read_from_local_database_async(key)
+				return nil
+			end
+
+			return read_from_local_database(key)
+		end
+	end
+end; end; end

data/lib/logstash/filters/plugin-logic.rb

@@ -0,0 +1,163 @@
+require 'time'
+require "concurrent"
+require_relative "classification-request"
+require_relative "field-handler"
+require_relative 'response'
+require_relative 'utils'
+
+module LogStash; module Filters; module Empow;
+	class PluginLogic
+		include LogStash::Util::Loggable
+
+		def initialize(classifer, field_handler, max_parking_time, max_parked_events, tag_on_timeout, tag_on_error)
+			@logger ||= self.logger
+			#@logger.info("initializing classifier")
+
+			@field_handler = field_handler
+
+			@max_parking_time = max_parking_time
+			@max_parked_events = max_parked_events
+			@tag_on_timeout = tag_on_timeout
+			@tag_on_error = tag_on_error
+
+			@classifer = classifer
+			@parked_events = Concurrent::Array.new
+		end
+
+		def close
+			@classifer.close
+		end
+
+		def classify(event)
+			request = @field_handler.event_to_classification_request(event)
+
+			if request.nil?
+				@tag_on_error.each{|tag| event.tag(tag)}
+				return event
+			end
+
+			if classify_event(request, event)
+				return event
+			else
+				park(event)
+
+				if @parked_events.length > @max_parked_events
+					tuple = @parked_events.shift
+					
+					if !tuple.nil?
+						# todo: consider classifying the unparked event one last time
+						return tuple[:event]
+					end
+				end
+
+				return nil
+			end
+		end
+
+		def flush(options = {})
+			# tag flushed events, 
+			events_to_flush = []
+
+      		if options[:final] # indicating "final flush" special event, flush everything
+      			while tuple = @parked_events.shift do
+      				events_to_flush << tuple[:event]
+      			end
+      		else
+      			@parked_events.delete_if do |tuple|
+      				process_parked_event(tuple, events_to_flush)
+      			end
+      		end
+
+      		return events_to_flush
+		end
+
+		private def process_parked_event(tuple, events_to_flush)
+			event = tuple[:event]
+			request = @field_handler.event_to_classification_request(event)
+
+			begin
+				res = @classifer.classify(request)
+
+				if (parking_time_expired(tuple) or is_valid_classification(res))
+					tag_event(res, event)
+
+					# if we're releasing this event based on time expiration, tag it with timeout
+					if res.nil? or !res.is_final
+						@tag_on_timeout.each{|tag| event.tag(tag)}
+					end
+
+					events_to_flush << event
+					return true
+				end
+				
+			rescue StandardError => e
+				@logger.error("an error occured while processing event, event flushed backed to the stream", :request => request, :backtrace => e.backtrace)
+				return true # so that this event will be flushed out of the plugin
+			end
+
+			return false
+		end
+
+		private
+		def is_unauthorized(classification)
+			return (!classification.nil? and classification.kind_of?(LogStash::Filters::Empow::UnauthorizedReponse))
+		end
+
+		private
+		def classify_event(request, event)
+			res = @classifer.classify(request)
+
+			if is_valid_classification(res)
+				tag_event(res, event)
+				return true
+			end
+
+			return false
+		end
+
+  		private
+  		def is_valid_classification(classification)
+			return (!classification.nil? and classification.is_final())
+  		end
+
+		private
+		def tag_event(classification, event)
+			return if classification.nil?
+
+			responseBody = classification.response
+
+			@logger.debug("classification response", :classification => responseBody)
+
+			response = responseBody["response"]
+
+			if !response.nil? && response.size > 0
+				response.each do |k, v|
+					event.set("[empow_classification_response][#{k}]", v)
+				end
+			end
+
+			if !classification.is_successful()
+				@tag_on_error.each{|tag| event.tag(tag)}
+
+				if (!responseBody.nil?)
+					LogStash::Filters::Empow::Utils.add_error(event, responseBody)
+				end
+			end
+		end
+
+		private
+		def park(event)
+			tuple = {}
+			tuple[:event] = event
+			tuple[:time] = Time.now
+
+			@parked_events << tuple
+		end
+		
+		private
+		def parking_time_expired(tuple)
+			return (Time.now - tuple[:time]) > @max_parking_time
+		end
+	end
+
+end; end; end

data/lib/logstash/filters/response.rb

@@ -0,0 +1,36 @@
+module LogStash
+	module Filters
+		module Empow
+			class AbstractResponse
+		  		attr_reader :response, :is_successful, :is_final
+
+			  	def initialize(response, is_successful, is_final)
+			  		@response = response
+			  		@is_successful = is_successful
+			  		@is_final = is_final
+			  	end
+			end
+
+			class FailureResponse < AbstractResponse
+			  	def initialize(response)
+			  		super(response, false, true)
+			  	end
+  			end
+
+  			class UnauthorizedReponse < FailureResponse
+  			end
+
+  			class SuccessfulResponse < AbstractResponse
+  				def initialize(response)
+  					super(response, true, true)
+  				end
+  			end
+
+  			class InProgressResponse < AbstractResponse
+  				def initialize(response)
+  					super(response, true, false)
+  				end
+  			end
+		end
+	end
+end

data/lib/logstash/filters/utils.rb

@@ -0,0 +1,46 @@
+module LogStash; module Filters; module Empow
+
+  class Utils
+    TRUTHY_VALUES = [true, 1, '1']
+    FALSEY_VALUES = [false, 0, '0']
+
+  	def self.is_blank_string(txt)
+  	  return (txt.nil? or txt.strip.length == 0)
+  	end
+
+    def self.convert_to_boolean(val)
+      return nil if val.nil?
+
+      return true if TRUTHY_VALUES.include?(val)
+
+      return false if FALSEY_VALUES.include?(val)
+
+      return true if (val.is_a?(String) and val.downcase.strip == 'true')
+
+      return false if (val.is_a?(String) and val.downcase.strip == 'false')
+
+      return nil
+    end
+
+  	def self.add_error(event, msg)
+  		tag_empow_messages(event, msg, 'empow_errors')
+  	end
+
+  	def self.add_warn(event, msg)
+  		tag_empow_messages(event, msg, 'empow_warnings')
+  	end
+
+  	private
+  	def self.tag_empow_messages(event, msg, block)
+  		messages = event.get(block)
+
+  		# using arrayinstead of set, as set raises a logstash exception:
+      # No enum constant org.logstash.bivalues.BiValues.ORG_JRUBY_RUBYOBJECTVAR0
+      messages ||= Array.new
+      messages << msg
+
+      event.set(block, messages.uniq)
+  	end
+  end
+
+end; end; end

data/logstash-filter-empowclassifier.gemspec

@@ -0,0 +1,38 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-filter-empowclassifier'
+  s.version       = '0.3.15'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Logstash intent classification plugin client for accessing empow''s classifiction cloud'
+  #s.description   = 'Write a longer description or delete this line.'
+  s.homepage      = 'http://www.empowcybersecurity.com'
+  s.authors       = ['empow', 'Assaf Abulafia', 'Rami Cohen']
+  s.email         = ''
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'rest-client', '~> 1.8', '>= 1.8.0'
+  s.add_runtime_dependency 'lru_redux', '~> 1.1', '>= 1.1.0'
+  s.add_runtime_dependency 'json', '~> 1.8', '>= 1.8'
+  #s.add_runtime_dependency 'rufus-scheduler'
+  s.add_runtime_dependency 'hashie'
+  #s.add_runtime_dependency "murmurhash3"
+  
+  s.add_development_dependency 'aws-sdk', '~> 3'
+
+  s.add_development_dependency 'logstash-devutils'
+#  s.add_runtime_dependency 'jwt', '~> 2.1', '>= 2.1.0'
+  s.add_development_dependency "timecop", "~> 0.7"
+  s.add_development_dependency "webmock", "~> 1.22", ">= 1.21.0"
+
+  s.add_development_dependency 'elasticsearch'
+end

data/spec/filters/bulk-processor_spec.rb

@@ -0,0 +1,92 @@
+require_relative '../spec_helper'
+require "logstash/filters/classifier"
+require "logstash/filters/local-classifier"
+require "logstash/filters/classification-request"
+require "logstash/filters/center-client"
+require "logstash/filters/response"
+require 'timecop'
+
+describe LogStash::Filters::Empow::Classification::BulkProcessor do
+#empow_user, empow_password, cache_size, ttl, async_local_db, elastic_hosts, elastic_index, elastic_username, elastic_password
+	let(:time_between_attempts) { 1 }
+	let(:batch_size) { 10 }
+    let(:max_retries) { 5 }
+
+    describe "test with mocked classifiers" do
+    	it "single failed log" do
+
+    		Timecop.freeze(Time.now)
+
+    		req1 = "request1"
+            val1 = {}
+			val1[:retries] = 1
+			val1[:task] = nil
+			val1[:request] = req1
+			val1[:last_executed] = Time.at(310953600)
+
+			requests = Hash.new
+            requests[req1] = val1
+
+            local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
+            allow(local_classifier).to receive(:classify).and_return(nil)
+            allow(local_classifier).to receive(:close)
+
+            center_result = {}
+            center_result[req1] = LogStash::Filters::Empow::FailureReponse.new("failure1")
+
+            online_classifer = instance_double(LogStash::Filters::Empow::ClassificationCenterClient)
+            allow(online_classifer).to receive(:classify).and_return(center_result)
+
+            bulk_processor = described_class.new(max_retries, batch_size, time_between_attempts, requests, online_classifer, local_classifier)
+
+            expect(online_classifer).to receive(:classify)
+            expect(local_classifier).to receive(:add_to_cache)
+
+            bulk_processor.execute
+
+            #expect(local_classifier).to receive(:add_to_cache)
+
+            # expect(res).to be_nil
+            #save_to_cache_and_db
+
+            expect(requests[req1]).to be_nil
+
+    		#Timecop.freeze(Time.now + time_between_attempts)
+    		#Timecop.freeze(Time.now + 1 + time_between_attempts)
+    	end
+
+    	it "single successful log" do
+
+    		Timecop.freeze(Time.now)
+
+    		req1 = "request1"
+            val1 = {}
+			val1[:retries] = 1
+			val1[:task] = nil
+			val1[:request] = req1
+			val1[:last_executed] = Time.at(310953600)
+
+			requests = Hash.new
+            requests[req1] = val1
+
+            local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
+            allow(local_classifier).to receive(:classify).and_return(nil)
+            allow(local_classifier).to receive(:close)
+
+            center_result = {}
+            center_result[req1] = LogStash::Filters::Empow::SuccessfulReponse.new("result1")
+
+            online_classifer = instance_double(LogStash::Filters::Empow::ClassificationCenterClient)
+            allow(online_classifer).to receive(:classify).and_return(center_result)
+
+            bulk_processor = described_class.new(max_retries, batch_size, time_between_attempts, requests, online_classifer, local_classifier)
+
+            expect(online_classifer).to receive(:classify)
+            expect(local_classifier).to receive(:save_to_cache_and_db)
+
+            bulk_processor.execute
+
+            expect(requests[req1]).to be_nil
+    	end
+    end
+end