logstash-filter-threats_classifier 1.0.4

data/spec/filters/classifier_spec.rb

@@ -0,0 +1,78 @@
+require_relative '../spec_helper'
+require "logstash/filters/classifier"
+require "logstash/filters/local-classifier"
+require "logstash/filters/classification-request"
+require "logstash/filters/center-client"
+
+describe LogStash::Filters::Empow::Classifier do
+#empow_user, empow_password, cache_size, ttl, async_local_db, elastic_hosts, elastic_index, elastic_username, elastic_password
+    describe "test with mocked classifiers" do
+    	it "log with no result" do
+
+            local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
+            allow(local_classifier).to receive(:classify).and_return(nil)
+            allow(local_classifier).to receive(:close)
+
+            online_classifer = instance_double(LogStash::Filters::Empow::ClassificationCenterClient)
+            allow(online_classifer).to receive(:classify).and_return(nil)
+
+            req = "request-1"
+
+            classifier = described_class.new(online_classifer, local_classifier)
+
+            expect(local_classifier).to receive(:classify).with(req)
+
+            expect(online_classifer).to receive(:classify)
+
+            res = classifier.classify(req)
+
+            sleep 10
+
+            expect(res).to be_nil
+
+    		classifier.close
+    	end
+
+
+        it "log w/o results locally, online classification arrives later" do
+
+            # local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
+            # allow(local_classifier).to receive(:classify).and_return(nil)
+            # allow(local_classifier).to receive(:close)
+
+            # online_classifer = instance_double(LogStash::Filters::Empow::ClassificationCenterClient)
+            # allow(online_classifer).to receive(:classify).and_return(nil)
+
+            # req = LogStash::Filters::Empow::ClassificationRequest.new('anti-malware', 'lastline', 'assaf.clicker', nil)
+
+            # #online_classifer, local_classifier, local_db_cache, async_local_db, online_classifier_threads
+            # classifier = described_class.new(online_classifer, local_classifier, nil, true, 1)
+
+            # expect(local_classifier).to receive(:classify).with(req.get_key_by_term())
+            # expect(local_classifier).not_to receive(:classify).with(req.get_key_by_hash())
+
+            # expect(online_classifer).to receive(:classify)
+
+            # res = classifier.classify(req)
+
+            # #allow(Time).to receive(:now).and_return(5555555)
+
+            # expect(res).to be_nil
+
+            # sleep 60
+
+            # i = 20
+
+            # while i < 0 do
+
+            #     result = classifier.classify(req)
+            #     p "i: #{i} result: #{result}"
+
+            #     sleep 5
+            #     i = i - 1
+            # end
+
+            # classifier.close
+        end
+    end
+end

data/spec/filters/cognito-client_spec.rb

@@ -0,0 +1,20 @@
+require 'aws-sdk'
+require_relative '../spec_helper'
+require "logstash/filters/cognito-client"
+
+describe LogStash::Filters::Empow::CognitoClient do
+
+  describe "cognito test" do
+    skip "test authenticate" do
+
+      aws_region = 'us-east-2'
+      aws_client_id = '8dljcvt4jfif762le0ald6j'
+      username = 'bad'
+      password = 'request'
+
+      client = described_class.new(username, password, aws_region, aws_client_id)
+
+      expect{ client.authenticate }.to raise_error(Aws::CognitoIdentityProvider::Errors::UserNotFoundException)
+    end
+  end
+end

data/spec/filters/field-handler_spec.rb

@@ -0,0 +1,101 @@
+require_relative '../spec_helper'
+require "logstash/filters/field-handler"
+require "logstash/event"
+
+describe LogStash::Filters::Empow::FieldHandler do
+
+	let(:handler) { described_class.new("product_type", "product", "term", "is_src_internal", "is_dst_internal") }
+
+    describe "init" do
+    	it "src internal field empty" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"})
+            res = handler.event_to_classification_request(event)
+            expect(res).not_to be_nil
+            expect(res['is_src_internal']).to be true
+            expect(event.get("empow_warnings")).to include("src_internal_wrong_value")
+    	end
+
+    	it "dst internal field empty" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
+    	end
+
+    	it "src internal field numeric value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => 1})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_src_internal']).to be true
+            expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
+    	end
+
+        it "src internal field wrong value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_src_internal" => 11)
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_src_internal']).to be true
+            expect(event.get("empow_warnings")).to include("src_internal_wrong_value")
+        end
+
+    	it "dst internal field numeric value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_dst_internal" => 1})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
+    	end
+
+        it "dst internal field wrong numeric value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_dst_internal" => 11)
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
+        end
+
+        it "dst internal field wrong value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_dst_internal" => [])
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
+        end
+
+    	it "src internal field valid values" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => true})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_src_internal']).to be true
+            expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
+
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => false})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_src_internal']).to be false
+            expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
+    	end
+
+    	it "dst internal field valid values" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_dst_internal" => true})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
+
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1",  "is_dst_internal" => false})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['is_dst_internal']).to be false
+            expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
+    	end
+
+        it "test nested threat structure" do
+            my_handler = described_class.new("product_type", "product", 'threat', "is_src_internal", "is_dst_internal")
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "snort", "threat" => {"signature" => "name1"})
+            res = my_handler.event_to_classification_request(event)
+            expect(res['term']['signature']).to eq('name1')
+        end
+    end
+end

data/spec/filters/local-classifier_spec.rb

@@ -0,0 +1,46 @@
+require_relative '../spec_helper'
+require "logstash/filters/local-classifier"
+require "logstash/filters/elastic-db"
+require "logstash/filters/classification-request"
+
+describe LogStash::Filters::Empow::LocalClassifier do
+
+    describe "sync'ed local database as a fallback" do
+    	it "value isn't in memory, later fetched from local db" do
+            local_db = instance_double(LogStash::Filters::Empow::PersistentKeyValueDB)
+            allow(local_db).to receive(:query).and_return(nil)
+            allow(local_db).to receive(:close)
+
+    		classifier = described_class.new(5, 300, false, local_db)
+
+            key = LogStash::Filters::Empow::ClassificationRequest.new("product_type", "product", "threat", true, true)
+
+            expect(classifier.classify(key)).to be_nil
+
+            allow(local_db).to receive(:query).and_return("intent")
+
+            # allow backend thread to process the request
+            res = nil
+
+            for i in 1..10 do
+                sleep 1
+
+                res = classifier.classify(key)
+
+                break if !res.nil?
+            end
+
+            expect(res).to eq("intent")
+    	end
+    end
+
+    describe "no local database configured" do
+        it "value isn't in memory" do
+            classifier = described_class.new(5, 300, false, nil)
+
+            key = "key-1"
+
+            expect(classifier.classify(key)).to be_nil
+        end
+    end
+end

data/spec/filters/plugin-logic_spec.rb

@@ -0,0 +1,127 @@
+require_relative '../spec_helper'
+require "logstash/event"
+require "logstash/filters/classifier"
+require "logstash/filters/plugin-logic"
+
+describe LogStash::Filters::Empow::PluginLogic do
+
+    let(:intent_res1) { {"p1" => "s1"} }
+    let(:response_body1) { {'response' => intent_res1 } }
+    let(:sample_response) { LogStash::Filters::Empow::SuccessfulReponse.new(response_body1) }
+
+    describe "test classification" do
+    	
+    	it "event with warm classification" do
+    	   event = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name1", "my_hash" => "hash1")
+
+            field_handler = instance_double(LogStash::Filters::Empow::FieldHandler)
+            allow(field_handler).to receive(:event_to_classification_request).and_return("request")
+
+            classifier = instance_double(LogStash::Filters::Empow::Classifier)
+            allow(classifier).to receive(:classify).and_return(sample_response)
+
+            plugin_logic = described_class.new(classifier, field_handler, 60, 1, ['_timeout'], ['_error'])
+
+            expect(field_handler).to receive(:event_to_classification_request)
+            expect(classifier).to receive(:classify)
+
+            classified_event = plugin_logic.classify(event)
+
+            expect(classified_event).to eq(event)
+            expect(classified_event.get("empow_intent")).to eq(intent_res1)
+    	end
+
+    	it "event with cold classification is parked and then unparked only once" do
+    		event = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name1", "my_hash" => "hash1")
+
+            field_handler = instance_double(LogStash::Filters::Empow::FieldHandler)
+            allow(field_handler).to receive(:event_to_classification_request).and_return("request")
+		    allow(Time).to receive(:now).and_return(10)
+
+            classifier = instance_double(LogStash::Filters::Empow::Classifier)
+            allow(classifier).to receive(:classify).and_return(nil, nil, sample_response)
+
+            plugin_logic = described_class.new(classifier, field_handler, 60, 1, ['_timeout'], ['_error'])
+
+            expect(classifier).to receive(:classify)
+
+            classified_event = plugin_logic.classify(event)
+            expect(classified_event).to be_nil
+
+            allow(Time).to receive(:now).and_return(20)
+
+            expect(classifier).to receive(:classify)
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).to be_empty
+
+            allow(Time).to receive(:now).and_return(30)
+            expect(classifier).to receive(:classify)
+
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).not_to be_empty
+    	end
+
+    	it "event unparked after time expired" do
+    		event = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name1", "my_hash" => "hash1")
+
+            field_handler = instance_double(LogStash::Filters::Empow::FieldHandler)
+            allow(field_handler).to receive(:event_to_classification_request).and_return("request")
+            allow(Time).to receive(:now).and_return(10)
+
+            classifier = instance_double(LogStash::Filters::Empow::Classifier)
+            allow(classifier).to receive(:classify).and_return(nil)
+
+            plugin_logic = described_class.new(classifier, field_handler, 60, 1, ['_timeout'], ['_error'])
+
+            expect(classifier).to receive(:classify)
+
+            classified_event = plugin_logic.classify(event)
+            expect(classified_event).to be_nil
+
+            allow(Time).to receive(:now).and_return(20)
+
+            expect(classifier).to receive(:classify)
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).to be_empty
+
+            allow(Time).to receive(:now).and_return(100)
+            expect(classifier).to receive(:classify)
+
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).not_to be_empty
+
+            insist { flushed_events[0].get("tags") }.include?("_timeout")
+    	end
+
+    	it "too many parked events" do
+            event1 = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name1", "my_hash" => "hash1")
+            event2 = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name2", "my_hash" => "hash2")
+
+            field_handler = instance_double(LogStash::Filters::Empow::FieldHandler)
+            allow(field_handler).to receive(:event_to_classification_request).and_return("request")
+            allow(Time).to receive(:now).and_return(10)
+
+            classifier = instance_double(LogStash::Filters::Empow::Classifier)
+            allow(classifier).to receive(:classify).and_return(nil)
+
+            plugin_logic = described_class.new(classifier, field_handler, 60, 1, ['_timeout'], ['_error'])
+
+            expect(classifier).to receive(:classify)
+
+            classified_event = plugin_logic.classify(event1)
+            expect(classified_event).to be_nil
+
+            classified_event = plugin_logic.classify(event2)
+            expect(classified_event).to eq(event1)
+
+            allow(Time).to receive(:now).and_return(20)
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).to be_empty
+
+            allow(Time).to receive(:now).and_return(100)
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).not_to be_empty
+            expect(flushed_events.length).to eq(1)
+    	end
+    end
+end

data/spec/filters/threats-classifier_spec.rb

@@ -0,0 +1,103 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+require "logstash/filters/threats_classifier"
+require "logstash/event"
+
+describe LogStash::Filters::Threats_Classifier do
+
+  before(:each) do
+    allow(LogStash::Filters::Empow::LocalClassifier).to receive(:new).and_return(nil)
+    allow(LogStash::Filters::Empow::ClassificationCenterClient).to receive(:new).and_return(nil)
+    allow(LogStash::Filters::Empow::Classifier).to receive(:new).and_return(nil)
+  end
+
+  describe "config w/o local db and with mocks for online classifier" do    
+
+    it "test empty flush" do
+
+      plugin_core = instance_double(LogStash::Filters::Empow::PluginLogic)
+      allow(LogStash::Filters::Empow::PluginLogic).to receive(:new).and_return(plugin_core)
+      allow(plugin_core).to receive(:classify).and_return(nil)
+      allow(plugin_core).to receive(:flush).and_return([])
+
+      empty_config = {}
+      subject = described_class.new(empty_config)
+      subject.register
+
+      event = LogStash::Event.new({"data" => "empty"})
+
+      res = subject.flush({})
+
+      expect(res).to eq([])
+    end
+
+
+    it "2 events filtered w/o an answer on receive, correct event is flushed out" do
+
+      event = LogStash::Event.new({"data" => 1})
+
+      plugin_core = instance_double(LogStash::Filters::Empow::PluginLogic)
+      allow(LogStash::Filters::Empow::PluginLogic).to receive(:new).and_return(plugin_core)
+      allow(plugin_core).to receive(:classify).and_return(nil)
+      allow(plugin_core).to receive(:flush).and_return([event])
+
+      empty_config = {}
+      subject = described_class.new(empty_config)
+      subject.register
+
+      expect(plugin_core).to receive(:classify)
+
+      res = subject.filter(event)
+
+      expect(res).to be_nil
+
+      res = subject.flush({})
+
+      expect(res.length).to eq(1)
+      expect(res[0].get("data")).to eq(event.get("data"))
+    end
+
+    it "test answer on filter" do
+
+      event = LogStash::Event.new({"data" => "empty"})
+
+      plugin_core = instance_double(LogStash::Filters::Empow::PluginLogic)
+      allow(plugin_core).to receive(:classify).and_return(event)
+      allow(LogStash::Filters::Empow::PluginLogic).to receive(:new).and_return(plugin_core)
+
+      empty_config = {}
+      subject = described_class.new(empty_config)
+      subject.register
+
+      expect(plugin_core).to receive(:classify)
+      expect(subject).to receive(:filter_matched)
+
+      subject.filter(event)
+    end
+
+    it "test tag on error" do
+
+      event = instance_double(LogStash::Event)
+      allow(event).to receive(:cancel).and_raise("exception")
+      allow(event).to receive(:tag)
+
+      # event = .new({"data" => "empty"})
+
+      plugin_core = instance_double(LogStash::Filters::Empow::PluginLogic)
+      allow(plugin_core).to receive(:classify).and_return(nil)
+      allow(LogStash::Filters::Empow::PluginLogic).to receive(:new).and_return(plugin_core)
+
+      empty_config = {}
+      subject = described_class.new(empty_config)
+      subject.register
+
+      expect(plugin_core).to receive(:classify)
+      expect(event).to receive(:cancel)
+      expect(event).to receive(:tag).with('_empow_classifer_error')
+
+      res = subject.filter(event)
+
+      expect(res).to be_nil
+    end
+  end
+end