logstash-filter-empowclassifier 0.3.15

data/spec/filters/center-client_spec.rb

@@ -0,0 +1,88 @@
+require_relative '../spec_helper'
+require "logstash/filters/center-client"
+require "logstash/filters/response"
+require 'webmock/rspec'
+
+describe LogStash::Filters::Empow::ClassificationCenterClient do
+
+  # before(:each) do
+  #   local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
+  #   allow(LogStash::Filters::Empow::LocalClassifier).to receive(:new).and_return(local_classifier)
+  # end
+
+  let(:url_base) { 'http://localhost:5000' }
+  let(:username) { 'myuser' }
+  let(:password) { 'mypassword' }
+  let(:pool_id) { 'mypassword' }
+
+  describe "classification center api" do
+  	before(:each) do
+    	WebMock.disable_net_connect!
+
+      stub_request(:post, "#{url_base}/login").
+        to_return(:body => "", :status => 200,
+          :headers => { 'authorization' => 'Bearer my-token' })
+
+      mocked_cognito = double(LogStash::Filters::Empow::CognitoClient)
+      allow(LogStash::Filters::Empow::CognitoClient).to receive(:new).and_return(mocked_cognito)
+      allow(mocked_cognito).to receive(:authenticate).and_return("dummy token")
+  	end
+
+  	after(:each) do
+    	WebMock.reset!
+    	WebMock.allow_net_connect!
+
+      allow(LogStash::Filters::Empow::CognitoClient).to receive(:new).and_call_original
+  	end
+
+
+    it "test missing ids request" do
+      stub_request(:post, "#{url_base}/classification/intent").
+        to_return(:body => "", :status => 204,
+          :headers => { 'Content-Length' => 0 })
+  		
+  		client = described_class.new(username, password, pool_id, url_base)
+
+  		client.authenticate
+
+  		res = client.classify(["req1"])
+
+  		expect(res["req1"]).to be_kind_of(LogStash::Filters::Empow::FailureReponse)
+    end
+
+    it "test existing ids request" do
+  		
+      response = '{"some":"data"}'
+
+      stub_request(:post, "#{url_base}/classification/intent").
+        to_return(:body => response, :status => 200)
+
+  		client = described_class.new(username, password, pool_id, url_base)
+
+  		client.authenticate
+
+      k1 = "req1"
+  		response_map = client.classify([k1])
+
+      res = response_map[k1].response
+
+      p "res: #{res}"
+
+      expect(res["some"]).to eq("data")
+    end
+
+    it "test http status 500 during request" do
+
+      stub_request(:post, "#{url_base}/classification/intent").
+        to_return(:body => "", :status => 500)
+
+      client = described_class.new(username, password, pool_id, url_base)
+
+      client.authenticate
+
+      res = client.classify("ids", "Snort", "1:2", nil)
+
+      expect(res).to be_nil
+    end
+  end
+end

data/spec/filters/classifier-cache_spec.rb

@@ -0,0 +1,44 @@
+require_relative '../spec_helper'
+require 'timecop'
+require "logstash/filters/classifier-cache"
+
+describe LogStash::Filters::Empow::ClassifierCache do
+
+    describe "initialize signaure test" do
+    	it "test expiration by cache default ttl" do
+    		cache = described_class.new(5, 60)
+
+            expect(cache.classify("k")).to be_nil
+
+            Timecop.freeze(Time.now)
+            
+            cache.put("k", "v", Time.now + 24*60*60)
+
+            Timecop.freeze(Time.now + 59)
+
+            expect(cache.classify("k")).to eq("v")
+
+            Timecop.freeze(Time.now + 61)
+
+            expect(cache.classify("k")).to be_nil
+    	end
+
+        it "test expiration by entry ttl" do
+            cache = described_class.new(5, 60)
+
+            expect(cache.classify("k")).to be_nil
+
+            Timecop.freeze(Time.now)
+            
+            cache.put("k", "v", Time.now + 30)
+
+            Timecop.freeze(Time.now + 29)
+
+            expect(cache.classify("k")).to eq("v")
+
+            Timecop.freeze(Time.now + 31)
+
+            expect(cache.classify("k")).to be_nil
+        end
+    end
+end

data/spec/filters/classifier_spec.rb

@@ -0,0 +1,78 @@
+require_relative '../spec_helper'
+require "logstash/filters/classifier"
+require "logstash/filters/local-classifier"
+require "logstash/filters/classification-request"
+require "logstash/filters/center-client"
+
+describe LogStash::Filters::Empow::Classifier do
+#empow_user, empow_password, cache_size, ttl, async_local_db, elastic_hosts, elastic_index, elastic_username, elastic_password
+    describe "test with mocked classifiers" do
+    	it "log with no result" do
+
+            local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
+            allow(local_classifier).to receive(:classify).and_return(nil)
+            allow(local_classifier).to receive(:close)
+
+            online_classifer = instance_double(LogStash::Filters::Empow::ClassificationCenterClient)
+            allow(online_classifer).to receive(:classify).and_return(nil)
+
+            req = "request-1"
+
+            classifier = described_class.new(online_classifer, local_classifier)
+
+            expect(local_classifier).to receive(:classify).with(req)
+
+            expect(online_classifer).to receive(:classify)
+
+            res = classifier.classify(req)
+
+            sleep 10
+
+            expect(res).to be_nil
+
+    		classifier.close
+    	end
+
+
+        it "log w/o results locally, online classification arrives later" do
+
+            # local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
+            # allow(local_classifier).to receive(:classify).and_return(nil)
+            # allow(local_classifier).to receive(:close)
+
+            # online_classifer = instance_double(LogStash::Filters::Empow::ClassificationCenterClient)
+            # allow(online_classifer).to receive(:classify).and_return(nil)
+
+            # req = LogStash::Filters::Empow::ClassificationRequest.new('anti-malware', 'lastline', 'assaf.clicker', nil)
+
+            # #online_classifer, local_classifier, local_db_cache, async_local_db, online_classifier_threads
+            # classifier = described_class.new(online_classifer, local_classifier, nil, true, 1)
+
+            # expect(local_classifier).to receive(:classify).with(req.get_key_by_term())
+            # expect(local_classifier).not_to receive(:classify).with(req.get_key_by_hash())
+
+            # expect(online_classifer).to receive(:classify)
+
+            # res = classifier.classify(req)
+
+            # #allow(Time).to receive(:now).and_return(5555555)
+
+            # expect(res).to be_nil
+
+            # sleep 60
+
+            # i = 20
+
+            # while i < 0 do
+
+            #     result = classifier.classify(req)
+            #     p "i: #{i} result: #{result}"
+
+            #     sleep 5
+            #     i = i - 1
+            # end
+
+            # classifier.close
+        end
+    end
+end

data/spec/filters/cognito-client_spec.rb

@@ -0,0 +1,20 @@
+require 'aws-sdk'
+require_relative '../spec_helper'
+require "logstash/filters/cognito-client"
+
+describe LogStash::Filters::Empow::CognitoClient do
+
+  describe "cognito test" do
+    skip "test authenticate" do
+
+      aws_region = 'us-east-2'
+      aws_client_id = '8dljcvt4jfif762le0ald6j'
+      username = 'bad'
+      password = 'request'
+
+      client = described_class.new(username, password, aws_region, aws_client_id)
+
+      expect{ client.authenticate }.to raise_error(Aws::CognitoIdentityProvider::Errors::UserNotFoundException)
+    end
+  end
+end

data/spec/filters/elastic-db_spec.rb

@@ -0,0 +1,44 @@
+# require_relative '../spec_helper'
+# require "logstash/filters/elastic-db"
+
+# describe LogStash::Filters::Empow::PersistentKeyValueDB do
+
+# 	let(:user) { 'user' }
+# 	let(:indexName) { 'key-val-8' }
+# 	let(:password) { 'pass' }
+# 	let(:elastic) { '192.168.3.24:9200' }
+
+# 	subject { described_class.new(elastic, user, password, indexName) }
+
+# 	after do
+#     	subject.close
+#   	end
+
+# 	describe "initialization" do
+#     	it "should be successful" do
+#     		expect { subject }.not_to raise_error
+#     	end
+#     end
+
+#     describe "read a value that doesn't exists" do
+#     	it "should return nil" do
+#     		res = subject.query "ids", "snort", "123:456:789"
+#     		expect(res).to be_nil
+#     	end
+#     end
+
+#     describe "write a value then read" do
+#     	let(:data) { "blob" }
+
+#     	it "write should be successful" do
+#     		expect { subject.save 1234, "am", "my-product", "not-my-name", 'something else' }.not_to raise_error
+#     		expect { subject.save 12345, "am", "my-product", "my-name", data }.not_to raise_error
+#     		sleep(2)
+#     	end
+
+#     	it "read the new value should succeed" do
+#     		res = subject.query "am", "my-product", "my-name"
+#     		expect(res).to eq(data)
+#     	end
+#     end
+# end

data/spec/filters/empowclassifier_spec.rb

@@ -0,0 +1,103 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+require "logstash/filters/empowclassifier"
+require "logstash/event"
+
+describe LogStash::Filters::EmpowClassifier do
+
+  before(:each) do
+    allow(LogStash::Filters::Empow::LocalClassifier).to receive(:new).and_return(nil)
+    allow(LogStash::Filters::Empow::ClassificationCenterClient).to receive(:new).and_return(nil)
+    allow(LogStash::Filters::Empow::Classifier).to receive(:new).and_return(nil)
+  end
+
+  describe "config w/o local db and with mocks for online classifier" do    
+
+    it "test empty flush" do
+
+      plugin_core = instance_double(LogStash::Filters::Empow::PluginLogic)
+      allow(LogStash::Filters::Empow::PluginLogic).to receive(:new).and_return(plugin_core)
+      allow(plugin_core).to receive(:classify).and_return(nil)
+      allow(plugin_core).to receive(:flush).and_return([])
+
+      empty_config = {}
+      subject = described_class.new(empty_config)
+      subject.register
+
+      event = LogStash::Event.new({"data" => "empty"})
+
+      res = subject.flush({})
+
+      expect(res).to eq([])
+    end
+
+
+    it "2 events filtered w/o an answer on receive, correct event is flushed out" do
+
+      event = LogStash::Event.new({"data" => 1})
+
+      plugin_core = instance_double(LogStash::Filters::Empow::PluginLogic)
+      allow(LogStash::Filters::Empow::PluginLogic).to receive(:new).and_return(plugin_core)
+      allow(plugin_core).to receive(:classify).and_return(nil)
+      allow(plugin_core).to receive(:flush).and_return([event])
+
+      empty_config = {}
+      subject = described_class.new(empty_config)
+      subject.register
+
+      expect(plugin_core).to receive(:classify)
+
+      res = subject.filter(event)
+
+      expect(res).to be_nil
+
+      res = subject.flush({})
+
+      expect(res.length).to eq(1)
+      expect(res[0].get("data")).to eq(event.get("data"))
+    end
+
+    it "test answer on filter" do
+
+      event = LogStash::Event.new({"data" => "empty"})
+
+      plugin_core = instance_double(LogStash::Filters::Empow::PluginLogic)
+      allow(plugin_core).to receive(:classify).and_return(event)
+      allow(LogStash::Filters::Empow::PluginLogic).to receive(:new).and_return(plugin_core)
+
+      empty_config = {}
+      subject = described_class.new(empty_config)
+      subject.register
+
+      expect(plugin_core).to receive(:classify)
+      expect(subject).to receive(:filter_matched)
+
+      subject.filter(event)
+    end
+
+    it "test tag on error" do
+
+      event = instance_double(LogStash::Event)
+      allow(event).to receive(:cancel).and_raise("exception")
+      allow(event).to receive(:tag)
+
+      # event = .new({"data" => "empty"})
+
+      plugin_core = instance_double(LogStash::Filters::Empow::PluginLogic)
+      allow(plugin_core).to receive(:classify).and_return(nil)
+      allow(LogStash::Filters::Empow::PluginLogic).to receive(:new).and_return(plugin_core)
+
+      empty_config = {}
+      subject = described_class.new(empty_config)
+      subject.register
+
+      expect(plugin_core).to receive(:classify)
+      expect(event).to receive(:cancel)
+      expect(event).to receive(:tag).with('_empow_classifer_error')
+
+      res = subject.filter(event)
+
+      expect(res).to be_nil
+    end
+  end
+end

data/spec/filters/field-handler_spec.rb

@@ -0,0 +1,101 @@
+require_relative '../spec_helper'
+require "logstash/filters/field-handler"
+require "logstash/event"
+
+describe LogStash::Filters::Empow::FieldHandler do
+
+	let(:handler) { described_class.new("product_type", "product", "term", "is_src_internal", "is_dst_internal") }
+
+    describe "init" do
+    	it "src internal field empty" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"})
+            res = handler.event_to_classification_request(event)
+            expect(res).not_to be_nil
+            expect(res['term']['is_src_internal']).to be true
+            expect(event.get("empow_warnings")).to include("src_internal_wrong_value")
+    	end
+
+    	it "dst internal field empty" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
+    	end
+
+    	it "src internal field numeric value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => 1})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_src_internal']).to be true
+            expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
+    	end
+
+        it "src internal field wrong value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_src_internal" => 11)
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_src_internal']).to be true
+            expect(event.get("empow_warnings")).to include("src_internal_wrong_value")
+        end
+
+    	it "dst internal field numeric value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_dst_internal" => 1})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
+    	end
+
+        it "dst internal field wrong numeric value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_dst_internal" => 11)
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
+        end
+
+        it "dst internal field wrong value" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_dst_internal" => [])
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
+        end
+
+    	it "src internal field valid values" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => true})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_src_internal']).to be true
+            expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
+
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => false})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_src_internal']).to be false
+            expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
+    	end
+
+    	it "dst internal field valid values" do
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_dst_internal" => true})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_dst_internal']).to be true
+            expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
+
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1",  "is_dst_internal" => false})
+            res = handler.event_to_classification_request(event)
+            expect(res.nil?).to be false
+            expect(res['term']['is_dst_internal']).to be false
+            expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
+    	end
+
+        it "test nested threat structure" do
+            my_handler = described_class.new("product_type", "product", 'threat', "is_src_internal", "is_dst_internal")
+            event = LogStash::Event.new("product_type" => "IDS", "product" => "snort", "threat" => {"signature" => "name1"})
+            res = my_handler.event_to_classification_request(event)
+            expect(res['term']['signature']).to eq('name1')
+        end
+    end
+end