logstash-filter-empowclassifier 0.3.15

data/spec/filters/local-classifier_spec.rb

@@ -0,0 +1,46 @@
+require_relative '../spec_helper'
+require "logstash/filters/local-classifier"
+require "logstash/filters/elastic-db"
+require "logstash/filters/classification-request"
+
+describe LogStash::Filters::Empow::LocalClassifier do
+
+    describe "sync'ed local database as a fallback" do
+    	it "value isn't in memory, later fetched from local db" do
+            local_db = instance_double(LogStash::Filters::Empow::PersistentKeyValueDB)
+            allow(local_db).to receive(:query).and_return(nil)
+            allow(local_db).to receive(:close)
+
+    		classifier = described_class.new(5, 300, false, local_db)
+
+            key = LogStash::Filters::Empow::ClassificationRequest.new("product_type", "product", "threat", true, true)
+
+            expect(classifier.classify(key)).to be_nil
+
+            allow(local_db).to receive(:query).and_return("intent")
+
+            # allow backend thread to process the request
+            res = nil
+
+            for i in 1..10 do
+                sleep 1
+
+                res = classifier.classify(key)
+
+                break if !res.nil?
+            end
+
+            expect(res).to eq("intent")
+    	end
+    end
+
+    describe "no local database configured" do
+        it "value isn't in memory" do
+            classifier = described_class.new(5, 300, false, nil)
+
+            key = "key-1"
+
+            expect(classifier.classify(key)).to be_nil
+        end
+    end
+end

data/spec/filters/plugin-logic_spec.rb

@@ -0,0 +1,127 @@
+require_relative '../spec_helper'
+require "logstash/event"
+require "logstash/filters/classifier"
+require "logstash/filters/plugin-logic"
+
+describe LogStash::Filters::Empow::PluginLogic do
+
+    let(:intent_res1) { {"p1" => "s1"} }
+    let(:response_body1) { {'response' => intent_res1 } }
+    let(:sample_response) { LogStash::Filters::Empow::SuccessfulReponse.new(response_body1) }
+
+    describe "test classification" do
+    	
+    	it "event with warm classification" do
+    	   event = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name1", "my_hash" => "hash1")
+
+            field_handler = instance_double(LogStash::Filters::Empow::FieldHandler)
+            allow(field_handler).to receive(:event_to_classification_request).and_return("request")
+
+            classifier = instance_double(LogStash::Filters::Empow::Classifier)
+            allow(classifier).to receive(:classify).and_return(sample_response)
+
+            plugin_logic = described_class.new(classifier, field_handler, 60, 1, ['_timeout'], ['_error'])
+
+            expect(field_handler).to receive(:event_to_classification_request)
+            expect(classifier).to receive(:classify)
+
+            classified_event = plugin_logic.classify(event)
+
+            expect(classified_event).to eq(event)
+            expect(classified_event.get("empow_intent")).to eq(intent_res1)
+    	end
+
+    	it "event with cold classification is parked and then unparked only once" do
+    		event = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name1", "my_hash" => "hash1")
+
+            field_handler = instance_double(LogStash::Filters::Empow::FieldHandler)
+            allow(field_handler).to receive(:event_to_classification_request).and_return("request")
+		    allow(Time).to receive(:now).and_return(10)
+
+            classifier = instance_double(LogStash::Filters::Empow::Classifier)
+            allow(classifier).to receive(:classify).and_return(nil, nil, sample_response)
+
+            plugin_logic = described_class.new(classifier, field_handler, 60, 1, ['_timeout'], ['_error'])
+
+            expect(classifier).to receive(:classify)
+
+            classified_event = plugin_logic.classify(event)
+            expect(classified_event).to be_nil
+
+            allow(Time).to receive(:now).and_return(20)
+
+            expect(classifier).to receive(:classify)
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).to be_empty
+
+            allow(Time).to receive(:now).and_return(30)
+            expect(classifier).to receive(:classify)
+
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).not_to be_empty
+    	end
+
+    	it "event unparked after time expired" do
+    		event = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name1", "my_hash" => "hash1")
+
+            field_handler = instance_double(LogStash::Filters::Empow::FieldHandler)
+            allow(field_handler).to receive(:event_to_classification_request).and_return("request")
+            allow(Time).to receive(:now).and_return(10)
+
+            classifier = instance_double(LogStash::Filters::Empow::Classifier)
+            allow(classifier).to receive(:classify).and_return(nil)
+
+            plugin_logic = described_class.new(classifier, field_handler, 60, 1, ['_timeout'], ['_error'])
+
+            expect(classifier).to receive(:classify)
+
+            classified_event = plugin_logic.classify(event)
+            expect(classified_event).to be_nil
+
+            allow(Time).to receive(:now).and_return(20)
+
+            expect(classifier).to receive(:classify)
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).to be_empty
+
+            allow(Time).to receive(:now).and_return(100)
+            expect(classifier).to receive(:classify)
+
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).not_to be_empty
+
+            insist { flushed_events[0].get("tags") }.include?("_timeout")
+    	end
+
+    	it "too many parked events" do
+            event1 = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name1", "my_hash" => "hash1")
+            event2 = LogStash::Event.new("my_product_type" => "ids", "my_product" => "some_av", "my_term" => "name2", "my_hash" => "hash2")
+
+            field_handler = instance_double(LogStash::Filters::Empow::FieldHandler)
+            allow(field_handler).to receive(:event_to_classification_request).and_return("request")
+            allow(Time).to receive(:now).and_return(10)
+
+            classifier = instance_double(LogStash::Filters::Empow::Classifier)
+            allow(classifier).to receive(:classify).and_return(nil)
+
+            plugin_logic = described_class.new(classifier, field_handler, 60, 1, ['_timeout'], ['_error'])
+
+            expect(classifier).to receive(:classify)
+
+            classified_event = plugin_logic.classify(event1)
+            expect(classified_event).to be_nil
+
+            classified_event = plugin_logic.classify(event2)
+            expect(classified_event).to eq(event1)
+
+            allow(Time).to receive(:now).and_return(20)
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).to be_empty
+
+            allow(Time).to receive(:now).and_return(100)
+            flushed_events = plugin_logic.flush
+            expect(flushed_events).not_to be_empty
+            expect(flushed_events.length).to eq(1)
+    	end
+    end
+end

data/spec/filters/utils_spec.rb

@@ -0,0 +1,74 @@
+require_relative '../spec_helper'
+require "logstash/event"
+require "logstash/filters/utils"
+
+describe LogStash::Filters::Empow::Utils do
+
+    describe "test internal tagging" do
+    	it "test error" do
+    		event = LogStash::Event.new("data" => "a b c")
+
+    		LogStash::Filters::Empow::Utils.add_error(event, "my_msg")
+
+            expect(event.get("empow_errors")).to contain_exactly("my_msg")
+    	end
+
+    	it "test warn" do
+    		event = LogStash::Event.new("data" => "a b c")
+
+    		LogStash::Filters::Empow::Utils.add_warn(event, "my_msg")
+
+            expect(event.get("empow_warnings")).to contain_exactly("my_msg")
+    	end
+    end
+#json = '{ "a": "True", "b": "true", "c": "1", "d": 1, "e": "False", "f": "0", "g": "TRUE" }'
+    describe "test is truthy" do
+        it "string TRUE" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean("TRUE")).to eq(true)
+        end
+
+        it "string true" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean("true")).to eq(true)
+        end
+
+        it "string 1" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean("1")).to eq(true)
+        end
+
+        it "string 11" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean("11")).to be_nil
+        end
+
+        it "string 0" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean('0')).to eq(false)
+        end
+
+        it "int 0" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean(0)).to eq(false)
+        end
+
+        it "int 1" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean(1)).to eq(true)
+        end
+
+        it "int 11" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean(11)).to be_nil
+        end
+
+        it "boolean true" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean(true)).to eq(true)
+        end
+
+        it "boolean false" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean(false)).to eq(false)
+        end
+
+        it "nil" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean(nil)).to be_nil
+        end
+
+        it "empty string" do
+            expect(LogStash::Filters::Empow::Utils.convert_to_boolean('')).to be_nil
+        end
+    end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,260 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-empowclassifier
+version: !ruby/object:Gem::Version
+  version: 0.3.15
+platform: ruby
+authors:
+- empow
+- Assaf Abulafia
+- Rami Cohen
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-01-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  name: rest-client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  name: lru_redux
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  name: json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: hashie
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  name: aws-sdk
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  name: timecop
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.22'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.21.0
+  name: webmock
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.22'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.21.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: elasticsearch
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email: ''
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/center-client.rb
+- lib/logstash/filters/classification-request.rb
+- lib/logstash/filters/classifier-cache.rb
+- lib/logstash/filters/classifier.rb
+- lib/logstash/filters/cognito-client.rb
+- lib/logstash/filters/elastic-db.rb
+- lib/logstash/filters/empowclassifier.rb
+- lib/logstash/filters/field-handler.rb
+- lib/logstash/filters/local-classifier.rb
+- lib/logstash/filters/plugin-logic.rb
+- lib/logstash/filters/response.rb
+- lib/logstash/filters/utils.rb
+- logstash-filter-empowclassifier.gemspec
+- spec/filters/bulk-processor_spec.rb
+- spec/filters/center-client_spec.rb
+- spec/filters/classifier-cache_spec.rb
+- spec/filters/classifier_spec.rb
+- spec/filters/cognito-client_spec.rb
+- spec/filters/elastic-db_spec.rb
+- spec/filters/empowclassifier_spec.rb
+- spec/filters/field-handler_spec.rb
+- spec/filters/local-classifier_spec.rb
+- spec/filters/plugin-logic_spec.rb
+- spec/filters/utils_spec.rb
+- spec/spec_helper.rb
+homepage: http://www.empowcybersecurity.com
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
+specification_version: 4
+summary: Logstash intent classification plugin client for accessing empows classifiction
+  cloud
+test_files:
+- spec/filters/bulk-processor_spec.rb
+- spec/filters/center-client_spec.rb
+- spec/filters/classifier-cache_spec.rb
+- spec/filters/classifier_spec.rb
+- spec/filters/cognito-client_spec.rb
+- spec/filters/elastic-db_spec.rb
+- spec/filters/empowclassifier_spec.rb
+- spec/filters/field-handler_spec.rb
+- spec/filters/local-classifier_spec.rb
+- spec/filters/plugin-logic_spec.rb
+- spec/filters/utils_spec.rb
+- spec/spec_helper.rb