logstash-filter-empowclassifier 0.3.23 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 247f7c74fd62e08857d7eab95a795432e4e83742a0a5cfceb089c16e0fc7fdae
-  data.tar.gz: 7331f3bf2ce795cf779104700ee2418fb7e1ecca56fa91671ee08d0e5b8303c0
+  metadata.gz: 74937d8d6662e3b4b2f2823984b751fd8b51446901c591af25b782e8e619c61d
+  data.tar.gz: 892c0f3f8f7c1a253b96d4fb52f6c8c752dfdbebf939f8d7ec3f99f46f1ac122
 SHA512:
-  metadata.gz: 52608ce0343e00cf4b2597815dcb2e7bafb1f32d858965a1cbf07ca5399ccde4f0372a5ce64b056a3f691dd5a0b57601bb38eca0e6bc19c74b3890f96745a740
-  data.tar.gz: cc91711eeaf7ff6420a8c01b6082e925080f3c9066177b8fae760ad05d4cba26d6116655c5ea53c63bb6469f9a5a269f614f453f2f64704a6b0c0f1042283150
+  metadata.gz: 7b37175bd121abb10e9a2904592b3d008b0c4217ee21c7750b8247a5622a74142159fb5363245d06330cd49d824301ed9b2b9a60d76bc6a0d2b6cb2232144091
+  data.tar.gz: 4827236e9939d9fdb4ef7ed6f5f667f0ce832a6b7e0c443bb8a122d27de5afa0f001605cef232f507623f46780decdf61c2b5d14b3e4d5afeb44df37118340ea

data/lib/logstash/filters/classification-request.rb

@@ -1,6 +1,6 @@
 module LogStash; module Filters; module Empow;
-  class LogStash::Filters::Empow::ClassificationRequest < Struct.new(:product_type, :product, :term)
-    def initialize(product_type, product, term)
+  class LogStash::Filters::Empow::ClassificationRequest < Struct.new(:product_type, :product, :term, :is_src_internal, :is_dst_internal)
+    def initialize(product_type, product, term, is_src_internal, is_dst_internal)
       if product_type.nil?
       	raise ArgumentError, 'product type cannot be empty' 
       end
@@ -11,7 +11,7 @@ module LogStash; module Filters; module Empow;
         product = product.downcase.strip
       end
 
-      super(product_type, product, term)
+      super(product_type, product, term, is_src_internal, is_dst_internal)
     end
   end
 end; end; end;

data/lib/logstash/filters/empowclassifier.rb

@@ -14,88 +14,70 @@ class LogStash::Filters::EmpowClassifier < LogStash::Filters::Base
 
   config_name "empowclassifier"
 
-  # The mail address used when registering to the classification center.
+  # The username (typically your email address), to access the classification center
   config :username, :validate => :string, :required => true
 
-  # The password for the classification center.
+  # The password to access the classification center
   config :password, :validate => :string, :required => true
 
-  # authentication hash. this parameter needs to be set only when using an entire empow stack, freeware users should leave this unchanged.
+  # Set this value only if using the complete empow stack; leave unchanged if using the empow Elastic open source plugin or module
   config :authentication_hash, :validate => :string, :default => '131n94ktfg7lj8hlpnnbkuiql1'
 
-  # Size of the local response cache
+  # The number of responses cached locally
   config :cache_size, :validate => :number, :default => 10000
 
-  # The maximum number of events that may wait in memory for a classification result from the classification center
+  # Max number of requests pending response from the classification center
   config :max_pending_requests, :validate => :number, :default => 10000
 
-  # Time to wait in seconds an event will wait for a classification before returning to the pipeline with no result
+  # Timeout for response from classification center (seconds)
   config :pending_request_timeout, :validate => :number, :default => 60
 
-  # Max number of concurrent threads classifying via the classification center
-  # These threads mostly wait on I/O during the web request, and aren't cpu intensive.
-  # Idle workers are closed after one minute, only one idle worker remains alive for incoming request on peace time.
+  # Maximum number of concurrent threads (workers) classifying logs using the classification center
   config :max_classification_center_workers, :validate => :number, :default => 5
 
-  # Classfication center bulk request size
+  # Classification center bulk request size (requests)
   config :bulk_request_size, :validate => :number, :default => 50
 
-  # Seconds to wait for batch to fill up before querying the classification center.
+  # Time (seconds) to wait for batch to fill on classifciation center, before querying for the response
   config :bulk_request_interval, :validate => :number, :default => 2
 
-  # Max number of times each request will be query the classification center.
+  # Max number of classification center request retries
   config :max_query_retries, :validate => :number, :default => 5
 
-  # Seconds to wait before reclassifying an in-progress request. In progress response will occur when the classification center is processing a new threat.
+  # Time (seconds) to wait between queries to the classification center for the final response to a request; the classification center will return an 'in-progress' response if queried before the final response is ready
   config :time_between_queries, :validate => :number, :default => 10
 
-  # Allows renaimg the log field containing the log's product type. Possible values are AM for Anti-Malware and IDS for Intrusion Detection systems.
-  # For example, if our log contained a 'log_type' field (instead of the expected product_type field),
-  # We would configure the plugin as follows:
+  # The name of the product type field in the log
+  # Example: If the log used log_type for the product type, configure the plugin like this:
   # [source,ruby]
-  #   filter {
-  #     empowclassifier {
-  #       username => "happy"
-  #       password => "festivus"
-  #       product_type_field => "log_type"
-  #     }
-  #   }
+  #    filter {
+  #      empowclassifier {
+  #        username => "happy"
+  #        password => "festivus"
+  #        product_type_field => "log_type"
+  #      }
+  #    }
   config :product_type_field, :validate => :string, :default => "product_type"
 
-  # Allows renaimg the log field containing the log's product name.
-  # Assuming our log contained a 'product' field (instead of the expected product_name field),
-  # We would configure the plugin as follows:
+  # The name of the product name field in the log
+  # Example: If the log used product for the product name, configure the plugin like this:
   # [source,ruby]
-  #   filter {
-  #     empowclassifier {
-  #       username => "happy"
-  #       password => "festivus"
-  #       product_type_field => "product"
-  #     }
-  #   }
+  #    filter {
+  #      empowclassifier {
+  #        username => "happy"
+  #        password => "festivus"
+  #        product_name_field => "product"
+  #      }
+  #    }
   config :product_name_field, :validate => :string, :default => "product_name"
+
+  # The name of the field containing the terms sent to the classification center
   config :threat_field, :validate => :string, :default => "threat"
 
-  # Configs the name of the field used to indicate whether the source described in the log was within the internal network.
-  # Example:
-  # [source,ruby]
-  #   filter {
-  #     empowclassifier {
-  #       ...
-  #       src_internal_field => "internal_src"
-  #     }
-  #   }
+  # Indicates whether the source field is internal to the user’s network (for example, an internal host/mail/user/app)
   config :src_internal_field, :validate => :string, :default => "is_src_internal"
 
-  # Configs the name of the field used to indicate whether the destination described in the log was within the internal network.
-  # Example:
-  # [source,ruby]
-  #   filter {
-  #     empowclassifier {
-  #       ...
-  #       dst_internal_field => "internal_dst"
-  #     }
-  #   }
+  # Indicates whether the dest field is internal to the user’s network (for example, an internal host/mail/user/app)
   config :dst_internal_field, :validate => :string, :default => "is_dst_internal"
 
   # changes the api root for customers of the commercial empow stack

data/lib/logstash/filters/field-handler.rb

@@ -24,6 +24,8 @@ class LogStash::Filters::Empow::FieldHandler
     @src_internal_field = @threat_field + '[' + src_internal_field + ']'
     @dst_internal_field = @threat_field + '[' + dst_internal_field + ']'
 
+    @blacklisted_fields = [src_internal_field, dst_internal_field]
+
     @hash_field = @threat_field + '[hash]'
   end
 
@@ -71,10 +73,7 @@ class LogStash::Filters::Empow::FieldHandler
       return nil
     end
 
-    threat['is_src_internal'] = is_src_internal
-    threat['is_dst_internal'] = is_dst_internal
-
-    return LogStash::Filters::Empow::ClassificationRequest.new(product_type, product, threat)
+    return LogStash::Filters::Empow::ClassificationRequest.new(product_type, product, threat, is_src_internal, is_dst_internal)
   end
 
   private
@@ -84,6 +83,7 @@ class LogStash::Filters::Empow::FieldHandler
     res = Hash.new
 
     threat.each do |k, v|
+      next if @blacklisted_fields.include?(k)
       res[k] = v
     end
 

data/logstash-filter-empowclassifier.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-empowclassifier'
-  s.version       = '0.3.23'
+  s.version       = '1.0.0'
   s.licenses      = ['Apache-2.0']
-  s.summary       = 'Logstash intent classification plugin client for accessing empow''s classifiction cloud'
+  s.summary       = 'Returns classification information for attacks from the empow classification center, based on information in log strings'
   #s.description   = 'Write a longer description or delete this line.'
   s.homepage      = 'http://www.empow.co'
   s.authors       = ['empow', 'Assaf Abulafia', 'Rami Cohen']

data/spec/filters/field-handler_spec.rb

@@ -11,7 +11,7 @@ describe LogStash::Filters::Empow::FieldHandler do
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"})
             res = handler.event_to_classification_request(event)
             expect(res).not_to be_nil
-            expect(res['term']['is_src_internal']).to be true
+            expect(res['is_src_internal']).to be true
             expect(event.get("empow_warnings")).to include("src_internal_wrong_value")
     	end
 
@@ -19,7 +19,7 @@ describe LogStash::Filters::Empow::FieldHandler do
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"})
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_dst_internal']).to be true
+            expect(res['is_dst_internal']).to be true
             expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
     	end
 
@@ -27,7 +27,7 @@ describe LogStash::Filters::Empow::FieldHandler do
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => 1})
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_src_internal']).to be true
+            expect(res['is_src_internal']).to be true
             expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
     	end
 
@@ -35,7 +35,7 @@ describe LogStash::Filters::Empow::FieldHandler do
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_src_internal" => 11)
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_src_internal']).to be true
+            expect(res['is_src_internal']).to be true
             expect(event.get("empow_warnings")).to include("src_internal_wrong_value")
         end
 
@@ -43,7 +43,7 @@ describe LogStash::Filters::Empow::FieldHandler do
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_dst_internal" => 1})
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_dst_internal']).to be true
+            expect(res['is_dst_internal']).to be true
             expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
     	end
 
@@ -51,7 +51,7 @@ describe LogStash::Filters::Empow::FieldHandler do
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_dst_internal" => 11)
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_dst_internal']).to be true
+            expect(res['is_dst_internal']).to be true
             expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
         end
 
@@ -59,7 +59,7 @@ describe LogStash::Filters::Empow::FieldHandler do
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_dst_internal" => [])
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_dst_internal']).to be true
+            expect(res['is_dst_internal']).to be true
             expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
         end
 
@@ -67,13 +67,13 @@ describe LogStash::Filters::Empow::FieldHandler do
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => true})
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_src_internal']).to be true
+            expect(res['is_src_internal']).to be true
             expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
 
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => false})
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_src_internal']).to be false
+            expect(res['is_src_internal']).to be false
             expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
     	end
 
@@ -81,13 +81,13 @@ describe LogStash::Filters::Empow::FieldHandler do
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_dst_internal" => true})
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_dst_internal']).to be true
+            expect(res['is_dst_internal']).to be true
             expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
 
             event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1",  "is_dst_internal" => false})
             res = handler.event_to_classification_request(event)
             expect(res.nil?).to be false
-            expect(res['term']['is_dst_internal']).to be false
+            expect(res['is_dst_internal']).to be false
             expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
     	end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-empowclassifier
 version: !ruby/object:Gem::Version
-  version: 0.3.23
+  version: 1.0.0
 platform: ruby
 authors:
 - empow
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-02-10 00:00:00.000000000 Z
+date: 2019-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -206,13 +206,10 @@ files:
 - lib/logstash/filters/response.rb
 - lib/logstash/filters/utils.rb
 - logstash-filter-empowclassifier.gemspec
-- spec/filters/assaf_spec.rb
 - spec/filters/bulk-processor_spec.rb
-- spec/filters/center-client_spec.rb
 - spec/filters/classifier-cache_spec.rb
 - spec/filters/classifier_spec.rb
 - spec/filters/cognito-client_spec.rb
-- spec/filters/elastic-db_spec.rb
 - spec/filters/empowclassifier_spec.rb
 - spec/filters/field-handler_spec.rb
 - spec/filters/local-classifier_spec.rb
@@ -244,16 +241,13 @@ rubyforge_project:
 rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
-summary: Logstash intent classification plugin client for accessing empows classifiction
-  cloud
+summary: Returns classification information for attacks from the empow classification
+  center, based on information in log strings
 test_files:
-- spec/filters/assaf_spec.rb
 - spec/filters/bulk-processor_spec.rb
-- spec/filters/center-client_spec.rb
 - spec/filters/classifier-cache_spec.rb
 - spec/filters/classifier_spec.rb
 - spec/filters/cognito-client_spec.rb
-- spec/filters/elastic-db_spec.rb
 - spec/filters/empowclassifier_spec.rb
 - spec/filters/field-handler_spec.rb
 - spec/filters/local-classifier_spec.rb

data/spec/filters/assaf_spec.rb

@@ -1,51 +0,0 @@
-require_relative '../spec_helper'
-require "logstash/filters/center-client"
-require "logstash/filters/response"
-require "logstash/filters/classification-request"
-
-# client = LogStash::Filters::Empow::ClassificationCenterClient.new('assaf', , , )
-
-describe LogStash::Filters::Empow::ClassificationCenterClient do
-
-  # before(:each) do
-  #   local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
-  #   allow(LogStash::Filters::Empow::LocalClassifier).to receive(:new).and_return(local_classifier)
-  # end
-
-  let(:url_base) { 'https://intent.cloud.empow.co' }
-  let(:username) { 'assafa@empownetworks.com' }
-  let(:password) { 'Empow2018!' }
-  let(:pool_id) { '131n94ktfg7lj8hlpnnbkuiql1' }
-
-  describe "classification center api" do
-    it "test missing ids request" do
-  		client = described_class.new(username, password, pool_id, url_base)
-
-  		client.authenticate
-
-      term = {}
-      term[:signature] = '1:238'
-      req1 = LogStash::Filters::Empow::ClassificationRequest.new('IDS', 'snort', term)
-
-      i = 0
-      while true do
-        i += 1
-  		  results = client.classify([req1])
-
-        results.each do |k,v|
-          p i
-          if !v.is_successful
-            p v
-            break
-          end
-        end
-      end
-
-      #p results
-
-      results.each do |res|
-        p "res: #{res}"
-      end
-    end
-  end
-end

data/spec/filters/center-client_spec.rb

@@ -1,88 +0,0 @@
-require_relative '../spec_helper'
-require "logstash/filters/center-client"
-require "logstash/filters/response"
-require 'webmock/rspec'
-
-describe LogStash::Filters::Empow::ClassificationCenterClient do
-
-  # before(:each) do
-  #   local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
-  #   allow(LogStash::Filters::Empow::LocalClassifier).to receive(:new).and_return(local_classifier)
-  # end
-
-  let(:url_base) { 'http://localhost:5000' }
-  let(:username) { 'myuser' }
-  let(:password) { 'mypassword' }
-  let(:pool_id) { 'mypassword' }
-
-  describe "classification center api" do
-  	before(:each) do
-    	WebMock.disable_net_connect!
-
-      stub_request(:post, "#{url_base}/login").
-        to_return(:body => "", :status => 200,
-          :headers => { 'authorization' => 'Bearer my-token' })
-
-      mocked_cognito = double(LogStash::Filters::Empow::CognitoClient)
-      allow(LogStash::Filters::Empow::CognitoClient).to receive(:new).and_return(mocked_cognito)
-      allow(mocked_cognito).to receive(:authenticate).and_return("dummy token")
-  	end
-
-  	after(:each) do
-    	WebMock.reset!
-    	WebMock.allow_net_connect!
-
-      allow(LogStash::Filters::Empow::CognitoClient).to receive(:new).and_call_original
-  	end
-
-
-    it "test missing ids request" do
-      stub_request(:post, "#{url_base}/classification/intent").
-        to_return(:body => "", :status => 204,
-          :headers => { 'Content-Length' => 0 })
-  		
-  		client = described_class.new(username, password, pool_id, url_base)
-
-  		client.authenticate
-
-  		res = client.classify(["req1"])
-
-  		expect(res["req1"]).to be_kind_of(LogStash::Filters::Empow::FailureReponse)
-    end
-
-    it "test existing ids request" do
-  		
-      response = '{"some":"data"}'
-
-      stub_request(:post, "#{url_base}/classification/intent").
-        to_return(:body => response, :status => 200)
-
-  		client = described_class.new(username, password, pool_id, url_base)
-
-  		client.authenticate
-
-      k1 = "req1"
-  		response_map = client.classify([k1])
-
-      res = response_map[k1].response
-
-      p "res: #{res}"
-
-      expect(res["some"]).to eq("data")
-    end
-
-    it "test http status 500 during request" do
-
-      stub_request(:post, "#{url_base}/classification/intent").
-        to_return(:body => "", :status => 500)
-
-      client = described_class.new(username, password, pool_id, url_base)
-
-      client.authenticate
-
-      res = client.classify("ids", "Snort", "1:2", nil)
-
-      expect(res).to be_nil
-    end
-  end
-end

data/spec/filters/elastic-db_spec.rb

@@ -1,44 +0,0 @@
-# require_relative '../spec_helper'
-# require "logstash/filters/elastic-db"
-
-# describe LogStash::Filters::Empow::PersistentKeyValueDB do
-
-# 	let(:user) { 'user' }
-# 	let(:indexName) { 'key-val-8' }
-# 	let(:password) { 'pass' }
-# 	let(:elastic) { '192.168.3.24:9200' }
-
-# 	subject { described_class.new(elastic, user, password, indexName) }
-
-# 	after do
-#     	subject.close
-#   	end
-
-# 	describe "initialization" do
-#     	it "should be successful" do
-#     		expect { subject }.not_to raise_error
-#     	end
-#     end
-
-#     describe "read a value that doesn't exists" do
-#     	it "should return nil" do
-#     		res = subject.query "ids", "snort", "123:456:789"
-#     		expect(res).to be_nil
-#     	end
-#     end
-
-#     describe "write a value then read" do
-#     	let(:data) { "blob" }
-
-#     	it "write should be successful" do
-#     		expect { subject.save 1234, "am", "my-product", "not-my-name", 'something else' }.not_to raise_error
-#     		expect { subject.save 12345, "am", "my-product", "my-name", data }.not_to raise_error
-#     		sleep(2)
-#     	end
-
-#     	it "read the new value should succeed" do
-#     		res = subject.query "am", "my-product", "my-name"
-#     		expect(res).to eq(data)
-#     	end
-#     end
-# end