logstash-filter-empowclassifier 0.3.23 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 247f7c74fd62e08857d7eab95a795432e4e83742a0a5cfceb089c16e0fc7fdae
4
- data.tar.gz: 7331f3bf2ce795cf779104700ee2418fb7e1ecca56fa91671ee08d0e5b8303c0
3
+ metadata.gz: 74937d8d6662e3b4b2f2823984b751fd8b51446901c591af25b782e8e619c61d
4
+ data.tar.gz: 892c0f3f8f7c1a253b96d4fb52f6c8c752dfdbebf939f8d7ec3f99f46f1ac122
5
5
  SHA512:
6
- metadata.gz: 52608ce0343e00cf4b2597815dcb2e7bafb1f32d858965a1cbf07ca5399ccde4f0372a5ce64b056a3f691dd5a0b57601bb38eca0e6bc19c74b3890f96745a740
7
- data.tar.gz: cc91711eeaf7ff6420a8c01b6082e925080f3c9066177b8fae760ad05d4cba26d6116655c5ea53c63bb6469f9a5a269f614f453f2f64704a6b0c0f1042283150
6
+ metadata.gz: 7b37175bd121abb10e9a2904592b3d008b0c4217ee21c7750b8247a5622a74142159fb5363245d06330cd49d824301ed9b2b9a60d76bc6a0d2b6cb2232144091
7
+ data.tar.gz: 4827236e9939d9fdb4ef7ed6f5f667f0ce832a6b7e0c443bb8a122d27de5afa0f001605cef232f507623f46780decdf61c2b5d14b3e4d5afeb44df37118340ea
@@ -1,6 +1,6 @@
1
1
  module LogStash; module Filters; module Empow;
2
- class LogStash::Filters::Empow::ClassificationRequest < Struct.new(:product_type, :product, :term)
3
- def initialize(product_type, product, term)
2
+ class LogStash::Filters::Empow::ClassificationRequest < Struct.new(:product_type, :product, :term, :is_src_internal, :is_dst_internal)
3
+ def initialize(product_type, product, term, is_src_internal, is_dst_internal)
4
4
  if product_type.nil?
5
5
  raise ArgumentError, 'product type cannot be empty'
6
6
  end
@@ -11,7 +11,7 @@ module LogStash; module Filters; module Empow;
11
11
  product = product.downcase.strip
12
12
  end
13
13
 
14
- super(product_type, product, term)
14
+ super(product_type, product, term, is_src_internal, is_dst_internal)
15
15
  end
16
16
  end
17
17
  end; end; end;
@@ -14,88 +14,70 @@ class LogStash::Filters::EmpowClassifier < LogStash::Filters::Base
14
14
 
15
15
  config_name "empowclassifier"
16
16
 
17
- # The mail address used when registering to the classification center.
17
+ # The username (typically your email address), to access the classification center
18
18
  config :username, :validate => :string, :required => true
19
19
 
20
- # The password for the classification center.
20
+ # The password to access the classification center
21
21
  config :password, :validate => :string, :required => true
22
22
 
23
- # authentication hash. this parameter needs to be set only when using an entire empow stack, freeware users should leave this unchanged.
23
+ # Set this value only if using the complete empow stack; leave unchanged if using the empow Elastic open source plugin or module
24
24
  config :authentication_hash, :validate => :string, :default => '131n94ktfg7lj8hlpnnbkuiql1'
25
25
 
26
- # Size of the local response cache
26
+ # The number of responses cached locally
27
27
  config :cache_size, :validate => :number, :default => 10000
28
28
 
29
- # The maximum number of events that may wait in memory for a classification result from the classification center
29
+ # Max number of requests pending response from the classification center
30
30
  config :max_pending_requests, :validate => :number, :default => 10000
31
31
 
32
- # Time to wait in seconds an event will wait for a classification before returning to the pipeline with no result
32
+ # Timeout for response from classification center (seconds)
33
33
  config :pending_request_timeout, :validate => :number, :default => 60
34
34
 
35
- # Max number of concurrent threads classifying via the classification center
36
- # These threads mostly wait on I/O during the web request, and aren't cpu intensive.
37
- # Idle workers are closed after one minute, only one idle worker remains alive for incoming request on peace time.
35
+ # Maximum number of concurrent threads (workers) classifying logs using the classification center
38
36
  config :max_classification_center_workers, :validate => :number, :default => 5
39
37
 
40
- # Classfication center bulk request size
38
+ # Classification center bulk request size (requests)
41
39
  config :bulk_request_size, :validate => :number, :default => 50
42
40
 
43
- # Seconds to wait for batch to fill up before querying the classification center.
41
+ # Time (seconds) to wait for batch to fill on classifciation center, before querying for the response
44
42
  config :bulk_request_interval, :validate => :number, :default => 2
45
43
 
46
- # Max number of times each request will be query the classification center.
44
+ # Max number of classification center request retries
47
45
  config :max_query_retries, :validate => :number, :default => 5
48
46
 
49
- # Seconds to wait before reclassifying an in-progress request. In progress response will occur when the classification center is processing a new threat.
47
+ # Time (seconds) to wait between queries to the classification center for the final response to a request; the classification center will return an 'in-progress' response if queried before the final response is ready
50
48
  config :time_between_queries, :validate => :number, :default => 10
51
49
 
52
- # Allows renaimg the log field containing the log's product type. Possible values are AM for Anti-Malware and IDS for Intrusion Detection systems.
53
- # For example, if our log contained a 'log_type' field (instead of the expected product_type field),
54
- # We would configure the plugin as follows:
50
+ # The name of the product type field in the log
51
+ # Example: If the log used log_type for the product type, configure the plugin like this:
55
52
  # [source,ruby]
56
- # filter {
57
- # empowclassifier {
58
- # username => "happy"
59
- # password => "festivus"
60
- # product_type_field => "log_type"
61
- # }
62
- # }
53
+ # filter {
54
+ # empowclassifier {
55
+ # username => "happy"
56
+ # password => "festivus"
57
+ # product_type_field => "log_type"
58
+ # }
59
+ # }
63
60
  config :product_type_field, :validate => :string, :default => "product_type"
64
61
 
65
- # Allows renaimg the log field containing the log's product name.
66
- # Assuming our log contained a 'product' field (instead of the expected product_name field),
67
- # We would configure the plugin as follows:
62
+ # The name of the product name field in the log
63
+ # Example: If the log used product for the product name, configure the plugin like this:
68
64
  # [source,ruby]
69
- # filter {
70
- # empowclassifier {
71
- # username => "happy"
72
- # password => "festivus"
73
- # product_type_field => "product"
74
- # }
75
- # }
65
+ # filter {
66
+ # empowclassifier {
67
+ # username => "happy"
68
+ # password => "festivus"
69
+ # product_name_field => "product"
70
+ # }
71
+ # }
76
72
  config :product_name_field, :validate => :string, :default => "product_name"
73
+
74
+ # The name of the field containing the terms sent to the classification center
77
75
  config :threat_field, :validate => :string, :default => "threat"
78
76
 
79
- # Configs the name of the field used to indicate whether the source described in the log was within the internal network.
80
- # Example:
81
- # [source,ruby]
82
- # filter {
83
- # empowclassifier {
84
- # ...
85
- # src_internal_field => "internal_src"
86
- # }
87
- # }
77
+ # Indicates whether the source field is internal to the user’s network (for example, an internal host/mail/user/app)
88
78
  config :src_internal_field, :validate => :string, :default => "is_src_internal"
89
79
 
90
- # Configs the name of the field used to indicate whether the destination described in the log was within the internal network.
91
- # Example:
92
- # [source,ruby]
93
- # filter {
94
- # empowclassifier {
95
- # ...
96
- # dst_internal_field => "internal_dst"
97
- # }
98
- # }
80
+ # Indicates whether the dest field is internal to the user’s network (for example, an internal host/mail/user/app)
99
81
  config :dst_internal_field, :validate => :string, :default => "is_dst_internal"
100
82
 
101
83
  # changes the api root for customers of the commercial empow stack
@@ -24,6 +24,8 @@ class LogStash::Filters::Empow::FieldHandler
24
24
  @src_internal_field = @threat_field + '[' + src_internal_field + ']'
25
25
  @dst_internal_field = @threat_field + '[' + dst_internal_field + ']'
26
26
 
27
+ @blacklisted_fields = [src_internal_field, dst_internal_field]
28
+
27
29
  @hash_field = @threat_field + '[hash]'
28
30
  end
29
31
 
@@ -71,10 +73,7 @@ class LogStash::Filters::Empow::FieldHandler
71
73
  return nil
72
74
  end
73
75
 
74
- threat['is_src_internal'] = is_src_internal
75
- threat['is_dst_internal'] = is_dst_internal
76
-
77
- return LogStash::Filters::Empow::ClassificationRequest.new(product_type, product, threat)
76
+ return LogStash::Filters::Empow::ClassificationRequest.new(product_type, product, threat, is_src_internal, is_dst_internal)
78
77
  end
79
78
 
80
79
  private
@@ -84,6 +83,7 @@ class LogStash::Filters::Empow::FieldHandler
84
83
  res = Hash.new
85
84
 
86
85
  threat.each do |k, v|
86
+ next if @blacklisted_fields.include?(k)
87
87
  res[k] = v
88
88
  end
89
89
 
@@ -1,8 +1,8 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-filter-empowclassifier'
3
- s.version = '0.3.23'
3
+ s.version = '1.0.0'
4
4
  s.licenses = ['Apache-2.0']
5
- s.summary = 'Logstash intent classification plugin client for accessing empow''s classifiction cloud'
5
+ s.summary = 'Returns classification information for attacks from the empow classification center, based on information in log strings'
6
6
  #s.description = 'Write a longer description or delete this line.'
7
7
  s.homepage = 'http://www.empow.co'
8
8
  s.authors = ['empow', 'Assaf Abulafia', 'Rami Cohen']
@@ -11,7 +11,7 @@ describe LogStash::Filters::Empow::FieldHandler do
11
11
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"})
12
12
  res = handler.event_to_classification_request(event)
13
13
  expect(res).not_to be_nil
14
- expect(res['term']['is_src_internal']).to be true
14
+ expect(res['is_src_internal']).to be true
15
15
  expect(event.get("empow_warnings")).to include("src_internal_wrong_value")
16
16
  end
17
17
 
@@ -19,7 +19,7 @@ describe LogStash::Filters::Empow::FieldHandler do
19
19
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"})
20
20
  res = handler.event_to_classification_request(event)
21
21
  expect(res.nil?).to be false
22
- expect(res['term']['is_dst_internal']).to be true
22
+ expect(res['is_dst_internal']).to be true
23
23
  expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
24
24
  end
25
25
 
@@ -27,7 +27,7 @@ describe LogStash::Filters::Empow::FieldHandler do
27
27
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => 1})
28
28
  res = handler.event_to_classification_request(event)
29
29
  expect(res.nil?).to be false
30
- expect(res['term']['is_src_internal']).to be true
30
+ expect(res['is_src_internal']).to be true
31
31
  expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
32
32
  end
33
33
 
@@ -35,7 +35,7 @@ describe LogStash::Filters::Empow::FieldHandler do
35
35
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_src_internal" => 11)
36
36
  res = handler.event_to_classification_request(event)
37
37
  expect(res.nil?).to be false
38
- expect(res['term']['is_src_internal']).to be true
38
+ expect(res['is_src_internal']).to be true
39
39
  expect(event.get("empow_warnings")).to include("src_internal_wrong_value")
40
40
  end
41
41
 
@@ -43,7 +43,7 @@ describe LogStash::Filters::Empow::FieldHandler do
43
43
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_dst_internal" => 1})
44
44
  res = handler.event_to_classification_request(event)
45
45
  expect(res.nil?).to be false
46
- expect(res['term']['is_dst_internal']).to be true
46
+ expect(res['is_dst_internal']).to be true
47
47
  expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
48
48
  end
49
49
 
@@ -51,7 +51,7 @@ describe LogStash::Filters::Empow::FieldHandler do
51
51
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_dst_internal" => 11)
52
52
  res = handler.event_to_classification_request(event)
53
53
  expect(res.nil?).to be false
54
- expect(res['term']['is_dst_internal']).to be true
54
+ expect(res['is_dst_internal']).to be true
55
55
  expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
56
56
  end
57
57
 
@@ -59,7 +59,7 @@ describe LogStash::Filters::Empow::FieldHandler do
59
59
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1"}, "is_dst_internal" => [])
60
60
  res = handler.event_to_classification_request(event)
61
61
  expect(res.nil?).to be false
62
- expect(res['term']['is_dst_internal']).to be true
62
+ expect(res['is_dst_internal']).to be true
63
63
  expect(event.get("empow_warnings")).to include("dst_internal_wrong_value")
64
64
  end
65
65
 
@@ -67,13 +67,13 @@ describe LogStash::Filters::Empow::FieldHandler do
67
67
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => true})
68
68
  res = handler.event_to_classification_request(event)
69
69
  expect(res.nil?).to be false
70
- expect(res['term']['is_src_internal']).to be true
70
+ expect(res['is_src_internal']).to be true
71
71
  expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
72
72
 
73
73
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_src_internal" => false})
74
74
  res = handler.event_to_classification_request(event)
75
75
  expect(res.nil?).to be false
76
- expect(res['term']['is_src_internal']).to be false
76
+ expect(res['is_src_internal']).to be false
77
77
  expect(event.get("empow_warnings")).not_to include("src_internal_wrong_value")
78
78
  end
79
79
 
@@ -81,13 +81,13 @@ describe LogStash::Filters::Empow::FieldHandler do
81
81
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_dst_internal" => true})
82
82
  res = handler.event_to_classification_request(event)
83
83
  expect(res.nil?).to be false
84
- expect(res['term']['is_dst_internal']).to be true
84
+ expect(res['is_dst_internal']).to be true
85
85
  expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
86
86
 
87
87
  event = LogStash::Event.new("product_type" => "IDS", "product" => "some_av", "term" => {"signature" => "name1", "is_dst_internal" => false})
88
88
  res = handler.event_to_classification_request(event)
89
89
  expect(res.nil?).to be false
90
- expect(res['term']['is_dst_internal']).to be false
90
+ expect(res['is_dst_internal']).to be false
91
91
  expect(event.get("empow_warnings")).not_to include("dst_internal_wrong_value")
92
92
  end
93
93
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-empowclassifier
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.23
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - empow
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2019-02-10 00:00:00.000000000 Z
13
+ date: 2019-02-13 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  requirement: !ruby/object:Gem::Requirement
@@ -206,13 +206,10 @@ files:
206
206
  - lib/logstash/filters/response.rb
207
207
  - lib/logstash/filters/utils.rb
208
208
  - logstash-filter-empowclassifier.gemspec
209
- - spec/filters/assaf_spec.rb
210
209
  - spec/filters/bulk-processor_spec.rb
211
- - spec/filters/center-client_spec.rb
212
210
  - spec/filters/classifier-cache_spec.rb
213
211
  - spec/filters/classifier_spec.rb
214
212
  - spec/filters/cognito-client_spec.rb
215
- - spec/filters/elastic-db_spec.rb
216
213
  - spec/filters/empowclassifier_spec.rb
217
214
  - spec/filters/field-handler_spec.rb
218
215
  - spec/filters/local-classifier_spec.rb
@@ -244,16 +241,13 @@ rubyforge_project:
244
241
  rubygems_version: 2.6.13
245
242
  signing_key:
246
243
  specification_version: 4
247
- summary: Logstash intent classification plugin client for accessing empows classifiction
248
- cloud
244
+ summary: Returns classification information for attacks from the empow classification
245
+ center, based on information in log strings
249
246
  test_files:
250
- - spec/filters/assaf_spec.rb
251
247
  - spec/filters/bulk-processor_spec.rb
252
- - spec/filters/center-client_spec.rb
253
248
  - spec/filters/classifier-cache_spec.rb
254
249
  - spec/filters/classifier_spec.rb
255
250
  - spec/filters/cognito-client_spec.rb
256
- - spec/filters/elastic-db_spec.rb
257
251
  - spec/filters/empowclassifier_spec.rb
258
252
  - spec/filters/field-handler_spec.rb
259
253
  - spec/filters/local-classifier_spec.rb
@@ -1,51 +0,0 @@
1
- require_relative '../spec_helper'
2
- require "logstash/filters/center-client"
3
- require "logstash/filters/response"
4
- require "logstash/filters/classification-request"
5
-
6
- # client = LogStash::Filters::Empow::ClassificationCenterClient.new('assaf', , , )
7
-
8
- describe LogStash::Filters::Empow::ClassificationCenterClient do
9
-
10
- # before(:each) do
11
- # local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
12
- # allow(LogStash::Filters::Empow::LocalClassifier).to receive(:new).and_return(local_classifier)
13
- # end
14
-
15
- let(:url_base) { 'https://intent.cloud.empow.co' }
16
- let(:username) { 'assafa@empownetworks.com' }
17
- let(:password) { 'Empow2018!' }
18
- let(:pool_id) { '131n94ktfg7lj8hlpnnbkuiql1' }
19
-
20
- describe "classification center api" do
21
- it "test missing ids request" do
22
- client = described_class.new(username, password, pool_id, url_base)
23
-
24
- client.authenticate
25
-
26
- term = {}
27
- term[:signature] = '1:238'
28
- req1 = LogStash::Filters::Empow::ClassificationRequest.new('IDS', 'snort', term)
29
-
30
- i = 0
31
- while true do
32
- i += 1
33
- results = client.classify([req1])
34
-
35
- results.each do |k,v|
36
- p i
37
- if !v.is_successful
38
- p v
39
- break
40
- end
41
- end
42
- end
43
-
44
- #p results
45
-
46
- results.each do |res|
47
- p "res: #{res}"
48
- end
49
- end
50
- end
51
- end
@@ -1,88 +0,0 @@
1
- require_relative '../spec_helper'
2
- require "logstash/filters/center-client"
3
- require "logstash/filters/response"
4
- require 'webmock/rspec'
5
-
6
- describe LogStash::Filters::Empow::ClassificationCenterClient do
7
-
8
- # before(:each) do
9
- # local_classifier = instance_double(LogStash::Filters::Empow::LocalClassifier)
10
- # allow(LogStash::Filters::Empow::LocalClassifier).to receive(:new).and_return(local_classifier)
11
- # end
12
-
13
- let(:url_base) { 'http://localhost:5000' }
14
- let(:username) { 'myuser' }
15
- let(:password) { 'mypassword' }
16
- let(:pool_id) { 'mypassword' }
17
-
18
- describe "classification center api" do
19
- before(:each) do
20
- WebMock.disable_net_connect!
21
-
22
- stub_request(:post, "#{url_base}/login").
23
- to_return(:body => "", :status => 200,
24
- :headers => { 'authorization' => 'Bearer my-token' })
25
-
26
- mocked_cognito = double(LogStash::Filters::Empow::CognitoClient)
27
- allow(LogStash::Filters::Empow::CognitoClient).to receive(:new).and_return(mocked_cognito)
28
- allow(mocked_cognito).to receive(:authenticate).and_return("dummy token")
29
- end
30
-
31
- after(:each) do
32
- WebMock.reset!
33
- WebMock.allow_net_connect!
34
-
35
- allow(LogStash::Filters::Empow::CognitoClient).to receive(:new).and_call_original
36
- end
37
-
38
-
39
- it "test missing ids request" do
40
- stub_request(:post, "#{url_base}/classification/intent").
41
- to_return(:body => "", :status => 204,
42
- :headers => { 'Content-Length' => 0 })
43
-
44
- client = described_class.new(username, password, pool_id, url_base)
45
-
46
- client.authenticate
47
-
48
- res = client.classify(["req1"])
49
-
50
- expect(res["req1"]).to be_kind_of(LogStash::Filters::Empow::FailureReponse)
51
- end
52
-
53
- it "test existing ids request" do
54
-
55
- response = '{"some":"data"}'
56
-
57
- stub_request(:post, "#{url_base}/classification/intent").
58
- to_return(:body => response, :status => 200)
59
-
60
- client = described_class.new(username, password, pool_id, url_base)
61
-
62
- client.authenticate
63
-
64
- k1 = "req1"
65
- response_map = client.classify([k1])
66
-
67
- res = response_map[k1].response
68
-
69
- p "res: #{res}"
70
-
71
- expect(res["some"]).to eq("data")
72
- end
73
-
74
- it "test http status 500 during request" do
75
-
76
- stub_request(:post, "#{url_base}/classification/intent").
77
- to_return(:body => "", :status => 500)
78
-
79
- client = described_class.new(username, password, pool_id, url_base)
80
-
81
- client.authenticate
82
-
83
- res = client.classify("ids", "Snort", "1:2", nil)
84
-
85
- expect(res).to be_nil
86
- end
87
- end
88
- end
@@ -1,44 +0,0 @@
1
- # require_relative '../spec_helper'
2
- # require "logstash/filters/elastic-db"
3
-
4
- # describe LogStash::Filters::Empow::PersistentKeyValueDB do
5
-
6
- # let(:user) { 'user' }
7
- # let(:indexName) { 'key-val-8' }
8
- # let(:password) { 'pass' }
9
- # let(:elastic) { '192.168.3.24:9200' }
10
-
11
- # subject { described_class.new(elastic, user, password, indexName) }
12
-
13
- # after do
14
- # subject.close
15
- # end
16
-
17
- # describe "initialization" do
18
- # it "should be successful" do
19
- # expect { subject }.not_to raise_error
20
- # end
21
- # end
22
-
23
- # describe "read a value that doesn't exists" do
24
- # it "should return nil" do
25
- # res = subject.query "ids", "snort", "123:456:789"
26
- # expect(res).to be_nil
27
- # end
28
- # end
29
-
30
- # describe "write a value then read" do
31
- # let(:data) { "blob" }
32
-
33
- # it "write should be successful" do
34
- # expect { subject.save 1234, "am", "my-product", "not-my-name", 'something else' }.not_to raise_error
35
- # expect { subject.save 12345, "am", "my-product", "my-name", data }.not_to raise_error
36
- # sleep(2)
37
- # end
38
-
39
- # it "read the new value should succeed" do
40
- # res = subject.query "am", "my-product", "my-name"
41
- # expect(res).to eq(data)
42
- # end
43
- # end
44
- # end