fluent-plugin-jfrog-siem 0.1.7 → 2.0.0

data/spec/position_file_spec.rb

@@ -0,0 +1,56 @@
+[
+  File.join(File.dirname(__FILE__), '..'),
+  File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'),
+  File.join(File.dirname(__FILE__), '..', 'spec'),
+].each do |dir|
+  $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
+end
+
+require 'position_file'
+require 'date'
+require 'rspec'
+
+
+RSpec.describe PositionFile do
+  describe "#processed?" do
+    let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} }
+
+    pos_file_date = DateTime.parse(Date.today.to_s).strftime("%Y-%m-%d")
+    temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos"
+
+    it "returns false when a violation has not been processed" do
+      pos_file = PositionFile.new(`pwd`)
+      allow(File).to receive(:open).and_yield []
+
+      expect(pos_file.processed?(JSON.parse(violation.to_json))).to be_falsey
+    end
+
+    it "returns true when a violation was found in the pos file" do
+      pos_file = PositionFile.new(`pwd`)
+
+      matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',')
+      another_violation = [violation[:created], "watch2", "12345"].join(',')
+      allow(File).to receive(:exist?).and_return true
+      allow(File).to receive(:open).and_yield [matching_violation, another_violation]
+
+      expect(pos_file.processed?(JSON.parse(violation.to_json))).to be_truthy
+    end
+
+  end
+
+  describe "#write" do
+    let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} }
+
+    it "returns false when a violation has not been processed" do
+      pos_file = PositionFile.new(`pwd`)
+
+      result = []
+      allow(File).to receive(:open).and_yield result
+
+      pos_file.write(JSON.parse(violation.to_json))
+
+      matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',')
+      expect(result.include? matching_violation).to be_truthy
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,111 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+
+[
+  File.join(File.dirname(__FILE__), '..'),
+  File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'),
+  File.join(File.dirname(__FILE__), '..', 'spec'),
+].each do |dir|
+  $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
+end
+
+require 'xray'
+
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

data/spec/xray_spec.rb

@@ -0,0 +1,135 @@
+[
+  File.join(File.dirname(__FILE__), '..'),
+  File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'),
+  File.join(File.dirname(__FILE__), '..', 'spec'),
+].each do |dir|
+  $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
+end
+
+require 'xray'
+require 'date'
+require 'rspec'
+require 'rest-client'
+
+
+RSpec.describe Xray do
+  describe "#violation_details" do
+
+    let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"),
+                        "watch_name": "watch1",
+                        "issue_id": "55444",
+                        "violation_details_url": "http://www.com"}
+                    }
+    let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"),
+                        "watch_name": "watch2",
+                        "issue_id": "55443",
+                        "violation_details_url": "http://www.com"}
+                    }
+
+    let(:violations) { Concurrent::Array.new }
+
+    it "creates a future for every violation" do
+      xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router, @tag)
+
+      (1..5).each do |i|
+        violations << i
+      end
+
+      promises = class_double("Concurrent::Promises").as_stubbed_const(:transfer_nested_constants => true)
+      expect(promises).to receive(:future).exactly(5).times
+
+      xray.violation_details(violations)
+    end
+
+    it "updates pos file for every violation (cannot do exactly tests since stubs with Concurrent ruby are broken)" do
+      router = double('router')
+      pos_file_path = `pwd`
+      xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, pos_file_path, router, @tag)
+
+      violations << JSON.parse(violation1.to_json)
+      violations << JSON.parse(violation2.to_json)
+
+      promises = class_double("Concurrent::Promises").as_stubbed_const(:transfer_nested_constants => true)
+      allow(promises).to receive(:future).and_yield(violation1).and_yield(violation2)
+
+      rest_client = double("RestClient::Request")
+      allow(RestClient::Request).to receive(:new).and_return rest_client
+      allow(rest_client).to receive(:execute).and_return(JSON.parse({'impacted_artifacts': []}.to_json))
+
+      pos_file = double(PositionFile)
+      allow(PositionFile).to receive(:new).and_return pos_file
+      allow(pos_file).to receive(:write)
+
+      fluent = class_double("Fluent::Engine").as_stubbed_const(:transfer_nested_constants => true)
+      allow(fluent).to receive(:now).and_return(DateTime.now)
+      allow(router).to receive(:emit)
+
+      xray.violation_details(JSON.parse(violations.to_json))
+    end
+  end
+
+  describe "#violations" do
+    it "runs a timer task to process violations periodically" do
+      batch_size = 25
+      pos_file_path = `pwd`
+      router = double('router')
+      xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, batch_size, pos_file_path, router, @tag)
+
+      channel = double(Concurrent::Channel)
+      expect(Concurrent::Channel).to receive(:new).with(capacity: batch_size).and_return(channel)
+
+      timer_task = double(Concurrent::TimerTask)
+      expect(Concurrent::TimerTask).to receive(:new).and_return timer_task
+
+      expect(timer_task).to receive(:execute)
+      xray.violations(Date.today)
+    end
+
+  end
+
+  describe "#process" do
+    let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"),
+                        "watch_name": "watch1",
+                        "issue_id": "55444",
+                        "violation_details_url": "http://www.com"}
+                    }
+    let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"),
+                        "watch_name": "watch2",
+                        "issue_id": "55443",
+                        "violation_details_url": "http://www.com"}
+                    }
+
+    let(:violations_channel) { Concurrent::Array.new }
+
+    it "skips processed violation" do
+      pos_file_path = `pwd`
+      xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router, @tag)
+
+      violations_channel << violation1
+
+      pos_file = double(PositionFile)
+      allow(PositionFile).to receive(:new).and_return pos_file
+      allow(pos_file).to receive(:processed?).and_return true
+
+      xray.process(violation1, violations_channel)
+
+      expect(violations_channel.size).to eq 1
+    end
+
+    it "adds unprocessed violation to the channel" do
+      pos_file_path = `pwd`
+      xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router, @tag)
+
+      violations_channel << violation1
+
+      pos_file = double(PositionFile)
+      allow(PositionFile).to receive(:new).and_return pos_file
+      allow(pos_file).to receive(:processed?).and_return false
+
+      xray.process(violation2, violations_channel)
+
+      expect(violations_channel.size).to eq 2
+    end
+  end
+
+end

data/test/helper.rb

@@ -4,5 +4,6 @@ require "fluent/test"
 require "fluent/test/driver/input"
 require "fluent/test/helpers"
 
+
 Test::Unit::TestCase.include(Fluent::Test::Helpers)
 Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_in_jfrog_siem.rb

@@ -11,13 +11,35 @@ class JfrogSiemInputTest < Test::Unit::TestCase
   end
 
   # Default configuration for tests
+  # CONFIG = %[
+  #   tag "partnership.test_tag"
+  #   jpd_url "https://partnership.jfrog.io/"
+  #   username "sudhindrar"
+  #   apikey "AKCp8ihpNg2JE5PV3nRXZQsmMGmzX9VTX6wN51hQBFRC1CXQWzGrKQvFL1tsw7aochjoQXAZq"
+  #   pos_file "test_pos.txt"
+  #   wait_interval 30
+  # ]
+
   CONFIG = %[
-    tag "test_tag"
-    jpd_url <jpd_url>
-    access_token <access_token>
-    pos_file "test_pos.txt"
+    tag "sudhindra-xray-rt.test_tag"
+    jpd_url "https://sudhindra-xray-rt.jfrog.tech/"
+    username "admin"
+    apikey "AKCp8jQd1zP4oKv43SNgewrNwikd1iAQznfhSfx3T249eVMkGnJnSjCpNsuv8vtHWChKLfJ1w"
+    wait_interval 1
+    from_date "2016-01-01"
   ]
 
+  # CONFIG = %[
+  #   tag "jfrog.xray.siem.vulnerabilities"
+  #   jpd_url "JPDURL"
+  #   username "admin"
+  #   apikey "APIKEY"
+  #   pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/"
+  #   wait_interval 10
+  #   from_date "2016-01-01"
+  #   batch_size 25
+  # ]
+
   private
 
   def create_driver(conf = CONFIG)
@@ -27,7 +49,11 @@ class JfrogSiemInputTest < Test::Unit::TestCase
   sub_test_case 'Testing' do
     test 'Testing plugin in_jfrog_siem' do
       d = create_driver(CONFIG)
-      d.run
+      begin
+        d.run
+      rescue => e
+        raise "Test failed due to #{e}"
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-siem
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 2.0.0
 platform: ruby
 authors:
 - John Peterson
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-21 00:00:00.000000000 Z
+date: 2021-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -68,33 +68,89 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: thread
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: 1.1.8
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: 1.1.8
 - !ruby/object:Gem::Dependency
-  name: thread
+  name: concurrent-ruby-edge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.10.0
+- !ruby/object:Gem::Dependency
+  name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.8
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby-edge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -124,14 +180,21 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".rspec"
+- CHANGELOG.md
 - Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
-- elastic.conf
 - fluent-plugin-jfrog-siem.gemspec
 - lib/fluent/plugin/in_jfrog_siem.rb
-- splunk.conf
+- lib/fluent/plugin/position_file.rb
+- lib/fluent/plugin/violations.json
+- lib/fluent/plugin/xray.rb
+- spec/position_file_spec.rb
+- spec/spec_helper.rb
+- spec/xray_spec.rb
 - test/helper.rb
 - test/plugin/test_in_jfrog_siem.rb
 homepage: https://github.com/jfrog/log-analytics
@@ -159,5 +222,8 @@ specification_version: 4
 summary: JFrog SIEM fluent input plugin will send the SIEM events from JFrog Xray
   to Fluentd
 test_files:
+- spec/position_file_spec.rb
+- spec/spec_helper.rb
+- spec/xray_spec.rb
 - test/helper.rb
 - test/plugin/test_in_jfrog_siem.rb