fluent-plugin-jfrog-siem 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 306eb5d59fd5e00e2e8feda0339a3b635f1e61aaabff9312ad372dc714c3ff8f
-  data.tar.gz: 469ea7950f9d96236a88159a797f17077b31f5c3d7ddc19ca91f4d6209963a9a
+  metadata.gz: e34dd9d1fb60e95b931f7c67bb508e635057b9e2f349f681eb2b54cb182c2396
+  data.tar.gz: b0acda337195accb81ce6bf3ee7bf05ef7e79630793dbc7553c0df5e541e26a3
 SHA512:
-  metadata.gz: 02db6faa97750196fd42a0b04a8a8f517dfa0a26ad585778b8283b4cce9814ff239d96eb0a9ca34ef02a2f66bab9bb632cf8c0194afe7a3e4902ea08f4d9fd77
-  data.tar.gz: 745bef31330a205aac78d2f49e2b25f6f924f52c3d8a0be35d3d1f4aee77f1c862b8e8124a95ed62d90befc3e5a351061cbe1b902482484e3778081404737527
+  metadata.gz: 9116c506cf562af1566e6001748f5464752b2c07aace2d9f6fe04573c5a97db4cbf0e638d048dea536387559a71fa493fad882dc55650f1597c4030ab13be962
+  data.tar.gz: 2aaeaa3f4a22753db0b97c35b93938c4049cb9037f56f3cc6199dc222034a9d26609e1906973c0b9f11960f856227b2cf93ab000e8bd677aee9045df0bd684b0

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/Gemfile.lock

@@ -0,0 +1,90 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-jfrog-siem (1.0.0)
+      concurrent-ruby (~> 1.1.8)
+      concurrent-ruby-edge
+      fluentd (>= 0.14.10, < 2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.1.8)
+    concurrent-ruby-edge (0.6.0)
+      concurrent-ruby (~> 1.1.6)
+    cool.io (1.7.1)
+    diff-lcs (1.4.4)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    fluentd (1.13.1)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
+      yajl-ruby (~> 1.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http_parser.rb (0.6.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2021.0225)
+    msgpack (1.4.2)
+    netrc (0.11.0)
+    power_assert (2.0.0)
+    rake (12.3.3)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    serverengine (2.2.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    test-unit (3.4.2)
+      power_assert
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    webrick (1.7.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  concurrent-ruby (~> 1.1.8)
+  concurrent-ruby-edge
+  fluent-plugin-jfrog-siem!
+  rake (~> 12.0)
+  rest-client (~> 2.0)
+  rspec (~> 3.10.0)
+  test-unit (~> 3.0)
+
+BUNDLED WITH
+   2.2.3

data/Rakefile

@@ -4,7 +4,7 @@ Bundler::GemHelper.install_tasks
 require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
-  t.libs.push("lib", "test")
+  t.libs.push("lib", "test", 'lib/fluent/plugin')
   t.test_files = FileList["test/**/test_*.rb"]
   t.verbose = true
   t.warning = false

data/fluent-plugin-jfrog-siem.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-siem"
-  spec.version = "1.0.0"
+  spec.version = "2.0.0"
   spec.authors = ["John Peterson", "Mahitha Byreddy"]
   spec.email   = ["johnp@jfrog.com", "mahithab@jfrog.com"]
 
@@ -24,7 +24,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_development_dependency "rest-client", "~> 2.0"
-  spec.add_development_dependency "thread", "~> 0.2.2"
-  spec.add_runtime_dependency "thread", "~> 0.2.2"
+  spec.add_development_dependency "concurrent-ruby", "~> 1.1.8"
+  spec.add_development_dependency "concurrent-ruby-edge", '>= 0'
+  spec.add_development_dependency 'rspec', '~> 3.10.0'
+
+  spec.add_runtime_dependency "rest-client", "~> 2.0"
+  spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8"
+  spec.add_runtime_dependency "concurrent-ruby-edge", '>= 0'
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/lib/fluent/plugin/in_jfrog_siem.rb

@@ -14,10 +14,10 @@
 # limitations under the License.
 require "fluent/plugin/input"
 require "rest-client"
-require "thread/pool"
-require "json"
 require "date"
 require "uri"
+require "fluent/plugin/xray"
+require "fluent/plugin/position_file"
 
 module Fluent
   module Plugin
@@ -32,11 +32,10 @@ module Fluent
       config_param :jpd_url, :string, default: ""
       config_param :username, :string, default: ""
       config_param :apikey, :string, default: ""
-      config_param :pos_file, :string, default: ""
       config_param :batch_size, :integer, default: 25
-      config_param :thread_count, :integer, default: 5
       config_param :wait_interval, :integer, default: 60
-
+      config_param :from_date, :string, default: ""
+      config_param :pos_file_path, :string, default: ""
 
       # `configure` is called before `start`.
       # 'conf' is a `Hash` that includes the configuration parameters.
@@ -59,22 +58,14 @@ module Fluent
           raise Fluent::ConfigError, "Must define the API Key to use for authentication."
         end
 
-        if @pos_file == ""
-          raise Fluent::ConfigError, "Must define a position file to record last SIEM violation pulled."
-        end
-
-        if @thread_count < 1
-          raise Fluent::ConfigError, "Must define at least one thread to process violation details."
-        end
-
-        if @thread_count > @batch_size
-          raise Fluent::ConfigError, "Violation detail url thread count exceeds batch size."
-        end
-
         if @wait_interval < 1
           raise Fluent::ConfigError, "Wait interval must be greater than 1 to wait between pulling new events."
         end
 
+        if @from_date == ""
+          puts "From date not specified, so getting violations from current date if pos_file doesn't exist"
+        end
+
       end
 
 
@@ -94,108 +85,19 @@ module Fluent
 
 
       def run
-        call_home(@jpd_url)
-        # runs the violation pull
-        last_created_date_string = get_last_item_create_date()
-        begin
-          last_created_date = DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
-        rescue
-          last_created_date = DateTime.parse("1970-01-01T00:00:00Z").strftime("%Y-%m-%dT%H:%M:%SZ")
-        end
-        offset_count=1
-        left_violations=0
-        waiting_for_violations = false
-        xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } }
-
-        while true
-          # Grab the batch of records
-          resp=get_xray_violations(xray_json, @jpd_url)
-          number_of_violations = JSON.parse(resp)['total_violations']
-          if left_violations <= 0
-            left_violations = number_of_violations
-          end
-
-          xray_violation_urls_list = []
-          for index in 0..JSON.parse(resp)['violations'].length-1 do
-            # Get the violation
-            item = JSON.parse(resp)['violations'][index]
-
-            # Get the created date and check if we should skip (already processed) or process this record.
-            created_date_string = item['created']
-            created_date = DateTime.parse(created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
+        # call_home(@jpd_url)
 
-            # Determine if we need to persist this record or not
-            persistItem = true
-            if waiting_for_violations
-              if created_date <= last_created_date
-                # "not persisting it - waiting for violations"
-                # waiting and same last timestamp (left violations in batch)
-                persistItem = false
-              end
-              if created_date > last_created_date
-                # new violation while waiting
-                persistItem = true
-                waiting_for_violations = false
-              end
-            else
-              if created_date < last_created_date
-                # "persisting everything"
-                persistItem = true
-              end
-            end
+        last_created_date = get_last_item_create_date()
 
-            # Publish the record to fluentd
-            if persistItem
-
-              now = Fluent::Engine.now
-              router.emit(@tag, now, item)
-
-              # write to the pos_file created_date_string
-              open(@pos_file, 'a') do |f|
-                f << "#{created_date_string}\n"
-              end
-
-              # Mark this as the last record successfully processed
-              last_created_date_string = created_date_string
-              last_created_date = created_date
-
-              # Grab violation detail url and add to url list to process w/ thread pool
-              xray_violation_details_url=item['violation_details_url']
-              xray_violation_urls_list.append(xray_violation_details_url)
-            end
-          end
-
-          # iterate over url array adding to thread pool each url.
-          # limit max workers to thread count to prevent overloading xray.
-          thread_pool = Thread.pool(thread_count)
-          thread_pool.process {
-            for xray_violation_url in xray_violation_urls_list do
-              pull_violation_details(xray_violation_url)
-            end
-          }
-          thread_pool.shutdown
-
-          # reduce left violations by jump size (not all batches have full item count??)
-          left_violations = left_violations - @batch_size
-          if left_violations <= 0
-            waiting_for_violations = true
-            sleep(@wait_interval)
-          else
-            # Grab the next record to process for the violation details url
-            waiting_for_violations = false
-            offset_count = offset_count + 1
-            xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } }
-          end
-        end
-      end
-
-
-      # pull the last item create date from the pos_file return created_date_string
-      def get_last_item_create_date()
-        if(!(File.exist?(@pos_file)))
-          @pos_file = File.new(@pos_file, "w")
+        if (@from_date != "")
+          last_created_date = DateTime.parse(@from_date).strftime("%Y-%m-%dT%H:%M:%SZ")
         end
-        return IO.readlines(@pos_file).last
+        date_since = last_created_date
+        puts "Getting queries from #{date_since}"
+        xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, router, @tag)
+        violations_channel = xray.violations(date_since)
+        xray.violation_details(violations_channel)
+        sleep 100
       end
 
       #call home functionality
@@ -209,119 +111,27 @@ module Fluent
             :password => @apikey,
             :headers => { :accept => :json, :content_type => :json}
         ).execute do |response, request, result|
-            puts "Posting call home information"
+          puts "Posting call home information"
         end
       end
 
-      # queries the xray API for violations based upon the input json
-      def get_xray_violations_detail(xray_violation_detail_url)
-        response = RestClient::Request.new(
-            :method => :get,
-            :url => xray_violation_detail_url,
-            :user => @username,
-            :password => @apikey
-        ).execute do |response, request, result|
-          case response.code
-          when 200
-            return response.to_str
-          else
-            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
-          end
-        end
-      end
-
-
-      # queries the xray API for violations based upon the input json
-      def get_xray_violations(xray_json, jpd_url)
-        response = RestClient::Request.new(
-            :method => :post,
-            :url => jpd_url + "/xray/api/v1/violations",
-            :payload => xray_json.to_json,
-            :user => @username,
-            :password => @apikey,
-            :headers => { :accept => :json, :content_type => :json}
-        ).execute do |response, request, result|
-          case response.code
-          when 200
-            return response.to_str
-          else
-            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
-          end
-        end
-      end
-
-      # normalizes Xray data according to common information models for all log-vendors
-      def data_normalization(detailResp)
-        detailResp_json = JSON.parse(detailResp)
-        cve = []
-        cvss_v2_list = []
-        cvss_v3_list = []
-        policy_list = []
-        rule_list = []
-        impacted_artifact_url_list = []
-        if detailResp_json.key?('properties')
-          properties = detailResp_json['properties']
-          for index in 0..properties.length-1 do
-            if properties[index].key?('cve')
-              cve.push(properties[index]['cve'])
-            end
-            if properties[index].key?('cvss_v2')
-              cvss_v2_list.push(properties[index]['cvss_v2'])
-            end
-            if properties[index].key?('cvss_v3')
-              cvss_v3_list.push(properties[index]['cvss_v3'])
-            end
-          end
-
-          detailResp_json["cve"] = cve.sort.reverse[0]
-          cvss_v2 = cvss_v2_list.sort.reverse[0]
-          cvss_v3 = cvss_v3_list.sort.reverse[0]
-          if !cvss_v3.nil?
-            cvss = cvss_v3
-          elsif !cvss_v2.nil?
-            cvss = cvss_v2
-          end
-          cvss_score = cvss[0..2]
-          cvss_version = cvss.split(':')[1][0..2]
-          detailResp_json["cvss_score"] = cvss_score
-          detailResp_json["cvss_version"] = cvss_version
-        end
-
-        if detailResp_json.key?('matched_policies')
-          matched_policies = detailResp_json['matched_policies']
-          for index in 0..matched_policies.length-1 do
-            if matched_policies[index].key?('policy')
-              policy_list.push(matched_policies[index]['policy'])
-            end
-            if matched_policies[index].key?('rule')
-              rule_list.push(matched_policies[index]['rule'])
-            end
-          end
-          detailResp_json['policies'] = policy_list
-          detailResp_json['rules'] = rule_list
-        end
-
-        impacted_artifacts = detailResp_json['impacted_artifacts']
-        for impacted_artifact in impacted_artifacts do
-          matchdata = impacted_artifact.match /default\/(?<repo_name>[^\/]*)\/(?<path>.*)/
-          impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " "
-          impacted_artifact_url_list.append(impacted_artifact_url)
+      # pull the last item create date from the pos_file return created_date_string
+      def get_last_item_create_date()
+        recent_pos_file = get_recent_pos_file()
+        if recent_pos_file != nil
+          last_created_date_string = IO.readlines(recent_pos_file).last
+          return DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
+        else
+          return DateTime.now.strftime("%Y-%m-%dT%H:%M:%SZ")
         end
-        detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list
-        return detailResp_json
       end
 
-      def pull_violation_details(xray_violation_detail_url)
-        begin
-          detailResp=get_xray_violations_detail(xray_violation_detail_url)
-          time = Fluent::Engine.now
-          detailResp_json = data_normalization(detailResp)
-          router.emit(@tag, time, detailResp_json)
-        rescue
-          raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}"
-        end
+      def get_recent_pos_file()
+        pos_file = @pos_file_path + "*.siem.pos"
+        return Dir.glob(pos_file).sort.last
       end
 
     end
   end
 end
+

data/lib/fluent/plugin/position_file.rb

@@ -0,0 +1,32 @@
+class PositionFile
+
+  def initialize(pos_file_path)
+    @pos_file_path = pos_file_path
+  end
+
+  def processed?(violation)
+    File.exist?(pos_file_name(violation)) && found?(violation)
+  end
+
+  def write(violation)
+    File.open(pos_file_name(violation), 'a') do |f|
+      f << violation_entry(violation)
+      f << "\n"
+    end
+  end
+
+  private
+  def found?(violation)
+    return File.open(pos_file_name(violation)) { |f| f.find { |line| line.include? violation_entry(violation) } }
+  end
+
+  def violation_entry(violation)
+    created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ")
+    [created_date, violation['watch_name'], violation['issue_id']].join(',')
+  end
+
+  def pos_file_name(violation)
+    pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d")
+    @pos_file_path + "jfrog_siem_log_#{pos_file_date}.siem.pos"
+  end
+end