fluent-plugin-jfrog-siem 2.0.5 → 2.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 602f632100f31aa06677f7f5ce98def8d1e51aa43d883cc18e6c8495f80b79dc
-  data.tar.gz: 8d28142729182b919f69ce6893928477e605aa658ae09f816f0d3f6e5dedace6
+  metadata.gz: fa44f9a815aa280de6977f666e2da6ea8be2bae24cc8c7365430cbfd6bc3496b
+  data.tar.gz: ad2eec36ca931720158e21e9abcf7d99e4c32618365ef300ca465df59339e307
 SHA512:
-  metadata.gz: 3b1e9b8c84c67e39bcc5eb935ff498bda659f1ee360caf2967b062e3b24a1874308beb831f9fcd57b985c0eb7dd865b286064eeba1ccc7f93b70f2864a87dc51
-  data.tar.gz: 4955cc838617021f7c3c84268af0840cee892ce17fdeafbaa41e67127fd08685cb0d20961c57d47c9b96a76b3d6371ad6ddb9b7ec5de27439a402439c97ff5b1
+  metadata.gz: f7cca01dc886183ad29fb99151f0c25f56900cce252f4c17eb88c5a4ec2e9c4dc9571049540b2dc5e1d36b5db612f4ee347d063e6c26a3d2506586a35b04ebf0
+  data.tar.gz: 7691c4e9dd2bdb3c6df6e562bd497842de79b086c46f28af0924b19c021350effa2b382e3cb8ab948af1a30ea6a52e470ba4467e4e9adcea38157fde9cb65dde

data/fluent-plugin-jfrog-siem.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-siem"
-  spec.version = "2.0.5"
+  spec.version = "2.0.6"
   spec.authors = ["Mahitha Byreddy", "Sudhindra Rao","Giridharan Ramasamy"]
   spec.email   = ["mahithab@jfrog.com", "sudhindrar@jfrog.com", "girir@jfrog.com"]
 
@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.10.0'
 
   spec.add_runtime_dependency "rest-client", "~> 2.0"
-  spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8"
+  spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8" , "< 1.1.10"
   spec.add_runtime_dependency "concurrent-ruby-edge", '>= 0'
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/lib/fluent/plugin/in_jfrog_siem.rb

@@ -118,9 +118,11 @@ module Fluent
       def get_last_item_create_date()
         recent_pos_file = get_recent_pos_file()
         if recent_pos_file != nil
+          puts "Position file already exists so pulling the latest create_date from it"
           last_created_date_string = IO.readlines(recent_pos_file).last
           return DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
         else
+          puts "Position file doesn't exist so fetching current DateTime to form a new position file"
           return DateTime.now.strftime("%Y-%m-%dT%H:%M:%SZ")
         end
       end

data/lib/fluent/plugin/xray.rb

@@ -1,186 +1,195 @@
-require 'concurrent'
-require 'concurrent-edge'
-require 'json'
-require "fluent/plugin/position_file"
-
-class Xray
-  def initialize(jpd_url, username, api_key, token, wait_interval, batch_size, pos_file_path, router, tag)
-    @jpd_url = jpd_url
-    @username = username
-    @api_key = api_key
-    @token = token
-    @wait_interval = wait_interval
-    @batch_size = batch_size
-    @pos_file_path = pos_file_path
-    @router = router
-    @tag = tag
-  end
-
-  def violations(date_since)
-    violations_channel = Concurrent::Channel.new(capacity: @batch_size)
-    page_number = 1
-    timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do
-      xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } }
-      resp = get_violations(xray_json)
-      page_violation_count = resp['violations'].length
-      puts "Total violations count is #{resp['total_violations']}"
-      if resp['total_violations'] > 0
-        puts "Number of Violations in page #{page_number} are #{page_violation_count}"
-        resp['violations'].each {|v| violations_channel = process(v, violations_channel) }
-        page_number += 1 if next_page?(page_violation_count)
-      end
-    end
-    timer_task.execute
-
-    violations_channel
-  end
-
-  def violation_details(violations_channel)
-    violations_channel.each do |v|
-      Concurrent::Promises.future(v) do |v|
-        pull_violation_details(v['violation_details_url'])
-        pos_file = PositionFile.new(@pos_file_path)
-        pos_file.write(v)
-      end
-    end
-  end
-
-  def pull_violation_details(xray_violation_detail_url)
-    begin
-      detailResp_json = data_normalization(get_violations_detail(xray_violation_detail_url))
-      time = Fluent::Engine.now
-      puts detailResp_json
-      @router.emit(@tag, time, detailResp_json)
-    rescue => e
-      puts "error: #{e}"
-      raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}"
-    end
-  end
-
-  def get_violations_detail(xray_violation_detail_url)
-    if !@token.nil? && @token != ''
-      response = RestClient::Request.new(
-          :method => :get,
-          :url => @jpd_url + xray_violation_detail_url[xray_violation_detail_url.index('/xray/'),xray_violation_detail_url.length],
-          :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + @token }
-      )
-    elsif !@api_key.nil? && @api_key != ''
-      response = RestClient::Request.new(
-          :method => :get,
-          :url => @jpd_url + xray_violation_detail_url[xray_violation_detail_url.index('/xray/'),xray_violation_detail_url.length],
-          :user => @username,
-          :password => @api_key
-      )
-    end
-
-    response.execute do |response, request, result|
-      case response.code
-      when 200
-        return JSON.parse(response.to_s)
-      else
-        puts "error: #{response.to_json}"
-        raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations details."
-      end
-    end
-  end
-
-  def data_normalization(detailResp_json)
-    cve = []
-    cvss_v2_list = []
-    cvss_v3_list = []
-    policy_list = []
-    rule_list = []
-    impacted_artifact_url_list = []
-    if detailResp_json.key?('properties')
-      properties = detailResp_json['properties']
-      for index in 0..properties.length-1 do
-        if properties[index].key?('cve')
-          cve.push(properties[index]['cve'])
-        end
-        if properties[index].key?('cvss_v2')
-          cvss_v2_list.push(properties[index]['cvss_v2'])
-        end
-        if properties[index].key?('cvss_v3')
-          cvss_v3_list.push(properties[index]['cvss_v3'])
-        end
-      end
-
-      detailResp_json["cve"] = cve.sort.reverse[0]
-      cvss_v2 = cvss_v2_list.sort.reverse[0]
-      cvss_v3 = cvss_v3_list.sort.reverse[0]
-      if !cvss_v3.nil?
-        cvss = cvss_v3
-      elsif !cvss_v2.nil?
-        cvss = cvss_v2
-      end
-      cvss_score = cvss[0..2]
-      cvss_version = cvss.split(':')[1][0..2]
-      detailResp_json["cvss_score"] = cvss_score
-      detailResp_json["cvss_version"] = cvss_version
-    end
-
-    if detailResp_json.key?('matched_policies')
-      matched_policies = detailResp_json['matched_policies']
-      for index in 0..matched_policies.length-1 do
-        if matched_policies[index].key?('policy')
-          policy_list.push(matched_policies[index]['policy'])
-        end
-        if matched_policies[index].key?('rule')
-          rule_list.push(matched_policies[index]['rule'])
-        end
-      end
-      detailResp_json['policies'] = policy_list
-      detailResp_json['rules'] = rule_list
-    end
-
-    detailResp_json['impacted_artifacts'].each do |impacted_artifact|
-      matchdata = impacted_artifact.match /default\/(?<repo_name>[^\/]*)\/(?<path>.*)/
-      impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " "
-      impacted_artifact_url_list.append(impacted_artifact_url)
-    end
-    detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list
-    return detailResp_json
-  end
-
-  def process(violation, violations_channel)
-    pos_file = PositionFile.new(@pos_file_path)
-    violations_channel << violation unless pos_file.processed?(violation)
-    violations_channel
-  end
-
-  private
-  def get_violations(xray_json)
-    if !@token.nil? && @token != ''
-      response = RestClient::Request.new(
-          :method => :post,
-          :url => @jpd_url + "/xray/api/v1/violations",
-          :payload => xray_json.to_json,
-          :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + @token }
-      )
-    elsif !@api_key.nil? && @api_key != ''
-      response = RestClient::Request.new(
-          :method => :post,
-          :url => @jpd_url + "/xray/api/v1/violations",
-          :payload => xray_json.to_json,
-          :user => @username,
-          :password => @api_key,
-          :headers => { :accept => :json, :content_type => :json }
-      )
-    end
-    response.execute do |response, request, result|
-      case response.code
-      when 200
-        return JSON.parse(response.to_str)
-      else
-        puts "error: #{response.to_json}"
-        raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}"
-      end
-    end
-  end
-
-  def next_page?(count)
-    count == @batch_size
-  end
-
-end
-
+require 'concurrent'
+require 'concurrent-edge'
+require 'json'
+require "fluent/plugin/position_file"
+
+class Xray
+  def initialize(jpd_url, username, api_key, token, wait_interval, batch_size, pos_file_path, router, tag)
+    @jpd_url = jpd_url
+    @username = username
+    @api_key = api_key
+    @token = token
+    @wait_interval = wait_interval
+    @batch_size = batch_size
+    @pos_file_path = pos_file_path
+    @router = router
+    @tag = tag
+  end
+
+  def violations(date_since)
+    violations_channel = Concurrent::Channel.new(capacity: @batch_size)
+    page_number = 1
+    timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do
+      xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } }
+      puts "Fetching Xray Violations with #{xray_json} parameters"
+      resp = get_violations(xray_json)
+      page_violation_count = resp['violations'].length
+      puts "Total violations count is #{resp['total_violations']}"
+      if resp['total_violations'] > 0
+        puts "Number of Violations in page #{page_number} are #{page_violation_count}"
+        resp['violations'].each {|v| violations_channel = process(v, violations_channel) }
+        page_number += 1 if next_page?(page_violation_count)
+      end
+    end
+    timer_task.execute
+
+    violations_channel
+  end
+
+  def violation_details(violations_channel)
+    violations_channel.each do |v|
+      Concurrent::Promises.future(v) do |v|
+        process_violation_details(v['violation_details_url'])
+        pos_file = PositionFile.new(@pos_file_path)
+        puts "Adding issue #{v['issue_id']} to position file at #{@pos_file_path}"
+        pos_file.write(v)
+      end
+    end
+  end
+
+  def process_violation_details(xray_violation_detail_url)
+    begin
+      detailResp_json = data_normalization(get_violations_detail(xray_violation_detail_url))
+      time = Fluent::Engine.now
+      puts "Emitting normalized Xray Violation #{detailResp_json['issue_id']}"
+      @router.emit(@tag, time, detailResp_json)
+    rescue => e
+      puts "Process Violation details error: #{e}"
+      raise Fluent::ConfigError, "Process Violation details error: #{e}"
+    end
+  end
+
+  def get_violations_detail(xray_violation_detail_url)
+    if !@token.nil? && @token != ''
+      response = RestClient::Request.new(
+          :method => :get,
+          :url => @jpd_url + xray_violation_detail_url[xray_violation_detail_url.index('/xray/'),xray_violation_detail_url.length],
+          :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + @token }
+      )
+    elsif !@api_key.nil? && @api_key != ''
+      response = RestClient::Request.new(
+          :method => :get,
+          :url => @jpd_url + xray_violation_detail_url[xray_violation_detail_url.index('/xray/'),xray_violation_detail_url.length],
+          :user => @username,
+          :password => @api_key
+      )
+    end
+
+    response.execute do |response, request, result|
+      case response.code
+      when 200
+        return JSON.parse(response.to_s)
+      else
+        puts "Validation failed error (cannot reach Artifactory to pull Xray Violation details): #{response.to_json}"
+        raise Fluent::ConfigError, "Validation failed error (cannot reach Artifactory to pull Xray Violation details): #{response.to_json}"
+      end
+    end
+  end
+
+  def data_normalization(detailResp_json)
+    cve = []
+    cvss_v2_list = []
+    cvss_v3_list = []
+    policy_list = []
+    rule_list = []
+    impacted_artifact_url_list = []
+    if detailResp_json.key?('properties')
+      properties = detailResp_json['properties']
+      for index in 0..properties.length-1 do
+        if properties[index].key?('cve')
+          cve.push(properties[index]['cve'])
+        end
+        if properties[index].key?('cvss_v2')
+          cvss_v2_list.push(properties[index]['cvss_v2'])
+        end
+        if properties[index].key?('cvss_v3')
+          cvss_v3_list.push(properties[index]['cvss_v3'])
+        end
+      end
+
+      detailResp_json["cve"] = cve.sort.reverse[0]
+      cvss_v2 = cvss_v2_list.sort.reverse[0]
+      cvss_v3 = cvss_v3_list.sort.reverse[0]
+      if !cvss_v3.nil?
+        cvss = cvss_v3
+      elsif !cvss_v2.nil?
+        cvss = cvss_v2
+      end
+      cvss_score = cvss[0..2]
+      cvss_version = cvss.split(':')[1][0..2]
+      detailResp_json["cvss_score"] = cvss_score
+      detailResp_json["cvss_version"] = cvss_version
+    end
+
+    if detailResp_json.key?('matched_policies')
+      matched_policies = detailResp_json['matched_policies']
+      for index in 0..matched_policies.length-1 do
+        if matched_policies[index].key?('policy')
+          policy_list.push(matched_policies[index]['policy'])
+        end
+        if matched_policies[index].key?('rule')
+          rule_list.push(matched_policies[index]['rule'])
+        end
+      end
+      detailResp_json['policies'] = policy_list
+      detailResp_json['rules'] = rule_list
+    end
+
+    detailResp_json['impacted_artifacts'].each do |impacted_artifact|
+      matchdata = impacted_artifact.match /default\/(?<repo_name>[^\/]*)\/(?<path>.*)/
+      impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " "
+      impacted_artifact_url_list.append(impacted_artifact_url)
+    end
+    detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list
+    return detailResp_json
+  end
+
+  def process(violation, violations_channel)
+    pos_file = PositionFile.new(@pos_file_path)
+    unless pos_file.processed?(violation)
+      violations_channel << violation
+    else
+      puts "Violation #{violation['issue_id']} is already processed"
+    end
+    #violations_channel << violation unless pos_file.processed?(violation)
+    violations_channel
+  end
+
+  private
+  def get_violations(xray_json)
+    if !@token.nil? && @token != ''
+      puts "Validating JPD access token and fetching violations"
+      response = RestClient::Request.new(
+          :method => :post,
+          :url => @jpd_url + "/xray/api/v1/violations",
+          :payload => xray_json.to_json,
+          :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + @token }
+      )
+    elsif !@api_key.nil? && @api_key != ''
+      puts "Validating JPD API Key and fetching violations"
+      response = RestClient::Request.new(
+          :method => :post,
+          :url => @jpd_url + "/xray/api/v1/violations",
+          :payload => xray_json.to_json,
+          :user => @username,
+          :password => @api_key,
+          :headers => { :accept => :json, :content_type => :json }
+      )
+    end
+    response.execute do |response, request, result|
+      case response.code
+      when 200
+        return JSON.parse(response.to_str)
+      else
+        puts "Validation failed error (cannot reach Artifactory to pull Xray Violations): #{response.to_json}"
+        raise Fluent::ConfigError, "Validation failed error (cannot reach Artifactory to pull Xray Violations): #{response.to_json}"
+      end
+    end
+  end
+
+  def next_page?(count)
+    count == @batch_size
+  end
+
+end
+

data/test/plugin/test_in_jfrog_siem.rb

@@ -1,41 +1,41 @@
-require "helper"
-require "fluent/plugin/in_jfrog_siem.rb"
-
-class JfrogSiemInputTest < Test::Unit::TestCase
-  setup do
-    Fluent::Test.setup
-  end
-
-  test "failure" do
-    #flunk
-  end
-
-  # Default configuration for tests
-  CONFIG = %[
-     tag "jfrog.xray.siem.vulnerabilities"
-     jpd_url "JPDURL"
-     username "admin"
-     apikey "APIKEY"
-     pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/"
-     wait_interval 10
-     from_date "2016-01-01"
-     batch_size 25
-   ]
-
-  private
-
-  def create_driver(conf = CONFIG)
-    Fluent::Test::Driver::Input.new(Fluent::Plugin::JfrogSiemInput).configure(conf)
-  end
-
-  sub_test_case 'Testing' do
-    test 'Testing plugin in_jfrog_siem' do
-      d = create_driver(CONFIG)
-      begin
-        d.run
-      rescue => e
-        raise "Test failed due to #{e}"
-      end
-    end
-  end
-end
+require "helper"
+require "fluent/plugin/in_jfrog_siem.rb"
+
+class JfrogSiemInputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    #flunk
+  end
+
+  # Default configuration for tests
+  CONFIG = %[
+     tag "jfrog.xray.siem.vulnerabilities"
+     jpd_url "JPDURL"
+     username "admin"
+     apikey "APIKEY"
+     pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/"
+     wait_interval 10
+     from_date "2016-01-01"
+     batch_size 25
+   ]
+
+  private
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::JfrogSiemInput).configure(conf)
+  end
+
+  sub_test_case 'Testing' do
+    test 'Testing plugin in_jfrog_siem' do
+      d = create_driver(CONFIG)
+      begin
+        d.run
+      rescue => e
+        raise "Test failed due to #{e}"
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-siem
 version: !ruby/object:Gem::Version
-  version: 2.0.5
+  version: 2.0.6
 platform: ruby
 authors:
 - Mahitha Byreddy
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-06 00:00:00.000000000 Z
+date: 2023-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -137,6 +137,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.1.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -144,6 +147,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.1.10
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby-edge
   requirement: !ruby/object:Gem::Requirement