fluent-plugin-jfrog-siem 0.1.3 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd2cc7532e266f9d58b4dbbf18479c1f34fd975649f7aa5ab9e5e671f449025b
-  data.tar.gz: fa8943afc154c4d6136ac1605f5bf85a240ce6ac58bf3ee61f42eb162f447aa6
+  metadata.gz: 4e222c0afdaf25a0aa38e4167236c2f06b5c7f9c889ca544f3a05677609c2c1e
+  data.tar.gz: '097fd2e7d1b8b0b8394e11efa9abbcfa5d0e67412d3284bdd5c5fe47badde641'
 SHA512:
-  metadata.gz: b47739ce004b1d48fddd026188aed933e4397b3730eb1d9bde1511f8bd623dc687246fa0ad85a91e814aa46175e5b0aee44213e12f18be97f078fb71b44904f7
-  data.tar.gz: a377d04e8b41cb108402036158d82632390db472cc4e7a97655110465e79a939e2bc90a25945b4932463f0561294c6f4af46fe70fa730141a403e1665a960a6e
+  metadata.gz: 18b38f238bef87f6e015aa6e6eda87ba58586e6e85ef5f29d58392b91ad1fc5c12da33777ce7b477f222d966df2c5eb7082d1cbe757cc712703adb4511945057
+  data.tar.gz: 49a13e9f1aeec783f9e7f35ca9750d3d3c4398d1e6facfa029dbfa6dec8a27d083941e995722cdb7ffa66937fb747bc079fe439e534e5dc23023e7543d56dc4b

data/Gemfile

@@ -1,2 +1,3 @@
 source "https://rubygems.org"
+
 gemspec

data/README.md

@@ -68,13 +68,19 @@ Splunk:
 
 Splunk setup can be found at [README.](https://github.com/jfrog/log-analytics-splunk/blob/master/README.md)
 ````text
-wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/splunk.conf
+wget https://raw.githubusercontent.com/jfrog/log-analytics-splunk/master/siem/splunk_siem.conf
 ````
 Elasticsearch: 
 
 Elasticsearch Kibana setup can be found at [README.](https://github.com/jfrog/log-analytics-elastic/blob/master/README.md)
 ````text
-wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/elastic.conf
+wget https://raw.githubusercontent.com/jfrog/log-analytics-elastic/master/siem/elastic_siem.conf
+````
+Datadog: 
+
+Datadog setup can be found at [README.](https://github.com/jfrog/log-analytics-datadog/blob/master/README.md)
+````text
+wget https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/siem/datadog_siem.conf
 ````
 
 #### Configuration parameters

data/fluent-plugin-jfrog-siem.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-siem"
-  spec.version = "0.1.3"
+  spec.version = "0.1.8"
   spec.authors = ["John Peterson", "Mahitha Byreddy"]
   spec.email   = ["johnp@jfrog.com", "mahithab@jfrog.com"]
 
@@ -24,5 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_development_dependency "rest-client", "~> 2.0"
+  spec.add_development_dependency "thread", "~> 0.2.2"
+  spec.add_runtime_dependency "thread", "~> 0.2.2"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/lib/fluent/plugin/in_jfrog_siem.rb

@@ -30,7 +30,8 @@ module Fluent
       # `:default` means that the parameter is optional.
       config_param :tag, :string, default: ""
       config_param :jpd_url, :string, default: ""
-      config_param :access_token, :string, default: ""
+      config_param :username, :string, default: ""
+      config_param :apikey, :string, default: ""
       config_param :pos_file, :string, default: ""
       config_param :batch_size, :integer, default: 25
       config_param :thread_count, :integer, default: 5
@@ -50,8 +51,12 @@ module Fluent
           raise Fluent::ConfigError, "Must define the JPD URL to pull Xray SIEM violations."
         end
 
-        if @access_token == ""
-          raise Fluent::ConfigError, "Must define the access token to use for authentication."
+        if @username == ""
+          raise Fluent::ConfigError, "Must define the username to use for authentication."
+        end
+
+        if @apikey == ""
+          raise Fluent::ConfigError, "Must define the API Key to use for authentication."
         end
 
         if @pos_file == ""
@@ -89,7 +94,7 @@ module Fluent
 
 
       def run
-        call_home(@jpd_url, @access_token)
+        call_home(@jpd_url)
         # runs the violation pull
         last_created_date_string = get_last_item_create_date()
         begin
@@ -99,11 +104,12 @@ module Fluent
         end
         offset_count=1
         left_violations=0
+        waiting_for_violations = false
         xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } }
 
         while true
           # Grab the batch of records
-          resp=get_xray_violations(xray_json, @jpd_url, @access_token)
+          resp=get_xray_violations(xray_json, @jpd_url)
           number_of_violations = JSON.parse(resp)['total_violations']
           if left_violations <= 0
             left_violations = number_of_violations
@@ -120,8 +126,16 @@ module Fluent
 
             # Determine if we need to persist this record or not
             persistItem = true
-            if created_date < last_created_date
-              persistItem = false
+            if waiting_for_violations
+              if created_date <= last_created_date
+                # "not persisting it - waiting for violations"
+                persistItem = false
+              end
+            else
+              if created_date < last_created_date
+                # "persisting everything"
+                persistItem = true
+              end
             end
 
             # Publish the record to fluentd
@@ -148,20 +162,21 @@ module Fluent
           # iterate over url array adding to thread pool each url.
           # limit max workers to thread count to prevent overloading xray.
           thread_pool = Thread.pool(thread_count)
-          for xray_violation_url in xray_violation_urls_list do
-            thread_pool.process {
-              pull_violation_details(xray_violation_url, @access_token)
-            }
-          end
-
+          thread_pool.process {
+            for xray_violation_url in xray_violation_urls_list do
+              pull_violation_details(xray_violation_url)
+            end
+          }
           thread_pool.shutdown
 
           # reduce left violations by jump size (not all batches have full item count??)
           left_violations = left_violations - @batch_size
           if left_violations <= 0
+            waiting_for_violations = true
             sleep(@wait_interval)
           else
             # Grab the next record to process for the violation details url
+            waiting_for_violations = false
             offset_count = offset_count + 1
             xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } }
           end
@@ -178,48 +193,53 @@ module Fluent
       end
 
       #call home functionality
-      def call_home(jpd_url, access_token)
+      def call_home(jpd_url)
         call_home_json = { "productId": "jfrogLogAnalytics/v0.5.1", "features": [ { "featureId": "Platform/Xray" }, { "featureId": "Channel/xrayeventsiem" } ] }
         response = RestClient::Request.new(
             :method => :post,
             :url => jpd_url + "/artifactory/api/system/usage",
             :payload => call_home_json.to_json,
-            :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + access_token }
+            :user => @username,
+            :password => @apikey,
+            :headers => { :accept => :json, :content_type => :json}
         ).execute do |response, request, result|
             puts "Posting call home information"
         end
       end
 
       # queries the xray API for violations based upon the input json
-      def get_xray_violations_detail(xray_violation_detail_url, access_token)
+      def get_xray_violations_detail(xray_violation_detail_url)
         response = RestClient::Request.new(
             :method => :get,
             :url => xray_violation_detail_url,
-            headers: {Authorization:'Bearer ' + access_token}
+            :user => @username,
+            :password => @apikey
         ).execute do |response, request, result|
           case response.code
           when 200
             return response.to_str
           else
-            raise Fluent::StandardError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
+            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
           end
         end
       end
 
 
       # queries the xray API for violations based upon the input json
-      def get_xray_violations(xray_json, jpd_url, access_token)
+      def get_xray_violations(xray_json, jpd_url)
         response = RestClient::Request.new(
             :method => :post,
             :url => jpd_url + "/xray/api/v1/violations",
             :payload => xray_json.to_json,
-            :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + access_token }
+            :user => @username,
+            :password => @apikey,
+            :headers => { :accept => :json, :content_type => :json}
         ).execute do |response, request, result|
           case response.code
           when 200
             return response.to_str
           else
-            raise Fluent::StandardError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
+            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
           end
         end
       end
@@ -227,44 +247,72 @@ module Fluent
       # normalizes Xray data according to common information models for all log-vendors
       def data_normalization(detailResp)
         detailResp_json = JSON.parse(detailResp)
-        properties = detailResp_json['properties']
         cve = []
         cvss_v2_list = []
         cvss_v3_list = []
-        for index in 0..properties.length-1 do
-          if properties[index].key?('cve')
-            cve.push(properties[index]['cve'])
+        policy_list = []
+        rule_list = []
+        impacted_artifact_url_list = []
+        if detailResp_json.key?('properties')
+          properties = detailResp_json['properties']
+          for index in 0..properties.length-1 do
+            if properties[index].key?('cve')
+              cve.push(properties[index]['cve'])
+            end
+            if properties[index].key?('cvss_v2')
+              cvss_v2_list.push(properties[index]['cvss_v2'])
+            end
+            if properties[index].key?('cvss_v3')
+              cvss_v3_list.push(properties[index]['cvss_v3'])
+            end
           end
-          if properties[index].key?('cvss_v2')
-            cvss_v2_list.push(properties[index]['cvss_v2'])
+
+          detailResp_json["cve"] = cve.sort.reverse[0]
+          cvss_v2 = cvss_v2_list.sort.reverse[0]
+          cvss_v3 = cvss_v3_list.sort.reverse[0]
+          if !cvss_v3.nil?
+            cvss = cvss_v3
+          elsif !cvss_v2.nil?
+            cvss = cvss_v2
           end
-          if properties[index].key?('cvss_v3')
-            cvss_v3_list.push(properties[index]['cvss_v3'])
+          cvss_score = cvss[0..2]
+          cvss_version = cvss.split(':')[1][0..2]
+          detailResp_json["cvss_score"] = cvss_score
+          detailResp_json["cvss_version"] = cvss_version
+        end
+
+        if detailResp_json.key?('matched_policies')
+          matched_policies = detailResp_json['matched_policies']
+          for index in 0..matched_policies.length-1 do
+            if matched_policies[index].key?('policy')
+              policy_list.push(matched_policies[index]['policy'])
+            end
+            if matched_policies[index].key?('rule')
+              rule_list.push(matched_policies[index]['rule'])
+            end
           end
+          detailResp_json['policies'] = policy_list
+          detailResp_json['rules'] = rule_list
         end
-        detailResp_json["cve"] = cve.sort.reverse[0]
-        cvss_v2 = cvss_v2_list.sort.reverse[0]
-        cvss_v3 = cvss_v3_list.sort.reverse[0]
-        if cvss_v3.length() > 0
-          cvss = cvss_v3
-        elsif cvss_v2.length() > 0
-          cvss = cvss_v2
+
+        impacted_artifacts = detailResp_json['impacted_artifacts']
+        for impacted_artifact in impacted_artifacts do
+          matchdata = impacted_artifact.match /default\/(?<repo_name>[^\/]*)\/(?<path>.*)/
+          impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " "
+          impacted_artifact_url_list.append(impacted_artifact_url)
         end
-        cvss_score = cvss[0..2]
-        cvss_version = cvss.split(':')[1][0..2]
-        detailResp_json["cvss_score"] = cvss_score
-        detailResp_json["cvss_version"] = cvss_version
+        detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list
         return detailResp_json
       end
 
-      def pull_violation_details(xray_violation_detail_url, access_token)
+      def pull_violation_details(xray_violation_detail_url)
         begin
-          detailResp=get_xray_violations_detail(xray_violation_detail_url, access_token)
+          detailResp=get_xray_violations_detail(xray_violation_detail_url)
           time = Fluent::Engine.now
           detailResp_json = data_normalization(detailResp)
           router.emit(@tag, time, detailResp_json)
         rescue
-          raise Fluent::StandardError, "Error pulling violation details url #{xray_violation_detail_url}"
+          raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}"
         end
       end
 

data/test/plugin/test_in_jfrog_siem.rb

@@ -13,8 +13,9 @@ class JfrogSiemInputTest < Test::Unit::TestCase
   # Default configuration for tests
   CONFIG = %[
     tag "test_tag"
-    jpd_url <jpd_url>
-    access_token <access_token>
+    jpd_url JPD_URL
+    username USER
+    apikey API_KEY
     pos_file "test_pos.txt"
   ]
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-siem
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.8
 platform: ruby
 authors:
 - John Peterson
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-21 00:00:00.000000000 Z
+date: 2021-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -67,6 +67,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement