fluent-plugin-jfrog-siem 0.1.4 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a6d55c86435d39f27aa0deaf46748366ac837ad4a2417f5a89bfe3711f62c3a
-  data.tar.gz: 8fc1d3286dab511cd5b6c2cb5358f4edc782dc694d0da052c191858de42057d6
+  metadata.gz: e45847f8ff6faf6103f383101c21b8534f1b0bc7c9d8626e8f3a5a172631fb9e
+  data.tar.gz: 3f45070df06e498421d9e64ea25a793b700bb6efab6b7a5e878a206214927889
 SHA512:
-  metadata.gz: 2fe48f82b2911228d5bf3073ed92d6d4991ee89a5e66c1d56516f6e63a8fe7b4037ec374cb4012b59630cb796435fb02db696582af472cb598ecdf304b670d4a
-  data.tar.gz: 0c8fa4396ccba483ac3d8931cecbfb6996d65c847a7fdfac749de7156be6d0f2dda4cc8a0e51a6672d210c57d683a5b2cb6d787eb1646ecd1c7efc006e2bc3bd
+  metadata.gz: 69f51b464049166d1930578447cd7ca2dd5bd273e013ea94e1eb682b2e664e83e1694db11358c929fda408c714cf3f5c2d3d272aee5b4bee88104c55441e556e
+  data.tar.gz: 0a94d81ed3741bbbffd41aa47e5781becb956337675261bb00eea4f4de2c9bcc41809fe212c6abb21a817b5626f9ff03db5be07e262ce0dde9093efa9c68c17a

data/README.md

@@ -68,13 +68,19 @@ Splunk:
 
 Splunk setup can be found at [README.](https://github.com/jfrog/log-analytics-splunk/blob/master/README.md)
 ````text
-wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/splunk.conf
+wget https://raw.githubusercontent.com/jfrog/log-analytics-splunk/master/siem/splunk_siem.conf
 ````
 Elasticsearch: 
 
 Elasticsearch Kibana setup can be found at [README.](https://github.com/jfrog/log-analytics-elastic/blob/master/README.md)
 ````text
-wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/elastic.conf
+wget https://raw.githubusercontent.com/jfrog/log-analytics-elastic/master/siem/elastic_siem.conf
+````
+Datadog: 
+
+Datadog setup can be found at [README.](https://github.com/jfrog/log-analytics-datadog/blob/master/README.md)
+````text
+wget https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/siem/datadog_siem.conf
 ````
 
 #### Configuration parameters

data/fluent-plugin-jfrog-siem.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-siem"
-  spec.version = "0.1.4"
+  spec.version = "0.1.9"
   spec.authors = ["John Peterson", "Mahitha Byreddy"]
   spec.email   = ["johnp@jfrog.com", "mahithab@jfrog.com"]
 

data/lib/fluent/plugin/in_jfrog_siem.rb

@@ -30,7 +30,8 @@ module Fluent
       # `:default` means that the parameter is optional.
       config_param :tag, :string, default: ""
       config_param :jpd_url, :string, default: ""
-      config_param :access_token, :string, default: ""
+      config_param :username, :string, default: ""
+      config_param :apikey, :string, default: ""
       config_param :pos_file, :string, default: ""
       config_param :batch_size, :integer, default: 25
       config_param :thread_count, :integer, default: 5
@@ -50,8 +51,12 @@ module Fluent
           raise Fluent::ConfigError, "Must define the JPD URL to pull Xray SIEM violations."
         end
 
-        if @access_token == ""
-          raise Fluent::ConfigError, "Must define the access token to use for authentication."
+        if @username == ""
+          raise Fluent::ConfigError, "Must define the username to use for authentication."
+        end
+
+        if @apikey == ""
+          raise Fluent::ConfigError, "Must define the API Key to use for authentication."
         end
 
         if @pos_file == ""
@@ -89,7 +94,7 @@ module Fluent
 
 
       def run
-        call_home(@jpd_url, @access_token)
+        call_home(@jpd_url)
         # runs the violation pull
         last_created_date_string = get_last_item_create_date()
         begin
@@ -99,11 +104,12 @@ module Fluent
         end
         offset_count=1
         left_violations=0
+        waiting_for_violations = false
         xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } }
 
         while true
           # Grab the batch of records
-          resp=get_xray_violations(xray_json, @jpd_url, @access_token)
+          resp=get_xray_violations(xray_json, @jpd_url)
           number_of_violations = JSON.parse(resp)['total_violations']
           if left_violations <= 0
             left_violations = number_of_violations
@@ -120,8 +126,22 @@ module Fluent
 
             # Determine if we need to persist this record or not
             persistItem = true
-            if created_date <= last_created_date
-              persistItem = false
+            if waiting_for_violations
+              if created_date <= last_created_date
+                # "not persisting it - waiting for violations"
+                # waiting and same last timestamp (left violations in batch)
+                persistItem = false
+              end
+              if created_date > last_created_date
+                # new violation while waiting
+                persistItem = true
+                waiting_for_violations = false
+              end
+            else
+              if created_date < last_created_date
+                # "persisting everything"
+                persistItem = true
+              end
             end
 
             # Publish the record to fluentd
@@ -150,7 +170,7 @@ module Fluent
           thread_pool = Thread.pool(thread_count)
           thread_pool.process {
             for xray_violation_url in xray_violation_urls_list do
-              pull_violation_details(xray_violation_url, @access_token)
+              pull_violation_details(xray_violation_url)
             end
           }
           thread_pool.shutdown
@@ -158,9 +178,11 @@ module Fluent
           # reduce left violations by jump size (not all batches have full item count??)
           left_violations = left_violations - @batch_size
           if left_violations <= 0
+            waiting_for_violations = true
             sleep(@wait_interval)
           else
             # Grab the next record to process for the violation details url
+            waiting_for_violations = false
             offset_count = offset_count + 1
             xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } }
           end
@@ -177,48 +199,53 @@ module Fluent
       end
 
       #call home functionality
-      def call_home(jpd_url, access_token)
+      def call_home(jpd_url)
         call_home_json = { "productId": "jfrogLogAnalytics/v0.5.1", "features": [ { "featureId": "Platform/Xray" }, { "featureId": "Channel/xrayeventsiem" } ] }
         response = RestClient::Request.new(
             :method => :post,
             :url => jpd_url + "/artifactory/api/system/usage",
             :payload => call_home_json.to_json,
-            :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + access_token }
+            :user => @username,
+            :password => @apikey,
+            :headers => { :accept => :json, :content_type => :json}
         ).execute do |response, request, result|
             puts "Posting call home information"
         end
       end
 
       # queries the xray API for violations based upon the input json
-      def get_xray_violations_detail(xray_violation_detail_url, access_token)
+      def get_xray_violations_detail(xray_violation_detail_url)
         response = RestClient::Request.new(
             :method => :get,
             :url => xray_violation_detail_url,
-            headers: {Authorization:'Bearer ' + access_token}
+            :user => @username,
+            :password => @apikey
         ).execute do |response, request, result|
           case response.code
           when 200
             return response.to_str
           else
-            raise Fluent::StandardError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
+            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
           end
         end
       end
 
 
       # queries the xray API for violations based upon the input json
-      def get_xray_violations(xray_json, jpd_url, access_token)
+      def get_xray_violations(xray_json, jpd_url)
         response = RestClient::Request.new(
             :method => :post,
             :url => jpd_url + "/xray/api/v1/violations",
             :payload => xray_json.to_json,
-            :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + access_token }
+            :user => @username,
+            :password => @apikey,
+            :headers => { :accept => :json, :content_type => :json}
         ).execute do |response, request, result|
           case response.code
           when 200
             return response.to_str
           else
-            raise Fluent::StandardError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
+            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
           end
         end
       end
@@ -226,44 +253,72 @@ module Fluent
       # normalizes Xray data according to common information models for all log-vendors
       def data_normalization(detailResp)
         detailResp_json = JSON.parse(detailResp)
-        properties = detailResp_json['properties']
         cve = []
         cvss_v2_list = []
         cvss_v3_list = []
-        for index in 0..properties.length-1 do
-          if properties[index].key?('cve')
-            cve.push(properties[index]['cve'])
+        policy_list = []
+        rule_list = []
+        impacted_artifact_url_list = []
+        if detailResp_json.key?('properties')
+          properties = detailResp_json['properties']
+          for index in 0..properties.length-1 do
+            if properties[index].key?('cve')
+              cve.push(properties[index]['cve'])
+            end
+            if properties[index].key?('cvss_v2')
+              cvss_v2_list.push(properties[index]['cvss_v2'])
+            end
+            if properties[index].key?('cvss_v3')
+              cvss_v3_list.push(properties[index]['cvss_v3'])
+            end
           end
-          if properties[index].key?('cvss_v2')
-            cvss_v2_list.push(properties[index]['cvss_v2'])
+
+          detailResp_json["cve"] = cve.sort.reverse[0]
+          cvss_v2 = cvss_v2_list.sort.reverse[0]
+          cvss_v3 = cvss_v3_list.sort.reverse[0]
+          if !cvss_v3.nil?
+            cvss = cvss_v3
+          elsif !cvss_v2.nil?
+            cvss = cvss_v2
           end
-          if properties[index].key?('cvss_v3')
-            cvss_v3_list.push(properties[index]['cvss_v3'])
+          cvss_score = cvss[0..2]
+          cvss_version = cvss.split(':')[1][0..2]
+          detailResp_json["cvss_score"] = cvss_score
+          detailResp_json["cvss_version"] = cvss_version
+        end
+
+        if detailResp_json.key?('matched_policies')
+          matched_policies = detailResp_json['matched_policies']
+          for index in 0..matched_policies.length-1 do
+            if matched_policies[index].key?('policy')
+              policy_list.push(matched_policies[index]['policy'])
+            end
+            if matched_policies[index].key?('rule')
+              rule_list.push(matched_policies[index]['rule'])
+            end
           end
+          detailResp_json['policies'] = policy_list
+          detailResp_json['rules'] = rule_list
         end
-        detailResp_json["cve"] = cve.sort.reverse[0]
-        cvss_v2 = cvss_v2_list.sort.reverse[0]
-        cvss_v3 = cvss_v3_list.sort.reverse[0]
-        if cvss_v3.length() > 0
-          cvss = cvss_v3
-        elsif cvss_v2.length() > 0
-          cvss = cvss_v2
+
+        impacted_artifacts = detailResp_json['impacted_artifacts']
+        for impacted_artifact in impacted_artifacts do
+          matchdata = impacted_artifact.match /default\/(?<repo_name>[^\/]*)\/(?<path>.*)/
+          impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " "
+          impacted_artifact_url_list.append(impacted_artifact_url)
         end
-        cvss_score = cvss[0..2]
-        cvss_version = cvss.split(':')[1][0..2]
-        detailResp_json["cvss_score"] = cvss_score
-        detailResp_json["cvss_version"] = cvss_version
+        detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list
         return detailResp_json
       end
 
-      def pull_violation_details(xray_violation_detail_url, access_token)
+      def pull_violation_details(xray_violation_detail_url)
         begin
-          detailResp=get_xray_violations_detail(xray_violation_detail_url, access_token)
+          detailResp=get_xray_violations_detail(xray_violation_detail_url)
           time = Fluent::Engine.now
           detailResp_json = data_normalization(detailResp)
           router.emit(@tag, time, detailResp_json)
         rescue
-          raise Fluent::StandardError, "Error pulling violation details url #{xray_violation_detail_url}"
+          raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}"
         end
       end
 

data/test/plugin/test_in_jfrog_siem.rb

@@ -13,8 +13,9 @@ class JfrogSiemInputTest < Test::Unit::TestCase
   # Default configuration for tests
   CONFIG = %[
     tag "test_tag"
-    jpd_url <jpd_url>
-    access_token <access_token>
+    jpd_url "http://dd-siem-unified-15-rt.jfrog.tech"
+    username "admin"
+    apikey "AKCp8jQ8tAb2tqC6jXH7J1qLShXnjorD5XhFkvqUhN78WMLmvfTx5GNuD9B8uEXnGyruBCzYk"
     pos_file "test_pos.txt"
   ]
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-siem
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.9
 platform: ruby
 authors:
 - John Peterson
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-02 00:00:00.000000000 Z
+date: 2021-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler