fluent-plugin-jfrog-siem 0.1.5 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6cfb13538b236dbfd917cf58038dab5e76d945fa0af419240fdd5508e976af1c
-  data.tar.gz: 9725242bcb3230a23f457b146bcf5e39cf562aad9abbcfbcc1ea390de2853093
+  metadata.gz: 306eb5d59fd5e00e2e8feda0339a3b635f1e61aaabff9312ad372dc714c3ff8f
+  data.tar.gz: 469ea7950f9d96236a88159a797f17077b31f5c3d7ddc19ca91f4d6209963a9a
 SHA512:
-  metadata.gz: e63137fd91a16f2ab65d3ecdc2e841d1c098e201a8a47b2d0687a80833e05bfa3fb0e4e6cf606245d58c884ef3740d7c4efa3fbbb9a4c095445e05ba3ee60b10
-  data.tar.gz: cc5dcb1ef463eb4d8c57bd246d4968b3a47639296f994913ebb75f01bb49496a44800197efe7bcc97e23bff0f84dc48570ba08d9157c0690715e00b9a0e5cfb5
+  metadata.gz: 02db6faa97750196fd42a0b04a8a8f517dfa0a26ad585778b8283b4cce9814ff239d96eb0a9ca34ef02a2f66bab9bb632cf8c0194afe7a3e4902ea08f4d9fd77
+  data.tar.gz: 745bef31330a205aac78d2f49e2b25f6f924f52c3d8a0be35d3d1f4aee77f1c862b8e8124a95ed62d90befc3e5a351061cbe1b902482484e3778081404737527

data/CHANGELOG.md

@@ -0,0 +1,36 @@
+# JFrog Fluentd SIEM Input Plugin Changelog
+All changes to the SIEM plugin will be documented in this file.
+
+## [1.0.0] - May 18, 2020
+* [BREAKING] Using JFrog API Key for authentication
+
+## [0.1.9] - May 17, 2021
+* Handling the case where violations are left in a batch to be processed 
+
+## [0.1.8] - May 10, 2021
+* Fixing persist, not persist item conditions
+
+## [0.1.7] - April 21, 2021
+* Adding policies and rules to payload
+
+## [0.1.6] - April 13, 2021
+* Adding additonal parameters to match with access logs for correlation
+
+## [0.1.5] - March 29, 2021 
+* Normalizing the format of Impacted Artifact, fixing properties not found case
+
+## [0.1.4] - February 02, 2021
+* Adding dependencies, gemspec updates
+
+## [0.1.3] - January 21, 2021 
+* Fixing thread pool issues (moving loop inside a thread pool)
+
+## [0.1.2] - November 17, 2020
+* Changes to better README
+
+## [0.1.1] - November 17, 2020
+* Adding dependencies to gemspec
+
+## [0.1.0] - October 05, 2020
+* Initial release of Jfrog Logs Analytic integration
+

data/README.md

@@ -68,20 +68,27 @@ Splunk:
 
 Splunk setup can be found at [README.](https://github.com/jfrog/log-analytics-splunk/blob/master/README.md)
 ````text
-wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/splunk.conf
+wget https://raw.githubusercontent.com/jfrog/log-analytics-splunk/master/siem/splunk_siem.conf
 ````
 Elasticsearch: 
 
 Elasticsearch Kibana setup can be found at [README.](https://github.com/jfrog/log-analytics-elastic/blob/master/README.md)
 ````text
-wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/elastic.conf
+wget https://raw.githubusercontent.com/jfrog/log-analytics-elastic/master/siem/elastic_siem.conf
+````
+Datadog: 
+
+Datadog setup can be found at [README.](https://github.com/jfrog/log-analytics-datadog/blob/master/README.md)
+````text
+wget https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/siem/datadog_siem.conf
 ````
 
 #### Configuration parameters
 Integration is done by setting up Xray. Obtain JPD url and access token for API. Configure the source directive parameters specified below
 * **tag** (string) (required): The value is the tag assigned to the generated events.
 * **jpd_url** (string) (required): JPD url required to pull Xray SIEM violations
-* **access_token** (string) (required): [Access token](https://www.jfrog.com/confluence/display/JFROG/Access+Tokens) to authenticate Xray
+* **apikey** (string) (required): API Key is the [Artifactory API Key](https://www.jfrog.com/confluence/display/JFROG/User+Profile#UserProfile-APIKey) for authentication
+* **username** (string) (required): USER is the Artifactory username for authentication
 * **pos_file** (string) (required): Position file to record last SIEM violation pulled
 * **batch_size** (integer) (optional): Batch size for processing violations
     * Default value: `25`

data/fluent-plugin-jfrog-siem.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-siem"
-  spec.version = "0.1.5"
+  spec.version = "1.0.0"
   spec.authors = ["John Peterson", "Mahitha Byreddy"]
   spec.email   = ["johnp@jfrog.com", "mahithab@jfrog.com"]
 

data/lib/fluent/plugin/in_jfrog_siem.rb

@@ -30,7 +30,8 @@ module Fluent
       # `:default` means that the parameter is optional.
       config_param :tag, :string, default: ""
       config_param :jpd_url, :string, default: ""
-      config_param :access_token, :string, default: ""
+      config_param :username, :string, default: ""
+      config_param :apikey, :string, default: ""
       config_param :pos_file, :string, default: ""
       config_param :batch_size, :integer, default: 25
       config_param :thread_count, :integer, default: 5
@@ -50,8 +51,12 @@ module Fluent
           raise Fluent::ConfigError, "Must define the JPD URL to pull Xray SIEM violations."
         end
 
-        if @access_token == ""
-          raise Fluent::ConfigError, "Must define the access token to use for authentication."
+        if @username == ""
+          raise Fluent::ConfigError, "Must define the username to use for authentication."
+        end
+
+        if @apikey == ""
+          raise Fluent::ConfigError, "Must define the API Key to use for authentication."
         end
 
         if @pos_file == ""
@@ -89,7 +94,7 @@ module Fluent
 
 
       def run
-        call_home(@jpd_url, @access_token)
+        call_home(@jpd_url)
         # runs the violation pull
         last_created_date_string = get_last_item_create_date()
         begin
@@ -104,7 +109,7 @@ module Fluent
 
         while true
           # Grab the batch of records
-          resp=get_xray_violations(xray_json, @jpd_url, @access_token)
+          resp=get_xray_violations(xray_json, @jpd_url)
           number_of_violations = JSON.parse(resp)['total_violations']
           if left_violations <= 0
             left_violations = number_of_violations
@@ -124,8 +129,14 @@ module Fluent
             if waiting_for_violations
               if created_date <= last_created_date
                 # "not persisting it - waiting for violations"
+                # waiting and same last timestamp (left violations in batch)
                 persistItem = false
               end
+              if created_date > last_created_date
+                # new violation while waiting
+                persistItem = true
+                waiting_for_violations = false
+              end
             else
               if created_date < last_created_date
                 # "persisting everything"
@@ -159,7 +170,7 @@ module Fluent
           thread_pool = Thread.pool(thread_count)
           thread_pool.process {
             for xray_violation_url in xray_violation_urls_list do
-              pull_violation_details(xray_violation_url, @access_token)
+              pull_violation_details(xray_violation_url)
             end
           }
           thread_pool.shutdown
@@ -188,48 +199,53 @@ module Fluent
       end
 
       #call home functionality
-      def call_home(jpd_url, access_token)
+      def call_home(jpd_url)
         call_home_json = { "productId": "jfrogLogAnalytics/v0.5.1", "features": [ { "featureId": "Platform/Xray" }, { "featureId": "Channel/xrayeventsiem" } ] }
         response = RestClient::Request.new(
             :method => :post,
             :url => jpd_url + "/artifactory/api/system/usage",
             :payload => call_home_json.to_json,
-            :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + access_token }
+            :user => @username,
+            :password => @apikey,
+            :headers => { :accept => :json, :content_type => :json}
         ).execute do |response, request, result|
             puts "Posting call home information"
         end
       end
 
       # queries the xray API for violations based upon the input json
-      def get_xray_violations_detail(xray_violation_detail_url, access_token)
+      def get_xray_violations_detail(xray_violation_detail_url)
         response = RestClient::Request.new(
             :method => :get,
             :url => xray_violation_detail_url,
-            headers: {Authorization:'Bearer ' + access_token}
+            :user => @username,
+            :password => @apikey
         ).execute do |response, request, result|
           case response.code
           when 200
             return response.to_str
           else
-            raise Fluent::StandardError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
+            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
           end
         end
       end
 
 
       # queries the xray API for violations based upon the input json
-      def get_xray_violations(xray_json, jpd_url, access_token)
+      def get_xray_violations(xray_json, jpd_url)
         response = RestClient::Request.new(
             :method => :post,
             :url => jpd_url + "/xray/api/v1/violations",
             :payload => xray_json.to_json,
-            :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + access_token }
+            :user => @username,
+            :password => @apikey,
+            :headers => { :accept => :json, :content_type => :json}
         ).execute do |response, request, result|
           case response.code
           when 200
             return response.to_str
           else
-            raise Fluent::StandardError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
+            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
           end
         end
       end
@@ -240,6 +256,8 @@ module Fluent
         cve = []
         cvss_v2_list = []
         cvss_v3_list = []
+        policy_list = []
+        rule_list = []
         impacted_artifact_url_list = []
         if detailResp_json.key?('properties')
           properties = detailResp_json['properties']
@@ -269,31 +287,38 @@ module Fluent
           detailResp_json["cvss_version"] = cvss_version
         end
 
+        if detailResp_json.key?('matched_policies')
+          matched_policies = detailResp_json['matched_policies']
+          for index in 0..matched_policies.length-1 do
+            if matched_policies[index].key?('policy')
+              policy_list.push(matched_policies[index]['policy'])
+            end
+            if matched_policies[index].key?('rule')
+              rule_list.push(matched_policies[index]['rule'])
+            end
+          end
+          detailResp_json['policies'] = policy_list
+          detailResp_json['rules'] = rule_list
+        end
+
         impacted_artifacts = detailResp_json['impacted_artifacts']
         for impacted_artifact in impacted_artifacts do
-          if impacted_artifact.split('/', -1)[-1] == "manifest.json"
-            #docker formatting
-            repo_name = impacted_artifact.split('/', -1)[1]
-            image_name = impacted_artifact.split('/', -1)[2]
-            tag_name = impacted_artifact.split('/', -1)[3]
-            impacted_artifact_url = "/api/docker/" + repo_name + "/v2/" + image_name + "/manifests/" + tag_name
-          else
-            impacted_artifact_url = impacted_artifact.gsub("default", "")
-          end
+          matchdata = impacted_artifact.match /default\/(?<repo_name>[^\/]*)\/(?<path>.*)/
+          impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " "
           impacted_artifact_url_list.append(impacted_artifact_url)
         end
         detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list
         return detailResp_json
       end
 
-      def pull_violation_details(xray_violation_detail_url, access_token)
+      def pull_violation_details(xray_violation_detail_url)
         begin
-          detailResp=get_xray_violations_detail(xray_violation_detail_url, access_token)
+          detailResp=get_xray_violations_detail(xray_violation_detail_url)
           time = Fluent::Engine.now
           detailResp_json = data_normalization(detailResp)
           router.emit(@tag, time, detailResp_json)
         rescue
-          raise Fluent::StandardError, "Error pulling violation details url #{xray_violation_detail_url}"
+          raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}"
         end
       end
 

data/test/plugin/test_in_jfrog_siem.rb

@@ -13,8 +13,9 @@ class JfrogSiemInputTest < Test::Unit::TestCase
   # Default configuration for tests
   CONFIG = %[
     tag "test_tag"
-    jpd_url <jpd_url>
-    access_token <access_token>
+    jpd_url JPD_URL
+    username USER
+    apikey API_KEY
     pos_file "test_pos.txt"
   ]
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-siem
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 1.0.0
 platform: ruby
 authors:
 - John Peterson
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-29 00:00:00.000000000 Z
+date: 2021-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -124,14 +124,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
-- elastic.conf
 - fluent-plugin-jfrog-siem.gemspec
 - lib/fluent/plugin/in_jfrog_siem.rb
-- splunk.conf
 - test/helper.rb
 - test/plugin/test_in_jfrog_siem.rb
 homepage: https://github.com/jfrog/log-analytics

data/elastic.conf

@@ -1,18 +0,0 @@
-<source>
-  @type jfrog_siem
-  tag elastic_jfrog
-  jpd_url <jpd_url>
-  access_token <access_token>
-  pos_file "elastic_pos.txt"
-</source>
-<match elastic*>
-  @type elasticsearch
-  @id elasticsearch
-  host elasticsearch
-  port 9200
-  user <username>
-  password <password>
-  index_name xray_siem
-  include_tag_key true
-  type_name fluentd
-</match>

data/splunk.conf

@@ -1,18 +0,0 @@
-<source>
-  @type jfrog_siem
-  tag splunk_jfrog
-  jpd_url <jpd_url>
-  access_token <access_token>
-  pos_file "splunk_pos.txt"
-</source>
-<match splunk*>
-  @type splunk_hec
-  host HEC_HOST
-  port HEC_PORT
-  token HEC_TOKEN
-  format json
-  sourcetype_key log_source
-  use_fluentd_time false
-  index violations
-  flush_interval 10s
-</match>