fluent-plugin-jfrog-siem 0.1.1 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b04492f8a59aaa298e6fa79245598e9ba727595663b38f1109b139db49ec98c
-  data.tar.gz: fda6f3902f9897f2681b34c01e98ad4a37f35d8e9706ec350acdf1b14e9ae022
+  metadata.gz: a804fe365772166ab8b36207d11e1ffc4b858c3e47a5792240e5d6e1567108bd
+  data.tar.gz: f0bc602534f2c109469688fe38b79d78fecc1eb3ed31e9e0b40874cbea0935e4
 SHA512:
-  metadata.gz: e77645f065b8d9802ab802fd85dffa028ee2cb6abe2a522e055fdb84bedd97e34df98e1fe02881613f09b895549fb5602b86e38e5e3f5fa0b18581f86a411f82
-  data.tar.gz: e211bbbe0c4640ee326b314ad0930a0ff25d5ab4c5dafefc6945ad8e117a19a726fbd7863b7e6da57a773f834642ebcf45a5f47075670d65b408887198d99563
+  metadata.gz: 39082ce3738d65195ce5c85f7670e7883fc003b9bd853ef9dbb74f4e74f9c2b9a9750a83b90bcc9bb9264a921c06a2548f9bd599794bb4504e78201accbf733c
+  data.tar.gz: 05dfcd3fe9e49d94e824f9265cf4b6a301911529520e479d36889328872e92943f876b2bfacc2a25203c6c8de3e0f93e3f7427b5a684687c4ed427c59d6bf5c0

data/README.md

@@ -19,19 +19,7 @@ bundle install
 This will install the gem shown below from source.
 
 
-## Installation
-
-### RubyGems
-
-```
-$ gem install rest-client
-```
-```
-$ gem install thread
-```
-```
-$ gem install fluent-plugin-jfrog-siem
-```
+## Development
 
 ### Bundler
 
@@ -47,7 +35,7 @@ And then execute:
 $ bundle
 ```
 
-## Configuration
+### Configuration
 
 You can generate configuration template:
 
@@ -57,23 +45,52 @@ $ fluent-plugin-config-format input jfrog-siem
 
 You can copy and paste generated documents here.
 
-###Setup & configuration parameters
+## Installation 
 
-Xray setup is required. Obtain JPD url and access token for API
+### RubyGems
+```
+$ gem install rest-client
+```
+```
+$ gem install thread
+```
+```
+$ gem install fluent-plugin-jfrog-siem
+```
+
+### Setup & configuration
+Fluentd is the supported log collector for this integration. 
+For Fluentd setup and information, read the JFrog log analytics repository's [README.](https://github.com/jfrog/log-analytics/blob/master/README.md)
+
+#### Fluentd Output
+Download fluentd conf for different log-vendors. For example
+Splunk: 
 
+Splunk setup can be found at [README.](https://github.com/jfrog/log-analytics-splunk/blob/master/README.md)
+````text
+wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/splunk.conf
+````
+Elasticsearch: 
+
+Elasticsearch Kibana setup can be found at [README.](https://github.com/jfrog/log-analytics-elastic/blob/master/README.md)
+````text
+wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/elastic.conf
+````
+
+#### Configuration parameters
+Integration is done by setting up Xray. Obtain JPD url and access token for API. Configure the source directive parameters specified below
 * **tag** (string) (required): The value is the tag assigned to the generated events.
 * **jpd_url** (string) (required): JPD url required to pull Xray SIEM violations
 * **access_token** (string) (required): [Access token](https://www.jfrog.com/confluence/display/JFROG/Access+Tokens) to authenticate Xray
 * **pos_file** (string) (required): Position file to record last SIEM violation pulled
 * **batch_size** (integer) (optional): Batch size for processing violations
-    * Default value: `25`.
+    * Default value: `25`
 * **thread_count** (integer) (optional): Number of workers to process violation records in thread pool
-    * Default value: `5`.
+    * Default value: `5`
 * **wait_interval** (integer) (optional): Wait interval between pulling new events
-    * Default value: `60`.
+    * Default value: `60`
     
 ## Copyright
-
 * Copyright(c) 2020 - JFrog
 * License
   * Apache License, Version 2.0

data/elastic.conf

@@ -0,0 +1,18 @@
+<source>
+  @type jfrog_siem
+  tag elastic_jfrog
+  jpd_url <jpd_url>
+  access_token <access_token>
+  pos_file "elastic_pos.txt"
+</source>
+<match elastic*>
+  @type elasticsearch
+  @id elasticsearch
+  host elasticsearch
+  port 9200
+  user <username>
+  password <password>
+  index_name xray_siem
+  include_tag_key true
+  type_name fluentd
+</match>

data/fluent-plugin-jfrog-siem.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-siem"
-  spec.version = "0.1.1"
+  spec.version = "0.1.6"
   spec.authors = ["John Peterson", "Mahitha Byreddy"]
   spec.email   = ["johnp@jfrog.com", "mahithab@jfrog.com"]
 
@@ -24,5 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_development_dependency "rest-client", "~> 2.0"
+  spec.add_development_dependency "thread", "~> 0.2.2"
+  spec.add_runtime_dependency "thread", "~> 0.2.2"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/lib/fluent/plugin/in_jfrog_siem.rb

@@ -99,6 +99,7 @@ module Fluent
         end
         offset_count=1
         left_violations=0
+        waiting_for_violations = false
         xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } }
 
         while true
@@ -120,8 +121,16 @@ module Fluent
 
             # Determine if we need to persist this record or not
             persistItem = true
-            if created_date < last_created_date
-              persistItem = false
+            if waiting_for_violations
+              if created_date <= last_created_date
+                # "not persisting it - waiting for violations"
+                persistItem = false
+              end
+            else
+              if created_date < last_created_date
+                # "persisting everything"
+                persistItem = true
+              end
             end
 
             # Publish the record to fluentd
@@ -148,20 +157,21 @@ module Fluent
           # iterate over url array adding to thread pool each url.
           # limit max workers to thread count to prevent overloading xray.
           thread_pool = Thread.pool(thread_count)
-          for xray_violation_url in xray_violation_urls_list do
-            thread_pool.process {
+          thread_pool.process {
+            for xray_violation_url in xray_violation_urls_list do
               pull_violation_details(xray_violation_url, @access_token)
-            }
-          end
-
+            end
+          }
           thread_pool.shutdown
 
           # reduce left violations by jump size (not all batches have full item count??)
           left_violations = left_violations - @batch_size
           if left_violations <= 0
+            waiting_for_violations = true
             sleep(@wait_interval)
           else
             # Grab the next record to process for the violation details url
+            waiting_for_violations = false
             offset_count = offset_count + 1
             xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } }
           end
@@ -227,33 +237,45 @@ module Fluent
       # normalizes Xray data according to common information models for all log-vendors
       def data_normalization(detailResp)
         detailResp_json = JSON.parse(detailResp)
-        properties = detailResp_json['properties']
         cve = []
         cvss_v2_list = []
         cvss_v3_list = []
-        for index in 0..properties.length-1 do
-          if properties[index].key?('cve')
-            cve.push(properties[index]['cve'])
-          end
-          if properties[index].key?('cvss_v2')
-            cvss_v2_list.push(properties[index]['cvss_v2'])
+        impacted_artifact_url_list = []
+        if detailResp_json.key?('properties')
+          properties = detailResp_json['properties']
+          for index in 0..properties.length-1 do
+            if properties[index].key?('cve')
+              cve.push(properties[index]['cve'])
+            end
+            if properties[index].key?('cvss_v2')
+              cvss_v2_list.push(properties[index]['cvss_v2'])
+            end
+            if properties[index].key?('cvss_v3')
+              cvss_v3_list.push(properties[index]['cvss_v3'])
+            end
           end
-          if properties[index].key?('cvss_v3')
-            cvss_v3_list.push(properties[index]['cvss_v3'])
+
+          detailResp_json["cve"] = cve.sort.reverse[0]
+          cvss_v2 = cvss_v2_list.sort.reverse[0]
+          cvss_v3 = cvss_v3_list.sort.reverse[0]
+          if !cvss_v3.nil?
+            cvss = cvss_v3
+          elsif !cvss_v2.nil?
+            cvss = cvss_v2
           end
+          cvss_score = cvss[0..2]
+          cvss_version = cvss.split(':')[1][0..2]
+          detailResp_json["cvss_score"] = cvss_score
+          detailResp_json["cvss_version"] = cvss_version
         end
-        detailResp_json["cve"] = cve.sort.reverse[0]
-        cvss_v2 = cvss_v2_list.sort.reverse[0]
-        cvss_v3 = cvss_v3_list.sort.reverse[0]
-        if cvss_v3.length() > 0
-          cvss = cvss_v3
-        elsif cvss_v2.length() > 0
-          cvss = cvss_v2
+
+        impacted_artifacts = detailResp_json['impacted_artifacts']
+        for impacted_artifact in impacted_artifacts do
+          matchdata = impacted_artifact.match /default\/(?<repo_name>[^\/]*)\/(?<path>.*)/
+          impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " "
+          impacted_artifact_url_list.append(impacted_artifact_url)
         end
-        cvss_score = cvss[0..2]
-        cvss_version = cvss.split(':')[1][0..2]
-        detailResp_json["cvss_score"] = cvss_score
-        detailResp_json["cvss_version"] = cvss_version
+        detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list
         return detailResp_json
       end
 

data/splunk.conf

@@ -0,0 +1,18 @@
+<source>
+  @type jfrog_siem
+  tag splunk_jfrog
+  jpd_url <jpd_url>
+  access_token <access_token>
+  pos_file "splunk_pos.txt"
+</source>
+<match splunk*>
+  @type splunk_hec
+  host HEC_HOST
+  port HEC_PORT
+  token HEC_TOKEN
+  format json
+  sourcetype_key log_source
+  use_fluentd_time false
+  index violations
+  flush_interval 10s
+</match>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jfrog-siem
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.6
 platform: ruby
 authors:
 - John Peterson
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-17 00:00:00.000000000 Z
+date: 2021-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -67,6 +67,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -100,8 +128,10 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- elastic.conf
 - fluent-plugin-jfrog-siem.gemspec
 - lib/fluent/plugin/in_jfrog_siem.rb
+- splunk.conf
 - test/helper.rb
 - test/plugin/test_in_jfrog_siem.rb
 homepage: https://github.com/jfrog/log-analytics