fluent-plugin-jfrog-siem 0.1.7 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d5598c727b3f7567c6ca432c96b12252e45387787ed75ec55ce98024f936725
-  data.tar.gz: e5d9aaed273061f7abbe7c41a337d6346bfac8896ffc88c182888b96770646d8
+  metadata.gz: e34dd9d1fb60e95b931f7c67bb508e635057b9e2f349f681eb2b54cb182c2396
+  data.tar.gz: b0acda337195accb81ce6bf3ee7bf05ef7e79630793dbc7553c0df5e541e26a3
 SHA512:
-  metadata.gz: a1b43ff8df04ecf7e89a30632e2174c7a55724b1ae6357ef564d4719ebfec699601ac5dd1936c8275758196ad2c1bfc260262c4f7e3490658c132b5f456eb2ca
-  data.tar.gz: 0a77aa48d288ad3ca06e3dfe2bef07fff81ad368fd834d686c55a3e39963ce332334389ff2f753692dcf06e65f97a83cc440f66c906f62093ba682fa7d55e232
+  metadata.gz: 9116c506cf562af1566e6001748f5464752b2c07aace2d9f6fe04573c5a97db4cbf0e638d048dea536387559a71fa493fad882dc55650f1597c4030ab13be962
+  data.tar.gz: 2aaeaa3f4a22753db0b97c35b93938c4049cb9037f56f3cc6199dc222034a9d26609e1906973c0b9f11960f856227b2cf93ab000e8bd677aee9045df0bd684b0

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/CHANGELOG.md

@@ -0,0 +1,36 @@
+# JFrog Fluentd SIEM Input Plugin Changelog
+All changes to the SIEM plugin will be documented in this file.
+
+## [1.0.0] - May 18, 2020
+* [BREAKING] Using JFrog API Key for authentication
+
+## [0.1.9] - May 17, 2021
+* Handling the case where violations are left in a batch to be processed 
+
+## [0.1.8] - May 10, 2021
+* Fixing persist, not persist item conditions
+
+## [0.1.7] - April 21, 2021
+* Adding policies and rules to payload
+
+## [0.1.6] - April 13, 2021
+* Adding additonal parameters to match with access logs for correlation
+
+## [0.1.5] - March 29, 2021 
+* Normalizing the format of Impacted Artifact, fixing properties not found case
+
+## [0.1.4] - February 02, 2021
+* Adding dependencies, gemspec updates
+
+## [0.1.3] - January 21, 2021 
+* Fixing thread pool issues (moving loop inside a thread pool)
+
+## [0.1.2] - November 17, 2020
+* Changes to better README
+
+## [0.1.1] - November 17, 2020
+* Adding dependencies to gemspec
+
+## [0.1.0] - October 05, 2020
+* Initial release of Jfrog Logs Analytic integration
+

data/Gemfile.lock

@@ -0,0 +1,90 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-jfrog-siem (1.0.0)
+      concurrent-ruby (~> 1.1.8)
+      concurrent-ruby-edge
+      fluentd (>= 0.14.10, < 2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.1.8)
+    concurrent-ruby-edge (0.6.0)
+      concurrent-ruby (~> 1.1.6)
+    cool.io (1.7.1)
+    diff-lcs (1.4.4)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    fluentd (1.13.1)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
+      yajl-ruby (~> 1.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http_parser.rb (0.6.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2021.0225)
+    msgpack (1.4.2)
+    netrc (0.11.0)
+    power_assert (2.0.0)
+    rake (12.3.3)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    serverengine (2.2.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    test-unit (3.4.2)
+      power_assert
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    webrick (1.7.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  concurrent-ruby (~> 1.1.8)
+  concurrent-ruby-edge
+  fluent-plugin-jfrog-siem!
+  rake (~> 12.0)
+  rest-client (~> 2.0)
+  rspec (~> 3.10.0)
+  test-unit (~> 3.0)
+
+BUNDLED WITH
+   2.2.3

data/README.md

@@ -68,20 +68,27 @@ Splunk:
 
 Splunk setup can be found at [README.](https://github.com/jfrog/log-analytics-splunk/blob/master/README.md)
 ````text
-wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/splunk.conf
+wget https://raw.githubusercontent.com/jfrog/log-analytics-splunk/master/siem/splunk_siem.conf
 ````
 Elasticsearch: 
 
 Elasticsearch Kibana setup can be found at [README.](https://github.com/jfrog/log-analytics-elastic/blob/master/README.md)
 ````text
-wget https://raw.githubusercontent.com/jfrog/log-analytics/master/fluentd/plugins/input/fluent-plugin-jfrog-siem/elastic.conf
+wget https://raw.githubusercontent.com/jfrog/log-analytics-elastic/master/siem/elastic_siem.conf
+````
+Datadog: 
+
+Datadog setup can be found at [README.](https://github.com/jfrog/log-analytics-datadog/blob/master/README.md)
+````text
+wget https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/siem/datadog_siem.conf
 ````
 
 #### Configuration parameters
 Integration is done by setting up Xray. Obtain JPD url and access token for API. Configure the source directive parameters specified below
 * **tag** (string) (required): The value is the tag assigned to the generated events.
 * **jpd_url** (string) (required): JPD url required to pull Xray SIEM violations
-* **access_token** (string) (required): [Access token](https://www.jfrog.com/confluence/display/JFROG/Access+Tokens) to authenticate Xray
+* **apikey** (string) (required): API Key is the [Artifactory API Key](https://www.jfrog.com/confluence/display/JFROG/User+Profile#UserProfile-APIKey) for authentication
+* **username** (string) (required): USER is the Artifactory username for authentication
 * **pos_file** (string) (required): Position file to record last SIEM violation pulled
 * **batch_size** (integer) (optional): Batch size for processing violations
     * Default value: `25`

data/Rakefile

@@ -4,7 +4,7 @@ Bundler::GemHelper.install_tasks
 require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
-  t.libs.push("lib", "test")
+  t.libs.push("lib", "test", 'lib/fluent/plugin')
   t.test_files = FileList["test/**/test_*.rb"]
   t.verbose = true
   t.warning = false

data/fluent-plugin-jfrog-siem.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-siem"
-  spec.version = "0.1.7"
+  spec.version = "2.0.0"
   spec.authors = ["John Peterson", "Mahitha Byreddy"]
   spec.email   = ["johnp@jfrog.com", "mahithab@jfrog.com"]
 
@@ -24,7 +24,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_development_dependency "rest-client", "~> 2.0"
-  spec.add_development_dependency "thread", "~> 0.2.2"
-  spec.add_runtime_dependency "thread", "~> 0.2.2"
+  spec.add_development_dependency "concurrent-ruby", "~> 1.1.8"
+  spec.add_development_dependency "concurrent-ruby-edge", '>= 0'
+  spec.add_development_dependency 'rspec', '~> 3.10.0'
+
+  spec.add_runtime_dependency "rest-client", "~> 2.0"
+  spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8"
+  spec.add_runtime_dependency "concurrent-ruby-edge", '>= 0'
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/lib/fluent/plugin/in_jfrog_siem.rb

@@ -14,10 +14,10 @@
 # limitations under the License.
 require "fluent/plugin/input"
 require "rest-client"
-require "thread/pool"
-require "json"
 require "date"
 require "uri"
+require "fluent/plugin/xray"
+require "fluent/plugin/position_file"
 
 module Fluent
   module Plugin
@@ -30,12 +30,12 @@ module Fluent
       # `:default` means that the parameter is optional.
       config_param :tag, :string, default: ""
       config_param :jpd_url, :string, default: ""
-      config_param :access_token, :string, default: ""
-      config_param :pos_file, :string, default: ""
+      config_param :username, :string, default: ""
+      config_param :apikey, :string, default: ""
       config_param :batch_size, :integer, default: 25
-      config_param :thread_count, :integer, default: 5
       config_param :wait_interval, :integer, default: 60
-
+      config_param :from_date, :string, default: ""
+      config_param :pos_file_path, :string, default: ""
 
       # `configure` is called before `start`.
       # 'conf' is a `Hash` that includes the configuration parameters.
@@ -50,26 +50,22 @@ module Fluent
           raise Fluent::ConfigError, "Must define the JPD URL to pull Xray SIEM violations."
         end
 
-        if @access_token == ""
-          raise Fluent::ConfigError, "Must define the access token to use for authentication."
-        end
-
-        if @pos_file == ""
-          raise Fluent::ConfigError, "Must define a position file to record last SIEM violation pulled."
+        if @username == ""
+          raise Fluent::ConfigError, "Must define the username to use for authentication."
         end
 
-        if @thread_count < 1
-          raise Fluent::ConfigError, "Must define at least one thread to process violation details."
-        end
-
-        if @thread_count > @batch_size
-          raise Fluent::ConfigError, "Violation detail url thread count exceeds batch size."
+        if @apikey == ""
+          raise Fluent::ConfigError, "Must define the API Key to use for authentication."
         end
 
         if @wait_interval < 1
           raise Fluent::ConfigError, "Wait interval must be greater than 1 to wait between pulling new events."
         end
 
+        if @from_date == ""
+          puts "From date not specified, so getting violations from current date if pos_file doesn't exist"
+        end
+
       end
 
 
@@ -89,223 +85,53 @@ module Fluent
 
 
       def run
-        call_home(@jpd_url, @access_token)
-        # runs the violation pull
-        last_created_date_string = get_last_item_create_date()
-        begin
-          last_created_date = DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
-        rescue
-          last_created_date = DateTime.parse("1970-01-01T00:00:00Z").strftime("%Y-%m-%dT%H:%M:%SZ")
-        end
-        offset_count=1
-        left_violations=0
-        waiting_for_violations = false
-        xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } }
-
-        while true
-          # Grab the batch of records
-          resp=get_xray_violations(xray_json, @jpd_url, @access_token)
-          number_of_violations = JSON.parse(resp)['total_violations']
-          if left_violations <= 0
-            left_violations = number_of_violations
-          end
-
-          xray_violation_urls_list = []
-          for index in 0..JSON.parse(resp)['violations'].length-1 do
-            # Get the violation
-            item = JSON.parse(resp)['violations'][index]
-
-            # Get the created date and check if we should skip (already processed) or process this record.
-            created_date_string = item['created']
-            created_date = DateTime.parse(created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
+        # call_home(@jpd_url)
 
-            # Determine if we need to persist this record or not
-            persistItem = true
-            if waiting_for_violations
-              if created_date <= last_created_date
-                # "not persisting it - waiting for violations"
-                persistItem = false
-              end
-            else
-              if created_date < last_created_date
-                # "persisting everything"
-                persistItem = true
-              end
-            end
+        last_created_date = get_last_item_create_date()
 
-            # Publish the record to fluentd
-            if persistItem
-
-              now = Fluent::Engine.now
-              router.emit(@tag, now, item)
-
-              # write to the pos_file created_date_string
-              open(@pos_file, 'a') do |f|
-                f << "#{created_date_string}\n"
-              end
-
-              # Mark this as the last record successfully processed
-              last_created_date_string = created_date_string
-              last_created_date = created_date
-
-              # Grab violation detail url and add to url list to process w/ thread pool
-              xray_violation_details_url=item['violation_details_url']
-              xray_violation_urls_list.append(xray_violation_details_url)
-            end
-          end
-
-          # iterate over url array adding to thread pool each url.
-          # limit max workers to thread count to prevent overloading xray.
-          thread_pool = Thread.pool(thread_count)
-          thread_pool.process {
-            for xray_violation_url in xray_violation_urls_list do
-              pull_violation_details(xray_violation_url, @access_token)
-            end
-          }
-          thread_pool.shutdown
-
-          # reduce left violations by jump size (not all batches have full item count??)
-          left_violations = left_violations - @batch_size
-          if left_violations <= 0
-            waiting_for_violations = true
-            sleep(@wait_interval)
-          else
-            # Grab the next record to process for the violation details url
-            waiting_for_violations = false
-            offset_count = offset_count + 1
-            xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } }
-          end
-        end
-      end
-
-
-      # pull the last item create date from the pos_file return created_date_string
-      def get_last_item_create_date()
-        if(!(File.exist?(@pos_file)))
-          @pos_file = File.new(@pos_file, "w")
+        if (@from_date != "")
+          last_created_date = DateTime.parse(@from_date).strftime("%Y-%m-%dT%H:%M:%SZ")
         end
-        return IO.readlines(@pos_file).last
+        date_since = last_created_date
+        puts "Getting queries from #{date_since}"
+        xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, router, @tag)
+        violations_channel = xray.violations(date_since)
+        xray.violation_details(violations_channel)
+        sleep 100
       end
 
       #call home functionality
-      def call_home(jpd_url, access_token)
+      def call_home(jpd_url)
         call_home_json = { "productId": "jfrogLogAnalytics/v0.5.1", "features": [ { "featureId": "Platform/Xray" }, { "featureId": "Channel/xrayeventsiem" } ] }
         response = RestClient::Request.new(
             :method => :post,
             :url => jpd_url + "/artifactory/api/system/usage",
             :payload => call_home_json.to_json,
-            :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + access_token }
-        ).execute do |response, request, result|
-            puts "Posting call home information"
-        end
-      end
-
-      # queries the xray API for violations based upon the input json
-      def get_xray_violations_detail(xray_violation_detail_url, access_token)
-        response = RestClient::Request.new(
-            :method => :get,
-            :url => xray_violation_detail_url,
-            headers: {Authorization:'Bearer ' + access_token}
-        ).execute do |response, request, result|
-          case response.code
-          when 200
-            return response.to_str
-          else
-            raise Fluent::StandardError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
-          end
-        end
-      end
-
-
-      # queries the xray API for violations based upon the input json
-      def get_xray_violations(xray_json, jpd_url, access_token)
-        response = RestClient::Request.new(
-            :method => :post,
-            :url => jpd_url + "/xray/api/v1/violations",
-            :payload => xray_json.to_json,
-            :headers => { :accept => :json, :content_type => :json, Authorization:'Bearer ' + access_token }
+            :user => @username,
+            :password => @apikey,
+            :headers => { :accept => :json, :content_type => :json}
         ).execute do |response, request, result|
-          case response.code
-          when 200
-            return response.to_str
-          else
-            raise Fluent::StandardError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
-          end
+          puts "Posting call home information"
         end
       end
 
-      # normalizes Xray data according to common information models for all log-vendors
-      def data_normalization(detailResp)
-        detailResp_json = JSON.parse(detailResp)
-        cve = []
-        cvss_v2_list = []
-        cvss_v3_list = []
-        policy_list = []
-        rule_list = []
-        impacted_artifact_url_list = []
-        if detailResp_json.key?('properties')
-          properties = detailResp_json['properties']
-          for index in 0..properties.length-1 do
-            if properties[index].key?('cve')
-              cve.push(properties[index]['cve'])
-            end
-            if properties[index].key?('cvss_v2')
-              cvss_v2_list.push(properties[index]['cvss_v2'])
-            end
-            if properties[index].key?('cvss_v3')
-              cvss_v3_list.push(properties[index]['cvss_v3'])
-            end
-          end
-
-          detailResp_json["cve"] = cve.sort.reverse[0]
-          cvss_v2 = cvss_v2_list.sort.reverse[0]
-          cvss_v3 = cvss_v3_list.sort.reverse[0]
-          if !cvss_v3.nil?
-            cvss = cvss_v3
-          elsif !cvss_v2.nil?
-            cvss = cvss_v2
-          end
-          cvss_score = cvss[0..2]
-          cvss_version = cvss.split(':')[1][0..2]
-          detailResp_json["cvss_score"] = cvss_score
-          detailResp_json["cvss_version"] = cvss_version
-        end
-
-        if detailResp_json.key?('matched_policies')
-          matched_policies = detailResp_json['matched_policies']
-          for index in 0..matched_policies.length-1 do
-            if matched_policies[index].key?('policy')
-              policy_list.push(matched_policies[index]['policy'])
-            end
-            if matched_policies[index].key?('rule')
-              rule_list.push(matched_policies[index]['rule'])
-            end
-          end
-          detailResp_json['policies'] = policy_list
-          detailResp_json['rules'] = rule_list
-        end
-
-        impacted_artifacts = detailResp_json['impacted_artifacts']
-        for impacted_artifact in impacted_artifacts do
-          matchdata = impacted_artifact.match /default\/(?<repo_name>[^\/]*)\/(?<path>.*)/
-          impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " "
-          impacted_artifact_url_list.append(impacted_artifact_url)
+      # pull the last item create date from the pos_file return created_date_string
+      def get_last_item_create_date()
+        recent_pos_file = get_recent_pos_file()
+        if recent_pos_file != nil
+          last_created_date_string = IO.readlines(recent_pos_file).last
+          return DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
+        else
+          return DateTime.now.strftime("%Y-%m-%dT%H:%M:%SZ")
         end
-        detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list
-        return detailResp_json
       end
 
-      def pull_violation_details(xray_violation_detail_url, access_token)
-        begin
-          detailResp=get_xray_violations_detail(xray_violation_detail_url, access_token)
-          time = Fluent::Engine.now
-          detailResp_json = data_normalization(detailResp)
-          router.emit(@tag, time, detailResp_json)
-        rescue
-          raise Fluent::StandardError, "Error pulling violation details url #{xray_violation_detail_url}"
-        end
+      def get_recent_pos_file()
+        pos_file = @pos_file_path + "*.siem.pos"
+        return Dir.glob(pos_file).sort.last
       end
 
     end
   end
 end
+