fluent-plugin-jfrog-siem 0.1.8 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e222c0afdaf25a0aa38e4167236c2f06b5c7f9c889ca544f3a05677609c2c1e
-  data.tar.gz: '097fd2e7d1b8b0b8394e11efa9abbcfa5d0e67412d3284bdd5c5fe47badde641'
+  metadata.gz: ea8dc8e85e1874da646a83a7aea1ab32a63057cfaedd0cce5b30a63461898b9c
+  data.tar.gz: 4d38c3858e8432b13cd25e1fde6a82619d037274037173fd0825b3b1236e811e
 SHA512:
-  metadata.gz: 18b38f238bef87f6e015aa6e6eda87ba58586e6e85ef5f29d58392b91ad1fc5c12da33777ce7b477f222d966df2c5eb7082d1cbe757cc712703adb4511945057
-  data.tar.gz: 49a13e9f1aeec783f9e7f35ca9750d3d3c4398d1e6facfa029dbfa6dec8a27d083941e995722cdb7ffa66937fb747bc079fe439e534e5dc23023e7543d56dc4b
+  metadata.gz: e9289b75176b973729f922e61b8a1c17481ad896f1bed3e9baae687f6387ecf8d968523ec229da36d22f6d6ef9b052ab8e9cab5d652310e2f4c8d0dfb8ef81f8
+  data.tar.gz: 9bf33d52a6265a39dceb9bb8e1c99d395fcc0e654b2586d180379c390ac76e86b0dcf680985d01cecf9daaa3237d34e7e9848a2d2a0f6b4461c3e59065a5ff2f

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/Gemfile.lock

@@ -0,0 +1,90 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-jfrog-siem (1.0.0)
+      concurrent-ruby (~> 1.1.8)
+      concurrent-ruby-edge
+      fluentd (>= 0.14.10, < 2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.1.8)
+    concurrent-ruby-edge (0.6.0)
+      concurrent-ruby (~> 1.1.6)
+    cool.io (1.7.1)
+    diff-lcs (1.4.4)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    fluentd (1.13.1)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
+      yajl-ruby (~> 1.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http_parser.rb (0.6.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2021.0225)
+    msgpack (1.4.2)
+    netrc (0.11.0)
+    power_assert (2.0.0)
+    rake (12.3.3)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    serverengine (2.2.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    test-unit (3.4.2)
+      power_assert
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    webrick (1.7.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  concurrent-ruby (~> 1.1.8)
+  concurrent-ruby-edge
+  fluent-plugin-jfrog-siem!
+  rake (~> 12.0)
+  rest-client (~> 2.0)
+  rspec (~> 3.10.0)
+  test-unit (~> 3.0)
+
+BUNDLED WITH
+   2.2.3

data/README.md

@@ -87,7 +87,8 @@ wget https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/siem/d
 Integration is done by setting up Xray. Obtain JPD url and access token for API. Configure the source directive parameters specified below
 * **tag** (string) (required): The value is the tag assigned to the generated events.
 * **jpd_url** (string) (required): JPD url required to pull Xray SIEM violations
-* **access_token** (string) (required): [Access token](https://www.jfrog.com/confluence/display/JFROG/Access+Tokens) to authenticate Xray
+* **apikey** (string) (required): API Key is the [Artifactory API Key](https://www.jfrog.com/confluence/display/JFROG/User+Profile#UserProfile-APIKey) for authentication
+* **username** (string) (required): USER is the Artifactory username for authentication
 * **pos_file** (string) (required): Position file to record last SIEM violation pulled
 * **batch_size** (integer) (optional): Batch size for processing violations
     * Default value: `25`

data/Rakefile

@@ -4,7 +4,7 @@ Bundler::GemHelper.install_tasks
 require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
-  t.libs.push("lib", "test")
+  t.libs.push("lib", "test", 'lib/fluent/plugin')
   t.test_files = FileList["test/**/test_*.rb"]
   t.verbose = true
   t.warning = false

data/fluent-plugin-jfrog-siem.gemspec

@@ -3,13 +3,13 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jfrog-siem"
-  spec.version = "0.1.8"
-  spec.authors = ["John Peterson", "Mahitha Byreddy"]
-  spec.email   = ["johnp@jfrog.com", "mahithab@jfrog.com"]
+  spec.version = "2.0.1"
+  spec.authors = ["Mahitha Byreddy", "Sudhindra Rao"]
+  spec.email   = ["mahithab@jfrog.com", "sudhindrar@jfrog.com"]
 
   spec.summary       = %q{JFrog SIEM fluent input plugin will send the SIEM events from JFrog Xray to Fluentd}
   spec.description   = %q{JFrog SIEM fluent input plugin will send the SIEM events from JFrog Xray to Fluentd which can then be delivered to whatever output plugin specified}
-  spec.homepage      = "https://github.com/jfrog/log-analytics"
+  spec.homepage      = "https://github.com/jfrog/fluent-plugin-jfrog-siem"
   spec.license       = "Apache-2.0"
 
   test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
@@ -24,7 +24,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_development_dependency "rest-client", "~> 2.0"
-  spec.add_development_dependency "thread", "~> 0.2.2"
-  spec.add_runtime_dependency "thread", "~> 0.2.2"
+  spec.add_development_dependency "concurrent-ruby", "~> 1.1.8"
+  spec.add_development_dependency "concurrent-ruby-edge", '>= 0'
+  spec.add_development_dependency 'rspec', '~> 3.10.0'
+
+  spec.add_runtime_dependency "rest-client", "~> 2.0"
+  spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8"
+  spec.add_runtime_dependency "concurrent-ruby-edge", '>= 0'
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/lib/fluent/plugin/in_jfrog_siem.rb

@@ -14,10 +14,10 @@
 # limitations under the License.
 require "fluent/plugin/input"
 require "rest-client"
-require "thread/pool"
-require "json"
 require "date"
 require "uri"
+require "fluent/plugin/xray"
+require "fluent/plugin/position_file"
 
 module Fluent
   module Plugin
@@ -32,11 +32,10 @@ module Fluent
       config_param :jpd_url, :string, default: ""
       config_param :username, :string, default: ""
       config_param :apikey, :string, default: ""
-      config_param :pos_file, :string, default: ""
       config_param :batch_size, :integer, default: 25
-      config_param :thread_count, :integer, default: 5
       config_param :wait_interval, :integer, default: 60
-
+      config_param :from_date, :string, default: ""
+      config_param :pos_file_path, :string, default: ""
 
       # `configure` is called before `start`.
       # 'conf' is a `Hash` that includes the configuration parameters.
@@ -59,22 +58,14 @@ module Fluent
           raise Fluent::ConfigError, "Must define the API Key to use for authentication."
         end
 
-        if @pos_file == ""
-          raise Fluent::ConfigError, "Must define a position file to record last SIEM violation pulled."
-        end
-
-        if @thread_count < 1
-          raise Fluent::ConfigError, "Must define at least one thread to process violation details."
-        end
-
-        if @thread_count > @batch_size
-          raise Fluent::ConfigError, "Violation detail url thread count exceeds batch size."
-        end
-
         if @wait_interval < 1
           raise Fluent::ConfigError, "Wait interval must be greater than 1 to wait between pulling new events."
         end
 
+        if @from_date == ""
+          puts "From date not specified, so getting violations from current date if pos_file doesn't exist"
+        end
+
       end
 
 
@@ -94,102 +85,19 @@ module Fluent
 
 
       def run
-        call_home(@jpd_url)
-        # runs the violation pull
-        last_created_date_string = get_last_item_create_date()
-        begin
-          last_created_date = DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
-        rescue
-          last_created_date = DateTime.parse("1970-01-01T00:00:00Z").strftime("%Y-%m-%dT%H:%M:%SZ")
-        end
-        offset_count=1
-        left_violations=0
-        waiting_for_violations = false
-        xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } }
-
-        while true
-          # Grab the batch of records
-          resp=get_xray_violations(xray_json, @jpd_url)
-          number_of_violations = JSON.parse(resp)['total_violations']
-          if left_violations <= 0
-            left_violations = number_of_violations
-          end
-
-          xray_violation_urls_list = []
-          for index in 0..JSON.parse(resp)['violations'].length-1 do
-            # Get the violation
-            item = JSON.parse(resp)['violations'][index]
-
-            # Get the created date and check if we should skip (already processed) or process this record.
-            created_date_string = item['created']
-            created_date = DateTime.parse(created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
+        # call_home(@jpd_url)
 
-            # Determine if we need to persist this record or not
-            persistItem = true
-            if waiting_for_violations
-              if created_date <= last_created_date
-                # "not persisting it - waiting for violations"
-                persistItem = false
-              end
-            else
-              if created_date < last_created_date
-                # "persisting everything"
-                persistItem = true
-              end
-            end
+        last_created_date = get_last_item_create_date()
 
-            # Publish the record to fluentd
-            if persistItem
-
-              now = Fluent::Engine.now
-              router.emit(@tag, now, item)
-
-              # write to the pos_file created_date_string
-              open(@pos_file, 'a') do |f|
-                f << "#{created_date_string}\n"
-              end
-
-              # Mark this as the last record successfully processed
-              last_created_date_string = created_date_string
-              last_created_date = created_date
-
-              # Grab violation detail url and add to url list to process w/ thread pool
-              xray_violation_details_url=item['violation_details_url']
-              xray_violation_urls_list.append(xray_violation_details_url)
-            end
-          end
-
-          # iterate over url array adding to thread pool each url.
-          # limit max workers to thread count to prevent overloading xray.
-          thread_pool = Thread.pool(thread_count)
-          thread_pool.process {
-            for xray_violation_url in xray_violation_urls_list do
-              pull_violation_details(xray_violation_url)
-            end
-          }
-          thread_pool.shutdown
-
-          # reduce left violations by jump size (not all batches have full item count??)
-          left_violations = left_violations - @batch_size
-          if left_violations <= 0
-            waiting_for_violations = true
-            sleep(@wait_interval)
-          else
-            # Grab the next record to process for the violation details url
-            waiting_for_violations = false
-            offset_count = offset_count + 1
-            xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } }
-          end
-        end
-      end
-
-
-      # pull the last item create date from the pos_file return created_date_string
-      def get_last_item_create_date()
-        if(!(File.exist?(@pos_file)))
-          @pos_file = File.new(@pos_file, "w")
+        if (@from_date != "")
+          last_created_date = DateTime.parse(@from_date).strftime("%Y-%m-%dT%H:%M:%SZ")
         end
-        return IO.readlines(@pos_file).last
+        date_since = last_created_date
+        puts "Getting queries from #{date_since}"
+        xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, router, @tag)
+        violations_channel = xray.violations(date_since)
+        xray.violation_details(violations_channel)
+        sleep 100
       end
 
       #call home functionality
@@ -203,119 +111,27 @@ module Fluent
             :password => @apikey,
             :headers => { :accept => :json, :content_type => :json}
         ).execute do |response, request, result|
-            puts "Posting call home information"
+          puts "Posting call home information"
         end
       end
 
-      # queries the xray API for violations based upon the input json
-      def get_xray_violations_detail(xray_violation_detail_url)
-        response = RestClient::Request.new(
-            :method => :get,
-            :url => xray_violation_detail_url,
-            :user => @username,
-            :password => @apikey
-        ).execute do |response, request, result|
-          case response.code
-          when 200
-            return response.to_str
-          else
-            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
-          end
-        end
-      end
-
-
-      # queries the xray API for violations based upon the input json
-      def get_xray_violations(xray_json, jpd_url)
-        response = RestClient::Request.new(
-            :method => :post,
-            :url => jpd_url + "/xray/api/v1/violations",
-            :payload => xray_json.to_json,
-            :user => @username,
-            :password => @apikey,
-            :headers => { :accept => :json, :content_type => :json}
-        ).execute do |response, request, result|
-          case response.code
-          when 200
-            return response.to_str
-          else
-            raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations."
-          end
-        end
-      end
-
-      # normalizes Xray data according to common information models for all log-vendors
-      def data_normalization(detailResp)
-        detailResp_json = JSON.parse(detailResp)
-        cve = []
-        cvss_v2_list = []
-        cvss_v3_list = []
-        policy_list = []
-        rule_list = []
-        impacted_artifact_url_list = []
-        if detailResp_json.key?('properties')
-          properties = detailResp_json['properties']
-          for index in 0..properties.length-1 do
-            if properties[index].key?('cve')
-              cve.push(properties[index]['cve'])
-            end
-            if properties[index].key?('cvss_v2')
-              cvss_v2_list.push(properties[index]['cvss_v2'])
-            end
-            if properties[index].key?('cvss_v3')
-              cvss_v3_list.push(properties[index]['cvss_v3'])
-            end
-          end
-
-          detailResp_json["cve"] = cve.sort.reverse[0]
-          cvss_v2 = cvss_v2_list.sort.reverse[0]
-          cvss_v3 = cvss_v3_list.sort.reverse[0]
-          if !cvss_v3.nil?
-            cvss = cvss_v3
-          elsif !cvss_v2.nil?
-            cvss = cvss_v2
-          end
-          cvss_score = cvss[0..2]
-          cvss_version = cvss.split(':')[1][0..2]
-          detailResp_json["cvss_score"] = cvss_score
-          detailResp_json["cvss_version"] = cvss_version
-        end
-
-        if detailResp_json.key?('matched_policies')
-          matched_policies = detailResp_json['matched_policies']
-          for index in 0..matched_policies.length-1 do
-            if matched_policies[index].key?('policy')
-              policy_list.push(matched_policies[index]['policy'])
-            end
-            if matched_policies[index].key?('rule')
-              rule_list.push(matched_policies[index]['rule'])
-            end
-          end
-          detailResp_json['policies'] = policy_list
-          detailResp_json['rules'] = rule_list
-        end
-
-        impacted_artifacts = detailResp_json['impacted_artifacts']
-        for impacted_artifact in impacted_artifacts do
-          matchdata = impacted_artifact.match /default\/(?<repo_name>[^\/]*)\/(?<path>.*)/
-          impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " "
-          impacted_artifact_url_list.append(impacted_artifact_url)
+      # pull the last item create date from the pos_file return created_date_string
+      def get_last_item_create_date()
+        recent_pos_file = get_recent_pos_file()
+        if recent_pos_file != nil
+          last_created_date_string = IO.readlines(recent_pos_file).last
+          return DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ")
+        else
+          return DateTime.now.strftime("%Y-%m-%dT%H:%M:%SZ")
         end
-        detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list
-        return detailResp_json
       end
 
-      def pull_violation_details(xray_violation_detail_url)
-        begin
-          detailResp=get_xray_violations_detail(xray_violation_detail_url)
-          time = Fluent::Engine.now
-          detailResp_json = data_normalization(detailResp)
-          router.emit(@tag, time, detailResp_json)
-        rescue
-          raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}"
-        end
+      def get_recent_pos_file()
+        pos_file = @pos_file_path + "*.siem.pos"
+        return Dir.glob(pos_file).sort.last
       end
 
     end
   end
 end
+