gitlab-secret_detection 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f72d7d8262749305fdf1811df74637af4f7ba4bbf693fc41d2cdb34d1ca25238
-  data.tar.gz: 6896b4eb32dc734640c0dc125dd75e79b765f27aa618f73fec81c83c0c20df93
+  metadata.gz: b011db6024a9386ec45e091c678b4b4ea43986065f846748f9b646caf2232c86
+  data.tar.gz: 9faabaf1da62a2470bc8c569a766f48867b0b59c701076b0c8e3d4fbd6ac7550
 SHA512:
-  metadata.gz: 2d35dfa136b9524f851a6ed20885aa6d50b9334f12166fd9d131ff523ed38bed1264bad953fd6fa58e63fe2c5f9c62be129d3be231258eef25990803b9b67d4e
-  data.tar.gz: 331a74ba7a779f275717c769c0f63785d7e74d25b07da5d07f44c08a1a0c7f343eec3ba7fd8285d549d298fac4d93bdc3f2bc122ea7cb30bd567e4c75e43efdb
+  metadata.gz: 7bfb7fcda514e6e5baf4d4aac12208629f9034ffea475d815f95644abf385e3d4983d8640524dd297021e5a441b5b9fd0189b26c2aa8db8e0c38f42d59a0e7fd
+  data.tar.gz: 188e1ceb321ed6b92535b704dc962d37d2f4aec6c67ecbddb64316d7ec2503f29975ef416fca3b792613a5a2be1c74fa7175c0f161122e0d5e485423ea4ea7c7

data/lib/gitlab/secret_detection/core/response.rb

@@ -8,7 +8,9 @@ module Gitlab
       # +status+:: One of values from Gitlab::SecretDetection::Core::Status indicating the scan operation's status
       # +results+:: Array of Gitlab::SecretDetection::Core::Finding values. Default value is nil.
       #   to embed more information on error.
-      # +applied_exclusions+:: Array of Exclusions that were applied during this scan.
+      # +applied_exclusions+:: Array of exclusions that were applied during this scan.
+      #   These can be either GRPC::Exclusions when used as a service, or `Security::ProjectSecurityExclusion
+      #   object when used as a gem.
       # +metadata+:: Hash object containing additional meta information about the response. It is currently used
       class Response
         attr_reader :status, :results, :applied_exclusions, :metadata

data/lib/gitlab/secret_detection/core/ruleset.rb

@@ -32,7 +32,7 @@ module Gitlab
           rules_data[:rules].freeze
         rescue StandardError => e
           logger.error "Failed to parse secret detection ruleset from '#{path}' path: #{e}"
-          raise Core::Scanner::RulesetParseError
+          raise Core::Scanner::RulesetParseError, e
         end
       end
     end

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -59,10 +59,13 @@ module Gitlab
         # +timeout+:: No of seconds(accepts floating point for smaller time values) to limit the total scan duration
         # +payload_timeout+:: No of seconds(accepts floating point for smaller time values) to limit
         #                  the scan duration on each payload
-        # +raw_value_exclusions:+:: Array of raw values to exclude from the scan.
-        # +rule_exclusions+:: Array of rules to exclude from the ruleset used for the scan. Each rule is represented
-        #           by its ID. For example: `gitlab_personal_access_token` for representing Gitlab Personal Access
-        #           Token. By default, no rule is excluded from the ruleset.
+        # +exclusions+:: Hash with keys: :raw_value, :rule and values of arrays of either
+        #           GRPC::Exclusion objects (when used as a standalone service)
+        #           or Security::ProjectSecurityExclusion objects (when used as gem).
+        #           :raw_value - Exclusions in the :raw array are the raw values to ignore.
+        #           :rule - Exclusions in the :rule array are the rules to exclude from the ruleset used for the scan.
+        #           Each rule is represented by its ID. For example: `gitlab_personal_access_token`
+        #           for representing Gitlab Personal Access Token. By default, no rule is excluded from the ruleset.
         # +tags+:: Array of tag values to filter from the default ruleset when determining the rules used for the scan.
         #           For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
         #           [`gitlab_blocking`] (+DEFAULT_PATTERN_MATCHER_TAGS+).
@@ -84,8 +87,7 @@ module Gitlab
           payloads,
           timeout: DEFAULT_SCAN_TIMEOUT_SECS,
           payload_timeout: DEFAULT_PAYLOAD_TIMEOUT_SECS,
-          raw_value_exclusions: [],
-          rule_exclusions: [],
+          exclusions: {},
           tags: DEFAULT_PATTERN_MATCHER_TAGS,
           subprocess: RUN_IN_SUBPROCESS
         )
@@ -108,8 +110,7 @@ module Gitlab
               payloads: matched_payloads,
               payload_timeout:,
               pattern_matcher: build_pattern_matcher(tags:),
-              raw_value_exclusions:,
-              rule_exclusions:
+              exclusions:
             }
 
             secrets, applied_exclusions = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
@@ -203,7 +204,7 @@ module Gitlab
             matched_payloads << payload
           end
 
-          matched_payloads.freeze
+          matched_payloads
         end
 
         # Runs the secret detection scan on the given list of payloads. It accepts
@@ -213,8 +214,7 @@ module Gitlab
           payloads:,
           payload_timeout:,
           pattern_matcher:,
-          raw_value_exclusions: [],
-          rule_exclusions: []
+          exclusions: {}
         )
           all_applied_exclusions = Set.new
 
@@ -223,8 +223,7 @@ module Gitlab
               findings, applied_exclusions = find_secrets_in_payload(
                 payload:,
                 pattern_matcher:,
-                raw_value_exclusions:,
-                rule_exclusions:
+                exclusions:
               )
               all_applied_exclusions.merge(applied_exclusions)
               findings
@@ -235,15 +234,14 @@ module Gitlab
             Core::Finding.new(payload.id,
               Core::Status::PAYLOAD_TIMEOUT)
           end
-          [all_findings.freeze, all_applied_exclusions.to_a.freeze]
+          [all_findings, all_applied_exclusions.to_a]
         end
 
         def run_scan_within_subprocess(
           payloads:,
           payload_timeout:,
           pattern_matcher:,
-          raw_value_exclusions: [],
-          rule_exclusions: []
+          exclusions: {}
         )
           all_applied_exclusions = Set.new
           payload_sizes = payloads.map(&:size)
@@ -261,7 +259,7 @@ module Gitlab
                 findings, applied_exclusions = find_secrets_in_payload(
                   payload:,
                   pattern_matcher:,
-                  raw_value_exclusions:, rule_exclusions:
+                  exclusions:
                 )
                 all_applied_exclusions.merge(applied_exclusions)
                 findings
@@ -273,25 +271,28 @@ module Gitlab
             end
           end
 
-          [found_secrets.freeze, all_applied_exclusions.to_a.freeze]
+          [found_secrets, all_applied_exclusions.to_a]
         end
 
         # Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan.
-        def find_secrets_in_payload(payload:, pattern_matcher:, raw_value_exclusions: [], rule_exclusions: [])
+        def find_secrets_in_payload(payload:, pattern_matcher:, exclusions: {})
           findings = []
           applied_exclusions = Set.new
 
           payload_offset = payload.respond_to?(:offset) ? payload.offset : 0
 
+          raw_value_exclusions = exclusions.fetch(:raw_value, [])
+          rule_exclusions = exclusions.fetch(:rule, [])
+
           payload.data
                  .each_line($INPUT_RECORD_SEPARATOR, chomp: true)
                  .each_with_index do |line, index|
             unless raw_value_exclusions.empty?
-              raw_value_exclusions.each do |value|
-                line.gsub!(value, '') # replace input that doesn't contain allowed value in it
-                applied_exclusions << value # TODO we need the id of the exclusion
+              raw_value_exclusions.each do |exclusion|
+                line.gsub!(exclusion.value, '') # replace input that doesn't contain allowed value in it
+                applied_exclusions << exclusion
               end
             end
 
@@ -322,7 +323,7 @@ module Gitlab
             end
           end
 
-          [findings.freeze, applied_exclusions]
+          [findings, applied_exclusions]
         rescue StandardError => e
           logger.error "Secret Detection scan failed on the payload(id:#{payload.id}): #{e}"
 
@@ -330,7 +331,7 @@ module Gitlab
         end
 
         def applied_rule_exclusion?(type, rule_exclusions, applied_exclusions)
-          applied_exclusion = rule_exclusions&.find { |rule_exclusion| rule_exclusion == type }
+          applied_exclusion = rule_exclusions&.find { |rule_exclusion| rule_exclusion.value == type }
           applied_exclusion && (applied_exclusions << applied_exclusion)
         end
 

data/lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb

@@ -5,7 +5,7 @@
 require 'google/protobuf'
 
 
-descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"Z\n\tExclusion\x12>\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32&.gitlab.secret_detection.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"\xc0\x02\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x36\n\nexclusions\x18\x04 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a\x43\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x13\n\x06offset\x18\x03 \x01(\x05H\x00\x88\x01\x01\x42\t\n\x07_offsetB\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xa2\x04\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12>\n\x12\x61pplied_exclusions\x18\x03 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08*f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitlab::SecretDetection::GRPCb\x06proto3"
+descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"Z\n\tExclusion\x12>\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32&.gitlab.secret_detection.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"\xc0\x02\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x36\n\nexclusions\x18\x04 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a\x43\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x13\n\x06offset\x18\x03 \x01(\x05H\x00\x88\x01\x01\x42\t\n\x07_offsetB\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xa2\x04\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12>\n\x12\x61pplied_exclusions\x18\x03 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08*\x7f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x12\x17\n\x13\x45XCLUSION_TYPE_PATH\x10\x03\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitlab::SecretDetection::GRPCb\x06proto3"
 
 pool = Google::Protobuf::DescriptorPool.generated_pool
 pool.add_serialized_file(descriptor_data)

data/lib/gitlab/secret_detection/grpc/scanner_service.rb

@@ -61,24 +61,25 @@ module Gitlab
           validate_request(request)
 
           payloads = request.payloads.to_a
+          exclusions = { raw_value: [], rule: [], path: [] }
 
-          raw_value_exclusions = []
-          rule_exclusions = []
-
-          request.exclusions&.each do |exclusion|
-            case exclusion.type
+          request.exclusions.each do |exclusion|
+            case exclusion.exclusion_type
             when :EXCLUSION_TYPE_RAW_VALUE
-              raw_value_exclusions << exclusion.value
+              exclusions[:raw] << exclusion
             when :EXCLUSION_TYPE_RULE
-              rule_exclusions << exclusion.value
+              exclusions[:rule] << exclusion
+            when :EXCLUSION_TYPE_PATH
+              exclusions[:path] << exclusion
+            else
+              logger.warn("Unknown exclusion type #{exclusion.exclusion_type}")
             end
           end
 
           begin
             result = scanner.secrets_scan(
               payloads,
-              raw_value_exclusions:,
-              rule_exclusions:,
+              exclusions:,
               tags: request.tags.to_a,
               timeout: request.timeout_secs,
               payload_timeout: request.payload_timeout_secs
@@ -94,7 +95,8 @@ module Gitlab
 
           Gitlab::SecretDetection::GRPC::ScanResponse.new(
             results: findings,
-            status: result.status
+            status: result.status,
+            applied_exclusions: result.applied_exclusions
           )
         end
 

data/proto/secret_detection.proto

@@ -17,6 +17,7 @@ enum ExclusionType {
   EXCLUSION_TYPE_UNSPECIFIED = 0;
   EXCLUSION_TYPE_RULE = 1; // Rule ID to exclude
   EXCLUSION_TYPE_RAW_VALUE = 2; // Raw value to exclude
+  EXCLUSION_TYPE_PATH = 3; // Specific file path to exclude
 }
 
 /* Request arg for triggering Scan/ScanStream method */

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-11 00:00:00.000000000 Z
+date: 2024-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc