gitlab-secret_detection 0.29.1 → 0.30.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 673a9afd30ab84aa8c163e417ec0fe729fc6ec39c07b15a903e666b3b0d36cc7
-  data.tar.gz: e213724b9cbbfdac19ecab74bf88d7a684396ad6ba875069f6e7dd99f9488617
+  metadata.gz: 65f45d17834681b9a97ff2bb4a896c5e7c227c7bc0f8f65fcf5ba0415a8d602b
+  data.tar.gz: a74588e9f8c97bac5c85cc443d51aa22e2f8000242a7bee00f2bd7896e2adb99
 SHA512:
-  metadata.gz: 5e6f2662182666ce60547c9a29bcccc0cd10ec0df4d5d3ebf0310f712e311a56b47a8efabc00592a243dbf71592d405e350f5b68b4cd4706ee29fdb3b1c14ccb
-  data.tar.gz: f06f4926180c559d22945608360a53217bb7ece7d64d8e1548d153745b28fa7fd53b1739c6eb1957008862fe673a8d489627065b4a9122c8b18d0f65492d5b6d
+  metadata.gz: f7637fe881e012b5a076cc1df31e2141a1f8d30cb6d255c5ef753bc7271a37c3688d6e7efc9e60d6405f3603654842570a40fed5eaed6c5a7221248c9c183291
+  data.tar.gz: 82dd43f806528787b3b97d9c4aedccf61f4b2ad2621a5f6cee29c913786490e8c5120fe0cb80f63203281e42b1b72895aa1852be66b5a112f64522fce93d0efd

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -24,7 +24,9 @@ module Gitlab
         # run the scan within a new subprocess.
         MIN_CHUNK_SIZE_PER_PROC_BYTES = 2_097_152 # 2MiB
         # Whether to run scan in subprocesses or not. Default is false.
-        RUN_IN_SUBPROCESS = false
+        RUN_IN_SUBPROCESS = ENV.fetch('GITLAB_SD_RUN_IN_SUBPROCESS', false)
+        # Default limit for max findings to be returned in the scan
+        DEFAULT_MAX_FINDINGS_LIMIT = 999
 
         # Initializes the instance with logger along with following operations:
         # 1. Extract keywords from the parsed ruleset to use it for matching keywords before regex operation.
@@ -61,6 +63,8 @@ module Gitlab
         # +tags+:: Array of tag values to filter from the default ruleset when determining the rules used for the scan.
         #           For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
         #           [`gitlab_blocking`] (+DEFAULT_PATTERN_MATCHER_TAGS+).
+        # +max_findings_limit+:: Integer to limit the number of findings to be returned in the scan. Defaults
+        #           to 999 (+DEFAULT_MAX_FINDINGS_LIMIT+).
         #
         # NOTE:
         # Running the scan in fork mode primarily focuses on reducing the memory consumption of the scan by
@@ -81,7 +85,8 @@ module Gitlab
           payload_timeout: DEFAULT_PAYLOAD_TIMEOUT_SECS,
           exclusions: {},
           tags: DEFAULT_PATTERN_MATCHER_TAGS,
-          subprocess: RUN_IN_SUBPROCESS
+          subprocess: RUN_IN_SUBPROCESS,
+          max_findings_limit: DEFAULT_MAX_FINDINGS_LIMIT
         )
           return Core::Response.new(status: Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
 
@@ -105,7 +110,8 @@ module Gitlab
               payload_timeout:,
               pattern_matcher:,
               exclusions:,
-              rules: active_rules
+              rules: active_rules,
+              max_findings_limit:
             }.freeze
 
             logger.info(
@@ -116,6 +122,7 @@ module Gitlab
               scannable_payloads_post_keyword_filter: matched_payloads.length,
               tags:,
               run_in_subprocess: subprocess,
+              max_findings_limit:,
               given_exclusions: format_exclusions_hash(exclusions)
             )
 
@@ -282,9 +289,9 @@ module Gitlab
           payloads:,
           payload_timeout:,
           pattern_matcher:,
+          max_findings_limit:,
           exclusions: {},
-          rules: []
-        )
+          rules: [])
           all_applied_exclusions = Set.new
 
           logger.info(
@@ -292,7 +299,7 @@ module Gitlab
             payload_timeout:
           )
 
-          all_findings = payloads.flat_map do |payload|
+          capped_findings = payloads.lazy.flat_map do |payload|
             Timeout.timeout(payload_timeout) do
               findings, applied_exclusions = find_secrets_in_payload(
                 payload:,
@@ -308,14 +315,16 @@ module Gitlab
 
             Core::Finding.new(payload.id,
               Core::Status::PAYLOAD_TIMEOUT)
-          end
-          [all_findings, all_applied_exclusions.to_a]
+          end.take(max_findings_limit).to_a
+
+          [capped_findings, all_applied_exclusions.to_a]
         end
 
         def run_scan_within_subprocess(
           payloads:,
           payload_timeout:,
           pattern_matcher:,
+          max_findings_limit:,
           exclusions: {},
           rules: []
         )
@@ -332,12 +341,16 @@ module Gitlab
             payload_timeout:
           )
 
-          found_secrets = Parallel.flat_map(
-            grouped_payloads,
-            in_processes: MAX_PROCS_PER_REQUEST,
-            isolation: true # do not reuse sub-processes
-          ) do |grouped_payload|
-            grouped_payload.flat_map do |payload|
+          found_secrets = []
+
+          grouped_payloads.each do |grouped_payload|
+            break if found_secrets.length >= max_findings_limit
+
+            batch_results = Parallel.map(
+              grouped_payload,
+              in_processes: MAX_PROCS_PER_REQUEST,
+              isolation: true # do not reuse sub-processes
+            ) do |payload|
               Timeout.timeout(payload_timeout) do
                 findings, applied_exclusions = find_secrets_in_payload(
                   payload:,
@@ -345,14 +358,23 @@ module Gitlab
                   exclusions:,
                   rules:
                 )
-                all_applied_exclusions.merge(applied_exclusions)
-                findings
+                [findings, applied_exclusions]
               end
             rescue Timeout::Error => e
               logger.warn "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
 
               Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
             end
+
+            # Process results and collect exclusions
+            batch_results.each do |findings, applied_exclusions|
+              all_applied_exclusions.merge(applied_exclusions)
+
+              remaining_slots = max_findings_limit - found_secrets.length
+              found_secrets.concat(findings.take(remaining_slots))
+
+              break if found_secrets.length >= max_findings_limit
+            end
           end
 
           [found_secrets, all_applied_exclusions.to_a]

data/lib/gitlab/secret_detection/version.rb

@@ -5,7 +5,7 @@ module Gitlab
     class Gem
       # Ensure to maintain the same version in CHANGELOG file.
       # More details available under 'Release Process' section in the README.md file.
-      VERSION = "0.29.1"
+      VERSION = "0.30.0"
 
       # SD_ENV env var is used to determine which environment the
       # server is running. This var is defined in `.runway/env-<env>.yml` files.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.29.1
+  version: 0.30.0
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-06-04 00:00:00.000000000 Z
+date: 2025-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc