gitlab-secret_detection 0.44.1 → 0.45.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f87ef52f7c7fcc5ca568db74d13514ae77976e7691cd09c3abaa84f0b9c05f19
-  data.tar.gz: 325da324f3ff92288d230522ea6fd7bf9457d81118c64cebccc2e0072dd564ff
+  metadata.gz: 0f9e6f5022798b252c15cce6bcbdc7ae84e9891af3a3552a1c727953868c1a27
+  data.tar.gz: e8ad9883653d36633fa58570b80d6da8d351feb6b9da91356f93db97f2d34466
 SHA512:
-  metadata.gz: fb248bf045bcf4fddf5e5a3c667d49dcb5c5ce6f030a88eb68058215b4e61dc85ccd8e3f55f4540a23c006f5f3fbb945258e3e3e9a4d5e1b147e3b28d90f4658
-  data.tar.gz: 6efaea4780addd93de280e24770a88b50b17b13df97bf0cfc84b650bc41905b07612189a8a76a4ae50feec470ae403ef130bfdfba4680a324a0463ea6ea157ca
+  metadata.gz: 40b41909555fd7b621805894913d305c2960793b5512f809a27521b17437839e84df43ae427f5d7106a076d6eef353d9a77864adeb1c26f94c5d0d2a718aa7a7
+  data.tar.gz: 7af02d50d1833419b2ae0d194bbf7ac5a8bc6cd7c0326d348e327a7cafad2fc98e454e518e04a7f6b0cb473488eac1535d1ab7831a68cae169d7e572cb3d7c0c

data/lib/gitlab/secret_detection/core/finding.rb

@@ -1,24 +1,28 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Gitlab
   module SecretDetection
     module Core
       # Finding is a data object representing a secret finding identified within a payload
       class Finding
-        attr_reader :payload_id, :status, :line_number, :type, :description
+        attr_reader :payload_id, :status, :line_number, :type, :description, :raw_value
 
-        def initialize(payload_id, status, line_number = nil, type = nil, description = nil)
+        def initialize(payload_id, status, line_number = nil, type = nil, description = nil, raw_value: nil)
           @payload_id = payload_id
           @status = status
           @line_number = line_number
           @type = type
           @description = description
+          @raw_value = raw_value
         end
 
         def ==(other)
           self.class == other.class && other.state == state
         end
 
+        # +raw_value+ is omitted here, and from +state+ and +inspect+, so a secret cannot leak.
         def to_h
           {
             payload_id:,
@@ -29,6 +33,22 @@ module Gitlab
           }
         end
 
+        def as_json(_options = nil)
+          to_h
+        end
+
+        def to_json(*args)
+          to_h.to_json(*args)
+        end
+
+        # The default +inspect+ would dump +raw_value+ in plain text, reaching `p`, error contexts,
+        # and anything holding a Finding (+Response+ included).
+        def inspect
+          "#<#{self.class.name} payload_id=#{payload_id.inspect} status=#{status.inspect} " \
+            "line_number=#{line_number.inspect} type=#{type.inspect} " \
+            "description=#{description.inspect} raw_value=#{raw_value.nil? ? 'nil' : '[FILTERED]'}>"
+        end
+
         protected
 
         def state

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -5,12 +5,15 @@ require 'logger'
 require 'timeout'
 require 'English'
 require 'parallel'
+require_relative '../utils/memoize'
 
 module Gitlab
   module SecretDetection
     module Core
       # Scan is responsible for running Secret Detection scan operation
       class Scanner
+        include ::Gitlab::SecretDetection::Utils::StrongMemoize
+
         # default time limit(in seconds) for running the scan operation per invocation
         DEFAULT_SCAN_TIMEOUT_SECS = 180 # 3 minutes
         # default time limit(in seconds) for running the scan operation on a single payload
@@ -25,6 +28,15 @@ module Gitlab
         RUN_IN_SUBPROCESS = ENV.fetch('GITLAB_SD_RUN_IN_SUBPROCESS', false)
         # Default limit for max findings to be returned in the scan
         DEFAULT_MAX_FINDINGS_LIMIT = 999
+        # Rules whose capture group is incorrectly capturing part of the secret
+        # instead of entire secret. We capture the whole match instead of captured
+        # match when extracting the secret
+        WHOLE_MATCH_RULE_IDS = Set[
+          'Adobe Client Secret',
+          'ContentfulPersonalAccessToken',
+          'Github App Token',
+          'Slack token'
+        ].freeze
 
         # Initializes the instance with logger along with following operations:
         # 1. Filter the parsed ruleset down to rules applicable to push protection based on
@@ -59,6 +71,13 @@ module Gitlab
         #           for backward compatibility with existing callers and will be removed.
         # +max_findings_limit+:: Integer to limit the number of findings to be returned in the scan. Defaults
         #           to 999 (+DEFAULT_MAX_FINDINGS_LIMIT+).
+        # +include_raw_value+:: Whether each finding should carry the substring it matched, on
+        #           +Finding#raw_value+. Defaults to false. Callers that opt in receive secrets in
+        #           memory and are responsible for keeping them out of logs and other sinks. In
+        #           subprocess mode the value travels back from the child through a +Parallel+ IPC pipe.
+        #
+        #           LIMITATION: one finding per rule per line, so two secrets of the *same* rule on one
+        #           line yield one +raw_value+, the leftmost. Not an exhaustive list of a line's secrets.
         #
         # NOTE:
         # Running the scan in fork mode primarily focuses on reducing the memory consumption of the scan by
@@ -80,7 +99,8 @@ module Gitlab
           exclusions: {},
           tags: [],
           subprocess: RUN_IN_SUBPROCESS,
-          max_findings_limit: DEFAULT_MAX_FINDINGS_LIMIT
+          max_findings_limit: DEFAULT_MAX_FINDINGS_LIMIT,
+          include_raw_value: false
         )
           return Core::Response.new(status: Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
 
@@ -97,6 +117,10 @@ module Gitlab
             )
           end
 
+          # Before the timeout, so the one-off compile is not charged to the scan budget, a bad
+          # pattern fails loudly, and forked children inherit the patterns instead of recompiling.
+          compiled_regexes if include_raw_value
+
           Timeout.timeout(timeout) do
             matched_payloads = filter_by_keywords(keyword_matcher, payloads)
 
@@ -108,7 +132,8 @@ module Gitlab
               pattern_matcher:,
               rules:,
               exclusions:,
-              max_findings_limit:
+              max_findings_limit:,
+              include_raw_value:
             }.freeze
 
             logger.info(
@@ -291,6 +316,7 @@ module Gitlab
           pattern_matcher:,
           max_findings_limit:,
           rules:,
+          include_raw_value:,
           exclusions: {})
           all_applied_exclusions = Set.new
 
@@ -305,7 +331,8 @@ module Gitlab
                 payload:,
                 pattern_matcher:,
                 exclusions:,
-                rules:
+                rules:,
+                include_raw_value:
               )
               all_applied_exclusions.merge(applied_exclusions)
               findings
@@ -326,6 +353,7 @@ module Gitlab
           pattern_matcher:,
           max_findings_limit:,
           rules:,
+          include_raw_value:,
           exclusions: {}
         )
           all_applied_exclusions = Set.new
@@ -356,14 +384,17 @@ module Gitlab
                   payload:,
                   pattern_matcher:,
                   exclusions:,
-                  rules:
+                  rules:,
+                  include_raw_value:
                 )
                 [findings, applied_exclusions]
               end
             rescue Timeout::Error => e
               logger.warn "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
 
-              Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
+              # Must match the happy path's shape: the consumer destructures each element into
+              # `findings, applied_exclusions`. A bare Finding made `Set#merge(nil)` raise.
+              [[Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)], []]
             end
 
             # Process results and collect exclusions
@@ -383,7 +414,7 @@ module Gitlab
         # Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan.
-        def find_secrets_in_payload(payload:, pattern_matcher:, rules:, exclusions: {})
+        def find_secrets_in_payload(payload:, pattern_matcher:, rules:, exclusions: {}, include_raw_value: false)
           findings = []
           applied_exclusions = Set.new
 
@@ -419,14 +450,8 @@ module Gitlab
 
               next if applied_rule_exclusion?(rule[:id], rule_exclusions, applied_exclusions)
 
-              title = rule[:title].nil? ? rule[:description] : rule[:title]
-
-              findings << Core::Finding.new(
-                payload.id,
-                Core::Status::FOUND,
-                line_no,
-                rule[:id],
-                title
+              findings << build_finding(
+                payload_id: payload.id, line_no:, rule:, rule_index: match_idx, line:, include_raw_value:
               )
             end
           end
@@ -445,6 +470,72 @@ module Gitlab
           [[Core::Finding.new(payload.id, Core::Status::SCAN_ERROR)], []]
         end
 
+        def build_finding(payload_id:, line_no:, rule:, rule_index:, line:, include_raw_value:)
+          secret = include_raw_value ? extract_raw_value(rule, rule_index, line) : nil
+          Core::Finding.new(
+            payload_id,
+            Core::Status::FOUND,
+            line_no,
+            rule[:id],
+            rule[:title] || rule[:description],
+            raw_value: secret
+          )
+        end
+
+        # Returns the substring of +line+ that the rule actually matched, or nil when it does not
+        # match. +line+ has already had raw value exclusions stripped out of it by the caller, so
+        # this can never return an allowlisted secret.
+        #
+        # RE2::Set reports which patterns hit but not where, so the pattern is run again to locate
+        # the match; with two secrets of the same rule on a line this returns the leftmost.
+        # +rule_index+ indexes +compiled_regexes+, which is built from the scanner's own +rules+,
+        # so callers must pass that same array as +rules:+ or extraction returns nil.
+        def extract_raw_value(rule, rule_index, line)
+          regex = compiled_regexes[rule_index]
+
+          # RE2 returns `true` rather than a match object when a pattern has no capturing groups and
+          # `submatches` is left at its default, so this argument is necessary. One more than the
+          # group count leaves `match[1]` nil for a group-less pattern.
+          match = regex.match(line, submatches: regex.number_of_capturing_groups + 1)
+          return unless match
+
+          return match[0] if WHOLE_MATCH_RULE_IDS.include?(rule[:id])
+
+          match[1] || match[0]
+        rescue StandardError => e
+          logger.warn(message: "Failed to extract the matched value", rule_id: rule[:id], error: e.class.name)
+          nil
+        end
+
+        # +RE2::Set+ does not expose its compiled patterns, so extraction needs its own +RE2::Regexp+
+        # per rule, in an Array aligned with the indices RE2::Set reports. Built once and frozen so it
+        # is fork-inheritable and not a shared mutable; lazy so opt-out scans pay nothing.
+        def compiled_regexes
+          strong_memoize(:compiled_regexes) do
+            build_compiled_regexes(rules)
+          end
+        end
+
+        # +RE2::Regexp.new+ does not raise on a bad pattern; it returns one whose +ok?+ is false and
+        # +number_of_capturing_groups+ is -1, which makes +submatches+ zero and flips +match+ to a
+        # boolean, silently nilling every raw_value for that rule. Fail loudly instead.
+        def build_compiled_regexes(rules)
+          compiled = rules.map { |rule| RE2::Regexp.new(rule[:regex]) }
+
+          invalid = rules.zip(compiled).reject { |_rule, regex| regex.ok? }
+          unless invalid.empty?
+            logger.error(
+              message: "Failed to compile individual secret detection rule patterns for raw value extraction",
+              rule_ids: invalid.map { |rule, _regex| rule[:id] },
+              errors: invalid.map { |_rule, regex| regex.error }
+            )
+
+            raise Core::Ruleset::RulesetCompilationError
+          end
+
+          compiled.freeze
+        end
+
         def applied_rule_exclusion?(type, rule_exclusions, applied_exclusions)
           applied_exclusion = rule_exclusions&.find { |rule_exclusion| rule_exclusion.value == type }
           applied_exclusion && (applied_exclusions << applied_exclusion)

data/lib/gitlab/secret_detection/version.rb

@@ -5,7 +5,7 @@ module Gitlab
     class Gem
       # Ensure to maintain the same version in CHANGELOG file.
       # More details available under 'Release Process' section in the README.md file.
-      VERSION = "0.44.1"
+      VERSION = "0.45.0"
 
       # SD_ENV env var is used to determine which environment the
       # server is running. This var is defined in `.runway/env-<env>.yml` files.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.44.1
+  version: 0.45.0
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-03 00:00:00.000000000 Z
+date: 2026-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc