gitlab-secret_detection 0.44.1 → 0.46.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f87ef52f7c7fcc5ca568db74d13514ae77976e7691cd09c3abaa84f0b9c05f19
-  data.tar.gz: 325da324f3ff92288d230522ea6fd7bf9457d81118c64cebccc2e0072dd564ff
+  metadata.gz: 31d3996d42590cabd29d5198d67cc9ee716f31232d207183f98986a3dc0f4bdb
+  data.tar.gz: 44811e65092f594a602a9cc2f399d98ac047569dc7bac7f359a18190fa18c54e
 SHA512:
-  metadata.gz: fb248bf045bcf4fddf5e5a3c667d49dcb5c5ce6f030a88eb68058215b4e61dc85ccd8e3f55f4540a23c006f5f3fbb945258e3e3e9a4d5e1b147e3b28d90f4658
-  data.tar.gz: 6efaea4780addd93de280e24770a88b50b17b13df97bf0cfc84b650bc41905b07612189a8a76a4ae50feec470ae403ef130bfdfba4680a324a0463ea6ea157ca
+  metadata.gz: 5ab9cea777bf959289694831001f055df76b80265df7af578e167b25096403952f4a659f442351f301d6b38a31865a579d47d9b83f68f5201dfbecd1f60679c3
+  data.tar.gz: 520ad2c9f1a97c9576460e0b28071564acf9cadf3516118051e91bb4c09e1ef103b86a054573efe8755d3def33a3b899c74baa01488e326c5a6fa32fcffedb0e

data/README.md

@@ -88,6 +88,33 @@ The command is called when running tests as well as when building the Dockerfile
 The secret_push_protection_rules.toml file is added to .gitignore to
 avoid checking in changes.
 
+### Custom ruleset file (gem only)
+
+A consumer of the ruby gem can use a custom ruleset file instead of the bundled
+secret_push_protection_rules.toml file. Set the `GITLAB_SD_RULESET_FILE_PATH` environment variable
+to the path of the custom file. Set it in the environment of the process that runs the scan.
+
+- The file must be a TOML file with a `.toml` extension.
+- A relative path is resolved against the working directory of the process. `~` is not expanded.
+- A `path:` argument to `Gitlab::SecretDetection::Core::Ruleset.new` has priority over the environment variable.
+- The rules are loaded one time for each `Ruleset` object. A change to the file has an effect only on a new load.
+- The gRPC service also reads this environment variable, but this use is not supported.
+
+The custom file must have a non-empty `[[rules]]` array. Each rule must have these fields:
+
+| Field | Required | Description |
+|---|---|---|
+| `id` | Yes | A non-empty string. Each `id` must be unique in the file. |
+| `regex` | Yes | A non-empty string that RE2 can compile. |
+| `title` or `description` | Yes | The scanner uses the `title` as the finding description. It uses the `description` only if the `title` is missing, empty, or has only white space. The value that the scanner uses must be a non-empty string. |
+| `keywords` | No | An array of non-empty strings. |
+
+The scanner uses the keyword prefilter only for the rules that have keywords. It matches the rules that have no keywords on all payloads.
+
+If the custom file cannot be loaded or fails validation, the gem logs an error with the message
+`Failed to load custom secret detection ruleset. Using the default ruleset.` and loads the bundled file.
+The log entry contains the error class and the full error message. For a validation error, the message lists each problem on its own line. For a TOML syntax error, the message contains the line of the file that has the error.
+
 ## Generating a ruby gem
 
 In the project directory, run `make gem` command in the terminal that builds a ruby gem(ex: `secret_detection-0.1.0.gem`) in the root of

data/lib/gitlab/secret_detection/core/finding.rb

@@ -1,24 +1,28 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Gitlab
   module SecretDetection
     module Core
       # Finding is a data object representing a secret finding identified within a payload
       class Finding
-        attr_reader :payload_id, :status, :line_number, :type, :description
+        attr_reader :payload_id, :status, :line_number, :type, :description, :raw_value
 
-        def initialize(payload_id, status, line_number = nil, type = nil, description = nil)
+        def initialize(payload_id, status, line_number = nil, type = nil, description = nil, raw_value: nil)
           @payload_id = payload_id
           @status = status
           @line_number = line_number
           @type = type
           @description = description
+          @raw_value = raw_value
         end
 
         def ==(other)
           self.class == other.class && other.state == state
         end
 
+        # +raw_value+ is omitted here, and from +state+ and +inspect+, so a secret cannot leak.
         def to_h
           {
             payload_id:,
@@ -29,6 +33,22 @@ module Gitlab
           }
         end
 
+        def as_json(_options = nil)
+          to_h
+        end
+
+        def to_json(*args)
+          to_h.to_json(*args)
+        end
+
+        # The default +inspect+ would dump +raw_value+ in plain text, reaching `p`, error contexts,
+        # and anything holding a Finding (+Response+ included).
+        def inspect
+          "#<#{self.class.name} payload_id=#{payload_id.inspect} status=#{status.inspect} " \
+            "line_number=#{line_number.inspect} type=#{type.inspect} " \
+            "description=#{description.inspect} raw_value=#{raw_value.nil? ? 'nil' : '[FILTERED]'}>"
+        end
+
         protected
 
         def state

data/lib/gitlab/secret_detection/core/ruleset.rb

@@ -2,11 +2,15 @@
 
 require 'toml-rb'
 require 'logger'
+require 're2'
+require_relative '../utils/memoize'
 
 module Gitlab
   module SecretDetection
     module Core
       class Ruleset
+        include ::Gitlab::SecretDetection::Utils::StrongMemoize
+
         # RulesetParseError is thrown when the code fails to parse the
         # ruleset file from the given path
         RulesetParseError = Class.new(StandardError)
@@ -15,10 +19,42 @@ module Gitlab
         # the predefined rulesets
         RulesetCompilationError = Class.new(StandardError)
 
-        # file path where the secrets ruleset file is located
+        ERROR_MESSAGES = {
+          version_extraction_failed: "Failed to extract Secret Detection Ruleset version from ruleset file",
+          parse_failed: "Failed to parse local secret detection ruleset: %<error>s",
+          fallback: "Failed to load custom secret detection ruleset. Using the default ruleset.",
+          validation_failed: "Ruleset validation failed:\n%<errors>s",
+          invalid_extension: "Ruleset file must have a .toml extension",
+          empty_rules: "Rules must be a non-empty Array of [[rules]] tables",
+          duplicate_ids: "Rule ids must be unique",
+          rule_error: "rules[%<index>d]: %<error>s",
+          rule_not_table: "Rule must be a toml table",
+          invalid_id: "Rule id must be a non-empty String",
+          empty_regex: "Rule regex must be a non-empty String",
+          invalid_regex: "Rule regex is not valid: %<error>s",
+          invalid_description: "Rule title or description must be a non-empty String",
+          invalid_keywords: "Rule keywords must be either empty or an Array of non-empty Strings"
+        }.freeze
+
+        # RulesetValidationError is raised when a custom ruleset file fails validation
+        class RulesetValidationError < RulesetParseError
+          def initialize(errors)
+            super(format(ERROR_MESSAGES[:validation_failed], errors: errors.join("\n")))
+          end
+        end
+
+        # Default ruleset file path
         RULESET_FILE_PATH = File.expand_path('secret_push_protection_rules.toml', __dir__)
 
-        def initialize(path: RULESET_FILE_PATH, logger: Logger.new($stdout))
+        RULESET_FILE_PATH_ENV = 'GITLAB_SD_RULESET_FILE_PATH'
+
+        # Returns the rule title, or the rule description when the title is nil or blank.
+        def self.finding_description(rule)
+          title = rule[:title]
+          title.is_a?(String) && !title.strip.empty? ? title : rule[:description]
+        end
+
+        def initialize(path: nil, logger: Logger.new($stdout))
           @path = path
           @logger = logger
         end
@@ -29,38 +65,138 @@ module Gitlab
           @rule_data = parse_ruleset
         end
 
+        # Before a load, it reads the file at resolved_path.
         def extract_ruleset_version
-          @ruleset_version ||= if File.readable?(RULESET_FILE_PATH)
-                                 first_line = File.open(RULESET_FILE_PATH, &:gets)
-                                 first_line&.split(":")&.[](1)&.strip
-                               end
+          file_path = loaded_path || resolved_path
+          return unless File.readable?(file_path)
+
+          first_line = File.open(file_path, &:gets)
+          first_line&.split(":")&.[](1)&.strip # Gets "0.24.3" from "# rule-set version: 0.24.3"
         rescue StandardError => e
-          logger.error(message: "Failed to extract Secret Detection Ruleset version from ruleset file: #{e.message}")
+          logger.error(message: ERROR_MESSAGES[:version_extraction_failed], error: e.message)
+          nil
         end
 
         private
 
-        attr_reader :path, :logger
+        attr_reader :logger, :loaded_path
 
-        # parses given ruleset file and returns the parsed rules
         def parse_ruleset
+          load_from(resolved_path)
+        rescue RulesetParseError => e
+          raise if resolved_path == RULESET_FILE_PATH
+
+          log_fallback(e)
+          load_from(RULESET_FILE_PATH)
+        end
+
+        def resolved_path
+          return File.absolute_path(@path) if @path
+
+          env_value = ENV.fetch(RULESET_FILE_PATH_ENV, '').strip
+          env_value.empty? ? RULESET_FILE_PATH : File.absolute_path(env_value)
+        end
+        strong_memoize_attr :resolved_path
+
+        def load_from(file_path)
+          custom = file_path != RULESET_FILE_PATH
+
           logger.info(
-            message: "Parsing local ruleset file",
-            ruleset_path: RULESET_FILE_PATH
+            message: "Parsing #{custom ? 'custom' : 'bundled'} ruleset file",
+            ruleset_path: file_path
           )
-          rules_data = TomlRB.load_file(path, symbolize_keys: true).freeze
-          ruleset_version = extract_ruleset_version
+          check_toml_extension!(file_path) if custom
+
+          rules_data = TomlRB.load_file(file_path, symbolize_keys: true).freeze
+          validate_rules!(rules_data) if custom
 
+          @loaded_path = file_path
           logger.info(
             message: "Ruleset details fetched for running Secret Detection scan",
+            ruleset_path: file_path,
             total_rules: rules_data[:rules]&.length,
-            ruleset_version:
+            ruleset_version: extract_ruleset_version,
+            custom_ruleset: custom
           )
           rules_data[:rules].freeze
-        rescue StandardError => e
-          logger.error(message: "Failed to parse local secret detection ruleset: #{e.message}")
+        rescue RulesetParseError
+          raise
+        # A deeply nested TOML file raises SystemStackError.
+        rescue StandardError, SystemStackError => e
+          logger.error(message: format(ERROR_MESSAGES[:parse_failed], error: e.message)) unless custom
           raise RulesetParseError, e
         end
+
+        def check_toml_extension!(file_path)
+          return if File.extname(file_path).casecmp?('.toml')
+
+          raise RulesetValidationError, [ERROR_MESSAGES[:invalid_extension]]
+        end
+
+        # Ruleset validation ensuring it has all the required
+        # fields. Applicable to custom ruleset input only.
+        def validate_rules!(rules_data)
+          rules = rules_data[:rules]
+          raise RulesetValidationError, [ERROR_MESSAGES[:empty_rules]] unless rules.is_a?(Array) && !rules.empty?
+
+          errors = rules.each_with_index.flat_map do |rule, index|
+            rule_errors(rule).map { |error| format(ERROR_MESSAGES[:rule_error], index:, error:) }
+          end
+
+          if errors.empty? && rules.map { |rule| rule[:id] }.uniq.size < rules.size
+            errors << ERROR_MESSAGES[:duplicate_ids]
+          end
+
+          raise RulesetValidationError, errors unless errors.empty?
+        end
+
+        def rule_errors(rule)
+          return [ERROR_MESSAGES[:rule_not_table]] unless rule.is_a?(Hash)
+
+          finding_description = self.class.finding_description(rule)
+          keywords = rule[:keywords]
+
+          errors = []
+          errors << ERROR_MESSAGES[:invalid_id] unless non_empty_string?(rule[:id])
+          errors << regex_error(rule[:regex])
+          errors << ERROR_MESSAGES[:invalid_description] unless non_empty_string?(finding_description)
+
+          unless keywords.nil? || (keywords.is_a?(Array) && keywords.all? { |keyword| non_empty_string?(keyword) })
+            errors << ERROR_MESSAGES[:invalid_keywords]
+          end
+
+          errors.compact
+        end
+
+        def regex_error(regex)
+          return ERROR_MESSAGES[:empty_regex] unless non_empty_string?(regex)
+
+          # Without log_errors: false, RE2 writes the pattern to STDERR.
+          compiled = RE2::Regexp.new(regex, log_errors: false)
+          format(ERROR_MESSAGES[:invalid_regex], error: compiled.error) unless compiled.ok?
+        end
+
+        def non_empty_string?(value)
+          value.is_a?(String) && !value.strip.empty?
+        end
+
+        def log_fallback(error)
+          logger.error(
+            message: ERROR_MESSAGES[:fallback],
+            custom_ruleset_path: resolved_path,
+            default_ruleset_path: RULESET_FILE_PATH,
+            ruleset_path_env: RULESET_FILE_PATH_ENV,
+            error_class: fallback_error_class(error),
+            error_message: error.message
+          )
+        end
+
+        # A validation error gets an unrelated cause when #rules runs inside a rescue block.
+        def fallback_error_class(error)
+          return error.class.name if error.is_a?(RulesetValidationError)
+
+          (error.cause || error).class.name
+        end
       end
     end
   end

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -5,12 +5,15 @@ require 'logger'
 require 'timeout'
 require 'English'
 require 'parallel'
+require_relative '../utils/memoize'
 
 module Gitlab
   module SecretDetection
     module Core
       # Scan is responsible for running Secret Detection scan operation
       class Scanner
+        include ::Gitlab::SecretDetection::Utils::StrongMemoize
+
         # default time limit(in seconds) for running the scan operation per invocation
         DEFAULT_SCAN_TIMEOUT_SECS = 180 # 3 minutes
         # default time limit(in seconds) for running the scan operation on a single payload
@@ -25,6 +28,15 @@ module Gitlab
         RUN_IN_SUBPROCESS = ENV.fetch('GITLAB_SD_RUN_IN_SUBPROCESS', false)
         # Default limit for max findings to be returned in the scan
         DEFAULT_MAX_FINDINGS_LIMIT = 999
+        # Rules whose capture group is incorrectly capturing part of the secret
+        # instead of entire secret. We capture the whole match instead of captured
+        # match when extracting the secret
+        WHOLE_MATCH_RULE_IDS = Set[
+          'Adobe Client Secret',
+          'ContentfulPersonalAccessToken',
+          'Github App Token',
+          'Slack token'
+        ].freeze
 
         # Initializes the instance with logger along with following operations:
         # 1. Filter the parsed ruleset down to rules applicable to push protection based on
@@ -34,10 +46,14 @@ module Gitlab
         # Raises +RulesetCompilationError+ in case the regex pattern compilation fails.
         def initialize(rules:, logger: Logger.new($stdout))
           @logger = logger
-          @rules = select_push_protection_rules(rules)
+          selected_rules = select_push_protection_rules(rules)
+          keywordless_rules, keyword_rules = selected_rules.partition { |rule| Array(rule[:keywords]).empty? }
+          # Rules with no keywords come first, so that the indexes of their pattern matcher are also indexes in @rules.
+          @rules = (keywordless_rules + keyword_rules).freeze
           @keywords = create_keywords(@rules)
-          @keyword_matcher = build_keyword_matcher(@rules)
+          @keyword_matcher = build_keyword_matcher(@keywords)
           @pattern_matcher = build_pattern_matcher(@rules)
+          @keywordless_pattern_matcher = build_pattern_matcher(keywordless_rules) unless keywordless_rules.empty?
         end
 
         # Runs Secret Detection scan on the list of given payloads. Both the total scan duration and
@@ -59,6 +75,13 @@ module Gitlab
         #           for backward compatibility with existing callers and will be removed.
         # +max_findings_limit+:: Integer to limit the number of findings to be returned in the scan. Defaults
         #           to 999 (+DEFAULT_MAX_FINDINGS_LIMIT+).
+        # +include_raw_value+:: Whether each finding should carry the substring it matched, on
+        #           +Finding#raw_value+. Defaults to false. Callers that opt in receive secrets in
+        #           memory and are responsible for keeping them out of logs and other sinks. In
+        #           subprocess mode the value travels back from the child through a +Parallel+ IPC pipe.
+        #
+        #           LIMITATION: one finding per rule per line, so two secrets of the *same* rule on one
+        #           line yield one +raw_value+, the leftmost. Not an exhaustive list of a line's secrets.
         #
         # NOTE:
         # Running the scan in fork mode primarily focuses on reducing the memory consumption of the scan by
@@ -80,7 +103,8 @@ module Gitlab
           exclusions: {},
           tags: [],
           subprocess: RUN_IN_SUBPROCESS,
-          max_findings_limit: DEFAULT_MAX_FINDINGS_LIMIT
+          max_findings_limit: DEFAULT_MAX_FINDINGS_LIMIT,
+          include_raw_value: false
         )
           return Core::Response.new(status: Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
 
@@ -97,18 +121,24 @@ module Gitlab
             )
           end
 
+          # Before the timeout, so the one-off compile is not charged to the scan budget, a bad
+          # pattern fails loudly, and forked children inherit the patterns instead of recompiling.
+          compiled_regexes if include_raw_value
+
           Timeout.timeout(timeout) do
             matched_payloads = filter_by_keywords(keyword_matcher, payloads)
+            # Rules with no keywords also apply to the payloads that the keyword prefilter drops.
+            unmatched_payloads = keywordless_pattern_matcher ? payloads - matched_payloads : []
 
-            next Core::Response.new(status: Core::Status::NOT_FOUND) if matched_payloads.empty?
+            if matched_payloads.empty? && unmatched_payloads.empty?
+              next Core::Response.new(status: Core::Status::NOT_FOUND)
+            end
 
             scan_args = {
-              payloads: matched_payloads,
               payload_timeout:,
-              pattern_matcher:,
               rules:,
               exclusions:,
-              max_findings_limit:
+              include_raw_value:
             }.freeze
 
             logger.info(
@@ -116,14 +146,17 @@ module Gitlab
               timeout:,
               payload_timeout:,
               given_total_payloads: payloads.length,
-              scannable_payloads_post_keyword_filter: matched_payloads.length,
+              keyword_matched_payloads: matched_payloads.length,
+              payloads_for_keywordless_rules: unmatched_payloads.length,
               active_rules: rules.length,
               run_in_subprocess: subprocess,
               max_findings_limit:,
               given_exclusions: format_exclusions_hash(exclusions)
             )
 
-            secrets, applied_exclusions = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
+            secrets, applied_exclusions = scan_payload_groups(
+              matched_payloads, unmatched_payloads, subprocess:, max_findings_limit:, **scan_args
+            )
 
             scan_status = overall_scan_status(secrets)
 
@@ -143,7 +176,7 @@ module Gitlab
 
         private
 
-        attr_reader :logger, :rules, :keywords, :pattern_matcher, :keyword_matcher
+        attr_reader :logger, :rules, :keywords, :pattern_matcher, :keyword_matcher, :keywordless_pattern_matcher
 
         # A rule applies to push protection when its structured `scanningCapabilities`
         # field includes `pushProtection`. Rules without the field are included for
@@ -220,36 +253,41 @@ module Gitlab
           secrets_keywords.freeze
         end
 
-        # Builds RE2 keyword matcher from the keywords of the given rules. Returns
-        # nil when the rules define no keywords, in which case the keyword-based
+        # Builds RE2 keyword matcher from the given set of keywords. Returns
+        # nil when the set is empty, in which case the keyword-based
         # payload prefilter is skipped.
-        def build_keyword_matcher(rules)
+        def build_keyword_matcher(keywords)
           logger.info(
             message: "Creating RE2 Keyword Matcher",
-            rules_count: rules.length
+            keywords_count: keywords.size
           )
 
-          include_keywords = Set.new
-
-          rules.each do |rule|
-            include_keywords.merge(rule[:keywords]) unless rule[:keywords].nil?
-          end
-
-          if include_keywords.empty?
+          if keywords.empty?
             logger.error(
               message: "No rule keywords found in the given rules, returning empty RE2 Keyword Matcher"
             )
             return nil
           end
 
-          keywords_regex = include_keywords.map { |keyword| RE2::Regexp.quote(keyword) }.join('|')
+          keywords_regex = keywords.map { |keyword| RE2::Regexp.quote(keyword) }.join('|')
 
           logger.debug(
             message: "Creating RE2 Keyword Matcher with set of rule keywords",
-            keywords: include_keywords.to_a
+            keywords: keywords.to_a
           )
 
-          RE2("(#{keywords_regex})")
+          # Without log_errors: false, RE2 writes the pattern to STDERR.
+          keyword_matcher = RE2::Regexp.new("(#{keywords_regex})", log_errors: false)
+          return keyword_matcher if keyword_matcher.ok?
+
+          # A matcher that does not compile matches no payload.
+          logger.error(
+            message: "Failed to compile the RE2 Keyword Matcher. The scan skips the keyword prefilter and directly " \
+              "scans all payloads with rule regex patterns.",
+            keywords_count: keywords.size,
+            error: keyword_matcher.error
+          )
+          nil
         end
 
         def filter_by_keywords(keyword_matcher, payloads)
@@ -282,6 +320,26 @@ module Gitlab
           matched_payloads
         end
 
+        # Scans the payloads that have a keyword with all rules, and the other payloads
+        # with only the rules that have no keywords.
+        def scan_payload_groups(matched_payloads, unmatched_payloads, subprocess:, max_findings_limit:, **scan_args)
+          secrets = []
+          applied_exclusions = []
+          groups = [[matched_payloads, pattern_matcher], [unmatched_payloads, keywordless_pattern_matcher]]
+
+          groups.each do |group, matcher|
+            remaining_limit = max_findings_limit - secrets.length
+            next if group.empty? || remaining_limit <= 0
+
+            args = scan_args.merge(payloads: group, pattern_matcher: matcher, max_findings_limit: remaining_limit)
+            found, excluded = subprocess ? run_scan_within_subprocess(**args) : run_scan(**args)
+            secrets.concat(found)
+            applied_exclusions |= excluded
+          end
+
+          [secrets, applied_exclusions]
+        end
+
         # Runs the secret detection scan on the given list of payloads. It accepts
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan when performed on the payloads.
@@ -291,6 +349,7 @@ module Gitlab
           pattern_matcher:,
           max_findings_limit:,
           rules:,
+          include_raw_value:,
           exclusions: {})
           all_applied_exclusions = Set.new
 
@@ -305,7 +364,8 @@ module Gitlab
                 payload:,
                 pattern_matcher:,
                 exclusions:,
-                rules:
+                rules:,
+                include_raw_value:
               )
               all_applied_exclusions.merge(applied_exclusions)
               findings
@@ -326,6 +386,7 @@ module Gitlab
           pattern_matcher:,
           max_findings_limit:,
           rules:,
+          include_raw_value:,
           exclusions: {}
         )
           all_applied_exclusions = Set.new
@@ -356,14 +417,17 @@ module Gitlab
                   payload:,
                   pattern_matcher:,
                   exclusions:,
-                  rules:
+                  rules:,
+                  include_raw_value:
                 )
                 [findings, applied_exclusions]
               end
             rescue Timeout::Error => e
               logger.warn "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
 
-              Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
+              # Must match the happy path's shape: the consumer destructures each element into
+              # `findings, applied_exclusions`. A bare Finding made `Set#merge(nil)` raise.
+              [[Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)], []]
             end
 
             # Process results and collect exclusions
@@ -383,7 +447,7 @@ module Gitlab
         # Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan.
-        def find_secrets_in_payload(payload:, pattern_matcher:, rules:, exclusions: {})
+        def find_secrets_in_payload(payload:, pattern_matcher:, rules:, exclusions: {}, include_raw_value: false)
           findings = []
           applied_exclusions = Set.new
 
@@ -419,14 +483,8 @@ module Gitlab
 
               next if applied_rule_exclusion?(rule[:id], rule_exclusions, applied_exclusions)
 
-              title = rule[:title].nil? ? rule[:description] : rule[:title]
-
-              findings << Core::Finding.new(
-                payload.id,
-                Core::Status::FOUND,
-                line_no,
-                rule[:id],
-                title
+              findings << build_finding(
+                payload_id: payload.id, line_no:, rule:, rule_index: match_idx, line:, include_raw_value:
               )
             end
           end
@@ -445,6 +503,72 @@ module Gitlab
           [[Core::Finding.new(payload.id, Core::Status::SCAN_ERROR)], []]
         end
 
+        def build_finding(payload_id:, line_no:, rule:, rule_index:, line:, include_raw_value:)
+          secret = include_raw_value ? extract_raw_value(rule, rule_index, line) : nil
+          Core::Finding.new(
+            payload_id,
+            Core::Status::FOUND,
+            line_no,
+            rule[:id],
+            Core::Ruleset.finding_description(rule),
+            raw_value: secret
+          )
+        end
+
+        # Returns the substring of +line+ that the rule actually matched, or nil when it does not
+        # match. +line+ has already had raw value exclusions stripped out of it by the caller, so
+        # this can never return an allowlisted secret.
+        #
+        # RE2::Set reports which patterns hit but not where, so the pattern is run again to locate
+        # the match; with two secrets of the same rule on a line this returns the leftmost.
+        # +rule_index+ indexes +compiled_regexes+, which is built from the scanner's own +rules+,
+        # so callers must pass that same array as +rules:+ or extraction returns nil.
+        def extract_raw_value(rule, rule_index, line)
+          regex = compiled_regexes[rule_index]
+
+          # RE2 returns `true` rather than a match object when a pattern has no capturing groups and
+          # `submatches` is left at its default, so this argument is necessary. One more than the
+          # group count leaves `match[1]` nil for a group-less pattern.
+          match = regex.match(line, submatches: regex.number_of_capturing_groups + 1)
+          return unless match
+
+          return match[0] if WHOLE_MATCH_RULE_IDS.include?(rule[:id])
+
+          match[1] || match[0]
+        rescue StandardError => e
+          logger.warn(message: "Failed to extract the matched value", rule_id: rule[:id], error: e.class.name)
+          nil
+        end
+
+        # +RE2::Set+ does not expose its compiled patterns, so extraction needs its own +RE2::Regexp+
+        # per rule, in an Array aligned with the indices RE2::Set reports. Built once and frozen so it
+        # is fork-inheritable and not a shared mutable; lazy so opt-out scans pay nothing.
+        def compiled_regexes
+          strong_memoize(:compiled_regexes) do
+            build_compiled_regexes(rules)
+          end
+        end
+
+        # +RE2::Regexp.new+ does not raise on a bad pattern; it returns one whose +ok?+ is false and
+        # +number_of_capturing_groups+ is -1, which makes +submatches+ zero and flips +match+ to a
+        # boolean, silently nilling every raw_value for that rule. Fail loudly instead.
+        def build_compiled_regexes(rules)
+          compiled = rules.map { |rule| RE2::Regexp.new(rule[:regex]) }
+
+          invalid = rules.zip(compiled).reject { |_rule, regex| regex.ok? }
+          unless invalid.empty?
+            logger.error(
+              message: "Failed to compile individual secret detection rule patterns for raw value extraction",
+              rule_ids: invalid.map { |rule, _regex| rule[:id] },
+              errors: invalid.map { |_rule, regex| regex.error }
+            )
+
+            raise Core::Ruleset::RulesetCompilationError
+          end
+
+          compiled.freeze
+        end
+
         def applied_rule_exclusion?(type, rule_exclusions, applied_exclusions)
           applied_exclusion = rule_exclusions&.find { |rule_exclusion| rule_exclusion.value == type }
           applied_exclusion && (applied_exclusions << applied_exclusion)

data/lib/gitlab/secret_detection/version.rb

@@ -5,7 +5,7 @@ module Gitlab
     class Gem
       # Ensure to maintain the same version in CHANGELOG file.
       # More details available under 'Release Process' section in the README.md file.
-      VERSION = "0.44.1"
+      VERSION = "0.46.0"
 
       # SD_ENV env var is used to determine which environment the
       # server is running. This var is defined in `.runway/env-<env>.yml` files.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.44.1
+  version: 0.46.0
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-03 00:00:00.000000000 Z
+date: 2026-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc