gitlab-secret_detection 0.45.0 → 0.46.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f9e6f5022798b252c15cce6bcbdc7ae84e9891af3a3552a1c727953868c1a27
-  data.tar.gz: e8ad9883653d36633fa58570b80d6da8d351feb6b9da91356f93db97f2d34466
+  metadata.gz: 31d3996d42590cabd29d5198d67cc9ee716f31232d207183f98986a3dc0f4bdb
+  data.tar.gz: 44811e65092f594a602a9cc2f399d98ac047569dc7bac7f359a18190fa18c54e
 SHA512:
-  metadata.gz: 40b41909555fd7b621805894913d305c2960793b5512f809a27521b17437839e84df43ae427f5d7106a076d6eef353d9a77864adeb1c26f94c5d0d2a718aa7a7
-  data.tar.gz: 7af02d50d1833419b2ae0d194bbf7ac5a8bc6cd7c0326d348e327a7cafad2fc98e454e518e04a7f6b0cb473488eac1535d1ab7831a68cae169d7e572cb3d7c0c
+  metadata.gz: 5ab9cea777bf959289694831001f055df76b80265df7af578e167b25096403952f4a659f442351f301d6b38a31865a579d47d9b83f68f5201dfbecd1f60679c3
+  data.tar.gz: 520ad2c9f1a97c9576460e0b28071564acf9cadf3516118051e91bb4c09e1ef103b86a054573efe8755d3def33a3b899c74baa01488e326c5a6fa32fcffedb0e

data/README.md

@@ -88,6 +88,33 @@ The command is called when running tests as well as when building the Dockerfile
 The secret_push_protection_rules.toml file is added to .gitignore to
 avoid checking in changes.
 
+### Custom ruleset file (gem only)
+
+A consumer of the ruby gem can use a custom ruleset file instead of the bundled
+secret_push_protection_rules.toml file. Set the `GITLAB_SD_RULESET_FILE_PATH` environment variable
+to the path of the custom file. Set it in the environment of the process that runs the scan.
+
+- The file must be a TOML file with a `.toml` extension.
+- A relative path is resolved against the working directory of the process. `~` is not expanded.
+- A `path:` argument to `Gitlab::SecretDetection::Core::Ruleset.new` has priority over the environment variable.
+- The rules are loaded one time for each `Ruleset` object. A change to the file has an effect only on a new load.
+- The gRPC service also reads this environment variable, but this use is not supported.
+
+The custom file must have a non-empty `[[rules]]` array. Each rule must have these fields:
+
+| Field | Required | Description |
+|---|---|---|
+| `id` | Yes | A non-empty string. Each `id` must be unique in the file. |
+| `regex` | Yes | A non-empty string that RE2 can compile. |
+| `title` or `description` | Yes | The scanner uses the `title` as the finding description. It uses the `description` only if the `title` is missing, empty, or has only white space. The value that the scanner uses must be a non-empty string. |
+| `keywords` | No | An array of non-empty strings. |
+
+The scanner uses the keyword prefilter only for the rules that have keywords. It matches the rules that have no keywords on all payloads.
+
+If the custom file cannot be loaded or fails validation, the gem logs an error with the message
+`Failed to load custom secret detection ruleset. Using the default ruleset.` and loads the bundled file.
+The log entry contains the error class and the full error message. For a validation error, the message lists each problem on its own line. For a TOML syntax error, the message contains the line of the file that has the error.
+
 ## Generating a ruby gem
 
 In the project directory, run `make gem` command in the terminal that builds a ruby gem(ex: `secret_detection-0.1.0.gem`) in the root of

data/lib/gitlab/secret_detection/core/ruleset.rb

@@ -2,11 +2,15 @@
 
 require 'toml-rb'
 require 'logger'
+require 're2'
+require_relative '../utils/memoize'
 
 module Gitlab
   module SecretDetection
     module Core
       class Ruleset
+        include ::Gitlab::SecretDetection::Utils::StrongMemoize
+
         # RulesetParseError is thrown when the code fails to parse the
         # ruleset file from the given path
         RulesetParseError = Class.new(StandardError)
@@ -15,10 +19,42 @@ module Gitlab
         # the predefined rulesets
         RulesetCompilationError = Class.new(StandardError)
 
-        # file path where the secrets ruleset file is located
+        ERROR_MESSAGES = {
+          version_extraction_failed: "Failed to extract Secret Detection Ruleset version from ruleset file",
+          parse_failed: "Failed to parse local secret detection ruleset: %<error>s",
+          fallback: "Failed to load custom secret detection ruleset. Using the default ruleset.",
+          validation_failed: "Ruleset validation failed:\n%<errors>s",
+          invalid_extension: "Ruleset file must have a .toml extension",
+          empty_rules: "Rules must be a non-empty Array of [[rules]] tables",
+          duplicate_ids: "Rule ids must be unique",
+          rule_error: "rules[%<index>d]: %<error>s",
+          rule_not_table: "Rule must be a toml table",
+          invalid_id: "Rule id must be a non-empty String",
+          empty_regex: "Rule regex must be a non-empty String",
+          invalid_regex: "Rule regex is not valid: %<error>s",
+          invalid_description: "Rule title or description must be a non-empty String",
+          invalid_keywords: "Rule keywords must be either empty or an Array of non-empty Strings"
+        }.freeze
+
+        # RulesetValidationError is raised when a custom ruleset file fails validation
+        class RulesetValidationError < RulesetParseError
+          def initialize(errors)
+            super(format(ERROR_MESSAGES[:validation_failed], errors: errors.join("\n")))
+          end
+        end
+
+        # Default ruleset file path
         RULESET_FILE_PATH = File.expand_path('secret_push_protection_rules.toml', __dir__)
 
-        def initialize(path: RULESET_FILE_PATH, logger: Logger.new($stdout))
+        RULESET_FILE_PATH_ENV = 'GITLAB_SD_RULESET_FILE_PATH'
+
+        # Returns the rule title, or the rule description when the title is nil or blank.
+        def self.finding_description(rule)
+          title = rule[:title]
+          title.is_a?(String) && !title.strip.empty? ? title : rule[:description]
+        end
+
+        def initialize(path: nil, logger: Logger.new($stdout))
           @path = path
           @logger = logger
         end
@@ -29,38 +65,138 @@ module Gitlab
           @rule_data = parse_ruleset
         end
 
+        # Before a load, it reads the file at resolved_path.
         def extract_ruleset_version
-          @ruleset_version ||= if File.readable?(RULESET_FILE_PATH)
-                                 first_line = File.open(RULESET_FILE_PATH, &:gets)
-                                 first_line&.split(":")&.[](1)&.strip
-                               end
+          file_path = loaded_path || resolved_path
+          return unless File.readable?(file_path)
+
+          first_line = File.open(file_path, &:gets)
+          first_line&.split(":")&.[](1)&.strip # Gets "0.24.3" from "# rule-set version: 0.24.3"
         rescue StandardError => e
-          logger.error(message: "Failed to extract Secret Detection Ruleset version from ruleset file: #{e.message}")
+          logger.error(message: ERROR_MESSAGES[:version_extraction_failed], error: e.message)
+          nil
         end
 
         private
 
-        attr_reader :path, :logger
+        attr_reader :logger, :loaded_path
 
-        # parses given ruleset file and returns the parsed rules
         def parse_ruleset
+          load_from(resolved_path)
+        rescue RulesetParseError => e
+          raise if resolved_path == RULESET_FILE_PATH
+
+          log_fallback(e)
+          load_from(RULESET_FILE_PATH)
+        end
+
+        def resolved_path
+          return File.absolute_path(@path) if @path
+
+          env_value = ENV.fetch(RULESET_FILE_PATH_ENV, '').strip
+          env_value.empty? ? RULESET_FILE_PATH : File.absolute_path(env_value)
+        end
+        strong_memoize_attr :resolved_path
+
+        def load_from(file_path)
+          custom = file_path != RULESET_FILE_PATH
+
           logger.info(
-            message: "Parsing local ruleset file",
-            ruleset_path: RULESET_FILE_PATH
+            message: "Parsing #{custom ? 'custom' : 'bundled'} ruleset file",
+            ruleset_path: file_path
           )
-          rules_data = TomlRB.load_file(path, symbolize_keys: true).freeze
-          ruleset_version = extract_ruleset_version
+          check_toml_extension!(file_path) if custom
+
+          rules_data = TomlRB.load_file(file_path, symbolize_keys: true).freeze
+          validate_rules!(rules_data) if custom
 
+          @loaded_path = file_path
           logger.info(
             message: "Ruleset details fetched for running Secret Detection scan",
+            ruleset_path: file_path,
             total_rules: rules_data[:rules]&.length,
-            ruleset_version:
+            ruleset_version: extract_ruleset_version,
+            custom_ruleset: custom
           )
           rules_data[:rules].freeze
-        rescue StandardError => e
-          logger.error(message: "Failed to parse local secret detection ruleset: #{e.message}")
+        rescue RulesetParseError
+          raise
+        # A deeply nested TOML file raises SystemStackError.
+        rescue StandardError, SystemStackError => e
+          logger.error(message: format(ERROR_MESSAGES[:parse_failed], error: e.message)) unless custom
           raise RulesetParseError, e
         end
+
+        def check_toml_extension!(file_path)
+          return if File.extname(file_path).casecmp?('.toml')
+
+          raise RulesetValidationError, [ERROR_MESSAGES[:invalid_extension]]
+        end
+
+        # Ruleset validation ensuring it has all the required
+        # fields. Applicable to custom ruleset input only.
+        def validate_rules!(rules_data)
+          rules = rules_data[:rules]
+          raise RulesetValidationError, [ERROR_MESSAGES[:empty_rules]] unless rules.is_a?(Array) && !rules.empty?
+
+          errors = rules.each_with_index.flat_map do |rule, index|
+            rule_errors(rule).map { |error| format(ERROR_MESSAGES[:rule_error], index:, error:) }
+          end
+
+          if errors.empty? && rules.map { |rule| rule[:id] }.uniq.size < rules.size
+            errors << ERROR_MESSAGES[:duplicate_ids]
+          end
+
+          raise RulesetValidationError, errors unless errors.empty?
+        end
+
+        def rule_errors(rule)
+          return [ERROR_MESSAGES[:rule_not_table]] unless rule.is_a?(Hash)
+
+          finding_description = self.class.finding_description(rule)
+          keywords = rule[:keywords]
+
+          errors = []
+          errors << ERROR_MESSAGES[:invalid_id] unless non_empty_string?(rule[:id])
+          errors << regex_error(rule[:regex])
+          errors << ERROR_MESSAGES[:invalid_description] unless non_empty_string?(finding_description)
+
+          unless keywords.nil? || (keywords.is_a?(Array) && keywords.all? { |keyword| non_empty_string?(keyword) })
+            errors << ERROR_MESSAGES[:invalid_keywords]
+          end
+
+          errors.compact
+        end
+
+        def regex_error(regex)
+          return ERROR_MESSAGES[:empty_regex] unless non_empty_string?(regex)
+
+          # Without log_errors: false, RE2 writes the pattern to STDERR.
+          compiled = RE2::Regexp.new(regex, log_errors: false)
+          format(ERROR_MESSAGES[:invalid_regex], error: compiled.error) unless compiled.ok?
+        end
+
+        def non_empty_string?(value)
+          value.is_a?(String) && !value.strip.empty?
+        end
+
+        def log_fallback(error)
+          logger.error(
+            message: ERROR_MESSAGES[:fallback],
+            custom_ruleset_path: resolved_path,
+            default_ruleset_path: RULESET_FILE_PATH,
+            ruleset_path_env: RULESET_FILE_PATH_ENV,
+            error_class: fallback_error_class(error),
+            error_message: error.message
+          )
+        end
+
+        # A validation error gets an unrelated cause when #rules runs inside a rescue block.
+        def fallback_error_class(error)
+          return error.class.name if error.is_a?(RulesetValidationError)
+
+          (error.cause || error).class.name
+        end
       end
     end
   end

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -46,10 +46,14 @@ module Gitlab
         # Raises +RulesetCompilationError+ in case the regex pattern compilation fails.
         def initialize(rules:, logger: Logger.new($stdout))
           @logger = logger
-          @rules = select_push_protection_rules(rules)
+          selected_rules = select_push_protection_rules(rules)
+          keywordless_rules, keyword_rules = selected_rules.partition { |rule| Array(rule[:keywords]).empty? }
+          # Rules with no keywords come first, so that the indexes of their pattern matcher are also indexes in @rules.
+          @rules = (keywordless_rules + keyword_rules).freeze
           @keywords = create_keywords(@rules)
-          @keyword_matcher = build_keyword_matcher(@rules)
+          @keyword_matcher = build_keyword_matcher(@keywords)
           @pattern_matcher = build_pattern_matcher(@rules)
+          @keywordless_pattern_matcher = build_pattern_matcher(keywordless_rules) unless keywordless_rules.empty?
         end
 
         # Runs Secret Detection scan on the list of given payloads. Both the total scan duration and
@@ -123,16 +127,17 @@ module Gitlab
 
           Timeout.timeout(timeout) do
             matched_payloads = filter_by_keywords(keyword_matcher, payloads)
+            # Rules with no keywords also apply to the payloads that the keyword prefilter drops.
+            unmatched_payloads = keywordless_pattern_matcher ? payloads - matched_payloads : []
 
-            next Core::Response.new(status: Core::Status::NOT_FOUND) if matched_payloads.empty?
+            if matched_payloads.empty? && unmatched_payloads.empty?
+              next Core::Response.new(status: Core::Status::NOT_FOUND)
+            end
 
             scan_args = {
-              payloads: matched_payloads,
               payload_timeout:,
-              pattern_matcher:,
               rules:,
               exclusions:,
-              max_findings_limit:,
               include_raw_value:
             }.freeze
 
@@ -141,14 +146,17 @@ module Gitlab
               timeout:,
               payload_timeout:,
               given_total_payloads: payloads.length,
-              scannable_payloads_post_keyword_filter: matched_payloads.length,
+              keyword_matched_payloads: matched_payloads.length,
+              payloads_for_keywordless_rules: unmatched_payloads.length,
               active_rules: rules.length,
               run_in_subprocess: subprocess,
               max_findings_limit:,
               given_exclusions: format_exclusions_hash(exclusions)
             )
 
-            secrets, applied_exclusions = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
+            secrets, applied_exclusions = scan_payload_groups(
+              matched_payloads, unmatched_payloads, subprocess:, max_findings_limit:, **scan_args
+            )
 
             scan_status = overall_scan_status(secrets)
 
@@ -168,7 +176,7 @@ module Gitlab
 
         private
 
-        attr_reader :logger, :rules, :keywords, :pattern_matcher, :keyword_matcher
+        attr_reader :logger, :rules, :keywords, :pattern_matcher, :keyword_matcher, :keywordless_pattern_matcher
 
         # A rule applies to push protection when its structured `scanningCapabilities`
         # field includes `pushProtection`. Rules without the field are included for
@@ -245,36 +253,41 @@ module Gitlab
           secrets_keywords.freeze
         end
 
-        # Builds RE2 keyword matcher from the keywords of the given rules. Returns
-        # nil when the rules define no keywords, in which case the keyword-based
+        # Builds RE2 keyword matcher from the given set of keywords. Returns
+        # nil when the set is empty, in which case the keyword-based
         # payload prefilter is skipped.
-        def build_keyword_matcher(rules)
+        def build_keyword_matcher(keywords)
           logger.info(
             message: "Creating RE2 Keyword Matcher",
-            rules_count: rules.length
+            keywords_count: keywords.size
           )
 
-          include_keywords = Set.new
-
-          rules.each do |rule|
-            include_keywords.merge(rule[:keywords]) unless rule[:keywords].nil?
-          end
-
-          if include_keywords.empty?
+          if keywords.empty?
             logger.error(
               message: "No rule keywords found in the given rules, returning empty RE2 Keyword Matcher"
             )
             return nil
           end
 
-          keywords_regex = include_keywords.map { |keyword| RE2::Regexp.quote(keyword) }.join('|')
+          keywords_regex = keywords.map { |keyword| RE2::Regexp.quote(keyword) }.join('|')
 
           logger.debug(
             message: "Creating RE2 Keyword Matcher with set of rule keywords",
-            keywords: include_keywords.to_a
+            keywords: keywords.to_a
           )
 
-          RE2("(#{keywords_regex})")
+          # Without log_errors: false, RE2 writes the pattern to STDERR.
+          keyword_matcher = RE2::Regexp.new("(#{keywords_regex})", log_errors: false)
+          return keyword_matcher if keyword_matcher.ok?
+
+          # A matcher that does not compile matches no payload.
+          logger.error(
+            message: "Failed to compile the RE2 Keyword Matcher. The scan skips the keyword prefilter and directly " \
+              "scans all payloads with rule regex patterns.",
+            keywords_count: keywords.size,
+            error: keyword_matcher.error
+          )
+          nil
         end
 
         def filter_by_keywords(keyword_matcher, payloads)
@@ -307,6 +320,26 @@ module Gitlab
           matched_payloads
         end
 
+        # Scans the payloads that have a keyword with all rules, and the other payloads
+        # with only the rules that have no keywords.
+        def scan_payload_groups(matched_payloads, unmatched_payloads, subprocess:, max_findings_limit:, **scan_args)
+          secrets = []
+          applied_exclusions = []
+          groups = [[matched_payloads, pattern_matcher], [unmatched_payloads, keywordless_pattern_matcher]]
+
+          groups.each do |group, matcher|
+            remaining_limit = max_findings_limit - secrets.length
+            next if group.empty? || remaining_limit <= 0
+
+            args = scan_args.merge(payloads: group, pattern_matcher: matcher, max_findings_limit: remaining_limit)
+            found, excluded = subprocess ? run_scan_within_subprocess(**args) : run_scan(**args)
+            secrets.concat(found)
+            applied_exclusions |= excluded
+          end
+
+          [secrets, applied_exclusions]
+        end
+
         # Runs the secret detection scan on the given list of payloads. It accepts
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan when performed on the payloads.
@@ -477,7 +510,7 @@ module Gitlab
             Core::Status::FOUND,
             line_no,
             rule[:id],
-            rule[:title] || rule[:description],
+            Core::Ruleset.finding_description(rule),
             raw_value: secret
           )
         end

data/lib/gitlab/secret_detection/version.rb

@@ -5,7 +5,7 @@ module Gitlab
     class Gem
       # Ensure to maintain the same version in CHANGELOG file.
       # More details available under 'Release Process' section in the README.md file.
-      VERSION = "0.45.0"
+      VERSION = "0.46.0"
 
       # SD_ENV env var is used to determine which environment the
       # server is running. This var is defined in `.runway/env-<env>.yml` files.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.45.0
+  version: 0.46.0
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-28 00:00:00.000000000 Z
+date: 2026-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc