gitlab-secret_detection 0.42.1 → 0.43.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d234e1b4ade6f71981fd1abafbf8f58a46714149f61d0e6a12ba8ed12e9bc18e
-  data.tar.gz: 00d2b291531cb3c7cb4d44ec4fb806940b8dc554c2efafbbac505b3c2cfe701a
+  metadata.gz: ee740db0824f6917fb06f4001fa9f01fa6f612a0e7e5c290341774a6fd81b600
+  data.tar.gz: d62ed242fa1d2df3e39d8af24baafeb8c3725b613f0e4b725f8cafca26c3f26f
 SHA512:
-  metadata.gz: 73afb2cf54396ed6fe3bb128d068a96327bfb7a83b21d4dd7621ecbf0ad7f2682309b6da868bc285572075eda281d247f415510539dadb1e9cd7db9527d66e75
-  data.tar.gz: 7fe5ab79f639286b1e15d1d4dd64a1722a55469f975340ce9c80c598b5cf01b507a7117b5a765ea0b54a14868f059d2492e05976dccd4e5bfbae6cde0950c91c
+  metadata.gz: a56e54761be8a6cf2b3885a7b072e1f65684aa4b0da542bf514834d388bfde6b3b840a7d45fb25e68c77f04c95f8a0d74484cb4be981eeb179d81a6f379c44db
+  data.tar.gz: ee2763b6627a6258caf220b4b2ca8cb68b24c6ff1161de107a75b0b0aeae65ecd201c5f71a52e8bf5c80bb86a985c3e0baa4289c6f7aa542aca3a5737f143110

data/README.md

@@ -202,7 +202,7 @@ You should see the following response as a result:
 </details>
 
 
-<details><summary>Example: Using Exclusions and Tags</summary>
+<details><summary>Example: Using Exclusions</summary>
 
 ```shell
 $ grpcurl -d @ \
@@ -233,9 +233,6 @@ $ grpcurl -d @ \
             "exclusion_type": "EXCLUSION_TYPE_RAW_VALUE",
             "value": "glrt-12345123451234512345"
         }
-    ],
-    "tags": [
-        "gitlab_blocking"
     ]
 }
 EOM

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -15,8 +15,6 @@ module Gitlab
         DEFAULT_SCAN_TIMEOUT_SECS = 180 # 3 minutes
         # default time limit(in seconds) for running the scan operation on a single payload
         DEFAULT_PAYLOAD_TIMEOUT_SECS = 30 # 30 seconds
-        # Tags used for creating default pattern matcher
-        DEFAULT_PATTERN_MATCHER_TAGS = ['gitlab_blocking'].freeze
         # Max no of child processes to spawn per request
         # ref: https://gitlab.com/gitlab-org/gitlab/-/issues/430160
         MAX_PROCS_PER_REQUEST = 5
@@ -29,21 +27,17 @@ module Gitlab
         DEFAULT_MAX_FINDINGS_LIMIT = 999
 
         # Initializes the instance with logger along with following operations:
-        # 1. Extract keywords from the parsed ruleset to use it for matching keywords before regex operation.
-        # 2. Build and Compile rule regex patterns obtained from the ruleset with +DEFAULT_PATTERN_MATCHER_TAGS+
-        # tags. Raises +RulesetCompilationError+ in case the regex pattern compilation fails.
+        # 1. Filter the parsed ruleset down to rules applicable to push protection based on
+        # their structured `scanningCapabilities` field.
+        # 2. Extract keywords from the applicable rules to use for matching keywords before regex operation.
+        # 3. Build and Compile rule regex patterns obtained from the applicable rules.
+        # Raises +RulesetCompilationError+ in case the regex pattern compilation fails.
         def initialize(rules:, logger: Logger.new($stdout))
           @logger = logger
-          @rules = rules
-          @keywords = create_keywords(rules)
-          @default_keyword_matcher = build_keyword_matcher(
-            tags: DEFAULT_PATTERN_MATCHER_TAGS,
-            include_missing_tags: false
-          )
-          @default_pattern_matcher, @default_rules = build_pattern_matcher(
-            tags: DEFAULT_PATTERN_MATCHER_TAGS,
-            include_missing_tags: false
-          ) # includes only gitlab_blocking rules
+          @rules = select_push_protection_rules(rules)
+          @keywords = create_keywords(@rules)
+          @keyword_matcher = build_keyword_matcher(@rules)
+          @pattern_matcher = build_pattern_matcher(@rules)
         end
 
         # Runs Secret Detection scan on the list of given payloads. Both the total scan duration and
@@ -60,9 +54,9 @@ module Gitlab
         #           :rule - Exclusions in the :rule array are the rules to exclude from the ruleset used for the scan.
         #           Each rule is represented by its ID. For example: `gitlab_personal_access_token`
         #           for representing Gitlab Personal Access Token. By default, no rule is excluded from the ruleset.
-        # +tags+:: Array of tag values to filter from the default ruleset when determining the rules used for the scan.
-        #           For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
-        #           [`gitlab_blocking`] (+DEFAULT_PATTERN_MATCHER_TAGS+).
+        # +tags+:: Deprecated and ignored. Rules are selected via their structured
+        #           `scanningCapabilities` field instead of tags. The argument is kept only
+        #           for backward compatibility with existing callers and will be removed.
         # +max_findings_limit+:: Integer to limit the number of findings to be returned in the scan. Defaults
         #           to 999 (+DEFAULT_MAX_FINDINGS_LIMIT+).
         #
@@ -84,7 +78,7 @@ module Gitlab
           timeout: DEFAULT_SCAN_TIMEOUT_SECS,
           payload_timeout: DEFAULT_PAYLOAD_TIMEOUT_SECS,
           exclusions: {},
-          tags: DEFAULT_PATTERN_MATCHER_TAGS,
+          tags: [],
           subprocess: RUN_IN_SUBPROCESS,
           max_findings_limit: DEFAULT_MAX_FINDINGS_LIMIT
         )
@@ -93,24 +87,27 @@ module Gitlab
           # assign defaults since grpc passing zero timeout value to `Timeout.timeout(..)` makes it effectively useless.
           timeout = DEFAULT_SCAN_TIMEOUT_SECS unless timeout.positive?
           payload_timeout = DEFAULT_PAYLOAD_TIMEOUT_SECS unless payload_timeout.positive?
-          tags = DEFAULT_PATTERN_MATCHER_TAGS if tags.empty?
 
-          Timeout.timeout(timeout) do
-            keyword_matcher = build_keyword_matcher(tags:)
+          unless tags.nil? || tags.empty? || @tags_deprecation_warned
+            @tags_deprecation_warned = true
+            logger.warn(
+              message: "The `tags` argument is deprecated and ignored. Rules are selected via their " \
+                "`scanningCapabilities` field instead.",
+              given_tags: tags
+            )
+          end
 
+          Timeout.timeout(timeout) do
             matched_payloads = filter_by_keywords(keyword_matcher, payloads)
 
             next Core::Response.new(status: Core::Status::NOT_FOUND) if matched_payloads.empty?
 
-            # the pattern matcher will filter rules by tags so we use the filtered rule list
-            pattern_matcher, active_rules = build_pattern_matcher(tags:)
-
             scan_args = {
               payloads: matched_payloads,
               payload_timeout:,
               pattern_matcher:,
+              rules:,
               exclusions:,
-              rules: active_rules,
               max_findings_limit:
             }.freeze
 
@@ -120,7 +117,7 @@ module Gitlab
               payload_timeout:,
               given_total_payloads: payloads.length,
               scannable_payloads_post_keyword_filter: matched_payloads.length,
-              tags:,
+              active_rules: rules.length,
               run_in_subprocess: subprocess,
               max_findings_limit:,
               given_exclusions: format_exclusions_hash(exclusions)
@@ -146,44 +143,57 @@ module Gitlab
 
         private
 
-        attr_reader :logger, :rules, :keywords, :default_pattern_matcher, :default_keyword_matcher, :default_rules
+        attr_reader :logger, :rules, :keywords, :pattern_matcher, :keyword_matcher
 
-        # Builds RE2::Set pattern matcher for the given combination of rules
-        # and tags. It also allows a choice(via `include_missing_tags`) to consider rules
-        # for pattern matching that do not have `tags` property defined. If the given tags
-        # are same as +DEFAULT_PATTERN_MATCHER_TAGS+ then returns the eagerly loaded default
-        # pattern matcher created during initialization.
-        def build_pattern_matcher(tags:, include_missing_tags: false)
-          if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_pattern_matcher.nil?
-            logger.info(
-              message: "Given tags input matches default matcher tags, using pre-defined RE2 Pattern Matcher"
+        # A rule applies to push protection when its structured `scanningCapabilities`
+        # field includes `pushProtection`. Rules without the field are included for
+        # backward compatibility: ruleset artifacts released before the structured
+        # fields refactor (https://gitlab.com/gitlab-org/gitlab/-/work_items/592827)
+        # do not carry it and are already limited to push protection rules when the
+        # artifact is generated.
+        def push_protection_rule?(rule)
+          capabilities = rule[:scanningCapabilities]
+
+          capabilities.nil? || capabilities[:pushProtection] == true
+        end
+
+        # Filters the given ruleset down to push protection rules and logs how
+        # it was narrowed, so that a malformed ruleset artifact is visible in
+        # the logs at startup.
+        def select_push_protection_rules(rules)
+          selected = rules.select { |rule| push_protection_rule?(rule) }.freeze
+          log_selected_rules(selected, total_rules_given: rules.length)
+          selected
+        end
+
+        # An empty selection is logged as an error because every scan would
+        # then return no findings.
+        def log_selected_rules(selected_rules, total_rules_given:)
+          if selected_rules.empty?
+            logger.error(
+              message: "No rules with the pushProtection scanning capability found in the given ruleset. " \
+                "All scans will return no findings.",
+              total_rules_given:
             )
-            return [default_pattern_matcher, default_rules]
+            return
           end
 
           logger.info(
-            message: "Creating a new RE2 Pattern Matcher with given tags",
-            tags:,
-            include_missing_tags:
+            message: "Selected push protection rules from the given ruleset",
+            total_rules_given:,
+            push_protection_rules_selected: selected_rules.length,
+            selected_without_scanning_capabilities: selected_rules.count { |rule| rule[:scanningCapabilities].nil? }
           )
-          active_rules = []
+        end
 
+        # Builds RE2::Set pattern matcher for the given rules. The matcher's
+        # pattern indices align with the order of the given rules.
+        def build_pattern_matcher(rules)
           matcher = RE2::Set.new
 
           begin
             rules.each do |rule|
-              rule_tags = rule[:tags]
-
-              include_rule = if tags.empty?
-                               true
-                             elsif rule_tags
-                               tags.intersect?(rule_tags)
-                             else
-                               include_missing_tags
-                             end
-
-              active_rules << rule if include_rule
-              matcher.add(rule[:regex]) if include_rule
+              matcher.add(rule[:regex])
             end
           rescue StandardError => e
             logger.error "Failed to add regex secret detection ruleset in RE::Set: #{e.message}"
@@ -196,7 +206,7 @@ module Gitlab
             raise Core::Ruleset::RulesetCompilationError
           end
 
-          [matcher, active_rules]
+          matcher
         end
 
         # Creates and returns the unique set of rule matching keywords
@@ -210,34 +220,24 @@ module Gitlab
           secrets_keywords.freeze
         end
 
-        def build_keyword_matcher(tags:, include_missing_tags: false)
-          if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_keyword_matcher.nil?
-            logger.info(
-              message: "Given tags input matches default tags, using pre-defined RE2 Keyword Matcher"
-            )
-            return default_keyword_matcher
-          end
-
+        # Builds RE2 keyword matcher from the keywords of the given rules. Returns
+        # nil when the rules define no keywords, in which case the keyword-based
+        # payload prefilter is skipped.
+        def build_keyword_matcher(rules)
           logger.info(
-            message: "Creating a new RE2 Keyword Matcher..",
-            tags:,
-            include_missing_tags:
+            message: "Creating RE2 Keyword Matcher",
+            rules_count: rules.length
           )
 
           include_keywords = Set.new
 
           rules.each do |rule|
-            rule_tags = rule.fetch(:tags, [])
-
-            next if rule_tags.empty? && !include_missing_tags
-            next unless rule_tags.intersect?(tags)
-
             include_keywords.merge(rule[:keywords]) unless rule[:keywords].nil?
           end
 
           if include_keywords.empty?
             logger.error(
-              message: "No rule keywords found a match with given rule tags, returning empty RE2 Keyword Matcher"
+              message: "No rule keywords found in the given rules, returning empty RE2 Keyword Matcher"
             )
             return nil
           end
@@ -290,8 +290,8 @@ module Gitlab
           payload_timeout:,
           pattern_matcher:,
           max_findings_limit:,
-          exclusions: {},
-          rules: [])
+          rules:,
+          exclusions: {})
           all_applied_exclusions = Set.new
 
           logger.info(
@@ -325,8 +325,8 @@ module Gitlab
           payload_timeout:,
           pattern_matcher:,
           max_findings_limit:,
-          exclusions: {},
-          rules: []
+          rules:,
+          exclusions: {}
         )
           all_applied_exclusions = Set.new
 
@@ -383,7 +383,7 @@ module Gitlab
         # Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan.
-        def find_secrets_in_payload(payload:, pattern_matcher:, exclusions: {}, rules: @default_rules)
+        def find_secrets_in_payload(payload:, pattern_matcher:, rules:, exclusions: {})
           findings = []
           applied_exclusions = Set.new
 

data/lib/gitlab/secret_detection/version.rb

@@ -5,7 +5,7 @@ module Gitlab
     class Gem
       # Ensure to maintain the same version in CHANGELOG file.
       # More details available under 'Release Process' section in the README.md file.
-      VERSION = "0.42.1"
+      VERSION = "0.43.0"
 
       # SD_ENV env var is used to determine which environment the
       # server is running. This var is defined in `.runway/env-<env>.yml` files.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.42.1
+  version: 0.43.0
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-25 00:00:00.000000000 Z
+date: 2026-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc