gitlab-secret_detection 0.19.0 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24a3afdfa8519bd53576f9fe18ffffc12d9ed32d80e27610d03300666423e8a7
-  data.tar.gz: f7b000df1c5c6e712e528f388a44506f3cfce79bd6044de44ad106087f59d52f
+  metadata.gz: edc523ab4e978a6870d4ec9d8055390cfe54b96bf5d2db9c5165dc860b8068a3
+  data.tar.gz: 55c39c2c1862db5f17a183eb77bebd00bb7faebcb10bfc83ea5956d69f718031
 SHA512:
-  metadata.gz: 4ca94bc1d02d099c7f7404b321abbc1bf7be811112e564dc44c1256b42e5aeea6d4ab207280d0ccb88ef2c62f7f17eb3fbb47f6eadc717ef5cd33f4e66cf5e26
-  data.tar.gz: f8815666aa5ff2f129c40eb42814449d9201755a91ee2b4f3bb33b48bbe3364a9e3d78b36ec8adf2fdd94e51a03d448d9d36fbf12326f4312a14cb31843bdb96
+  metadata.gz: afa785c41b6b0af5f7f462a80b7618874b8d6b408e0fd8921b06bf4a58ae9cf75bd2e23352fd67fc840475d047c9a760172de8278152a44c4daa99a33c584e9a
+  data.tar.gz: 71a2de7afb06997658c98cd14b2167b05c1a4ca191bb854aef7f309a5d40c0353de1feeba462f35012cd0fed47f3e2a41a95a527a8dfe5e172b03d28f2d532e3

data/README.md

@@ -62,20 +62,21 @@ the approach:
 
 Usage `make <command>`
 
-| Command             | Description                                                                                                                     |
-|---------------------|---------------------------------------------------------------------------------------------------------------------------------|
-| `install_secret_detection_rules` | Downloads secret-detection-rules based on package version defined in RULES_VERSION                                 |
-| `install`           | Installs ruby gems in the project using Ruby bundler                                                                            |
-| `lint_fix`          | Fixes all the fixable Rubocop lint offenses                                                                                     |
-| `gem_clean`         | Cleans existing gem file(if any) generated through gem build process                                                            |
-| `gem_build`         | Builds Ruby gem file wrapping secret detection logic (lib directory)                                                            |
-| `generate_proto`    | Generates ruby(.rb) files for the Protobud Service Definition files(.proto)                                                     |
-| `grpc_docker_build` | Builds a docker container image for gRPC server                                                                                 |
-| `grpc_docker_serve` | Runs gRPC server via docker container listening on port 8080. Run `grpc_docker_build` make command before running this command. |
-| `grpc_serve`        | Runs gRPC server on the CLI listening on port 50001. Run `install` make command before running this command.                    |
-| `run_core_tests`    | Runs RSpec tests for Secret Detection core logic                                                                                |
-| `run_grpc_tests`    | Runs RSpec tests for Secret Detection gRPC endpoints                                                                            |
-| `run_all_tests`     | Runs all the RSpec tests in the project                                                                                         |
+| Command                          | Description                                                                                                                     |
+|----------------------------------|---------------------------------------------------------------------------------------------------------------------------------|
+| `install_secret_detection_rules` | Downloads secret-detection-rules based on package version defined in RULES_VERSION                                              |
+| `install`                        | Installs ruby gems in the project using Ruby bundler                                                                            |
+| `lint_fix`                       | Fixes all the fixable Rubocop lint offenses                                                                                     |
+| `gem_clean`                      | Cleans existing gem file(if any) generated through gem build process                                                            |
+| `gem_build`                      | Builds Ruby gem file wrapping secret detection logic (lib directory)                                                            |
+| `generate_proto`                 | Generates ruby(.rb) files for the Protobud Service Definition files(.proto)                                                     |
+| `grpc_docker_build`              | Builds a docker container image for gRPC server                                                                                 |
+| `grpc_docker_serve`              | Runs gRPC server via docker container listening on port 8080. Run `grpc_docker_build` make command before running this command. |
+| `grpc_serve`                     | Runs gRPC server on the CLI listening on port 50001. Run `install` make command before running this command.                    |
+| `run_core_tests`                 | Runs RSpec tests for Secret Detection core logic                                                                                |
+| `run_grpc_tests`                 | Runs RSpec tests for Secret Detection gRPC endpoints                                                                            |
+| `run_utils_tests`                | Runs RSpec tests for Secret Detection utilities                                                                                 |
+| `run_all_tests`                  | Runs all the RSpec tests in the project                                                                                         |
 
 
 ## Secret Detection Rules
@@ -166,9 +167,9 @@ You should see the following response as a result:
 
 
 ```shell
-$ grpcurl -d @ \
-  localhost:50001 \
+grpcurl -plaintext -d @ \
   -rpc-header 'x-sd-auth:12345' \
+  localhost:50001 \
   gitlab.secret_detection.Scanner/Scan <<EOM
 {
   "payloads": [
@@ -335,15 +336,22 @@ Run `ruby examples/sample-client/sample_client.rb` on your terminal to run the s
 
 RPC service is benchmarked using [`ghz`](https://ghz.sh), a powerful CLI-based tool for load testing and benchmarking gRPC services. More details added [here](https://gitlab.com/gitlab-org/gitlab/-/work_items/468107).
 
-## Project Status
+## Release Process
+
+We do three primary actions for every merge to `main` branch:
+
+- **Build and Publish SD ruby gem to RubyGems.org**:
+  - The latest version for releasing Secret Detection gem is pulled from `Gitlab::SecretDetection::Gem::VERSION` (located at`lib/gitlab/secret_detection/version.rb`).
+  - We build a ruby gem for the code snapshot and tag it to the extract release version.
+  - The script for publising the gem to RubyGems.org is available [here](ci/scripts/publish_ruby_gem.sh).
 
-Secret Detection service's status can be tracked here: https://gitlab.com/gitlab-org/gitlab/-/issues/467531
+- **Deploy SD gRPC server to GCP using Runway**:
+  - We build a docker container for the current code snapshot and tag it under `$CI_REGISTRY_IMAGE/image:$CI_COMMIT_SHORT_SHA` container registry path.
+  - The same container registry path is given as input to the Runway CI downstream which takes it forward for deploying in Staging and Production environments.
 
-#### Changes made in the secret detection logic that were previously not present in the Gem
+- **Make a GitLab Release**:
+  - We use a modified version of [`upsert git tag`](https://gitlab.com/gitlab-org/security-products/ci-templates/-/blob/master/includes-dev/upsert-git-tag.ym) job where instead of fetching the version from the first changelog entry, we fetch it from `Gitlab::SecretDetection::Gem::VERSION`. The rest of the behaviour is retained i.e., creating a tag from the version and then creating a new GitLab release against that tag.
+  - The job pulls the description of the latest version entry from the [`CHANGELOG.md`](CHANGELOG.md) and uses it for the Release description.
+  - The script for creating a git tag and making GitLab release is available [here](ci/scripts/make_gitlab_release.sh).
 
-- [Gitlab::SecretDetection::Core::Scanner#initialize(...)](lib/gitlab/secret_detection/core/scanner.rb): To reuse the logic of ruleset parsing from a file source, we parse the ruleset file at once and pass the parsed rules around. So,
-the `initialize()` method now accepts parsed rules instead of ruleset file path
-- [Gitlab::SecretDetection::Core::Status](lib/gitlab/secret_detection/core/status.rb): `NOT_FOUND` status moved from `0` to `7` since
-gRPC reserves `0` for enums. We need to reflect this change on the Rails side too
-- [Gitlab::SecretDetection::Core::Scanner#scan(...)](lib/gitlab/secret_detection/core/scanner.rb): Introduced `rule_exclusions`, `raw_value_exclusions` and `tags` args to `scan(..)`
-method to suport [exclusions](https://gitlab.com/groups/gitlab-org/-/epics/14315) feature.
+*NOTE: There is no logical requirement for the versions defined in `Gitlab::SecretDetection::Gem::VERSION` and latest entry of `CHANGELOG.md` to be the same. However, we expect them to be the same to keep it consistent. We've added a CI job([`validate version sync`](ci/templates/validate.yml)) that ensures the version sync between them.*

data/lib/gitlab/secret_detection/core/ruleset.rb

@@ -7,6 +7,14 @@ module Gitlab
   module SecretDetection
     module Core
       class Ruleset
+        # RulesetParseError is thrown when the code fails to parse the
+        # ruleset file from the given path
+        RulesetParseError = Class.new(StandardError)
+
+        # RulesetCompilationError is thrown when the code fails to compile
+        # the predefined rulesets
+        RulesetCompilationError = Class.new(StandardError)
+
         # file path where the secrets ruleset file is located
         RULESET_FILE_PATH = File.expand_path('secret_push_protection_rules.toml', __dir__)
 
@@ -21,18 +29,37 @@ module Gitlab
           @rule_data = parse_ruleset
         end
 
+        def extract_ruleset_version
+          @ruleset_version ||= if File.readable?(RULESET_FILE_PATH)
+                                 first_line = File.open(RULESET_FILE_PATH, &:gets)
+                                 first_line&.split(":")&.[](1)&.strip
+                               end
+        rescue StandardError => e
+          logger.error(message: "Failed to extract Secret Detection Ruleset version from ruleset file: #{e.message}")
+        end
+
         private
 
         attr_reader :path, :logger
 
         # parses given ruleset file and returns the parsed rules
         def parse_ruleset
-          # rule_file_content = File.read(path)
+          logger.info(
+            message: "Parsing local ruleset file",
+            ruleset_path: RULESET_FILE_PATH
+          )
           rules_data = TomlRB.load_file(path, symbolize_keys: true).freeze
+          ruleset_version = extract_ruleset_version
+
+          logger.info(
+            message: "Ruleset details fetched for running Secret Detection scan",
+            total_rules: rules_data[:rules]&.length,
+            ruleset_version:
+          )
           rules_data[:rules].freeze
         rescue StandardError => e
-          logger.error "Failed to parse secret detection ruleset from '#{path}' path: #{e}"
-          raise Core::Scanner::RulesetParseError, e
+          logger.error(message: "Failed to parse local secret detection ruleset: #{e.message}")
+          raise RulesetParseError, e
         end
       end
     end

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -11,14 +11,6 @@ module Gitlab
     module Core
       # Scan is responsible for running Secret Detection scan operation
       class Scanner
-        # RulesetParseError is thrown when the code fails to parse the
-        # ruleset file from the given path
-        RulesetParseError = Class.new(StandardError)
-
-        # RulesetCompilationError is thrown when the code fails to compile
-        # the predefined rulesets
-        RulesetCompilationError = Class.new(StandardError)
-
         # default time limit(in seconds) for running the scan operation per invocation
         DEFAULT_SCAN_TIMEOUT_SECS = 180 # 3 minutes
         # default time limit(in seconds) for running the scan operation on a single payload
@@ -46,7 +38,7 @@ module Gitlab
             tags: DEFAULT_PATTERN_MATCHER_TAGS,
             include_missing_tags: false
           )
-          @default_pattern_matcher = build_pattern_matcher(
+          @default_pattern_matcher, @default_rules = build_pattern_matcher(
             tags: DEFAULT_PATTERN_MATCHER_TAGS,
             include_missing_tags: false
           ) # includes only gitlab_blocking rules
@@ -91,7 +83,6 @@ module Gitlab
           tags: DEFAULT_PATTERN_MATCHER_TAGS,
           subprocess: RUN_IN_SUBPROCESS
         )
-
           return Core::Response.new(status: Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
 
           # assign defaults since grpc passing zero timeout value to `Timeout.timeout(..)` makes it effectively useless.
@@ -106,17 +97,38 @@ module Gitlab
 
             next Core::Response.new(status: Core::Status::NOT_FOUND) if matched_payloads.empty?
 
+            # the pattern matcher will filter rules by tags so we use the filtered rule list
+            pattern_matcher, active_rules = build_pattern_matcher(tags:)
+
             scan_args = {
               payloads: matched_payloads,
               payload_timeout:,
-              pattern_matcher: build_pattern_matcher(tags:),
-              exclusions:
-            }
+              pattern_matcher:,
+              exclusions:,
+              rules: active_rules
+            }.freeze
+
+            logger.info(
+              message: "Scan input parameters for running Secret Detection scan",
+              timeout:,
+              payload_timeout:,
+              given_total_payloads: payloads.length,
+              scannable_payloads_post_keyword_filter: matched_payloads.length,
+              tags:,
+              run_in_subprocess: subprocess,
+              given_exclusions: format_exclusions_hash(exclusions)
+            )
 
             secrets, applied_exclusions = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
 
             scan_status = overall_scan_status(secrets)
 
+            logger.info(
+              message: "Secret Detection scan completed with #{secrets.length} secrets detected in the given payloads",
+              detected_secrets_metadata: format_detected_secrets_metadata(secrets),
+              applied_exclusions: format_exclusions_arr(applied_exclusions)
+            )
+
             Core::Response.new(status: scan_status, results: secrets, applied_exclusions:)
           end
         rescue Timeout::Error => e
@@ -127,7 +139,7 @@ module Gitlab
 
         private
 
-        attr_reader :logger, :rules, :keywords, :default_pattern_matcher, :default_keyword_matcher
+        attr_reader :logger, :rules, :keywords, :default_pattern_matcher, :default_keyword_matcher, :default_rules
 
         # Builds RE2::Set pattern matcher for the given combination of rules
         # and tags. It also allows a choice(via `include_missing_tags`) to consider rules
@@ -135,31 +147,49 @@ module Gitlab
         # are same as +DEFAULT_PATTERN_MATCHER_TAGS+ then returns the eagerly loaded default
         # pattern matcher created during initialization.
         def build_pattern_matcher(tags:, include_missing_tags: false)
-          return default_pattern_matcher if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_pattern_matcher.nil?
+          if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_pattern_matcher.nil?
+            logger.info(
+              message: "Given tags input matches default matcher tags, using pre-defined RE2 Pattern Matcher"
+            )
+            return [default_pattern_matcher, default_rules]
+          end
+
+          logger.info(
+            message: "Creating a new RE2 Pattern Matcher with given tags",
+            tags:,
+            include_missing_tags:
+          )
+          active_rules = []
 
           matcher = RE2::Set.new
 
-          rules.each do |rule|
-            rule_tags = rule[:tags]
+          begin
+            rules.each do |rule|
+              rule_tags = rule[:tags]
 
-            include_rule = if tags.empty?
-                             true
-                           elsif rule_tags
-                             tags.intersect?(rule_tags)
-                           else
-                             include_missing_tags
-                           end
+              include_rule = if tags.empty?
+                               true
+                             elsif rule_tags
+                               tags.intersect?(rule_tags)
+                             else
+                               include_missing_tags
+                             end
 
-            matcher.add(rule[:regex]) if include_rule
+              active_rules << rule if include_rule
+              matcher.add(rule[:regex]) if include_rule
+            end
+          rescue StandardError => e
+            logger.error "Failed to add regex secret detection ruleset in RE::Set: #{e.message}"
+            raise Core::Ruleset::RulesetCompilationError, cause: e
           end
 
           unless matcher.compile
-            logger.error "Failed to compile secret detection rulesets in RE::Set"
+            logger.error "Failed to compile secret detection ruleset in RE::Set"
 
-            raise RulesetCompilationError
+            raise Core::Ruleset::RulesetCompilationError
           end
 
-          matcher
+          [matcher, active_rules]
         end
 
         # Creates and returns the unique set of rule matching keywords
@@ -174,7 +204,18 @@ module Gitlab
         end
 
         def build_keyword_matcher(tags:, include_missing_tags: false)
-          return default_keyword_matcher if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_keyword_matcher.nil?
+          if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_keyword_matcher.nil?
+            logger.info(
+              message: "Given tags input matches default tags, using pre-defined RE2 Keyword Matcher"
+            )
+            return default_keyword_matcher
+          end
+
+          logger.info(
+            message: "Creating a new RE2 Keyword Matcher..",
+            tags:,
+            include_missing_tags:
+          )
 
           include_keywords = Set.new
 
@@ -187,15 +228,28 @@ module Gitlab
             include_keywords.merge(rule[:keywords]) unless rule[:keywords].nil?
           end
 
-          return nil if include_keywords.empty?
+          if include_keywords.empty?
+            logger.error(
+              message: "No rule keywords found a match with given rule tags, returning empty RE2 Keyword Matcher"
+            )
+            return nil
+          end
 
           keywords_regex = include_keywords.join('|')
 
+          logger.debug(
+            message: "Creating RE2 Keyword Matcher with set of rule keywords",
+            keywords: include_keywords.to_a
+          )
+
           RE2("\\b(#{keywords_regex})")
         end
 
         def filter_by_keywords(keyword_matcher, payloads)
-          return [] if keyword_matcher.nil?
+          if keyword_matcher.nil?
+            logger.warn "No RE2 Keyword Matcher instance available, skipping payload filter by rule keywords step.."
+            return payloads
+          end
 
           matched_payloads = []
           payloads.each do |payload|
@@ -204,6 +258,20 @@ module Gitlab
             matched_payloads << payload
           end
 
+          total_payloads_retained = matched_payloads.length == payloads.length ? 'all' : matched_payloads.length
+          log_message = if matched_payloads.empty?
+                          "No payloads available to scan further after keyword-matching, exiting Secret Detection scan"
+                        else
+                          "Retained #{total_payloads_retained} payloads to scan further after keyword-matching step"
+                        end
+
+          logger.info(
+            message: log_message,
+            given_total_payloads: payloads.length,
+            matched_payloads: matched_payloads.length,
+            payloads_to_scan_further: matched_payloads.map(&:id)
+          )
+
           matched_payloads
         end
 
@@ -214,22 +282,29 @@ module Gitlab
           payloads:,
           payload_timeout:,
           pattern_matcher:,
-          exclusions: {}
+          exclusions: {},
+          rules: []
         )
           all_applied_exclusions = Set.new
 
+          logger.info(
+            message: "Running Secret Detection scan sequentially",
+            payload_timeout:
+          )
+
           all_findings = payloads.flat_map do |payload|
             Timeout.timeout(payload_timeout) do
               findings, applied_exclusions = find_secrets_in_payload(
                 payload:,
                 pattern_matcher:,
-                exclusions:
+                exclusions:,
+                rules:
               )
               all_applied_exclusions.merge(applied_exclusions)
               findings
             end
           rescue Timeout::Error => e
-            logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
+            logger.warn "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
 
             Core::Finding.new(payload.id,
               Core::Status::PAYLOAD_TIMEOUT)
@@ -241,14 +316,22 @@ module Gitlab
           payloads:,
           payload_timeout:,
           pattern_matcher:,
-          exclusions: {}
+          exclusions: {},
+          rules: []
         )
           all_applied_exclusions = Set.new
+
           payload_sizes = payloads.map(&:size)
           grouped_payload_indices = group_by_chunk_size(payload_sizes)
 
           grouped_payloads = grouped_payload_indices.map { |idx_arr| idx_arr.map { |i| payloads[i] } }
 
+          logger.info(
+            message: "Running Secret Detection scan within a subprocess",
+            grouped_payloads: grouped_payloads.length,
+            payload_timeout:
+          )
+
           found_secrets = Parallel.flat_map(
             grouped_payloads,
             in_processes: MAX_PROCS_PER_REQUEST,
@@ -259,13 +342,14 @@ module Gitlab
                 findings, applied_exclusions = find_secrets_in_payload(
                   payload:,
                   pattern_matcher:,
-                  exclusions:
+                  exclusions:,
+                  rules:
                 )
                 all_applied_exclusions.merge(applied_exclusions)
                 findings
               end
             rescue Timeout::Error => e
-              logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
+              logger.warn "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
 
               Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
             end
@@ -277,7 +361,7 @@ module Gitlab
         # Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan.
-        def find_secrets_in_payload(payload:, pattern_matcher:, exclusions: {})
+        def find_secrets_in_payload(payload:, pattern_matcher:, exclusions: {}, rules: @default_rules)
           findings = []
           applied_exclusions = Set.new
 
@@ -291,8 +375,10 @@ module Gitlab
                  .each_with_index do |line, index|
             unless raw_value_exclusions.empty?
               raw_value_exclusions.each do |exclusion|
-                line.gsub!(exclusion.value, '') # replace input that doesn't contain allowed value in it
-                applied_exclusions << exclusion
+                # replace input that doesn't contain allowed value in it
+                # replace exclusion value, `.gsub!` returns 'self' if replaced otherwise 'nil'
+                excl_replaced = !!line.gsub!(exclusion.value, '')
+                applied_exclusions << exclusion if excl_replaced
               end
             end
 
@@ -323,6 +409,13 @@ module Gitlab
             end
           end
 
+          logger.info(
+            message: "Secret Detection scan found #{findings.length} secret leaks in the payload(id:#{payload.id})",
+            payload_id: payload.id,
+            detected_rules: findings.map { |f| "#{f.type}:#{f.line_number}" },
+            applied_exclusions: format_exclusions_arr(applied_exclusions)
+          )
+
           [findings, applied_exclusions]
         rescue StandardError => e
           logger.error "Secret Detection scan failed on the payload(id:#{payload.id}): #{e}"
@@ -338,10 +431,20 @@ module Gitlab
         # Validates the given payloads by verifying the type and
         # presence of `id` and `data` fields necessary for the scan
         def validate_scan_input(payloads)
-          return false if payloads.nil? || !payloads.instance_of?(Array)
+          if payloads.nil? || !payloads.instance_of?(Array)
+            logger.debug(message: "Scan input validation error: payloads arg is empty or not instance of array")
+            return false
+          end
 
           payloads.all? do |payload|
-            payload.respond_to?(:id) && payload.respond_to?(:data)
+            has_valid_fields = payload.respond_to?(:id) && payload.respond_to?(:data) && payload.data.is_a?(String)
+            unless has_valid_fields
+              logger.debug(
+                message: "Scan input validation error: one of the payloads does not respond to `id` or `data`"
+              )
+            end
+
+            has_valid_fields
           end
         end
 
@@ -390,6 +493,75 @@ module Gitlab
 
           chunk_indexes
         end
+
+        # Returns array of strings with each representing a masked exclusion
+        #
+        # Example: For given arg exclusions = {
+        #     rule: ["gitlab_personal_access_token", "aws_key"],
+        #     path: ["test.py"],
+        #     raw_value: ["ABC123XYZ"]
+        # }
+        #
+        # The output will look like the following:
+        # [
+        #   "rule=gitlab_personal_access_token,aws_key",
+        #   "raw_value=AB*****YZ",
+        #   "paths=test.py"
+        # ]
+        def format_exclusions_hash(exclusions = {})
+          masked_raw_values = exclusions.fetch(:raw_value, []).map do |exclusion|
+            Gitlab::SecretDetection::Utils::Masker.mask_secret(exclusion.value)
+          end.join(", ")
+          paths = exclusions.fetch(:path, []).map(&:value).join(", ")
+          rules = exclusions.fetch(:rule, []).map(&:value).join(", ")
+
+          out = []
+
+          out << "rules=#{rules}" unless rules.empty?
+          out << "raw_values=#{masked_raw_values}" unless masked_raw_values.empty?
+          out << "paths=#{paths}" unless paths.empty?
+
+          out
+        end
+
+        def format_exclusions_arr(exclusions = [])
+          return [] if exclusions.empty?
+
+          masked_raw_values = Set.new
+          paths = Set.new
+          rules = Set.new
+
+          exclusions.each do |exclusion|
+            case exclusion.exclusion_type
+            when :EXCLUSION_TYPE_RAW_VALUE
+              masked_raw_values << Gitlab::SecretDetection::Utils::Masker.mask_secret(exclusion.value)
+            when :EXCLUSION_TYPE_RULE
+              rules << exclusion.value
+            when :EXCLUSION_TYPE_PATH
+              paths << exclusion.value
+            else
+              logger.warn("Unknown exclusion type #{exclusion.exclusion_type}")
+            end
+          end
+
+          out = []
+
+          out << "rules=#{rules.join(',')}" unless rules.empty?
+          out << "raw_values=#{masked_raw_values.join(',')}" unless masked_raw_values.empty?
+          out << "paths=#{paths.join(',')}" unless paths.empty?
+
+          out
+        end
+
+        def format_detected_secrets_metadata(findings = [])
+          return [] if findings.empty?
+
+          found_secrets = findings.filter do |f|
+            f.status == Core::Status::FOUND
+          end
+
+          found_secrets.map { |f| "#{f.payload_id}=>#{f.type}:#{f.line_number}" }
+        end
       end
     end
   end