gitlab-secret_detection 0.18.0 → 0.19.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 42440db9db11a739e80d2ca1d50fa29d5cb3570d7fe803b8d4598eca8c3eac86
-  data.tar.gz: 7a329f9a814fe10a694876d835510e118cca91a7abc6d3787ef7b975c2105bda
+  metadata.gz: 92079cc4159944de4812acba9f6c0f7ec7f36a0a1c4f0770849fc2e35740d7da
+  data.tar.gz: 5cda8188ccc3d46d47b074a3126543db9215fd5d99def8082bae2f8f720d5454
 SHA512:
-  metadata.gz: 873cb85a55a82e2561dfce6f8a8ba5fa4e36977542096a693c6585f713d5d56038872af97f461c7e8bd52196ebaf98159db92ab7ed09a6590677c75ab45fe31a
-  data.tar.gz: 266e4ff452bcee7968bf9da72f6260146b6278c419c28c7aee30306c97682603c1dfa544fc34d276aaa6ea25b52e6fb986746931adcbc081858e3794253fdc7f
+  metadata.gz: d0a74f47adcfbf6e8ec0ab72366d5151b2860788e4adba51944e7ddc03b73eef119b9b8f10890454273845ca2c94c8d74be8a5519adc6ff97a3ca485e47464b7
+  data.tar.gz: 399a729fa667c6174e185c10eef898e51f6f1827c0584242560033d0233dc19ce307d9a75c97eb18acf18e19f647d20d31efe1e11fb984ace4640d6dd7ddd85b

data/README.md

@@ -62,20 +62,21 @@ the approach:
 
 Usage `make <command>`
 
-| Command             | Description                                                                                                                     |
-|---------------------|---------------------------------------------------------------------------------------------------------------------------------|
-| `install_secret_detection_rules` | Downloads secret-detection-rules based on package version defined in RULES_VERSION                                 |
-| `install`           | Installs ruby gems in the project using Ruby bundler                                                                            |
-| `lint_fix`          | Fixes all the fixable Rubocop lint offenses                                                                                     |
-| `gem_clean`         | Cleans existing gem file(if any) generated through gem build process                                                            |
-| `gem_build`         | Builds Ruby gem file wrapping secret detection logic (lib directory)                                                            |
-| `generate_proto`    | Generates ruby(.rb) files for the Protobud Service Definition files(.proto)                                                     |
-| `grpc_docker_build` | Builds a docker container image for gRPC server                                                                                 |
-| `grpc_docker_serve` | Runs gRPC server via docker container listening on port 8080. Run `grpc_docker_build` make command before running this command. |
-| `grpc_serve`        | Runs gRPC server on the CLI listening on port 50001. Run `install` make command before running this command.                    |
-| `run_core_tests`    | Runs RSpec tests for Secret Detection core logic                                                                                |
-| `run_grpc_tests`    | Runs RSpec tests for Secret Detection gRPC endpoints                                                                            |
-| `run_all_tests`     | Runs all the RSpec tests in the project                                                                                         |
+| Command                          | Description                                                                                                                     |
+|----------------------------------|---------------------------------------------------------------------------------------------------------------------------------|
+| `install_secret_detection_rules` | Downloads secret-detection-rules based on package version defined in RULES_VERSION                                              |
+| `install`                        | Installs ruby gems in the project using Ruby bundler                                                                            |
+| `lint_fix`                       | Fixes all the fixable Rubocop lint offenses                                                                                     |
+| `gem_clean`                      | Cleans existing gem file(if any) generated through gem build process                                                            |
+| `gem_build`                      | Builds Ruby gem file wrapping secret detection logic (lib directory)                                                            |
+| `generate_proto`                 | Generates ruby(.rb) files for the Protobud Service Definition files(.proto)                                                     |
+| `grpc_docker_build`              | Builds a docker container image for gRPC server                                                                                 |
+| `grpc_docker_serve`              | Runs gRPC server via docker container listening on port 8080. Run `grpc_docker_build` make command before running this command. |
+| `grpc_serve`                     | Runs gRPC server on the CLI listening on port 50001. Run `install` make command before running this command.                    |
+| `run_core_tests`                 | Runs RSpec tests for Secret Detection core logic                                                                                |
+| `run_grpc_tests`                 | Runs RSpec tests for Secret Detection gRPC endpoints                                                                            |
+| `run_utils_tests`                | Runs RSpec tests for Secret Detection utilities                                                                                 |
+| `run_all_tests`                  | Runs all the RSpec tests in the project                                                                                         |
 
 
 ## Secret Detection Rules

data/lib/gitlab/secret_detection/core/ruleset.rb

@@ -27,13 +27,32 @@ module Gitlab
 
         # parses given ruleset file and returns the parsed rules
         def parse_ruleset
-          # rule_file_content = File.read(path)
+          logger.info(
+            message: "Parsing local ruleset file",
+            ruleset_path: RULESET_FILE_PATH
+          )
           rules_data = TomlRB.load_file(path, symbolize_keys: true).freeze
+          ruleset_version = extract_ruleset_version
+
+          logger.info(
+            message: "Ruleset details fetched for running Secret Detection scan",
+            total_rules: rules_data[:rules]&.length,
+            ruleset_version:
+          )
           rules_data[:rules].freeze
         rescue StandardError => e
-          logger.error "Failed to parse secret detection ruleset from '#{path}' path: #{e}"
+          logger.error(message: "Failed to parse local secret detection ruleset: #{e.message}")
           raise Core::Scanner::RulesetParseError, e
         end
+
+        def extract_ruleset_version
+          @ruleset_version ||= if File.readable?(RULESET_FILE_PATH)
+                                 first_line = File.open(RULESET_FILE_PATH, &:gets)
+                                 first_line&.split(":")&.[](1)&.strip
+                               end
+        rescue StandardError => e
+          logger.error(message: "Failed to extract Secret Detection Ruleset version from ruleset file: #{e.message}")
+        end
       end
     end
   end

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -111,12 +111,29 @@ module Gitlab
               payload_timeout:,
               pattern_matcher: build_pattern_matcher(tags:),
               exclusions:
-            }
+            }.freeze
+
+            logger.info(
+              message: "Scan input parameters for running Secret Detection scan",
+              timeout:,
+              payload_timeout:,
+              given_total_payloads: payloads.length,
+              scannable_payloads_post_keyword_filter: matched_payloads.length,
+              tags:,
+              run_in_subprocess: subprocess,
+              given_exclusions: format_exclusions_hash(exclusions)
+            )
 
             secrets, applied_exclusions = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
 
             scan_status = overall_scan_status(secrets)
 
+            logger.info(
+              message: "Secret Detection scan completed with #{secrets.length} secrets detected in the given payloads",
+              detected_secrets_metadata: format_detected_secrets_metadata(secrets),
+              applied_exclusions: format_exclusions_arr(applied_exclusions)
+            )
+
             Core::Response.new(status: scan_status, results: secrets, applied_exclusions:)
           end
         rescue Timeout::Error => e
@@ -135,7 +152,18 @@ module Gitlab
         # are same as +DEFAULT_PATTERN_MATCHER_TAGS+ then returns the eagerly loaded default
         # pattern matcher created during initialization.
         def build_pattern_matcher(tags:, include_missing_tags: false)
-          return default_pattern_matcher if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_pattern_matcher.nil?
+          if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_pattern_matcher.nil?
+            logger.info(
+              message: "Given tags input matches default matcher tags, using pre-defined RE2 Pattern Matcher"
+            )
+            return default_pattern_matcher
+          end
+
+          logger.info(
+            message: "Creating a new RE2 Pattern Matcher with given tags",
+            tags:,
+            include_missing_tags:
+          )
 
           matcher = RE2::Set.new
 
@@ -154,7 +182,7 @@ module Gitlab
           end
 
           unless matcher.compile
-            logger.error "Failed to compile secret detection rulesets in RE::Set"
+            logger.error "Failed to compile secret detection ruleset in RE::Set"
 
             raise RulesetCompilationError
           end
@@ -174,7 +202,18 @@ module Gitlab
         end
 
         def build_keyword_matcher(tags:, include_missing_tags: false)
-          return default_keyword_matcher if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_keyword_matcher.nil?
+          if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_keyword_matcher.nil?
+            logger.info(
+              message: "Given tags input matches default tags, using pre-defined RE2 Keyword Matcher"
+            )
+            return default_keyword_matcher
+          end
+
+          logger.info(
+            message: "Creating a new RE2 Keyword Matcher..",
+            tags:,
+            include_missing_tags:
+          )
 
           include_keywords = Set.new
 
@@ -187,15 +226,28 @@ module Gitlab
             include_keywords.merge(rule[:keywords]) unless rule[:keywords].nil?
           end
 
-          return nil if include_keywords.empty?
+          if include_keywords.empty?
+            logger.error(
+              message: "No rule keywords found a match with given rule tags, returning empty RE2 Keyword Matcher"
+            )
+            return nil
+          end
 
           keywords_regex = include_keywords.join('|')
 
+          logger.debug(
+            message: "Creating RE2 Keyword Matcher with set of rule keywords",
+            keywords: include_keywords.to_a
+          )
+
           RE2("\\b(#{keywords_regex})")
         end
 
         def filter_by_keywords(keyword_matcher, payloads)
-          return [] if keyword_matcher.nil?
+          if keyword_matcher.nil?
+            logger.warn "No RE2 Keyword Matcher instance available, skipping payload filter by rule keywords step.."
+            return payloads
+          end
 
           matched_payloads = []
           payloads.each do |payload|
@@ -204,6 +256,20 @@ module Gitlab
             matched_payloads << payload
           end
 
+          total_payloads_retained = matched_payloads.length == payloads.length ? 'all' : matched_payloads.length
+          log_message = if matched_payloads.empty?
+                          "No payloads available to scan further after keyword-matching, exiting Secret Detection scan"
+                        else
+                          "Retained #{total_payloads_retained} payloads to scan further after keyword-matching step"
+                        end
+
+          logger.info(
+            message: log_message,
+            given_total_payloads: payloads.length,
+            matched_payloads: matched_payloads.length,
+            payloads_to_scan_further: matched_payloads.map(&:id)
+          )
+
           matched_payloads
         end
 
@@ -218,6 +284,11 @@ module Gitlab
         )
           all_applied_exclusions = Set.new
 
+          logger.info(
+            message: "Running Secret Detection scan sequentially",
+            payload_timeout:
+          )
+
           all_findings = payloads.flat_map do |payload|
             Timeout.timeout(payload_timeout) do
               findings, applied_exclusions = find_secrets_in_payload(
@@ -249,6 +320,12 @@ module Gitlab
 
           grouped_payloads = grouped_payload_indices.map { |idx_arr| idx_arr.map { |i| payloads[i] } }
 
+          logger.info(
+            message: "Running Secret Detection scan within a subprocess",
+            grouped_payloads: grouped_payloads.length,
+            payload_timeout:
+          )
+
           found_secrets = Parallel.flat_map(
             grouped_payloads,
             in_processes: MAX_PROCS_PER_REQUEST,
@@ -291,8 +368,10 @@ module Gitlab
                  .each_with_index do |line, index|
             unless raw_value_exclusions.empty?
               raw_value_exclusions.each do |exclusion|
-                line.gsub!(exclusion.value, '') # replace input that doesn't contain allowed value in it
-                applied_exclusions << exclusion
+                # replace input that doesn't contain allowed value in it
+                # replace exclusion value, `.gsub!` returns 'self' if replaced otherwise 'nil'
+                excl_replaced = !!line.gsub!(exclusion.value, '')
+                applied_exclusions << exclusion if excl_replaced
               end
             end
 
@@ -323,6 +402,13 @@ module Gitlab
             end
           end
 
+          logger.info(
+            message: "Secret Detection scan found #{findings.length} secret leaks in the payload(id:#{payload.id})",
+            payload_id: payload.id,
+            detected_rules: findings.map { |f| "#{f.type}:#{f.line_number}" },
+            applied_exclusions: format_exclusions_arr(applied_exclusions)
+          )
+
           [findings, applied_exclusions]
         rescue StandardError => e
           logger.error "Secret Detection scan failed on the payload(id:#{payload.id}): #{e}"
@@ -338,10 +424,20 @@ module Gitlab
         # Validates the given payloads by verifying the type and
         # presence of `id` and `data` fields necessary for the scan
         def validate_scan_input(payloads)
-          return false if payloads.nil? || !payloads.instance_of?(Array)
+          if payloads.nil? || !payloads.instance_of?(Array)
+            logger.debug(message: "Scan input validation error: payloads arg is empty or not instance of array")
+            return false
+          end
 
           payloads.all? do |payload|
-            payload.respond_to?(:id) && payload.respond_to?(:data)
+            has_valid_fields = payload.respond_to?(:id) && payload.respond_to?(:data)
+            unless has_valid_fields
+              logger.debug(
+                message: "Scan input validation error: one of the payloads does not respond to `id` or `data`"
+              )
+            end
+
+            has_valid_fields
           end
         end
 
@@ -390,6 +486,75 @@ module Gitlab
 
           chunk_indexes
         end
+
+        # Returns array of strings with each representing a masked exclusion
+        #
+        # Example: For given arg exclusions = {
+        #     rule: ["gitlab_personal_access_token", "aws_key"],
+        #     path: ["test.py"],
+        #     raw_value: ["ABC123XYZ"]
+        # }
+        #
+        # The output will look like the following:
+        # [
+        #   "rule=gitlab_personal_access_token,aws_key",
+        #   "raw_value=AB*****YZ",
+        #   "paths=test.py"
+        # ]
+        def format_exclusions_hash(exclusions = {})
+          masked_raw_values = exclusions.fetch(:raw_value, []).map do |exclusion|
+            Gitlab::SecretDetection::Utils::Masker.mask_secret(exclusion.value)
+          end.join(", ")
+          paths = exclusions.fetch(:path, []).map(&:value).join(", ")
+          rules = exclusions.fetch(:rule, []).map(&:value).join(", ")
+
+          out = []
+
+          out << "rules=#{rules}" unless rules.empty?
+          out << "raw_values=#{masked_raw_values}" unless masked_raw_values.empty?
+          out << "paths=#{paths}" unless paths.empty?
+
+          out
+        end
+
+        def format_exclusions_arr(exclusions = [])
+          return [] if exclusions.empty?
+
+          masked_raw_values = Set.new
+          paths = Set.new
+          rules = Set.new
+
+          exclusions.each do |exclusion|
+            case exclusion.exclusion_type
+            when :EXCLUSION_TYPE_RAW_VALUE
+              masked_raw_values << Gitlab::SecretDetection::Utils::Masker.mask_secret(exclusion.value)
+            when :EXCLUSION_TYPE_RULE
+              rules << exclusion.value
+            when :EXCLUSION_TYPE_PATH
+              paths << exclusion.value
+            else
+              logger.warn("Unknown exclusion type #{exclusion.exclusion_type}")
+            end
+          end
+
+          out = []
+
+          out << "rules=#{rules.join(',')}" unless rules.empty?
+          out << "raw_values=#{masked_raw_values.join(',')}" unless masked_raw_values.empty?
+          out << "paths=#{paths.join(',')}" unless paths.empty?
+
+          out
+        end
+
+        def format_detected_secrets_metadata(findings = [])
+          return [] if findings.empty?
+
+          found_secrets = findings.filter do |f|
+            f.status == Core::Status::FOUND
+          end
+
+          found_secrets.map { |f| "#{f.payload_id}=>#{f.type}:#{f.line_number}" }
+        end
       end
     end
   end

data/lib/gitlab/secret_detection/grpc/scanner_service.rb

@@ -45,19 +45,34 @@ module Gitlab
         }.freeze
 
         # Implementation for /Scan RPC method
-        def scan(request, _call)
-          scan_request_action(request)
+        def scan(request, call)
+          scan_request_action(request, call)
         end
 
         # Implementation for /ScanStream RPC method
-        def scan_stream(requests, _call)
-          request_action = ->(r) { scan_request_action(r) }
+        def scan_stream(requests, call)
+          request_action = ->(r) { scan_request_action(r, call) }
           StreamEnumerator.new(requests, request_action).each_item
         end
 
         private
 
-        def scan_request_action(request)
+        def scan_request_action(request, call)
+          if request.nil?
+            logger.error(
+              message: "FATAL: Secret Detection gRPC scan request is `nil`",
+              deadline: call.deadline,
+              cancelled: call.cancelled?
+            )
+            return Gitlab::SecretDetection::GRPC::ScanResponse.new(
+              results: [],
+              status: Gitlab::SecretDetection::GRPC::ScanResponse::Status::STATUS_INPUT_ERROR,
+              applied_exclusions: []
+            )
+          end
+
+          logger.info(message: "Secret Detection gRPC scan request received")
+
           validate_request(request)
 
           payloads = request.payloads.to_a
@@ -66,7 +81,7 @@ module Gitlab
           request.exclusions.each do |exclusion|
             case exclusion.exclusion_type
             when :EXCLUSION_TYPE_RAW_VALUE
-              exclusions[:raw] << exclusion
+              exclusions[:raw_value] << exclusion
             when :EXCLUSION_TYPE_RULE
               exclusions[:rule] << exclusion
             when :EXCLUSION_TYPE_PATH
@@ -85,7 +100,8 @@ module Gitlab
               payload_timeout: request.payload_timeout_secs
             )
           rescue StandardError => e
-            logger.error("Failed to run the scan: #{e}")
+            logger.error(message: "Failed to run the secret detection scan", exception: e)
+            logger.error(e.backtrace&.join("\n"))
             raise ::GRPC::Unknown, e.message
           end
 

data/lib/gitlab/secret_detection/utils/masker.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SecretDetection
+    module Utils
+      class Masker
+        DEFAULT_VISIBLE_CHAR_COUNT = 3
+        DEFAULT_MASK_CHAR_COUNT = 5
+        DEFAULT_MASK_CHAR = '*'
+
+        class << self
+          def mask_secret(
+            raw_secret_value,
+            mask_char: DEFAULT_MASK_CHAR,
+            visible_chars_count: DEFAULT_VISIBLE_CHAR_COUNT,
+            mask_chars_count: DEFAULT_MASK_CHAR_COUNT
+          )
+            return '' if raw_secret_value.nil? || raw_secret_value.empty?
+            return raw_secret_value if raw_secret_value.length <= visible_chars_count # Too short to mask
+
+            chars = raw_secret_value.chars
+            position = 0
+
+            while position < chars.length
+              # Show 'visible_chars_count' characters
+              position += visible_chars_count
+
+              # Mask next 'mask_chars' characters if available
+              mask_chars_count.times do
+                break if position >= chars.length
+
+                chars[position] = mask_char
+                position += 1
+              end
+            end
+
+            chars.join
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/utils.rb

@@ -2,6 +2,7 @@
 
 require_relative 'utils/certificate'
 require_relative 'utils/memoize'
+require_relative 'utils/masker'
 
 module Gitlab
   module SecretDetection

data/lib/gitlab/secret_detection/version.rb

@@ -8,7 +8,7 @@ module Gitlab
       #  https://gitlab.com/gitlab-org/gitlab/-/issues/514015
       #
       # Ensure to maintain the same version in CHANGELOG file.
-      VERSION = "0.18.0"
+      VERSION = "0.19.1"
 
       # SD_ENV env var is used to determine which environment the
       # server is running. This var is defined in `.runway/env-<env>.yml` files.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.18.0
+  version: 0.19.1
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-20 00:00:00.000000000 Z
+date: 2025-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc
@@ -127,6 +127,7 @@ files:
 - lib/gitlab/secret_detection/grpc/scanner_service.rb
 - lib/gitlab/secret_detection/utils.rb
 - lib/gitlab/secret_detection/utils/certificate.rb
+- lib/gitlab/secret_detection/utils/masker.rb
 - lib/gitlab/secret_detection/utils/memoize.rb
 - lib/gitlab/secret_detection/version.rb
 - proto/secret_detection.proto