gitlab-secret_detection 0.11.1 → 0.39.2

data/lib/gitlab/secret_detection/core/status.rb

@@ -5,6 +5,8 @@ module Gitlab
     module Core
       # All the possible statuses emitted by the scan operation
       class Status
+        # These values must stay in-sync with the GRPC::ScanResponse::Status values
+        UNSPECIFIED = 0 # to match the GRPC::Status values
         FOUND = 1 # When scan operation completes with one or more findings
         FOUND_WITH_ERRORS = 2 # When scan operation completes with one or more findings along with some errors
         SCAN_TIMEOUT = 3 # When the scan operation runs beyond given time out
@@ -13,6 +15,38 @@ module Gitlab
         INPUT_ERROR = 6 # When the scan operation fails due to invalid input
         NOT_FOUND = 7 # When scan operation completes with zero findings
         AUTH_ERROR = 8 # When authentication fails
+
+        # Maps values to constants
+        @values_map = {}
+
+        # Using class instance variables and singleton methods
+        class << self
+          attr_reader :values_map
+
+          # Register constants and their values in the map
+          def const_set(name, value)
+            const = super
+            @values_map[value] = name
+            const
+          end
+
+          # Look up a constant by its value
+          def find_by_value(value)
+            const_name = @values_map[value]
+            const_name ? const_get(const_name) : nil
+          end
+
+          # Get the name of a constant by its value
+          def name_by_value(value)
+            @values_map[value]
+          end
+        end
+
+        # Initialize the values map with existing constants
+        constants.each do |const_name|
+          const_value = const_get(const_name)
+          @values_map[const_value] = const_name
+        end
       end
     end
   end

data/lib/gitlab/secret_detection/grpc/client/grpc_client.rb

@@ -17,10 +17,14 @@ module Gitlab
         # Time to wait for the response from the service
         REQUEST_TIMEOUT_SECONDS = 10 # 10 seconds
 
-        def initialize(host, secure: false, compression: true)
+        # Total payload size limit allowed per scan request
+        MAX_PAYLOAD_SIZE_PER_REQUEST = 4_000_000 # 3.8MiB (0.2MiB buffer for other request props)
+
+        def initialize(host, secure: false, compression: true, logger: nil)
           @host = host
           @secure = secure
           @compression = compression
+          @logger = logger.nil? ? LOGGER : logger
         end
 
         # Triggers Secret Detection service's `/Scan` gRPC endpoint. To keep it consistent with SDS gem interface,
@@ -29,13 +33,27 @@ module Gitlab
         # +Gitlab::SecretDetection::Core::Response+ type by assiging a appropriate +status+ value to it.
         def run_scan(request:, auth_token:, extra_headers: {})
           with_rescued_errors do
+            payload_size = calculate_payload_size(request)
+            if payload_size >= MAX_PAYLOAD_SIZE_PER_REQUEST
+              @logger.info(
+                message: "Skipping to send Scan Request to Secret Detection server due to request size overlimit",
+                payload_size:
+              )
+
+              next Gitlab::SecretDetection::GRPC::ScanResponse.new(
+                results: [],
+                status: SecretDetection::Core::Status::INPUT_ERROR,
+                applied_exclusions: []
+              )
+            end
+
             grpc_response = stub.scan(
               request,
               metadata: build_metadata(auth_token, extra_headers),
               deadline: request_deadline
             )
 
-            convert_to_core_response(grpc_response)
+            grpc_response
           end
         end
 
@@ -51,16 +69,28 @@ module Gitlab
           request_stream = Gitlab::SecretDetection::GRPC::StreamRequestEnumerator.new(requests)
           results = []
           with_rescued_errors do
+            has_oversized_request = requests.any? do |request|
+              payload_size = calculate_payload_size(request)
+              payload_size >= MAX_PAYLOAD_SIZE_PER_REQUEST
+            end
+
+            if has_oversized_request
+              @logger.info("Skipping to send Scan Request to Secret Detection server due to request size overlimit")
+              response = Gitlab::SecretDetection::GRPC::ScanResponse.new(
+                status: SecretDetection::Core::Status::INPUT_ERROR
+              )
+              next (block_given? ? response : [response])
+            end
+
             stub.scan_stream(
               request_stream.each_item,
               metadata: build_metadata(auth_token, extra_headers),
               deadline: request_deadline
             ).each do |grpc_response|
-              response = convert_to_core_response(grpc_response)
               if block_given?
-                yield response
+                yield grpc_response
               else
-                results << response
+                results << grpc_response
               end
             end
             results
@@ -116,28 +146,29 @@ module Gitlab
         def with_rescued_errors
           yield
         rescue ::GRPC::Unauthenticated
-          SecretDetection::Core::Response.new(SecretDetection::Core::Status::AUTH_ERROR)
+          SecretDetection::Core::Response.new(status: SecretDetection::Core::Status::AUTH_ERROR)
         rescue ::GRPC::InvalidArgument => e
           SecretDetection::Core::Response.new(
-            SecretDetection::Core::Status::INPUT_ERROR, nil, { message: e.details, **e.metadata }
+            status: SecretDetection::Core::Status::INPUT_ERROR,
+            results: nil,
+            metadata: { message: e.details, **e.metadata }
+          )
+        rescue ::GRPC::ResourceExhausted => e
+          @logger.error(message: "Secret Detection Server resource exhausted: #{e.details}", **e.metadata)
+          SecretDetection::Core::Response.new(
+            status: SecretDetection::Core::Status::SCAN_ERROR,
+            metadata: { message: e.details, **e.metadata }
           )
         rescue ::GRPC::Unknown, ::GRPC::BadStatus => e
           SecretDetection::Core::Response.new(
-            SecretDetection::Core::Status::SCAN_ERROR, nil, { message: e.details }
+            status: SecretDetection::Core::Status::SCAN_ERROR,
+            results: nil,
+            metadata: { message: e.details, **e.metadata }
           )
         end
 
-        def convert_to_core_response(grpc_response)
-          response = grpc_response.to_h
-
-          SecretDetection::Core::Response.new(
-            response[:status],
-            response[:results],
-            response[:metadata]
-          )
-        rescue StandardError => e
-          logger.error("Failed to convert to core response: #{e}")
-          SecretDetection::Core::Response.new(SecretDetection::Core::Status::SCAN_ERROR)
+        def calculate_payload_size(request)
+          request&.payloads&.reduce(0) { |total, p| total + p.data.size + p.id.size }
         end
       end
     end

data/lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb

@@ -5,7 +5,7 @@
 require 'google/protobuf'
 
 
-descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"Z\n\tExclusion\x12>\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32&.gitlab.secret_detection.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"\xc0\x02\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x36\n\nexclusions\x18\x04 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a\x43\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x13\n\x06offset\x18\x03 \x01(\x05H\x00\x88\x01\x01\x42\t\n\x07_offsetB\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xa2\x04\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12>\n\x12\x61pplied_exclusions\x18\x03 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08*f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitlab::SecretDetection::GRPCb\x06proto3"
+descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"Z\n\tExclusion\x12>\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32&.gitlab.secret_detection.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"\xc0\x02\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x36\n\nexclusions\x18\x04 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a\x43\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x13\n\x06offset\x18\x03 \x01(\x05H\x00\x88\x01\x01\x42\t\n\x07_offsetB\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xa2\x04\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12>\n\x12\x61pplied_exclusions\x18\x03 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08*\xa1\x01\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x12\x17\n\x13\x45XCLUSION_TYPE_PATH\x10\x03\x12 \n\x1c\x45XCLUSION_TYPE_REGEX_PATTERN\x10\x04\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitlab::SecretDetection::GRPCb\x06proto3"
 
 pool = Google::Protobuf::DescriptorPool.generated_pool
 pool.add_serialized_file(descriptor_data)

data/lib/gitlab/secret_detection/grpc/integrated_error_tracking.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'sentry-ruby'
+
+require_relative '../../../../lib/gitlab/secret_detection/core/ruleset'
+
+module Gitlab
+  module SecretDetection
+    module GRPC
+      module IntegratedErrorTracking
+        extend self
+
+        def track_exception(exception, args = {})
+          unless Sentry.initialized?
+            logger.warn(message: "Cannot track exception in Error Tracking as Sentry is not initialized")
+            return
+          end
+
+          args[:ruleset_version] = ruleset_version
+
+          Sentry.capture_exception(exception, **args)
+        end
+
+        def setup(logger: Logger.new($stdout))
+          if Sentry.initialized?
+            logger.warn(message: "Sentry is already initialized, skipping re-setup")
+            return
+          end
+
+          logger.info(message: "Initializing Sentry SDK for Integrated Error Tracking..")
+
+          unless can_setup_sentry?
+            logger.warn(message: "Integrated Error Tracking not available, skipping Sentry SDK initialization")
+            return false
+          end
+
+          Sentry.init do |config|
+            config.dsn = ENV.fetch('SD_TRACKING_DSN')
+            config.environment = ENV.fetch('SD_ENV')
+            config.release = Gitlab::SecretDetection::Gem::VERSION
+            config.send_default_pii = true
+            config.send_modules = false
+            config.traces_sample_rate = 0.2 if ENV.fetch('ENABLE_SENTRY_PERFORMANCE_MONITORING', 'false') == 'true'
+          end
+
+          Sentry.set_context('ruleset', { version: ruleset_version })
+
+          true
+        rescue StandardError => e
+          logger.error(message: "Failed to initialize Sentry SDK for Integrated Error Tracking: #{e}")
+          raise e
+        end
+
+        def ruleset_version
+          @ruleset_version ||= Gitlab::SecretDetection::Core::Ruleset.new.extract_ruleset_version || 'unknown'
+        end
+
+        def can_setup_sentry?
+          ENV.fetch('SD_ENV', '') == 'production' && ENV.fetch('SD_TRACKING_DSN', '') != ''
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc/scanner_service.rb

@@ -32,6 +32,7 @@ module Gitlab
     module GRPC
       class ScannerService < Scanner::Service
         include SDLogger
+        include IntegratedErrorTracking
 
         # Maximum timeout value that can be given as the input. This guards
         # against the misuse of timeouts.
@@ -45,46 +46,63 @@ module Gitlab
         }.freeze
 
         # Implementation for /Scan RPC method
-        def scan(request, _call)
-          scan_request_action(request)
+        def scan(request, call)
+          scan_request_action(request, call)
         end
 
         # Implementation for /ScanStream RPC method
-        def scan_stream(requests, _call)
-          request_action = ->(r) { scan_request_action(r) }
+        def scan_stream(requests, call)
+          request_action = ->(r) { scan_request_action(r, call) }
           StreamEnumerator.new(requests, request_action).each_item
         end
 
         private
 
-        def scan_request_action(request)
+        def scan_request_action(request, call)
+          if request.nil?
+            logger.error(
+              message: "FATAL: Secret Detection gRPC scan request is `nil`",
+              deadline: call.deadline,
+              cancelled: call.cancelled?
+            )
+            return Gitlab::SecretDetection::GRPC::ScanResponse.new(
+              results: [],
+              status: Gitlab::SecretDetection::GRPC::ScanResponse::Status::STATUS_INPUT_ERROR,
+              applied_exclusions: []
+            )
+          end
+
+          logger.info(message: "Secret Detection gRPC scan request received")
+
           validate_request(request)
 
           payloads = request.payloads.to_a
+          exclusions = { raw_value: [], rule: [], path: [] }
 
-          raw_value_exclusions = []
-          rule_exclusions = []
-
-          request.exclusions&.each do |exclusion|
-            case exclusion.type
+          request.exclusions.each do |exclusion|
+            case exclusion.exclusion_type
             when :EXCLUSION_TYPE_RAW_VALUE
-              raw_value_exclusions << exclusion.value
+              exclusions[:raw_value] << exclusion
             when :EXCLUSION_TYPE_RULE
-              rule_exclusions << exclusion.value
+              exclusions[:rule] << exclusion
+            when :EXCLUSION_TYPE_PATH
+              exclusions[:path] << exclusion
+            else
+              logger.warn("Unknown exclusion type #{exclusion.exclusion_type}")
             end
           end
 
           begin
             result = scanner.secrets_scan(
               payloads,
-              raw_value_exclusions:,
-              rule_exclusions:,
+              exclusions:,
               tags: request.tags.to_a,
               timeout: request.timeout_secs,
               payload_timeout: request.payload_timeout_secs
             )
           rescue StandardError => e
-            logger.error("Failed to run the scan: #{e}")
+            logger.error(message: "Failed to run the secret detection scan", exception: e.message)
+            track_exception(e)
             raise ::GRPC::Unknown, e.message
           end
 
@@ -94,7 +112,8 @@ module Gitlab
 
           Gitlab::SecretDetection::GRPC::ScanResponse.new(
             results: findings,
-            status: result.status
+            status: result.status,
+            applied_exclusions: result.applied_exclusions
           )
         end
 

data/lib/gitlab/secret_detection/grpc.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'grpc/integrated_error_tracking'
 require_relative 'grpc/scanner_service'
 require_relative 'grpc/client/stream_request_enumerator'
 require_relative 'grpc/client/grpc_client'

data/lib/gitlab/secret_detection/utils/masker.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SecretDetection
+    module Utils
+      class Masker
+        DEFAULT_VISIBLE_CHAR_COUNT = 3
+        DEFAULT_MASK_CHAR_COUNT = 5
+        DEFAULT_MASK_CHAR = '*'
+
+        class << self
+          def mask_secret(
+            raw_secret_value,
+            mask_char: DEFAULT_MASK_CHAR,
+            visible_chars_count: DEFAULT_VISIBLE_CHAR_COUNT,
+            mask_chars_count: DEFAULT_MASK_CHAR_COUNT
+          )
+            return '' if raw_secret_value.nil? || raw_secret_value.empty?
+            return raw_secret_value if raw_secret_value.length <= visible_chars_count # Too short to mask
+
+            chars = raw_secret_value.chars
+            position = 0
+
+            while position < chars.length
+              # Show 'visible_chars_count' characters
+              position += visible_chars_count
+
+              # Mask next 'mask_chars' characters if available
+              mask_chars_count.times do
+                break if position >= chars.length
+
+                chars[position] = mask_char
+                position += 1
+              end
+            end
+
+            chars.join
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/utils.rb

@@ -2,6 +2,7 @@
 
 require_relative 'utils/certificate'
 require_relative 'utils/memoize'
+require_relative 'utils/masker'
 
 module Gitlab
   module SecretDetection

data/lib/gitlab/secret_detection/version.rb

@@ -3,23 +3,9 @@
 module Gitlab
   module SecretDetection
     class Gem
-      DEFAULT_VERSION = "0.0.1"
-
-      SEMVER_REGEX = /^\d+\.\d+\.\d+(?:-[a-zA-Z0-9\-\.]+)?(?:\+[a-zA-Z0-9\-\.]+)?$/
-
-      def self.get_release_version
-        release_version = ENV.fetch("SD_GEM_RELEASE_VERSION", "")
-
-        if release_version.empty?
-          raise LoadError("Missing SD_GEM_RELEASE_VERSION environment variable.") unless local_env?
-
-          "#{DEFAULT_VERSION}-debug"
-        elsif release_version.match?(SEMVER_REGEX)
-          release_version
-        else
-          "#{DEFAULT_VERSION}-#{release_version}"
-        end
-      end
+      # Ensure to maintain the same version in CHANGELOG file.
+      # More details available under 'Release Process' section in the README.md file.
+      VERSION = "0.39.2"
 
       # SD_ENV env var is used to determine which environment the
       # server is running. This var is defined in `.runway/env-<env>.yml` files.

data/proto/secret_detection.proto

@@ -17,6 +17,8 @@ enum ExclusionType {
   EXCLUSION_TYPE_UNSPECIFIED = 0;
   EXCLUSION_TYPE_RULE = 1; // Rule ID to exclude
   EXCLUSION_TYPE_RAW_VALUE = 2; // Raw value to exclude
+  EXCLUSION_TYPE_PATH = 3; // Specific file path to exclude
+  EXCLUSION_TYPE_REGEX_PATTERN = 4; // Regular expression to exclude
 }
 
 /* Request arg for triggering Scan/ScanStream method */
@@ -39,6 +41,7 @@ message ScanRequest {
 }
 
 /* Response from Scan/ScanStream method */
+/* Any changes to these definitions must be matched by changes in the Core classes that correspond to them */
 message ScanResponse {
   // Represents a secret finding identified within a payload
   message Finding {