gitlab-secret_detection 0.19.1 → 0.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92079cc4159944de4812acba9f6c0f7ec7f36a0a1c4f0770849fc2e35740d7da
-  data.tar.gz: 5cda8188ccc3d46d47b074a3126543db9215fd5d99def8082bae2f8f720d5454
+  metadata.gz: 64291ae0fc59c36ec4e951eba1887416f8febbe2d89ff87b4690d3fcde46c14c
+  data.tar.gz: 230c2d0623f5879edfba4cbe57a1ce6cb421e4cefa7223f9214ed40d241e616c
 SHA512:
-  metadata.gz: d0a74f47adcfbf6e8ec0ab72366d5151b2860788e4adba51944e7ddc03b73eef119b9b8f10890454273845ca2c94c8d74be8a5519adc6ff97a3ca485e47464b7
-  data.tar.gz: 399a729fa667c6174e185c10eef898e51f6f1827c0584242560033d0233dc19ce307d9a75c97eb18acf18e19f647d20d31efe1e11fb984ace4640d6dd7ddd85b
+  metadata.gz: 80a65bff589d277515df5ca04670180224a1996b0ed65f8dba79170c2dce312870f5e56122d1d7ae100b649a0d7818d51c87eae9d0c35ad5e3ce50844af191d3
+  data.tar.gz: 1a311f585f0231650e1041dae2c280a62927cde9e8e22ac18fcd81216f25dcaaff64c24defe970adc9ec63b533c3834b0221286c58f8967c0422fb75b64fd45d

data/lib/gitlab/secret_detection/core/ruleset.rb

@@ -7,6 +7,14 @@ module Gitlab
   module SecretDetection
     module Core
       class Ruleset
+        # RulesetParseError is thrown when the code fails to parse the
+        # ruleset file from the given path
+        RulesetParseError = Class.new(StandardError)
+
+        # RulesetCompilationError is thrown when the code fails to compile
+        # the predefined rulesets
+        RulesetCompilationError = Class.new(StandardError)
+
         # file path where the secrets ruleset file is located
         RULESET_FILE_PATH = File.expand_path('secret_push_protection_rules.toml', __dir__)
 
@@ -21,6 +29,15 @@ module Gitlab
           @rule_data = parse_ruleset
         end
 
+        def extract_ruleset_version
+          @ruleset_version ||= if File.readable?(RULESET_FILE_PATH)
+                                 first_line = File.open(RULESET_FILE_PATH, &:gets)
+                                 first_line&.split(":")&.[](1)&.strip
+                               end
+        rescue StandardError => e
+          logger.error(message: "Failed to extract Secret Detection Ruleset version from ruleset file: #{e.message}")
+        end
+
         private
 
         attr_reader :path, :logger
@@ -44,15 +61,6 @@ module Gitlab
           logger.error(message: "Failed to parse local secret detection ruleset: #{e.message}")
           raise Core::Scanner::RulesetParseError, e
         end
-
-        def extract_ruleset_version
-          @ruleset_version ||= if File.readable?(RULESET_FILE_PATH)
-                                 first_line = File.open(RULESET_FILE_PATH, &:gets)
-                                 first_line&.split(":")&.[](1)&.strip
-                               end
-        rescue StandardError => e
-          logger.error(message: "Failed to extract Secret Detection Ruleset version from ruleset file: #{e.message}")
-        end
       end
     end
   end

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -11,14 +11,6 @@ module Gitlab
     module Core
       # Scan is responsible for running Secret Detection scan operation
       class Scanner
-        # RulesetParseError is thrown when the code fails to parse the
-        # ruleset file from the given path
-        RulesetParseError = Class.new(StandardError)
-
-        # RulesetCompilationError is thrown when the code fails to compile
-        # the predefined rulesets
-        RulesetCompilationError = Class.new(StandardError)
-
         # default time limit(in seconds) for running the scan operation per invocation
         DEFAULT_SCAN_TIMEOUT_SECS = 180 # 3 minutes
         # default time limit(in seconds) for running the scan operation on a single payload
@@ -91,7 +83,6 @@ module Gitlab
           tags: DEFAULT_PATTERN_MATCHER_TAGS,
           subprocess: RUN_IN_SUBPROCESS
         )
-
           return Core::Response.new(status: Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
 
           # assign defaults since grpc passing zero timeout value to `Timeout.timeout(..)` makes it effectively useless.
@@ -184,7 +175,7 @@ module Gitlab
           unless matcher.compile
             logger.error "Failed to compile secret detection ruleset in RE::Set"
 
-            raise RulesetCompilationError
+            raise Core::Ruleset::RulesetCompilationError
           end
 
           matcher
@@ -300,7 +291,7 @@ module Gitlab
               findings
             end
           rescue Timeout::Error => e
-            logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
+            logger.warn "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
 
             Core::Finding.new(payload.id,
               Core::Status::PAYLOAD_TIMEOUT)
@@ -342,7 +333,7 @@ module Gitlab
                 findings
               end
             rescue Timeout::Error => e
-              logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
+              logger.warn "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
 
               Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
             end

data/lib/gitlab/secret_detection/core/status.rb

@@ -5,6 +5,8 @@ module Gitlab
     module Core
       # All the possible statuses emitted by the scan operation
       class Status
+        # These values must stay in-sync with the GRPC::ScanResponse::Status values
+        UNSPECIFIED = 0 # to match the GRPC::Status values
         FOUND = 1 # When scan operation completes with one or more findings
         FOUND_WITH_ERRORS = 2 # When scan operation completes with one or more findings along with some errors
         SCAN_TIMEOUT = 3 # When the scan operation runs beyond given time out
@@ -13,6 +15,38 @@ module Gitlab
         INPUT_ERROR = 6 # When the scan operation fails due to invalid input
         NOT_FOUND = 7 # When scan operation completes with zero findings
         AUTH_ERROR = 8 # When authentication fails
+
+        # Maps values to constants
+        @values_map = {}
+
+        # Using class instance variables and singleton methods
+        class << self
+          attr_reader :values_map
+
+          # Register constants and their values in the map
+          def const_set(name, value)
+            const = super
+            @values_map[value] = name
+            const
+          end
+
+          # Look up a constant by its value
+          def find_by_value(value)
+            const_name = @values_map[value]
+            const_name ? const_get(const_name) : nil
+          end
+
+          # Get the name of a constant by its value
+          def name_by_value(value)
+            @values_map[value]
+          end
+        end
+
+        # Initialize the values map with existing constants
+        constants.each do |const_name|
+          const_value = const_get(const_name)
+          @values_map[const_value] = const_name
+        end
       end
     end
   end

data/lib/gitlab/secret_detection/grpc/client/grpc_client.rb

@@ -36,7 +36,7 @@ module Gitlab
               deadline: request_deadline
             )
 
-            convert_to_core_response(grpc_response)
+            grpc_response
           end
         end
 
@@ -57,11 +57,10 @@ module Gitlab
               metadata: build_metadata(auth_token, extra_headers),
               deadline: request_deadline
             ).each do |grpc_response|
-              response = convert_to_core_response(grpc_response)
               if block_given?
-                yield response
+                yield grpc_response
               else
-                results << response
+                results << grpc_response
               end
             end
             results
@@ -131,19 +130,6 @@ module Gitlab
             metadata: { message: e.details }
           )
         end
-
-        def convert_to_core_response(grpc_response)
-          response = grpc_response.to_h
-
-          SecretDetection::Core::Response.new(
-            status: response[:status],
-            results: response[:results],
-            metadata: response[:metadata]
-          )
-        rescue StandardError => e
-          @logger.error("Failed to convert to core response: #{e}")
-          SecretDetection::Core::Response.new(status: SecretDetection::Core::Status::SCAN_ERROR)
-        end
       end
     end
   end

data/lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb

@@ -5,7 +5,7 @@
 require 'google/protobuf'
 
 
-descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"Z\n\tExclusion\x12>\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32&.gitlab.secret_detection.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"\xc0\x02\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x36\n\nexclusions\x18\x04 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a\x43\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x13\n\x06offset\x18\x03 \x01(\x05H\x00\x88\x01\x01\x42\t\n\x07_offsetB\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xa2\x04\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12>\n\x12\x61pplied_exclusions\x18\x03 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08*\x7f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x12\x17\n\x13\x45XCLUSION_TYPE_PATH\x10\x03\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitlab::SecretDetection::GRPCb\x06proto3"
+descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"Z\n\tExclusion\x12>\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32&.gitlab.secret_detection.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"\xc0\x02\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x36\n\nexclusions\x18\x04 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a\x43\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x13\n\x06offset\x18\x03 \x01(\x05H\x00\x88\x01\x01\x42\t\n\x07_offsetB\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xa2\x04\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12>\n\x12\x61pplied_exclusions\x18\x03 \x03(\x0b\x32\".gitlab.secret_detection.Exclusion\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08*\xa1\x01\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x12\x17\n\x13\x45XCLUSION_TYPE_PATH\x10\x03\x12 \n\x1c\x45XCLUSION_TYPE_REGEX_PATTERN\x10\x04\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitlab::SecretDetection::GRPCb\x06proto3"
 
 pool = Google::Protobuf::DescriptorPool.generated_pool
 pool.add_serialized_file(descriptor_data)

data/lib/gitlab/secret_detection/grpc/integrated_error_tracking.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'sentry-ruby'
+
+require_relative '../../../../lib/gitlab/secret_detection/core/ruleset'
+
+module Gitlab
+  module SecretDetection
+    module GRPC
+      module IntegratedErrorTracking
+        extend self
+
+        def track_exception(exception, args = {})
+          unless Sentry.initialized?
+            logger.warn(message: "Cannot track exception in Error Tracking as Sentry is not initialized")
+            return
+          end
+
+          args[:ruleset_version] = ruleset_version
+
+          Sentry.capture_exception(exception, **args)
+        end
+
+        def setup(logger: Logger.new($stdout))
+          if Sentry.initialized?
+            logger.warn(message: "Sentry is already initialized, skipping re-setup")
+            return
+          end
+
+          logger.info(message: "Initializing Sentry SDK for Integrated Error Tracking..")
+
+          unless can_setup_sentry?
+            logger.warn(message: "Integrated Error Tracking not available, skipping Sentry SDK initialization")
+            return false
+          end
+
+          Sentry.init do |config|
+            config.dsn = ENV.fetch('SD_TRACKING_DSN')
+            config.environment = ENV.fetch('SD_ENV')
+            config.release = Gitlab::SecretDetection::Gem::VERSION
+            config.send_default_pii = true
+            config.send_modules = false
+            config.traces_sample_rate = 0.2 if ENV.fetch('ENABLE_SENTRY_PERFORMANCE_MONITORING', 'false') == 'true'
+          end
+
+          Sentry.set_context('ruleset', { version: ruleset_version })
+
+          true
+        rescue StandardError => e
+          logger.error(message: "Failed to initialize Sentry SDK for Integrated Error Tracking: #{e}")
+          raise e
+        end
+
+        def ruleset_version
+          @ruleset_version ||= Gitlab::SecretDetection::Core::Ruleset.new.extract_ruleset_version || 'unknown'
+        end
+
+        def can_setup_sentry?
+          ENV.fetch('SD_ENV', '') == 'production' && ENV.fetch('SD_TRACKING_DSN', '') != ''
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc/scanner_service.rb

@@ -32,6 +32,7 @@ module Gitlab
     module GRPC
       class ScannerService < Scanner::Service
         include SDLogger
+        include IntegratedErrorTracking
 
         # Maximum timeout value that can be given as the input. This guards
         # against the misuse of timeouts.
@@ -100,8 +101,8 @@ module Gitlab
               payload_timeout: request.payload_timeout_secs
             )
           rescue StandardError => e
-            logger.error(message: "Failed to run the secret detection scan", exception: e)
-            logger.error(e.backtrace&.join("\n"))
+            logger.error(message: "Failed to run the secret detection scan", exception: e.message)
+            track_exception(e)
             raise ::GRPC::Unknown, e.message
           end
 

data/lib/gitlab/secret_detection/grpc.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'grpc/integrated_error_tracking'
 require_relative 'grpc/scanner_service'
 require_relative 'grpc/client/stream_request_enumerator'
 require_relative 'grpc/client/grpc_client'

data/lib/gitlab/secret_detection/version.rb

@@ -8,7 +8,7 @@ module Gitlab
       #  https://gitlab.com/gitlab-org/gitlab/-/issues/514015
       #
       # Ensure to maintain the same version in CHANGELOG file.
-      VERSION = "0.19.1"
+      VERSION = "0.20.1"
 
       # SD_ENV env var is used to determine which environment the
       # server is running. This var is defined in `.runway/env-<env>.yml` files.

data/proto/secret_detection.proto

@@ -18,6 +18,7 @@ enum ExclusionType {
   EXCLUSION_TYPE_RULE = 1; // Rule ID to exclude
   EXCLUSION_TYPE_RAW_VALUE = 2; // Raw value to exclude
   EXCLUSION_TYPE_PATH = 3; // Specific file path to exclude
+  EXCLUSION_TYPE_REGEX_PATTERN = 4; // Regular expression to exclude
 }
 
 /* Request arg for triggering Scan/ScanStream method */
@@ -40,6 +41,7 @@ message ScanRequest {
 }
 
 /* Response from Scan/ScanStream method */
+/* Any changes to these definitions must be matched by changes in the Core classes that correspond to them */
 message ScanResponse {
   // Represents a secret finding identified within a payload
   message Finding {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.19.1
+  version: 0.20.1
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-19 00:00:00.000000000 Z
+date: 2025-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc
@@ -82,6 +82,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: sentry-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.22'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.22'
+- !ruby/object:Gem::Dependency
+  name: stackprof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.27
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.27
 - !ruby/object:Gem::Dependency
   name: toml-rb
   requirement: !ruby/object:Gem::Requirement
@@ -124,6 +152,7 @@ files:
 - lib/gitlab/secret_detection/grpc/generated/.gitkeep
 - lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb
 - lib/gitlab/secret_detection/grpc/generated/secret_detection_services_pb.rb
+- lib/gitlab/secret_detection/grpc/integrated_error_tracking.rb
 - lib/gitlab/secret_detection/grpc/scanner_service.rb
 - lib/gitlab/secret_detection/utils.rb
 - lib/gitlab/secret_detection/utils/certificate.rb