gitlab-secret_detection 0.4.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4b61c4602691e857833407badf2d2556d493679db2a5897c098eaff7aa96847
-  data.tar.gz: f803fcbae2d2988aace3f62e141500e175a75664b3a21f28ef5ef8c2cf89fcb7
+  metadata.gz: 648b6d5277ac8e7948762533af39dc2f4ff0ae4c62fbdcc6d5d32615a44ab815
+  data.tar.gz: afd2a580cb0a73bb84a401616ec622e1e14d2b75021036a959f933ed75864cf6
 SHA512:
-  metadata.gz: 0b7b61e27e95f816d6d8d260669eb0b54606170910ca18eb67937ca9f05b44333b7f6c8dccdf5c04aa6896d4d68a355e284745e91b309120f3a3fc789c9b05fb
-  data.tar.gz: f91c75463925727308c61166ac89e4f48f61c2e0a7cb4d37cc4e0a411db4fcb88245cbc355ed49577d2fe22fcb5d1cb9bd08a734e3125dd4815ef86e44f7b578
+  metadata.gz: 7c49b71891f6e13d8dc252936f18df838d8e9d42000cbc1f4f3d7fe56b258a7e2c17bce7a7f0e771bc16991fa46bacf641a636a1d1efd3ef23de6b0a31a011b8
+  data.tar.gz: af93c82a0025be12bc82fe6ff62b290fe193a2338ae2f98d83bc2729d1c7cc35a223330a3a9907a9f715c009127e93ba2d13fc33982d2f13a179d8548f5cc36e

data/README.md

@@ -42,6 +42,7 @@ the approach:
 │           ├── core/..                  # Secret detection logic (most of it pulled from existing gem)
 │           └── grpc
 │               ├── generated/..         # gRPC generated files and secret detection gRPC service
+│               ├── client/..            # gRPC client to invoke secret detection service's RPC endpoints 
 │               └── scanner_service.rb   # Secret detection gRPC service implementation
 ├── examples
 │   └── sample-client/..                 # Sample Ruby RPC client that connects with gRPC server and calls RPC scan
@@ -320,7 +321,7 @@ Run `ruby examples/sample-client/sample_client.rb` on your terminal to run the s
 
 ## Benchmark
 
-RPC service is benchmarked using [`ghz`](https://ghz.sh), a powerful CLI-based tool for load testing and benchmarking gRPC services. More details added [here](benchmark/README.md).
+RPC service is benchmarked using [`ghz`](https://ghz.sh), a powerful CLI-based tool for load testing and benchmarking gRPC services. More details added [here](https://gitlab.com/gitlab-org/gitlab/-/work_items/468107).
 
 ## Project Status
 

data/lib/gitlab/secret_detection/core/response.rb

@@ -7,12 +7,15 @@ module GitLab
       #
       # +status+:: One of values from GitLab::SecretDetection::Core::Status indicating the scan operation's status
       # +results+:: Array of GitLab::SecretDetection::Core::Finding values. Default value is nil.
+      # +metadata+:: Hash object containing additional meta information about the response. It is currently used
+      #   to embed more information on error.
       class Response
-        attr_reader :status, :results
+        attr_reader :status, :results, :metadata
 
-        def initialize(status, results = [])
+        def initialize(status, results = [], metadata = {})
           @status = status
           @results = results
+          @metadata = metadata
         end
 
         def ==(other)
@@ -22,6 +25,7 @@ module GitLab
         def to_h
           {
             status:,
+            metadata:,
             results: results&.map(&:to_h)
           }
         end
@@ -29,7 +33,7 @@ module GitLab
         protected
 
         def state
-          [status, results]
+          [status, metadata, results]
         end
       end
     end

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -4,6 +4,7 @@ require 're2'
 require 'logger'
 require 'timeout'
 require 'English'
+require 'parallel'
 
 module GitLab
   module SecretDetection
@@ -24,6 +25,14 @@ module GitLab
         DEFAULT_PAYLOAD_TIMEOUT_SECS = 30 # 30 seconds
         # Tags used for creating default pattern matcher
         DEFAULT_PATTERN_MATCHER_TAGS = ['gitlab_blocking'].freeze
+        # Max no of child processes to spawn per request
+        # ref: https://gitlab.com/gitlab-org/gitlab/-/issues/430160
+        MAX_PROCS_PER_REQUEST = 5
+        # Minimum cumulative size of the payloads required to spawn and
+        # run the scan within a new subprocess.
+        MIN_CHUNK_SIZE_PER_PROC_BYTES = 2_097_152 # 2MiB
+        # Whether to run scan in subprocesses or not. Default is false.
+        RUN_IN_SUBPROCESS = false
 
         # Initializes the instance with logger along with following operations:
         # 1. Extract keywords from the parsed ruleset to use it for matching keywords before regex operation.
@@ -58,6 +67,13 @@ module GitLab
         #           For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
         #           [`gitlab_blocking`] (+DEFAULT_PATTERN_MATCHER_TAGS+).
         #
+        # NOTE:
+        # Running the scan in fork mode primarily focuses on reducing the memory consumption of the scan by
+        # offloading regex operations on large payloads to sub-processes. However, it does not assure the improvement
+        # in the overall latency of the scan, specifically in the case of smaller payloads, where the overhead of
+        # forking a new process adds to the overall latency of the scan instead. More reference on Subprocess-based
+        # execution is found here: https://gitlab.com/gitlab-org/gitlab/-/issues/430160.
+        #
         # Returns an instance of GitLab::SecretDetection::Core::Response by following below structure:
         # {
         #     status: One of the Core::Status values
@@ -70,7 +86,8 @@ module GitLab
           payload_timeout: DEFAULT_PAYLOAD_TIMEOUT_SECS,
           raw_value_exclusions: [],
           rule_exclusions: [],
-          tags: DEFAULT_PATTERN_MATCHER_TAGS
+          tags: DEFAULT_PATTERN_MATCHER_TAGS,
+          subprocess: RUN_IN_SUBPROCESS
         )
 
           return Core::Response.new(Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
@@ -87,11 +104,13 @@ module GitLab
 
             next Core::Response.new(Core::Status::NOT_FOUND) if matched_payloads.empty?
 
-            secrets = run_scan(
+            scan_args = {
               payloads: matched_payloads, payload_timeout:,
               pattern_matcher: build_pattern_matcher(tags:),
               raw_value_exclusions:, rule_exclusions:
-            )
+            }
+
+            secrets = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
 
             scan_status = overall_scan_status(secrets)
 
@@ -205,6 +224,36 @@ module GitLab
           end
         end
 
+        def run_scan_within_subprocess(
+          payloads:, payload_timeout:, pattern_matcher:, raw_value_exclusions: [],
+          rule_exclusions: [])
+          payload_sizes = payloads.map(&:size)
+          grouped_payload_indices = group_by_chunk_size(payload_sizes)
+
+          grouped_payloads = grouped_payload_indices.map { |idx_arr| idx_arr.map { |i| payloads[i] } }
+
+          found_secrets = Parallel.flat_map(
+            grouped_payloads,
+            in_processes: MAX_PROCS_PER_REQUEST,
+            isolation: true # do not reuse sub-processes
+          ) do |grouped_payload|
+            grouped_payload.flat_map do |payload|
+              Timeout.timeout(payload_timeout) do
+                find_secrets_in_payload(
+                  payload:,
+                  pattern_matcher:,
+                  raw_value_exclusions:, rule_exclusions:
+                )
+              end
+            rescue Timeout::Error => e
+              logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
+              Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
+            end
+          end
+
+          found_secrets.freeze
+        end
+
         # Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan.
@@ -268,6 +317,35 @@ module GitLab
             Core::Status::FOUND_WITH_ERRORS
           end
         end
+
+        # This method accepts an array of payload sizes(in bytes) and groups them into an array
+        # of arrays structure where each element is the group of indices of the input
+        # array whose cumulative payload sizes has at least +MIN_CHUNK_SIZE_PER_PROC_BYTES+
+        def group_by_chunk_size(payload_size_arr)
+          cumulative_size = 0
+          chunk_indexes = []
+          chunk_idx_start = 0
+
+          payload_size_arr.each_with_index do |size, index|
+            cumulative_size += size
+            next unless cumulative_size >= MIN_CHUNK_SIZE_PER_PROC_BYTES
+
+            chunk_indexes << (chunk_idx_start..index).to_a
+
+            chunk_idx_start = index + 1
+            cumulative_size = 0
+          end
+
+          if cumulative_size.positive? && (chunk_idx_start < payload_size_arr.length)
+            chunk_indexes << if chunk_idx_start == payload_size_arr.length - 1
+                               [chunk_idx_start]
+                             else
+                               (chunk_idx_start..payload_size_arr.length - 1).to_a
+                             end
+          end
+
+          chunk_indexes
+        end
       end
     end
   end

data/lib/gitlab/secret_detection/core/status.rb

@@ -12,6 +12,7 @@ module GitLab
         SCAN_ERROR = 5 # When the scan operation fails due to regex error
         INPUT_ERROR = 6 # When the scan operation fails due to invalid input
         NOT_FOUND = 7 # When scan operation completes with zero findings
+        AUTH_ERROR = 8 # When authentication fails
       end
     end
   end

data/lib/gitlab/secret_detection/grpc/client/grpc_client.rb

@@ -1,19 +1,143 @@
 # frozen_string_literal: true
 
-require_relative '../generated/secret_detection_pb'
-require_relative '../generated/secret_detection_services_pb'
+require 'grpc'
+require_relative '../../grpc/scanner_service'
+require_relative '../../core/response'
+require_relative '../../core/status'
+require_relative '../../utils'
+require_relative './stream_request_enumerator'
 
 module GitLab
   module SecretDetection
     module GRPC
       class Client
-        # TODO add implementation
-        def scan
-          raise NotImplementedError
+        include SecretDetection::Utils::StrongMemoize
+        include SDLogger
+
+        # Time to wait for the response from the service
+        REQUEST_TIMEOUT_SECONDS = 10 # 10 seconds
+
+        def initialize(host, secure: false, compression: true)
+          @host = host
+          @secure = secure
+          @compression = compression
+        end
+
+        # Triggers Secret Detection service's `/Scan` gRPC endpoint. To keep it consistent with SDS gem interface,
+        # this method transforms the gRPC response to +GitLab::SecretDetection::Core::Response+.
+        # Furthermore, any errors that are raised by the service will be translated to
+        # +GitLab::SecretDetection::Core::Response+ type by assiging a appropriate +status+ value to it.
+        def run_scan(request:, auth_token:, extra_headers: {})
+          with_rescued_errors do
+            grpc_response = stub.scan(
+              request,
+              metadata: build_metadata(auth_token, extra_headers),
+              deadline: request_deadline
+            )
+
+            convert_to_core_response(grpc_response)
+          end
+        end
+
+        # Triggers Secret Detection service's `/ScanStream` gRPC endpoint.
+        #
+        # To keep it consistent with SDS gem interface, this method transforms the gRPC response to
+        # +GitLab::SecretDetection::Core::Response+ type. Furthermore, any errors that are raised by the service will be
+        # translated to +GitLab::SecretDetection::Core::Response+ type by assiging a appropriate +status+ value to it.
+        #
+        # Note: If one of the stream requests result in an error, the stream will end immediately without processing the
+        # remaining requests.
+        def run_scan_stream(requests:, auth_token:, extra_headers: {})
+          request_stream = GitLab::SecretDetection::GRPC::StreamRequestEnumerator.new(requests)
+          results = []
+          with_rescued_errors do
+            stub.scan_stream(
+              request_stream.each_item,
+              metadata: build_metadata(auth_token, extra_headers),
+              deadline: request_deadline
+            ).each do |grpc_response|
+              response = convert_to_core_response(grpc_response)
+              if block_given?
+                yield response
+              else
+                results << response
+              end
+            end
+            results
+          end
+        end
+
+        private
+
+        attr_reader :secure, :host, :compression
+
+        def stub
+          GitLab::SecretDetection::GRPC::Scanner::Stub.new(
+            host,
+            channel_credentials,
+            channel_args:
+          )
+        end
+
+        strong_memoize_attr :stub
+
+        def channel_args
+          default_options = {
+            'grpc.keepalive_permit_without_calls' => 1,
+            'grpc.keepalive_time_ms' => 30000, # 30 seconds
+            'grpc.keepalive_timeout_ms' => 10000 # 10 seconds timeout for keepalive response
+          }
+
+          compression_options = ::GRPC::Core::CompressionOptions
+                                  .new(default_algorithm: :gzip)
+                                  .to_channel_arg_hash
+
+          default_options.merge!(compression_options) if compression
+
+          default_options.freeze
+        end
+
+        def channel_credentials
+          return :this_channel_is_insecure unless secure
+
+          certs = GitLab::SecretDetection::Utils::X509::Certificate.ca_certs_bundle
+
+          ::GRPC::Core::ChannelCredentials.new(certs)
         end
 
-        def scan_stream
-          raise NotImplementedError
+        def build_metadata(token, extra_headers = {})
+          { 'x-sd-auth' => token }.merge!(extra_headers).freeze
+        end
+
+        def request_deadline
+          Time.now + REQUEST_TIMEOUT_SECONDS
+        end
+
+        def with_rescued_errors
+          yield
+        rescue ::GRPC::Unauthenticated
+          SecretDetection::Core::Response.new(SecretDetection::Core::Status::AUTH_ERROR)
+        rescue ::GRPC::InvalidArgument => e
+          SecretDetection::Core::Response.new(
+            SecretDetection::Core::Status::INPUT_ERROR, nil, { message: e.details, **e.metadata }
+          )
+        rescue ::GRPC::Unknown, ::GRPC::BadStatus => e
+          SecretDetection::Core::Response.new(
+            SecretDetection::Core::Status::SCAN_ERROR, nil, { message: e.details }
+          )
+        end
+
+        def convert_to_core_response(grpc_response)
+          response = grpc_response.to_h
+
+          SecretDetection::Core::Response.new(
+            response[:status],
+            response[:results],
+            response[:metadata]
+          )
+        rescue StandardError => e
+          logger.error("Failed to convert to core response: #{e}")
+          SecretDetection::Core::Response.new(SecretDetection::Core::Status::SCAN_ERROR)
         end
       end
     end

data/lib/gitlab/secret_detection/grpc/client/stream_request_enumerator.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module GitLab
+  module SecretDetection
+    module GRPC
+      class StreamRequestEnumerator
+        def initialize(requests = [])
+          @requests = requests
+        end
+
+        # yields a request, waiting between 0 and 1 seconds between requests
+        #
+        # @return an Enumerable that yields a request input
+        def each_item
+          return enum_for(:each_item) unless block_given?
+
+          @requests.each do |request|
+            yield request
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb

@@ -5,7 +5,7 @@
 require 'google/protobuf'
 
 
-descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xfc\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\nexclusions\x18\x04 \x03(\x0b\x32..gitlab.secret_detection.ScanRequest.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a\x66\n\tExclusion\x12J\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32\x32.gitlab.secret_detection.ScanRequest.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe3\x04\n\x0cScanResponse\x12\x12\n\x05\x65rror\x18\x01 \x01(\tH\x00\x88\x01\x01\x12>\n\x07results\x18\x02 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12<\n\x06status\x18\x03 \x01(\x0e\x32,.gitlab.secret_detection.ScanResponse.Status\x1a\xe9\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12<\n\x06status\x18\x02 \x01(\x0e\x32,.gitlab.secret_detection.ScanResponse.Status\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05\x65rror\x18\x06 \x01(\tH\x03\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_numberB\x08\n\x06_error\"\xca\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x42\x08\n\x06_error2\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitLab::SecretDetection::GRPCb\x06proto3"
+descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xfc\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\nexclusions\x18\x04 \x03(\x0b\x32..gitlab.secret_detection.ScanRequest.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a\x66\n\tExclusion\x12J\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32\x32.gitlab.secret_detection.ScanRequest.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe2\x03\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitLab::SecretDetection::GRPCb\x06proto3"
 
 pool = Google::Protobuf::DescriptorPool.generated_pool
 pool.add_serialized_file(descriptor_data)

data/lib/gitlab/secret_detection/grpc/scanner_service.rb

@@ -74,14 +74,19 @@ module GitLab
             end
           end
 
-          result = scanner.secrets_scan(
-            payloads,
-            raw_value_exclusions:,
-            rule_exclusions:,
-            tags: request.tags.to_a,
-            timeout: request.timeout_secs,
-            payload_timeout: request.payload_timeout_secs
-          )
+          begin
+            result = scanner.secrets_scan(
+              payloads,
+              raw_value_exclusions:,
+              rule_exclusions:,
+              tags: request.tags.to_a,
+              timeout: request.timeout_secs,
+              payload_timeout: request.payload_timeout_secs
+            )
+          rescue StandardError => e
+            logger.error("Failed to run the scan: #{e}")
+            raise ::GRPC::Unknown, e.message
+          end
 
           findings = result.results&.map do |finding|
             GitLab::SecretDetection::GRPC::ScanResponse::Finding.new(**finding.to_h)

data/lib/gitlab/secret_detection/grpc.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'grpc/scanner_service'
+require_relative 'grpc/client/stream_request_enumerator'
 require_relative 'grpc/client/grpc_client'
 
 module GitLab

data/lib/gitlab/secret_detection/utils/certificate.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require_relative 'memoize'
+
+module GitLab
+  module SecretDetection
+    module Utils
+      module X509
+        # Pulled from GitLab.com source
+        # Link: https://gitlab.com/gitlab-org/gitlab/-/blob/4713a798f997389f04e442db3d1d8349a39d5d46/lib/gitlab/x509/certificate.rb
+        class Certificate
+          CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/
+
+          attr_reader :key, :cert, :ca_certs
+
+          def self.default_cert_dir
+            strong_memoize(:default_cert_dir) do
+              ENV.fetch('SSL_CERT_DIR', OpenSSL::X509::DEFAULT_CERT_DIR)
+            end
+          end
+
+          def self.default_cert_file
+            strong_memoize(:default_cert_file) do
+              ENV.fetch('SSL_CERT_FILE', OpenSSL::X509::DEFAULT_CERT_FILE)
+            end
+          end
+
+          def self.from_strings(key_string, cert_string, ca_certs_string = nil)
+            key = OpenSSL::PKey::RSA.new(key_string)
+            cert = OpenSSL::X509::Certificate.new(cert_string)
+            ca_certs = load_ca_certs_bundle(ca_certs_string)
+
+            new(key, cert, ca_certs)
+          end
+
+          def self.from_files(key_path, cert_path, ca_certs_path = nil)
+            ca_certs_string = File.read(ca_certs_path) if ca_certs_path
+
+            from_strings(File.read(key_path), File.read(cert_path), ca_certs_string)
+          end
+
+          # Returns all top-level, readable files in the default CA cert directory
+          def self.ca_certs_paths
+            cert_paths = Dir["#{default_cert_dir}/*"].select do |path|
+              !File.directory?(path) && File.readable?(path)
+            end
+            cert_paths << default_cert_file if File.exist? default_cert_file
+            cert_paths
+          end
+
+          # Returns a concatenated array of Strings, each being a PEM-coded CA certificate.
+          def self.ca_certs_bundle
+            strong_memoize(:ca_certs_bundle) do
+              ca_certs_paths.flat_map do |cert_file|
+                load_ca_certs_bundle(File.read(cert_file))
+              end.uniq.join("\n")
+            end
+          end
+
+          def self.reset_ca_certs_bundle
+            clear_memoization(:ca_certs_bundle)
+          end
+
+          def self.reset_default_cert_paths
+            clear_memoization(:default_cert_dir)
+            clear_memoization(:default_cert_file)
+          end
+
+          # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found
+          #
+          # Ruby OpenSSL::X509::Certificate.new will only load the first
+          # certificate if a bundle is presented, this allows to parse multiple certs
+          # in the same file
+          def self.load_ca_certs_bundle(ca_certs_string)
+            return [] unless ca_certs_string
+
+            ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string|
+              OpenSSL::X509::Certificate.new(ca_cert_string)
+            end
+          end
+
+          def initialize(key, cert, ca_certs = nil)
+            @key = key
+            @cert = cert
+            @ca_certs = ca_certs
+          end
+
+          def key_string
+            key.to_s
+          end
+
+          def cert_string
+            cert.to_pem
+          end
+
+          def ca_certs_string
+            ca_certs&.map(&:to_pem)&.join('\n') unless ca_certs.blank?
+          end
+
+          class << self
+            include ::GitLab::SecretDetection::Utils::StrongMemoize
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/utils/memoize.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module GitLab
+  module SecretDetection
+    module Utils
+      # Pulled from GitLab.com source
+      # Link: https://gitlab.com/gitlab-org/gitlab/-/blob/4713a798f997389f04e442db3d1d8349a39d5d46/gems/gitlab-utils/lib/gitlab/utils/strong_memoize.rb
+      module StrongMemoize
+        # Instead of writing patterns like this:
+        #
+        #     def trigger_from_token
+        #       return @trigger if defined?(@trigger)
+        #
+        #       @trigger = Ci::Trigger.find_by_token(params[:token].to_s)
+        #     end
+        #
+        # We could write it like:
+        #
+        #     include GitLab::SecretDetection::Utils::StrongMemoize
+        #
+        #     def trigger_from_token
+        #       Ci::Trigger.find_by_token(params[:token].to_s)
+        #     end
+        #     strong_memoize_attr :trigger_from_token
+        #
+        #     def enabled?
+        #       Feature.enabled?(:some_feature)
+        #     end
+        #     strong_memoize_attr :enabled?
+        #
+        def strong_memoize(name)
+          key = ivar(name)
+
+          if instance_variable_defined?(key)
+            instance_variable_get(key)
+          else
+            instance_variable_set(key, yield)
+          end
+        end
+
+        # Works the same way as "strong_memoize" but takes
+        # a second argument - expire_in. This allows invalidate
+        # the data after specified number of seconds
+        def strong_memoize_with_expiration(name, expire_in)
+          key = ivar(name)
+          expiration_key = "#{key}_expired_at"
+
+          if instance_variable_defined?(expiration_key)
+            expire_at = instance_variable_get(expiration_key)
+            clear_memoization(name) if expire_at.past?
+          end
+
+          if instance_variable_defined?(key)
+            instance_variable_get(key)
+          else
+            value = instance_variable_set(key, yield)
+            instance_variable_set(expiration_key, Time.current + expire_in)
+            value
+          end
+        end
+
+        def strong_memoize_with(name, *args)
+          container = strong_memoize(name) { {} }
+
+          if container.key?(args)
+            container[args]
+          else
+            container[args] = yield
+          end
+        end
+
+        def strong_memoized?(name)
+          key = ivar(StrongMemoize.normalize_key(name))
+          instance_variable_defined?(key)
+        end
+
+        def clear_memoization(name)
+          key = ivar(StrongMemoize.normalize_key(name))
+          remove_instance_variable(key) if instance_variable_defined?(key)
+        end
+
+        module StrongMemoizeClassMethods
+          def strong_memoize_attr(method_name)
+            member_name = StrongMemoize.normalize_key(method_name)
+
+            StrongMemoize.send(:do_strong_memoize, self, method_name, member_name) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        def self.included(base)
+          base.singleton_class.prepend(StrongMemoizeClassMethods)
+        end
+
+        private
+
+        # Convert `"name"`/`:name` into `:@name`
+        #
+        # Depending on a type ensure that there's a single memory allocation
+        def ivar(name)
+          case name
+          when Symbol
+            name.to_s.prepend("@").to_sym
+          when String
+            :"@#{name}"
+          else
+            raise ArgumentError, "Invalid type of '#{name}'"
+          end
+        end
+
+        class << self
+          def normalize_key(key)
+            return key unless key.end_with?('!', '?')
+
+            # Replace invalid chars like `!` and `?` with allowed Unicode codeparts.
+            key.to_s.tr('!?', "\uFF01\uFF1F")
+          end
+
+          private
+
+          def do_strong_memoize(klass, method_name, member_name)
+            method = klass.instance_method(method_name)
+
+            unless method.arity.zero?
+              raise <<~ERROR
+                Using `strong_memoize_attr` on methods with parameters is not supported.
+
+                Use `strong_memoize_with` instead.
+                See https://docs.gitlab.com/ee/development/utilities.html#strongmemoize
+              ERROR
+            end
+
+            # Methods defined within a class method are already public by default, so we don't need to
+            # explicitly make them public.
+            scope = %i[private protected].find do |scope|
+              klass.send(:"#{scope}_instance_methods") # rubocop:disable GitlabSecurity/PublicSend
+                   .include? method_name
+            end
+
+            klass.define_method(method_name) do |&block|
+              strong_memoize(member_name) do
+                method.bind_call(self, &block)
+              end
+            end
+
+            klass.send(scope, method_name) if scope # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/utils.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'utils/certificate'
+require_relative 'utils/memoize'
+
+module GitLab
+  module SecretDetection
+    module Utils
+    end
+  end
+end

data/lib/gitlab/secret_detection.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'secret_detection/utils'
 require_relative 'secret_detection/core'
 require_relative 'secret_detection/grpc'
 require_relative 'secret_detection/version'

data/proto/secret_detection.proto

@@ -5,7 +5,7 @@ package gitlab.secret_detection;
 /* We keep generated files within grpc namespace i.e GitLab::SecretDetection::GRPC
  * so that these files are exported too in the Ruby Gem along with Core and GRPC logic.
  */
-option ruby_package="GitLab::SecretDetection::GRPC";
+option ruby_package = "GitLab::SecretDetection::GRPC";
 
 /* Request arg for triggering Scan/ScanStream method */
 message ScanRequest {
@@ -42,11 +42,10 @@ message ScanResponse {
   // Represents a secret finding identified within a payload
   message Finding {
     string payload_id = 1;
-    Status status = 2;
+    int32 status = 2;
     optional string type = 3;
     optional string description = 4;
     optional int32 line_number = 5;
-    optional string error = 6;
   }
 
   // Return status code in sync with ::SecretDetection::Status
@@ -59,18 +58,18 @@ message ScanResponse {
     STATUS_SCAN_ERROR = 5;         // internal scan failure
     STATUS_INPUT_ERROR = 6;        // invalid input failure
     STATUS_NOT_FOUND = 7;          // zero findings
+    STATUS_AUTH_ERROR = 8;         // authentication failure
   }
 
-  optional string error = 1;
-  repeated Finding results = 2;
-  Status status = 3;
+  repeated Finding results = 1;
+  int32 status = 2;
 }
 
 /* Scanner service that scans given payloads and returns findings */
 service Scanner {
   // Runs secret detection scan for the given request
-  rpc Scan(ScanRequest) returns (ScanResponse) { }
+  rpc Scan(ScanRequest) returns (ScanResponse) {}
 
   // Runs bi-directional streaming of scans for the given stream of requests with a stream of responses
-  rpc ScanStream(stream ScanRequest) returns (stream ScanResponse) { }
+  rpc ScanStream(stream ScanRequest) returns (stream ScanResponse) {}
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.6.0
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-01 00:00:00.000000000 Z
+date: 2024-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc
@@ -28,18 +28,32 @@ dependencies:
         version: 1.63.0
 - !ruby/object:Gem::Dependency
   name: grpc-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.63.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.63.0
+- !ruby/object:Gem::Dependency
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.66'
+        version: '1.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.66'
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: re2
   requirement: !ruby/object:Gem::Requirement
@@ -93,10 +107,14 @@ files:
 - lib/gitlab/secret_detection/core/status.rb
 - lib/gitlab/secret_detection/grpc.rb
 - lib/gitlab/secret_detection/grpc/client/grpc_client.rb
+- lib/gitlab/secret_detection/grpc/client/stream_request_enumerator.rb
 - lib/gitlab/secret_detection/grpc/generated/.gitkeep
 - lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb
 - lib/gitlab/secret_detection/grpc/generated/secret_detection_services_pb.rb
 - lib/gitlab/secret_detection/grpc/scanner_service.rb
+- lib/gitlab/secret_detection/utils.rb
+- lib/gitlab/secret_detection/utils/certificate.rb
+- lib/gitlab/secret_detection/utils/memoize.rb
 - lib/gitlab/secret_detection/version.rb
 - proto/secret_detection.proto
 homepage: https://gitlab.com/gitlab-org/security-products/secret-detection/secret-detection-service