gitlab-secret_detection 0.1.0 → 0.7.0

data/lib/gitlab/secret_detection/grpc/generated/secret_detection_services_pb.rb

@@ -0,0 +1,30 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: secret_detection.proto for package 'Gitlab.SecretDetection.GRPC'
+
+require 'grpc'
+require 'secret_detection_pb'
+
+module Gitlab
+  module SecretDetection
+    module GRPC
+      module Scanner
+        # Scanner service that scans given payloads and returns findings 
+        class Service
+
+          include ::GRPC::GenericService
+
+          self.marshal_class_method = :encode
+          self.unmarshal_class_method = :decode
+          self.service_name = 'gitlab.secret_detection.Scanner'
+
+          # Runs secret detection scan for the given request
+          rpc :Scan, ::Gitlab::SecretDetection::GRPC::ScanRequest, ::Gitlab::SecretDetection::GRPC::ScanResponse
+          # Runs bi-directional streaming of scans for the given stream of requests with a stream of responses
+          rpc :ScanStream, stream(::Gitlab::SecretDetection::GRPC::ScanRequest), stream(::Gitlab::SecretDetection::GRPC::ScanResponse)
+        end
+
+        Stub = Service.rpc_stub_class
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc/scanner_service.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('generated', __dir__))
+
+require 'grpc'
+
+require_relative 'generated/secret_detection_pb'
+require_relative 'generated/secret_detection_services_pb'
+
+require_relative '../core'
+require_relative '../../../../config/log'
+
+# StreamEnumerator is used for Bi-directional streaming
+# of requests by returning stream of responses.
+class StreamEnumerator
+  def initialize(requests, action)
+    @requests = requests
+    @request_action = action
+  end
+
+  def each_item
+    return enum_for(:each_item) unless block_given?
+
+    @requests.each do |req|
+      yield @request_action.call(req)
+    end
+  end
+end
+
+module Gitlab
+  module SecretDetection
+    module GRPC
+      class ScannerService < Scanner::Service
+        include SDLogger
+
+        # Maximum timeout value that can be given as the input. This guards
+        # against the misuse of timeouts.
+        MAX_ALLOWED_TIMEOUT_SECONDS = 600
+
+        ERROR_MESSAGES = {
+          invalid_payload_fields: "Payload should not contain empty `id` and `data` fields",
+          exclusion_empty_value: "Exclusion value cannot be empty",
+          exclusion_invalid_type: "Invalid exclusion type",
+          invalid_timeout_range: "Timeout value should be > 0 and <= #{MAX_ALLOWED_TIMEOUT_SECONDS} seconds"
+        }.freeze
+
+        # Implementation for /Scan RPC method
+        def scan(request, _call)
+          scan_request_action(request)
+        end
+
+        # Implementation for /ScanStream RPC method
+        def scan_stream(requests, _call)
+          request_action = ->(r) { scan_request_action(r) }
+          StreamEnumerator.new(requests, request_action).each_item
+        end
+
+        private
+
+        def scan_request_action(request)
+          validate_request(request)
+
+          payloads = request.payloads.to_a
+
+          raw_value_exclusions = []
+          rule_exclusions = []
+
+          request.exclusions&.each do |exclusion|
+            case exclusion.type
+            when :EXCLUSION_TYPE_RAW_VALUE
+              raw_value_exclusions << exclusion.value
+            when :EXCLUSION_TYPE_RULE
+              rule_exclusions << exclusion.value
+            end
+          end
+
+          begin
+            result = scanner.secrets_scan(
+              payloads,
+              raw_value_exclusions:,
+              rule_exclusions:,
+              tags: request.tags.to_a,
+              timeout: request.timeout_secs,
+              payload_timeout: request.payload_timeout_secs
+            )
+          rescue StandardError => e
+            logger.error("Failed to run the scan: #{e}")
+            raise ::GRPC::Unknown, e.message
+          end
+
+          findings = result.results&.map do |finding|
+            Gitlab::SecretDetection::GRPC::ScanResponse::Finding.new(**finding.to_h)
+          end
+
+          Gitlab::SecretDetection::GRPC::ScanResponse.new(
+            results: findings,
+            status: result.status
+          )
+        end
+
+        def scanner
+          @scanner ||= Gitlab::SecretDetection::Core::Scanner.new(rules:, logger:)
+        end
+
+        def rules
+          Gitlab::SecretDetection::Core::Ruleset.new.rules
+        end
+
+        # validates grpc request body
+        def validate_request(request)
+          # check for non-blank values and allowed types
+          request.exclusions&.each do |exclusion|
+            if exclusion.value.empty?
+              raise ::GRPC::InvalidArgument.new(ERROR_MESSAGES[:exclusion_empty_value],
+                { field: "exclusion.value" })
+            end
+          end
+
+          unless valid_timeout_range?(request.timeout_secs)
+            raise ::GRPC::InvalidArgument.new(ERROR_MESSAGES[:invalid_timeout_range],
+              { field: "timeout_secs" })
+          end
+
+          unless valid_timeout_range?(request.payload_timeout_secs)
+            raise ::GRPC::InvalidArgument.new(ERROR_MESSAGES[:invalid_timeout_range],
+              { field: "payload_timeout_secs" })
+          end
+
+          # check for required payload fields
+          request.payloads.to_a.each_with_index do |payload, index|
+            if !payload.respond_to?(:id) || payload.id.empty?
+              raise ::GRPC::InvalidArgument.new(
+                ERROR_MESSAGES[:invalid_payload_fields],
+                { field: "payloads[#{index}].id" }
+              )
+            end
+
+            unless payload.respond_to?(:data) # rubocop:disable Style/Next
+              raise ::GRPC::InvalidArgument.new(
+                ERROR_MESSAGES[:invalid_payload_fields],
+                { field: "payloads[#{index}].data" }
+              )
+            end
+          end
+        end
+
+        # checks if the given timeout value is within range
+        def valid_timeout_range?(timeout_value)
+          timeout_value >= 0 && timeout_value <= MAX_ALLOWED_TIMEOUT_SECONDS
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative 'grpc/scanner_service'
+require_relative 'grpc/client/stream_request_enumerator'
+require_relative 'grpc/client/grpc_client'
+
+module Gitlab
+  module SecretDetection
+    module GRPC
+    end
+  end
+end

data/lib/gitlab/secret_detection/utils/certificate.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require_relative 'memoize'
+
+module Gitlab
+  module SecretDetection
+    module Utils
+      module X509
+        # Pulled from Gitlab.com source
+        # Link: https://gitlab.com/gitlab-org/gitlab/-/blob/4713a798f997389f04e442db3d1d8349a39d5d46/lib/gitlab/x509/certificate.rb
+        class Certificate
+          CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/
+
+          attr_reader :key, :cert, :ca_certs
+
+          def self.default_cert_dir
+            strong_memoize(:default_cert_dir) do
+              ENV.fetch('SSL_CERT_DIR', OpenSSL::X509::DEFAULT_CERT_DIR)
+            end
+          end
+
+          def self.default_cert_file
+            strong_memoize(:default_cert_file) do
+              ENV.fetch('SSL_CERT_FILE', OpenSSL::X509::DEFAULT_CERT_FILE)
+            end
+          end
+
+          def self.from_strings(key_string, cert_string, ca_certs_string = nil)
+            key = OpenSSL::PKey::RSA.new(key_string)
+            cert = OpenSSL::X509::Certificate.new(cert_string)
+            ca_certs = load_ca_certs_bundle(ca_certs_string)
+
+            new(key, cert, ca_certs)
+          end
+
+          def self.from_files(key_path, cert_path, ca_certs_path = nil)
+            ca_certs_string = File.read(ca_certs_path) if ca_certs_path
+
+            from_strings(File.read(key_path), File.read(cert_path), ca_certs_string)
+          end
+
+          # Returns all top-level, readable files in the default CA cert directory
+          def self.ca_certs_paths
+            cert_paths = Dir["#{default_cert_dir}/*"].select do |path|
+              !File.directory?(path) && File.readable?(path)
+            end
+            cert_paths << default_cert_file if File.exist? default_cert_file
+            cert_paths
+          end
+
+          # Returns a concatenated array of Strings, each being a PEM-coded CA certificate.
+          def self.ca_certs_bundle
+            strong_memoize(:ca_certs_bundle) do
+              ca_certs_paths.flat_map do |cert_file|
+                load_ca_certs_bundle(File.read(cert_file))
+              end.uniq.join("\n")
+            end
+          end
+
+          def self.reset_ca_certs_bundle
+            clear_memoization(:ca_certs_bundle)
+          end
+
+          def self.reset_default_cert_paths
+            clear_memoization(:default_cert_dir)
+            clear_memoization(:default_cert_file)
+          end
+
+          # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found
+          #
+          # Ruby OpenSSL::X509::Certificate.new will only load the first
+          # certificate if a bundle is presented, this allows to parse multiple certs
+          # in the same file
+          def self.load_ca_certs_bundle(ca_certs_string)
+            return [] unless ca_certs_string
+
+            ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string|
+              OpenSSL::X509::Certificate.new(ca_cert_string)
+            end
+          end
+
+          def initialize(key, cert, ca_certs = nil)
+            @key = key
+            @cert = cert
+            @ca_certs = ca_certs
+          end
+
+          def key_string
+            key.to_s
+          end
+
+          def cert_string
+            cert.to_pem
+          end
+
+          def ca_certs_string
+            ca_certs&.map(&:to_pem)&.join('\n') unless ca_certs.blank?
+          end
+
+          class << self
+            include ::Gitlab::SecretDetection::Utils::StrongMemoize
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/utils/memoize.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SecretDetection
+    module Utils
+      # Pulled from GitLab.com source
+      # Link: https://gitlab.com/gitlab-org/gitlab/-/blob/4713a798f997389f04e442db3d1d8349a39d5d46/gems/gitlab-utils/lib/gitlab/utils/strong_memoize.rb
+      module StrongMemoize
+        # Instead of writing patterns like this:
+        #
+        #     def trigger_from_token
+        #       return @trigger if defined?(@trigger)
+        #
+        #       @trigger = Ci::Trigger.find_by_token(params[:token].to_s)
+        #     end
+        #
+        # We could write it like:
+        #
+        #     include Gitlab::SecretDetection::Utils::StrongMemoize
+        #
+        #     def trigger_from_token
+        #       Ci::Trigger.find_by_token(params[:token].to_s)
+        #     end
+        #     strong_memoize_attr :trigger_from_token
+        #
+        #     def enabled?
+        #       Feature.enabled?(:some_feature)
+        #     end
+        #     strong_memoize_attr :enabled?
+        #
+        def strong_memoize(name)
+          key = ivar(name)
+
+          if instance_variable_defined?(key)
+            instance_variable_get(key)
+          else
+            instance_variable_set(key, yield)
+          end
+        end
+
+        # Works the same way as "strong_memoize" but takes
+        # a second argument - expire_in. This allows invalidate
+        # the data after specified number of seconds
+        def strong_memoize_with_expiration(name, expire_in)
+          key = ivar(name)
+          expiration_key = "#{key}_expired_at"
+
+          if instance_variable_defined?(expiration_key)
+            expire_at = instance_variable_get(expiration_key)
+            clear_memoization(name) if expire_at.past?
+          end
+
+          if instance_variable_defined?(key)
+            instance_variable_get(key)
+          else
+            value = instance_variable_set(key, yield)
+            instance_variable_set(expiration_key, Time.current + expire_in)
+            value
+          end
+        end
+
+        def strong_memoize_with(name, *args)
+          container = strong_memoize(name) { {} }
+
+          if container.key?(args)
+            container[args]
+          else
+            container[args] = yield
+          end
+        end
+
+        def strong_memoized?(name)
+          key = ivar(StrongMemoize.normalize_key(name))
+          instance_variable_defined?(key)
+        end
+
+        def clear_memoization(name)
+          key = ivar(StrongMemoize.normalize_key(name))
+          remove_instance_variable(key) if instance_variable_defined?(key)
+        end
+
+        module StrongMemoizeClassMethods
+          def strong_memoize_attr(method_name)
+            member_name = StrongMemoize.normalize_key(method_name)
+
+            StrongMemoize.send(:do_strong_memoize, self, method_name, member_name) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        def self.included(base)
+          base.singleton_class.prepend(StrongMemoizeClassMethods)
+        end
+
+        private
+
+        # Convert `"name"`/`:name` into `:@name`
+        #
+        # Depending on a type ensure that there's a single memory allocation
+        def ivar(name)
+          case name
+          when Symbol
+            name.to_s.prepend("@").to_sym
+          when String
+            :"@#{name}"
+          else
+            raise ArgumentError, "Invalid type of '#{name}'"
+          end
+        end
+
+        class << self
+          def normalize_key(key)
+            return key unless key.end_with?('!', '?')
+
+            # Replace invalid chars like `!` and `?` with allowed Unicode codeparts.
+            key.to_s.tr('!?', "\uFF01\uFF1F")
+          end
+
+          private
+
+          def do_strong_memoize(klass, method_name, member_name)
+            method = klass.instance_method(method_name)
+
+            unless method.arity.zero?
+              raise <<~ERROR
+                Using `strong_memoize_attr` on methods with parameters is not supported.
+
+                Use `strong_memoize_with` instead.
+                See https://docs.gitlab.com/ee/development/utilities.html#strongmemoize
+              ERROR
+            end
+
+            # Methods defined within a class method are already public by default, so we don't need to
+            # explicitly make them public.
+            scope = %i[private protected].find do |scope|
+              klass.send(:"#{scope}_instance_methods") # rubocop:disable GitlabSecurity/PublicSend
+                   .include? method_name
+            end
+
+            klass.define_method(method_name) do |&block|
+              strong_memoize(member_name) do
+                method.bind_call(self, &block)
+              end
+            end
+
+            klass.send(scope, method_name) if scope # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/utils.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'utils/certificate'
+require_relative 'utils/memoize'
+
+module Gitlab
+  module SecretDetection
+    module Utils
+    end
+  end
+end

data/lib/gitlab/secret_detection/version.rb

@@ -2,6 +2,30 @@
 
 module Gitlab
   module SecretDetection
-    VERSION = "0.1.0"
+    class Gem
+      DEFAULT_VERSION = "0.0.1"
+
+      SEMVER_REGEX = /^\d+\.\d+\.\d+(?:-[a-zA-Z0-9\-\.]+)?(?:\+[a-zA-Z0-9\-\.]+)?$/
+
+      def self.get_release_version
+        release_version = ENV.fetch("SD_GEM_RELEASE_VERSION", "")
+
+        if release_version.empty?
+          raise LoadError("Missing SD_GEM_RELEASE_VERSION environment variable.") unless local_env?
+
+          "#{DEFAULT_VERSION}-debug"
+        elsif release_version.match?(SEMVER_REGEX)
+          release_version
+        else
+          "#{DEFAULT_VERSION}-#{release_version}"
+        end
+      end
+
+      # SD_ENV env var is used to determine which environment the
+      # server is running. This var is defined in `.runway/env-<env>.yml` files.
+      def self.local_env?
+        ENV.fetch('SD_ENV', 'localhost') == 'localhost'
+      end
+    end
   end
 end

data/lib/gitlab/secret_detection.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-require_relative "secret_detection/version"
+require_relative 'secret_detection/utils'
+require_relative 'secret_detection/core'
+require_relative 'secret_detection/grpc'
+require_relative 'secret_detection/version'
 
 module Gitlab
   module SecretDetection
-    class Error < StandardError; end
-    # Your code goes here...
   end
 end

data/lib/gitlab.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require_relative 'gitlab/secret_detection'
+
+module Gitlab
+end

data/proto/secret_detection.proto

@@ -0,0 +1,75 @@
+syntax = "proto3";
+
+package gitlab.secret_detection;
+
+/* We keep generated files within grpc namespace i.e Gitlab::SecretDetection::GRPC
+ * so that these files are exported too in the Ruby Gem along with Core and GRPC logic.
+ */
+option ruby_package = "Gitlab::SecretDetection::GRPC";
+
+/* Request arg for triggering Scan/ScanStream method */
+message ScanRequest {
+  message Payload {
+    string id = 1;
+    string data = 2;
+  }
+
+  // Either provide rule type or a particular value to allow during the scan
+  message Exclusion {
+    ExclusionType exclusion_type = 1;
+    string value = 2;
+  }
+
+  enum ExclusionType {
+    EXCLUSION_TYPE_UNSPECIFIED = 0;
+    EXCLUSION_TYPE_RULE = 1; // Rule ID to exclude
+    EXCLUSION_TYPE_RAW_VALUE = 2; // Raw value to exclude
+  }
+
+  repeated Payload payloads = 1; // Array of payloads to scan
+  // Scan timeout on the entire request. Value is represented in seconds, accepts float values to represent
+  // smaller unit values. Default is 180 seconds.
+  optional float timeout_secs = 2;
+  // Scan timeout on each payload . Value is represented in seconds, accepts float values to represent smaller
+  // unit values. Default is 30 seconds.
+  optional float payload_timeout_secs = 3;
+  repeated Exclusion exclusions = 4; // Optional. Array of rule-types/raw-values to exclude from being considered during scan.
+  repeated string tags = 5; // Optional. Array of rule tags to consider for scan. Ex: ["gitlab_blocking"]
+}
+
+/* Response from Scan/ScanStream method */
+message ScanResponse {
+  // Represents a secret finding identified within a payload
+  message Finding {
+    string payload_id = 1;
+    int32 status = 2;
+    optional string type = 3;
+    optional string description = 4;
+    optional int32 line_number = 5;
+  }
+
+  // Return status code in sync with ::SecretDetection::Status
+  enum Status {
+    STATUS_UNSPECIFIED = 0;
+    STATUS_FOUND = 1;              // one or more findings
+    STATUS_FOUND_WITH_ERRORS = 2;  // one or more findings along with some errors
+    STATUS_SCAN_TIMEOUT = 3;       // whole scan timeout
+    STATUS_PAYLOAD_TIMEOUT = 4;       // single payload timeout
+    STATUS_SCAN_ERROR = 5;         // internal scan failure
+    STATUS_INPUT_ERROR = 6;        // invalid input failure
+    STATUS_NOT_FOUND = 7;          // zero findings
+    STATUS_AUTH_ERROR = 8;         // authentication failure
+  }
+
+  repeated Finding results = 1;
+  int32 status = 2;
+}
+
+/* Scanner service that scans given payloads and returns findings */
+service Scanner {
+  // Runs secret detection scan for the given request
+  rpc Scan(ScanRequest) returns (ScanResponse) {}
+
+  // Runs bi-directional streaming of scans for the given stream of requests with a stream of responses
+  rpc ScanStream(stream ScanRequest) returns (stream ScanResponse) {}
+}