gitlab-secret_detection 0.1.0 → 0.3.0

data/lib/gitlab/secret_detection/core/response.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module GitLab
+  module SecretDetection
+    module Core
+      # Response is the data object returned by the scan operation with the following structure
+      #
+      # +status+:: One of values from GitLab::SecretDetection::Core::Status indicating the scan operation's status
+      # +results+:: Array of GitLab::SecretDetection::Core::Finding values. Default value is nil.
+      class Response
+        attr_reader :status, :results
+
+        def initialize(status, results = [])
+          @status = status
+          @results = results
+        end
+
+        def ==(other)
+          self.class == other.class && other.state == state
+        end
+
+        def to_h
+          {
+            status:,
+            results: results&.map(&:to_h)
+          }
+        end
+
+        protected
+
+        def state
+          [status, results]
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/core/ruleset.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'toml-rb'
+require 'logger'
+
+module GitLab
+  module SecretDetection
+    module Core
+      class Ruleset
+        # file path where the secrets ruleset file is located
+        RULESET_FILE_PATH = File.expand_path('gitleaks.toml', __dir__)
+
+        def initialize(path: RULESET_FILE_PATH)
+          @path = path
+        end
+
+        def rules(force_fetch: false)
+          return @rule_data unless @rule_data.nil? || force_fetch
+
+          @rule_data ||= parse_ruleset
+        end
+
+        private
+
+        attr_reader :path
+
+        # parses given ruleset file and returns the parsed rules
+        def parse_ruleset
+          # rule_file_content = File.read(path)
+          rules_data = TomlRB.load_file(path, symbolize_keys: true).freeze
+          rules_data[:rules].freeze
+        rescue StandardError => e
+          logger.error "Failed to parse secret detection ruleset from '#{path}' path: #{e}"
+          raise Core::Scanner::RulesetParseError
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+require 're2'
+require 'logger'
+require 'timeout'
+require 'English'
+
+module GitLab
+  module SecretDetection
+    module Core
+      # Scan is responsible for running Secret Detection scan operation
+      class Scanner
+        # RulesetParseError is thrown when the code fails to parse the
+        # ruleset file from the given path
+        RulesetParseError = Class.new(StandardError)
+
+        # RulesetCompilationError is thrown when the code fails to compile
+        # the predefined rulesets
+        RulesetCompilationError = Class.new(StandardError)
+
+        # default time limit(in seconds) for running the scan operation per invocation
+        DEFAULT_SCAN_TIMEOUT_SECS = 180 # 3 minutes
+        # default time limit(in seconds) for running the scan operation on a single payload
+        DEFAULT_PAYLOAD_TIMEOUT_SECS = 30 # 30 seconds
+        # Tags used for creating default pattern matcher
+        DEFAULT_PATTERN_MATCHER_TAGS = ['gitlab_blocking'].freeze
+
+        # Initializes the instance with logger along with following operations:
+        # 1. Extract keywords from the parsed ruleset to use it for matching keywords before regex operation.
+        # 2. Build and Compile rule regex patterns obtained from the ruleset with +DEFAULT_PATTERN_MATCHER_TAGS+
+        # tags. Raises +RulesetCompilationError+ in case the regex pattern compilation fails.
+        def initialize(rules:, logger: Logger.new($stdout))
+          @logger = logger
+          @rules = rules
+          @keywords = create_keywords(rules)
+          @default_keyword_matcher = build_keyword_matcher(
+            tags: DEFAULT_PATTERN_MATCHER_TAGS,
+            include_missing_tags: false
+          )
+          @default_pattern_matcher = build_pattern_matcher(
+            tags: DEFAULT_PATTERN_MATCHER_TAGS,
+            include_missing_tags: false
+          ) # includes only gitlab_blocking rules
+        end
+
+        # Runs Secret Detection scan on the list of given payloads. Both the total scan duration and
+        # the duration for each payload is time bound via +timeout+ and +payload_timeout+ respectively.
+        #
+        # +payloads+:: Array of payloads where each payload should have `id` and `data` properties.
+        # +timeout+:: No of seconds(accepts floating point for smaller time values) to limit the total scan duration
+        # +payload_timeout+:: No of seconds(accepts floating point for smaller time values) to limit
+        #                  the scan duration on each payload
+        # +rule_exclusions+:: Array of rules to exclude from the ruleset used for the scan. Each rule is represented
+        #           by its ID. For example: `gitlab_personal_access_token` for representing GitLab Personal Access
+        #           Token. By default, no rule is excluded from the ruleset.
+        # +allow_values+:: Array of raw values to exclude from the scan.
+        # +tags+:: Array of tag values to filter from the default ruleset when determining the rules used for the scan.
+        #           For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
+        #           [`gitlab_blocking`] (+DEFAULT_PATTERN_MATCHER_TAGS+).
+        #
+        # Returns an instance of GitLab::SecretDetection::Core::Response by following below structure:
+        # {
+        #     status: One of the Core::Status values
+        #     results: [SecretDetection::Finding]
+        # }
+        #
+        def secrets_scan(
+          payloads,
+          timeout: DEFAULT_SCAN_TIMEOUT_SECS,
+          payload_timeout: DEFAULT_PAYLOAD_TIMEOUT_SECS,
+          rule_exclusions: [],
+          allow_values: [],
+          tags: DEFAULT_PATTERN_MATCHER_TAGS
+        )
+
+          return Core::Response.new(Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
+
+          # assign defaults since grpc passing zero timeout value to `Timeout.timeout(..)` makes it effectively useless.
+          timeout = DEFAULT_SCAN_TIMEOUT_SECS unless timeout.positive?
+          payload_timeout = DEFAULT_PAYLOAD_TIMEOUT_SECS unless payload_timeout.positive?
+          tags = DEFAULT_PATTERN_MATCHER_TAGS if tags.empty?
+
+          Timeout.timeout(timeout) do
+            keyword_matcher = build_keyword_matcher(tags:)
+
+            matched_payloads = filter_by_keywords(keyword_matcher, payloads)
+
+            next Core::Response.new(Core::Status::NOT_FOUND) if matched_payloads.empty?
+
+            secrets = run_scan(
+              payloads: matched_payloads, payload_timeout:,
+              pattern_matcher: build_pattern_matcher(tags:),
+              allow_values:, rule_exclusions:
+            )
+
+            scan_status = overall_scan_status(secrets)
+
+            Core::Response.new(scan_status, secrets)
+          end
+        rescue Timeout::Error => e
+          logger.error "Secret detection operation timed out: #{e}"
+
+          Core::Response.new(Core::Status::SCAN_TIMEOUT)
+        end
+
+        private
+
+        attr_reader :logger, :rules, :keywords, :default_pattern_matcher, :default_keyword_matcher
+
+        # Builds RE2::Set pattern matcher for the given combination of rules
+        # and tags. It also allows a choice(via `include_missing_tags`) to consider rules
+        # for pattern matching that do not have `tags` property defined. If the given tags
+        # are same as +DEFAULT_PATTERN_MATCHER_TAGS+ then returns the eagerly loaded default
+        # pattern matcher created during initialization.
+        def build_pattern_matcher(tags:, include_missing_tags: false)
+          return default_pattern_matcher if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_pattern_matcher.nil?
+
+          matcher = RE2::Set.new
+
+          rules.each do |rule|
+            rule_tags = rule[:tags]
+
+            include_rule = if tags.empty?
+                             true
+                           elsif rule_tags
+                             tags.intersect?(rule_tags)
+                           else
+                             include_missing_tags
+                           end
+
+            matcher.add(rule[:regex]) if include_rule
+          end
+
+          unless matcher.compile
+            logger.error "Failed to compile secret detection rulesets in RE::Set"
+
+            raise RulesetCompilationError
+          end
+
+          matcher
+        end
+
+        # Creates and returns the unique set of rule matching keywords
+        def create_keywords(rules)
+          secrets_keywords = Set.new
+
+          rules.each do |rule|
+            secrets_keywords.merge rule[:keywords] unless rule[:keywords].nil?
+          end
+
+          secrets_keywords.freeze
+        end
+
+        def build_keyword_matcher(tags:, include_missing_tags: false)
+          return default_keyword_matcher if tags.eql?(DEFAULT_PATTERN_MATCHER_TAGS) && !default_keyword_matcher.nil?
+
+          include_keywords = Set.new
+
+          rules.each do |rule|
+            rule_tags = rule.fetch(:tags, [])
+
+            next if rule_tags.empty? && !include_missing_tags
+            next unless rule_tags.intersect?(tags)
+
+            include_keywords.merge(rule[:keywords]) unless rule[:keywords].nil?
+          end
+
+          return nil if include_keywords.empty?
+
+          keywords_regex = include_keywords.join('|')
+
+          RE2("\\b(#{keywords_regex})")
+        end
+
+        def filter_by_keywords(keyword_matcher, payloads)
+          return [] if keyword_matcher.nil?
+
+          matched_payloads = []
+          payloads.each do |payload|
+            next unless keyword_matcher.partial_match?(payload.data)
+
+            matched_payloads << payload
+          end
+
+          matched_payloads.freeze
+        end
+
+        # Runs the secret detection scan on the given list of payloads. It accepts
+        # literal values to exclude from the input before the scan, also SD rules to exclude during
+        # the scan when performed on the payloads.
+        def run_scan(
+          payloads:, payload_timeout:, pattern_matcher:, allow_values: [], rule_exclusions: [])
+          payloads.flat_map do |payload|
+            Timeout.timeout(payload_timeout) do
+              find_secrets_in_payload(
+                payload:,
+                pattern_matcher:,
+                allow_values:, rule_exclusions:
+              )
+            end
+          rescue Timeout::Error => e
+            logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
+            Core::Finding.new(payload.id,
+              Core::Status::PAYLOAD_TIMEOUT)
+          end
+        end
+
+        # Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
+        # literal values to exclude from the input before the scan, also SD rules to exclude during
+        # the scan.
+        def find_secrets_in_payload(payload:, pattern_matcher:, allow_values: [], rule_exclusions: [])
+          findings = []
+
+          payload.data
+                 .each_line($INPUT_RECORD_SEPARATOR, chomp: true)
+                 .each_with_index do |line, index|
+            unless allow_values.empty?
+              allow_values.each do |value|
+                line.gsub!(value, '') # replace input that doesn't contain allowed value in it
+              end
+            end
+
+            next if line.empty?
+
+            line_no = index + 1
+
+            matches = pattern_matcher.match(line, exception: false) # returns indices of matched patterns
+
+            matches.each do |match_idx|
+              rule = rules[match_idx]
+
+              next if rule_exclusions.include?(rule[:id])
+
+              findings << Core::Finding.new(payload.id, Core::Status::FOUND, line_no, rule[:id], rule[:description])
+            end
+          end
+
+          findings.freeze
+        rescue StandardError => e
+          logger.error "Secret Detection scan failed on the payload(id:#{payload.id}): #{e}"
+
+          Core::Finding.new(payload.id, Core::Status::SCAN_ERROR)
+        end
+
+        # Validates the given payloads by verifying the type and
+        # presence of `id` and `data` fields necessary for the scan
+        def validate_scan_input(payloads)
+          return false if payloads.nil? || !payloads.instance_of?(Array)
+
+          payloads.all? do |payload|
+            payload.respond_to?(:id) && payload.respond_to?(:data)
+          end
+        end
+
+        # Returns the status of the overall scan request
+        # based on the detected secret findings found in the input payloads
+        def overall_scan_status(found_secrets)
+          return Core::Status::NOT_FOUND if found_secrets.empty?
+
+          timed_out_payloads = found_secrets.count { |el| el.status == Core::Status::PAYLOAD_TIMEOUT }
+
+          case timed_out_payloads
+          when 0
+            Core::Status::FOUND
+          when found_secrets.length
+            Core::Status::SCAN_TIMEOUT
+          else
+            Core::Status::FOUND_WITH_ERRORS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/core/status.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module GitLab
+  module SecretDetection
+    module Core
+      # All the possible statuses emitted by the scan operation
+      class Status
+        FOUND = 1 # When scan operation completes with one or more findings
+        FOUND_WITH_ERRORS = 2 # When scan operation completes with one or more findings along with some errors
+        SCAN_TIMEOUT = 3 # When the scan operation runs beyond given time out
+        PAYLOAD_TIMEOUT = 4 # When the scan operation on a payload runs beyond given time out
+        SCAN_ERROR = 5 # When the scan operation fails due to regex error
+        INPUT_ERROR = 6 # When the scan operation fails due to invalid input
+        NOT_FOUND = 7 # When scan operation completes with zero findings
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/core.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative 'core/finding'
+require_relative 'core/response'
+require_relative 'core/status'
+require_relative 'core/scanner'
+require_relative 'core/ruleset'
+
+module GitLab
+  module SecretDetection
+    module Core
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc/client/grpc_client.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative '../generated/secret_detection_pb'
+require_relative '../generated/secret_detection_services_pb'
+
+module GitLab
+  module SecretDetection
+    module GRPC
+      class Client
+        # TODO add implementation
+        def scan
+          raise NotImplementedError
+        end
+
+        def scan_stream
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: secret_detection.proto
+
+require 'google/protobuf'
+
+
+descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xdb\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\tallowlist\x18\x04 \x03(\x0b\x32/.gitlab.secret_detection.ScanRequest.AllowEntry\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a_\n\nAllowEntry\x12\x42\n\nallow_type\x18\x01 \x01(\x0e\x32..gitlab.secret_detection.ScanRequest.AllowType\x12\r\n\x05value\x18\x02 \x01(\t\"L\n\tAllowType\x12\x15\n\x11\x41LLOW_UNSPECIFIED\x10\x00\x12\x13\n\x0f\x41LLOW_RULE_TYPE\x10\x01\x12\x13\n\x0f\x41LLOW_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe3\x04\n\x0cScanResponse\x12\x12\n\x05\x65rror\x18\x01 \x01(\tH\x00\x88\x01\x01\x12>\n\x07results\x18\x02 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12<\n\x06status\x18\x03 \x01(\x0e\x32,.gitlab.secret_detection.ScanResponse.Status\x1a\xe9\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12<\n\x06status\x18\x02 \x01(\x0e\x32,.gitlab.secret_detection.ScanResponse.Status\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05\x65rror\x18\x06 \x01(\tH\x03\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_numberB\x08\n\x06_error\"\xca\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x42\x08\n\x06_error2\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitLab::SecretDetection::GRPCb\x06proto3"
+
+pool = Google::Protobuf::DescriptorPool.generated_pool
+pool.add_serialized_file(descriptor_data)
+
+module GitLab
+  module SecretDetection
+    module GRPC
+      ScanRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest").msgclass
+      ScanRequest::Payload = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest.Payload").msgclass
+      ScanRequest::AllowEntry = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest.AllowEntry").msgclass
+      ScanRequest::AllowType = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest.AllowType").enummodule
+      ScanResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanResponse").msgclass
+      ScanResponse::Finding = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanResponse.Finding").msgclass
+      ScanResponse::Status = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanResponse.Status").enummodule
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc/generated/secret_detection_services_pb.rb

@@ -0,0 +1,30 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: secret_detection.proto for package 'GitLab.SecretDetection.GRPC'
+
+require 'grpc'
+require 'secret_detection_pb'
+
+module GitLab
+  module SecretDetection
+    module GRPC
+      module Scanner
+        # Scanner service that scans given payloads and returns findings 
+        class Service
+
+          include ::GRPC::GenericService
+
+          self.marshal_class_method = :encode
+          self.unmarshal_class_method = :decode
+          self.service_name = 'gitlab.secret_detection.Scanner'
+
+          # Runs secret detection scan for the given request
+          rpc :Scan, ::GitLab::SecretDetection::GRPC::ScanRequest, ::GitLab::SecretDetection::GRPC::ScanResponse
+          # Runs bi-directional streaming of scans for the given stream of requests with a stream of responses
+          rpc :ScanStream, stream(::GitLab::SecretDetection::GRPC::ScanRequest), stream(::GitLab::SecretDetection::GRPC::ScanResponse)
+        end
+
+        Stub = Service.rpc_stub_class
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc/scanner_service.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('generated', __dir__))
+
+require 'grpc'
+
+require_relative 'generated/secret_detection_pb'
+require_relative 'generated/secret_detection_services_pb'
+
+require_relative '../core'
+require_relative '../../../../config/log'
+
+# StreamEnumerator is used for Bi-directional streaming
+# of requests by returning stream of responses.
+class StreamEnumerator
+  def initialize(requests, action)
+    @requests = requests
+    @request_action = action
+  end
+
+  def each_item
+    return enum_for(:each_item) unless block_given?
+
+    @requests.each do |req|
+      yield @request_action.call(req)
+    end
+  end
+end
+
+module GitLab
+  module SecretDetection
+    module GRPC
+      class ScannerService < Scanner::Service
+        include SDLogger
+
+        # Maximum timeout value that can be given as the input. This guards
+        # against the misuse of timeouts.
+        MAX_ALLOWED_TIMEOUT_SECONDS = 600
+
+        ERROR_MESSAGES = {
+          invalid_payload_fields: "Payload should not contain empty `id` and `data` fields",
+          allowlist_empty_value: "Allowlist entry value cannot be empty",
+          allowlist_invalid_type: "Invalid Allowlist entry type",
+          invalid_timeout_range: "Timeout value should be > 0 and <= #{MAX_ALLOWED_TIMEOUT_SECONDS} seconds"
+        }.freeze
+
+        # Implementation for /Scan RPC method
+        def scan(request, _call)
+          scan_request_action(request)
+        end
+
+        # Implementation for /ScanStream RPC method
+        def scan_stream(requests, _call)
+          request_action = ->(r) { scan_request_action(r) }
+          StreamEnumerator.new(requests, request_action).each_item
+        end
+
+        private
+
+        def scan_request_action(request)
+          validate_request(request)
+
+          payloads = request.payloads.to_a
+
+          rule_exclusions = []
+          allow_values = []
+          request.allowlist&.each do |entry|
+            case entry.allow_type
+            when :ALLOW_RULE_TYPE
+              rule_exclusions << entry.value
+            when :ALLOW_RAW_VALUE
+              allow_values << entry.value
+            end
+          end
+
+          result = scanner.secrets_scan(
+            payloads,
+            rule_exclusions:,
+            allow_values:,
+            tags: request.tags.to_a,
+            timeout: request.timeout_secs,
+            payload_timeout: request.payload_timeout_secs
+          )
+
+          findings = result.results&.map do |finding|
+            GitLab::SecretDetection::GRPC::ScanResponse::Finding.new(**finding.to_h)
+          end
+
+          GitLab::SecretDetection::GRPC::ScanResponse.new(
+            results: findings,
+            status: result.status
+          )
+        end
+
+        def scanner
+          @scanner ||= GitLab::SecretDetection::Core::Scanner.new(rules:, logger:)
+        end
+
+        def rules
+          GitLab::SecretDetection::Core::Ruleset.new.rules
+        end
+
+        # validates grpc request body
+        def validate_request(request)
+          # check for non-blank values and allowed types
+          request.allowlist&.each do |entry|
+            if entry.value.empty?
+              raise ::GRPC::InvalidArgument.new(ERROR_MESSAGES[:allowlist_empty_value],
+                { field: "allowlist.value" })
+            end
+          end
+
+          unless valid_timeout_range?(request.timeout_secs)
+            raise ::GRPC::InvalidArgument.new(ERROR_MESSAGES[:invalid_timeout_range],
+              { field: "timeout_secs" })
+          end
+
+          unless valid_timeout_range?(request.payload_timeout_secs)
+            raise ::GRPC::InvalidArgument.new(ERROR_MESSAGES[:invalid_timeout_range],
+              { field: "payload_timeout_secs" })
+          end
+
+          # check for required payload fields
+          request.payloads.to_a.each_with_index do |payload, index|
+            if !payload.respond_to?(:id) || payload.id.empty?
+              raise ::GRPC::InvalidArgument.new(
+                ERROR_MESSAGES[:invalid_payload_fields],
+                { field: "payloads[#{index}].id" }
+              )
+            end
+
+            unless payload.respond_to?(:data) # rubocop:disable Style/Next
+              raise ::GRPC::InvalidArgument.new(
+                ERROR_MESSAGES[:invalid_payload_fields],
+                { field: "payloads[#{index}].data" }
+              )
+            end
+          end
+        end
+
+        # checks if the given timeout value is within range
+        def valid_timeout_range?(timeout_value)
+          timeout_value >= 0 && timeout_value <= MAX_ALLOWED_TIMEOUT_SECONDS
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/secret_detection/grpc.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'grpc/scanner_service'
+require_relative 'grpc/client/grpc_client'
+
+module GitLab
+  module SecretDetection
+    module GRPC
+    end
+  end
+end

data/lib/gitlab/secret_detection/version.rb

@@ -1,7 +1,31 @@
 # frozen_string_literal: true
 
-module Gitlab
+module GitLab
   module SecretDetection
-    VERSION = "0.1.0"
+    class Gem
+      DEFAULT_VERSION = "0.0.1"
+
+      SEMVER_REGEX = /^\d+\.\d+\.\d+(?:-[a-zA-Z0-9\-\.]+)?(?:\+[a-zA-Z0-9\-\.]+)?$/
+
+      def self.get_release_version
+        release_version = ENV.fetch("SD_GEM_RELEASE_VERSION", "")
+
+        if release_version.empty?
+          raise LoadError("Missing SD_GEM_RELEASE_VERSION environment variable.") unless local_env?
+
+          "#{DEFAULT_VERSION}-debug"
+        elsif release_version.match?(SEMVER_REGEX)
+          release_version
+        else
+          "#{DEFAULT_VERSION}-#{release_version}"
+        end
+      end
+
+      # SD_ENV env var is used to determine which environment the
+      # server is running. This var is defined in `.runway/env-<env>.yml` files.
+      def self.local_env?
+        ENV.fetch('SD_ENV', 'localhost') == 'localhost'
+      end
+    end
   end
 end

data/lib/gitlab/secret_detection.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require_relative "secret_detection/version"
+require_relative 'secret_detection/core'
+require_relative 'secret_detection/grpc'
+require_relative 'secret_detection/version'
 
-module Gitlab
+module GitLab
   module SecretDetection
-    class Error < StandardError; end
-    # Your code goes here...
   end
 end

data/lib/gitlab.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require_relative 'gitlab/secret_detection'
+
+module GitLab
+end