gitlab-labkit 1.11.1 → 1.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8c136c0ed8617c5023574687e1b1284621726be410583eb7ee17586fe56c20e
-  data.tar.gz: f9f496d1b6ea56d986b577052f8f50c7a91f1114307af3bd9856734c482afac1
+  metadata.gz: 8db8c324d6fe60ad1b16ab51dd87207d4a5d08e1d83ad99a3ae900392bd0739c
+  data.tar.gz: 4487ae2ff2bacac400f753c92012cd7b118525f0bed143794ca4fd12c14cca53
 SHA512:
-  metadata.gz: caed2e7c544c73856cd6a47a0f63655038f636674cf5070a0cb4b5330c2beda0c07ff0a7352b5d367f293ae818ecf08aa414316485f5cc6a262595c5240f334c
-  data.tar.gz: 7c3b49fc037060c21ea36e49e4878f99640e241b5a7358665fddf6bdde354ac26ddda80c87b11f5de183c4a259a9b2d6c8629818a852078393487c1c78ef6ba1
+  metadata.gz: 7292067a1a5d8e231776b65b381ba161a2123c6363df5ebd4269ae65b596871ebddb5bd610e5ab33da0939cdeab4639a51bcb2636046cfd8cb941ac6f6baa9bb
+  data.tar.gz: 771b7fded81f969ee414ef737b74fa9c22d620762f0e56c669e89de9c4ebf6c022fdbf91513fe630a580acfc8408d20a0eb55e3f05e0c97c412ab6ab818a75bf

data/.copier-answers.yml

@@ -3,7 +3,7 @@
 # See the project for instructions on how to update the project
 #
 # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY
-_commit: v1.45.0
+_commit: v1.46.0
 _src_path: https://gitlab.com/gitlab-com/gl-infra/common-template-copier.git
 ee_licensed: false
 golang: false

data/CODEOWNERS

@@ -1,4 +1,4 @@
 # CODEOWNERS is used to lookup assignees for
 # Renovate Bot dependency change Merge Requests.
 # https://docs.renovatebot.com/configuration-options/#assigneesfromcodeowners
-* @reprazent @andrewn @mkaeppler @ayufan @hmerscher @d.barrett @splattael @e_forbes @M_Alvarez
+* @reprazent @andrewn @mkaeppler @ayufan @hmerscher @d.barrett @splattael @e_forbes @M_Alvarez @mwoolf

data/lib/gitlab-labkit.rb

@@ -18,6 +18,7 @@ module Labkit
   autoload :Metrics, "labkit/metrics"
   autoload :Middleware, "labkit/middleware"
   autoload :Fields, "labkit/fields"
+  autoload :RateLimit, "labkit/rate_limit"
 
   # Publishers to publish notifications whenever a HTTP reqeust is made.
   # A broadcasted notification's payload in topic "request.external_http" includes:

data/lib/labkit/fields.rb

@@ -102,6 +102,22 @@ module Labkit
     # Name of the service or component emitting the log event.
     SERVICE_NAME = "service_name"
 
+    # GitLab organization numeric ID.
+    GL_ORGANIZATION_ID = "gl_organization_id"
+
+    # GitLab project path.
+    GL_PROJECT_PATH = "gl_project_path"
+
+    # The endpoint_id of the current endpoint to be passed as caller_id when
+    # calling another service.
+    ENDPOINT_ID = "endpoint_id"
+
+    # The ID of the caller of the current service.
+    CALLER_ID = "caller_id"
+
+    # The endpoint_id of the original caller at the root of the call chain.
+    ROOT_CALLER_ID = "root_caller_id"
+
     # Get the constant name for a field value
     # @param field_value [String] The field value (e.g., "gl_user_id")
     # @return [String, nil] The constant name (e.g., "GL_USER_ID") or nil if not found
@@ -135,6 +151,8 @@ module Labkit
         Fields::LOG_MESSAGE => %w[msg custom_message extra.message fields.message graphql.message reason color_message exception.gitaly],
         Fields::CLASS_NAME => %w[class author_class exception.class extra.class extra.class_name],
         Fields::SERVICE_NAME => %w[service grpc.service_name auth_service type component subcomponent],
+        Fields::GL_ORGANIZATION_ID => %w[organization_id],
+        Fields::GL_PROJECT_PATH => %w[project_path full_path root_pipeline_project_path requested_project_path auth_project_path extra.gl_project_path],
       }.freeze
 
       class << self

data/lib/labkit/rate_limit/evaluator.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require "openssl"
+require "labkit/logging/json_logger"
+
+module Labkit
+  module RateLimit
+    # Evaluator contains the core rule-matching + Redis counter logic.
+    class Evaluator
+      KNOWN_CHARACTERISTICS = [:user, :ip, :namespace, :plan, :endpoint].freeze
+      KNOWN_ACTIONS = [:block, :log].freeze
+      REDIS_KEY_PREFIX = "labkit:rl"
+      CHAR_VALUE_MAX_LENGTH = 200
+      UNKNOWN_SENTINEL = "unknown_characteristic"
+      CALL_SITE_PATTERN = /\A[a-z0-9_]+\z/
+
+      def initialize(call_site:, identifier:, rules:, redis:, logger: nil)
+        @call_site = call_site
+        @identifier = identifier
+        @rules = rules
+        @redis = redis
+        @logger = logger || build_default_logger
+      end
+
+      def evaluate
+        validate_call_site!
+        evaluate_rules
+      rescue ArgumentError
+        raise
+      rescue StandardError => e
+        # Intentionally broad: fail-open applies to any unexpected error (network,
+        # timeout, OOM, etc.), not only Redis protocol errors.
+        log_evaluate_error(e)
+        :allow
+      end
+
+      private
+
+      def evaluate_rules
+        aggregate = :allow
+
+        @rules.each_with_index do |rule, index|
+          next unless rule_matches?(rule, @identifier)
+
+          result = evaluate_rule(rule, index)
+          aggregate = :block if result == :block
+        end
+
+        aggregate
+      end
+
+      def validate_call_site!
+        return if CALL_SITE_PATTERN.match?(@call_site)
+
+        raise ArgumentError, "Invalid call_site: #{@call_site.inspect}. Must match /\\A[a-z0-9_]+\\z/" if dev_or_test?
+
+        sanitized = @call_site.gsub(/[^a-z0-9_]/, "_")
+        @logger.warn(
+          message: "rate_limit_invalid_call_site",
+          call_site: @call_site,
+          sanitized: sanitized
+        )
+        @call_site = sanitized
+      end
+
+      def rule_matches?(rule, identifier)
+        rule.match.all? do |key, value|
+          identifier[key] == value
+        end
+      end
+
+      def evaluate_rule(rule, index)
+        exceeded = false
+
+        rule.characteristics.each do |char|
+          char_value = resolve_characteristic(char, @identifier)
+
+          if char_value.nil?
+            log_skipped_characteristic(rule, index, char)
+            next
+          end
+
+          redis_key = build_redis_key(@call_site, index, char, char_value)
+
+          count = incr_with_ttl(redis_key, rule.period)
+          rule_exceeded = count > rule.limit
+
+          exceeded = true if rule_exceeded
+
+          log_rule(rule, index, count, redis_key, rule_exceeded)
+        end
+
+        exceeded && rule.action == :block ? :block : :allow
+      end
+
+      def resolve_characteristic(char, identifier)
+        unless KNOWN_CHARACTERISTICS.include?(char)
+          raise ArgumentError, "Unknown characteristic: #{char.inspect}. Known: #{KNOWN_CHARACTERISTICS.inspect}" if dev_or_test?
+
+          @logger.warn(
+            message: "rate_limit_unknown_characteristic",
+            characteristic: char
+          )
+          return UNKNOWN_SENTINEL
+        end
+
+        value = identifier[char]
+
+        # Normalize endpoint: strip query string
+        value = Identifier.normalize_endpoint(value) if char == :endpoint
+
+        # Treat nil and empty-string the same: anonymous traffic must not collide on a shared bucket.
+        return nil if value.nil? || value.to_s.empty?
+
+        value.to_s
+      end
+
+      def build_redis_key(call_site, rule_index, char, char_value)
+        safe_value = encode_char_value(char_value.to_s)
+        "#{REDIS_KEY_PREFIX}:#{call_site}:#{rule_index}:#{char}:#{safe_value}"
+      end
+
+      def encode_char_value(value)
+        if value.length > CHAR_VALUE_MAX_LENGTH
+          OpenSSL::Digest::SHA256.hexdigest(value)
+        else
+          value
+        end
+      end
+
+      def incr_with_ttl(redis_key, period)
+        count = @redis.incr(redis_key)
+        # Set expiry only on first write to avoid resetting TTL on each call
+        @redis.expire(redis_key, period) if count == 1
+        count
+      end
+
+      def log_rule(rule, index, count, redis_key, exceeded)
+        @logger.info(
+          message: "rate_limit_check",
+          call_site: @call_site,
+          rule_index: index,
+          action: rule.action.to_s,
+          limit: rule.limit,
+          period: rule.period,
+          count: count,
+          matched: true,
+          exceeded: exceeded,
+          identifier: @identifier.to_h,
+          redis_key: redis_key
+        )
+      end
+
+      def log_skipped_characteristic(rule, index, char)
+        @logger.info(
+          message: "rate_limit_check",
+          call_site: @call_site,
+          rule_index: index,
+          action: rule.action.to_s,
+          limit: rule.limit,
+          period: rule.period,
+          characteristic: char,
+          matched: true,
+          skipped: true,
+          identifier: @identifier.to_h
+        )
+      end
+
+      def log_evaluate_error(error)
+        @logger.warn(
+          message: "rate_limit_redis_error",
+          call_site: @call_site,
+          error: error.class.to_s,
+          result: "allow"
+        )
+      end
+
+      def dev_or_test?
+        # Memoized: ENV access is not free under concurrency.
+        return @dev_or_test unless @dev_or_test.nil?
+
+        env = ENV.fetch("LABKIT_ENV", nil)
+        @dev_or_test = env == "test" || env == "development"
+      end
+
+      def build_default_logger
+        Labkit::Logging::JsonLogger.new($stdout)
+      end
+    end
+  end
+end

data/lib/labkit/rate_limit/identifier.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Labkit
+  module RateLimit
+    # Identifier is a value object wrapping a hash of key-value pairs that
+    # describe the caller (e.g. user, ip, endpoint).
+    class Identifier
+      # Normalize an endpoint value: strip query string.
+      def self.normalize_endpoint(value)
+        return value unless value.is_a?(String)
+
+        value.split("?", 2).first
+      end
+
+      attr_reader :attributes
+
+      def initialize(attributes = {})
+        @attributes = attributes.transform_keys(&:to_sym).freeze
+      end
+
+      # Return the value for a characteristic key.
+      def [](key)
+        @attributes[key.to_sym]
+      end
+
+      # Serialize to a plain Hash suitable for JSON logging.
+      def to_h
+        @attributes.transform_keys(&:to_s)
+      end
+
+      def ==(other)
+        other.is_a?(Identifier) && other.attributes == @attributes
+      end
+    end
+  end
+end

data/lib/labkit/rate_limit/rule.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Labkit
+  module RateLimit
+    # Rule is a value object describing a single rate limit rule.
+    Rule = Data.define(:match, :limit, :period, :action, :characteristics) do
+      def initialize(limit:, period:, characteristics:, match: {}, action: :block)
+        super(
+          match: match.transform_keys(&:to_sym).freeze,
+          limit: limit,
+          period: period,
+          action: action.to_sym,
+          characteristics: Array(characteristics).map(&:to_sym).freeze
+        )
+      end
+    end
+  end
+end

data/lib/labkit/rate_limit.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Labkit
+  # RateLimit provides a simple rules-based rate limiting API backed by Redis counters.
+  module RateLimit
+    autoload :Identifier, "labkit/rate_limit/identifier"
+    autoload :Rule, "labkit/rate_limit/rule"
+    autoload :Evaluator, "labkit/rate_limit/evaluator"
+
+    # Defined independently to avoid forcing eager load of Evaluator at module load time.
+    # Must stay in sync with Evaluator::KNOWN_CHARACTERISTICS.
+    KNOWN_CHARACTERISTICS = [:user, :ip, :namespace, :plan, :endpoint].freeze
+
+    # Check whether the given call_site + identifier combination is within the
+    # configured rules.
+    #
+    # @param call_site [String] machine-readable name of the call site
+    # @param identifier [Identifier, Hash] caller attributes
+    # @param rules [Array<Rule>] ordered list of rate limit rules
+    # @param redis [Object] Redis client (must respond to #incr and #expire)
+    # @param logger [Logger, nil] optional logger override
+    # @return [:allow, :block]
+    def self.check(call_site:, identifier:, rules:, redis:, logger: nil)
+      id = identifier.is_a?(Identifier) ? identifier : Identifier.new(identifier)
+      Evaluator.new(
+        call_site: call_site,
+        identifier: id,
+        rules: rules,
+        redis: redis,
+        logger: logger
+      ).evaluate
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 1.11.1
+  version: 1.13.0
 platform: ruby
 authors:
 - Andrew Newdigate
@@ -598,6 +598,10 @@ files:
 - lib/labkit/middleware/sidekiq/user_experience_sli/client.rb
 - lib/labkit/middleware/sidekiq/user_experience_sli/server.rb
 - lib/labkit/net_http_publisher.rb
+- lib/labkit/rate_limit.rb
+- lib/labkit/rate_limit/evaluator.rb
+- lib/labkit/rate_limit/identifier.rb
+- lib/labkit/rate_limit/rule.rb
 - lib/labkit/rspec/README.md
 - lib/labkit/rspec/matchers.rb
 - lib/labkit/rspec/matchers/user_experience_matchers.rb