gitlab-labkit 2.8.1 → 4.6.0

data/lib/labkit/rate_limit/limiter.rb

@@ -15,7 +15,7 @@ module Labkit
     #     rules: [Labkit::RateLimit::Rule.new(name: "api_user", limit: 100, period: 60, characteristics: [:user])]
     #   )
     #   result = limiter.check({ user: 42, ip: "1.2.3.4" })
-    #   render_429 if result.exceeded? && result.action == :block
+    #   render_429 if result.action == :block
     class Limiter
       NAME_PATTERN = /\A[a-z0-9_]+\z/
 
@@ -45,7 +45,7 @@ module Labkit
       #   the rule definition and the call site colocated.
       # @return [Result]
       def check(identifier, cost: 1, rule_context: nil)
-        id = identifier.is_a?(Identifier) ? identifier : Identifier.new(identifier)
+        id = to_identifier(identifier)
         @evaluator.check(id, cost: cost, rule_context: rule_context)
       end
 
@@ -65,12 +65,33 @@ module Labkit
       # @param rule_context [Hash, nil] see {#check}
       # @return [Result]
       def peek(identifier, rule_context: nil)
-        id = identifier.is_a?(Identifier) ? identifier : Identifier.new(identifier)
+        id = to_identifier(identifier)
         @evaluator.peek(id, rule_context: rule_context)
       end
 
+      # Discards this limiter's state for one identifier: every rule's counter,
+      # and any ban written by a rule carrying ban_for.
+      #
+      # For call sites where a later success should wipe earlier failures, such
+      # as an authentication ban cleared by a valid login. Counters otherwise
+      # only expire with their window; this is the only way to end one early.
+      #
+      # Scoped to this limiter and identifier, not to a single rule: a caller
+      # clearing after a success knows who succeeded, not which rules matched.
+      #
+      # @param identifier [Identifier, Hash] caller attributes
+      # @return [Integer] number of Redis keys removed; on a Redis error, those
+      #   removed before it failed
+      def clear(identifier)
+        @evaluator.clear(to_identifier(identifier))
+      end
+
       private
 
+      def to_identifier(identifier)
+        identifier.is_a?(Identifier) ? identifier : Identifier.new(identifier)
+      end
+
       def validate_name!(name)
         raise ArgumentError, "name must be a non-empty String" unless name.is_a?(String) && !name.empty?
         return name if NAME_PATTERN.match?(name)

data/lib/labkit/rate_limit/matcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "set"
+
 module Labkit
   module RateLimit
     # Matcher is the internal representation of a single key/value predicate in
@@ -11,6 +13,14 @@ module Labkit
     #   - a Regexp instance                              -> :re matcher (Ruby convenience)
     #   - { eq: <value> }                                -> :eq matcher (canonical, YAML-compatible)
     #   - { re: <String|Regexp> }                        -> :re matcher (canonical, YAML-compatible)
+    #   - { oneOf: <Array> }                             -> :oneOf matcher (set membership, YAML-compatible)
+    #
+    # A bare Array is rejected rather than treated as :oneOf, so a value that
+    # was meant as a single matcher hash but arrived as an Array fails loudly
+    # instead of silently becoming a membership test.
+    #
+    # :oneOf membership is Set#include? (eql?/hash equality), not :eq's ==, so
+    # cross-type numerics differ (1 == 1.0, but {oneOf: [1]} does not match 1.0).
     #
     # Hash-key naming follows the metrics-catalog selector pattern. Glob,
     # prefix, and other matcher kinds are intentionally out of scope here; see
@@ -20,29 +30,38 @@ module Labkit
     # the regex, so callers can match non-String identifier values such as
     # Integer status codes (e.g. {status: { re: "^5" }} against status: 503).
     class Matcher < Data.define(:type, :value)
-      KNOWN_HASH_KEYS = %i[eq re].freeze
+      KNOWN_HASH_KEYS = %i[eq re oneOf].freeze
       MAX_REGEX_SOURCE_LENGTH = 200
       ERROR_INSPECT_LIMIT = 80
 
+      # Wall-clock budget for a single #match? call. Without it a match is
+      # bounded only by whatever global the host sets (40s in GitLab Rails,
+      # unbounded elsewhere), and a timeout reaches Evaluator's fail-open
+      # rescue, so the request goes unlimited.
+      #
+      # 50ms is a safety net against catastrophic backtracking, not a
+      # performance bound: real matches are far cheaper (median 0.087ms, worst
+      # 1.484ms across 6935 paths that timed out in production at 5ms). Those
+      # timeouts were wall-clock stalls, since Ruby counts real time and GC
+      # pauses on GitLab's git fleet have a p50 of ~72ms. Under 4x CPU
+      # oversubscription 50ms still fires on 0.16% of those matches, 5ms 1.33%.
+      MATCH_TIMEOUT_SECONDS = 0.05
+
       def self.build(input)
         case input
         when Regexp
-          new(type: :re, value: input)
+          new(type: :re, value: with_match_timeout(input))
         when Hash
           from_hash(input)
         when Array
-          raise ArgumentError,
-            "rate-limit match value must be a single-key Hash like {re: \"...\"} or {eq: ...}, got #{truncate_for_error(input)}"
+          raise ArgumentError, invalid_shape_error(input)
         else
           new(type: :eq, value: input)
         end
       end
 
       def self.from_hash(input)
-        if input.size != 1
-          raise ArgumentError,
-            "rate-limit match value must be a single-key Hash like {re: \"...\"} or {eq: ...}, got #{truncate_for_error(input)}"
-        end
+        raise ArgumentError, invalid_shape_error(input) if input.size != 1
 
         type, source = input.first
         type_sym = type.to_sym
@@ -60,6 +79,13 @@ module Labkit
         case type_sym
         when :eq
           new(type: :eq, value: source)
+        when :oneOf
+          unless source.is_a?(Array)
+            raise ArgumentError,
+              "rate-limit match value {oneOf: ...} must be an Array, got #{truncate_for_error(source)}"
+          end
+
+          new(type: :oneOf, value: Set.new(source).freeze)
         when :re
           if source.to_s.length > MAX_REGEX_SOURCE_LENGTH
             raise ArgumentError,
@@ -67,7 +93,7 @@ module Labkit
           end
 
           begin
-            new(type: :re, value: Regexp.new(source))
+            new(type: :re, value: with_match_timeout(source))
           rescue RegexpError, TypeError => e
             raise ArgumentError,
               "rate-limit match value {re: #{truncate_for_error(source)}} failed to compile: #{e.message}"
@@ -76,6 +102,38 @@ module Labkit
       end
       private_class_method :compile
 
+      # Compiles +source+ (a String pattern or an existing Regexp) into a
+      # Regexp bounded by MATCH_TIMEOUT_SECONDS.
+      #
+      # The timeout is set per-Regexp rather than via the global
+      # +Regexp.timeout=+: labkit is a library, and a global would silently
+      # change regex behaviour throughout the host application, including
+      # code unrelated to rate limiting.
+      #
+      # A Regexp that already carries its own timeout is returned untouched -
+      # an explicit choice by the rule author wins over our default.
+      #
+      # Recompiling an existing Regexp preserves its source and options
+      # (including the fixed-encoding flags) and therefore +#==+; only object
+      # identity changes.
+      def self.with_match_timeout(source)
+        return source if source.is_a?(Regexp) && source.timeout
+
+        if source.is_a?(Regexp)
+          Regexp.new(source.source, source.options, timeout: MATCH_TIMEOUT_SECONDS)
+        else
+          Regexp.new(source, timeout: MATCH_TIMEOUT_SECONDS)
+        end
+      end
+
+      private_class_method :with_match_timeout
+
+      def self.invalid_shape_error(input)
+        "rate-limit match value must be a single-key Hash like {re: \"...\"}, {eq: ...} or {oneOf: [...]}, " \
+          "got #{truncate_for_error(input)}"
+      end
+      private_class_method :invalid_shape_error
+
       def self.truncate_for_error(value)
         s = value.inspect
         s.length > ERROR_INSPECT_LIMIT ? "#{s[0, ERROR_INSPECT_LIMIT]}...(truncated)" : s
@@ -86,6 +144,8 @@ module Labkit
         case type
         when :eq
           value == identifier_value
+        when :oneOf
+          value.include?(identifier_value)
         when :re
           value.match?(identifier_value.to_s)
         else

data/lib/labkit/rate_limit/metrics.rb

@@ -1,13 +1,60 @@
 # frozen_string_literal: true
 
+require "concurrent-ruby"
+
 module Labkit
   module RateLimit
     module Metrics
+      # Process-wide once-latch for log_failure; specs reset via make_false.
+      FAILURE_LOGGED = Concurrent::AtomicBoolean.new(false)
+
       module_function
 
-      # :log rules are non-terminating: a check that matched only :log rules
-      # increments calls_total once per matched :log rule AND once with
-      # rule="unmatched", action="allow", since no terminating decision was made.
+      # Metric emission must never affect the caller's outcome. Resolving the
+      # metric by name keeps a raising getter inside the rescue.
+      def safe_increment(counter, labels, logger: nil)
+        public_send(counter).increment(**labels) # rubocop:disable GitlabSecurity/PublicSend
+      rescue StandardError => e
+        log_failure(e, counter, labels, logger)
+      end
+
+      # Gauge counterpart of safe_increment.
+      def safe_set(gauge, labels, value, logger: nil)
+        public_send(gauge).set(labels, value) # rubocop:disable GitlabSecurity/PublicSend
+      rescue StandardError => e
+        log_failure(e, gauge, labels, logger)
+      end
+
+      # make_true returns true only for the flipping caller, so exactly one
+      # warn per process. Never raises - a raise here would defeat the
+      # callers' rescues.
+      def log_failure(error, metric, labels, logger)
+        return unless logger && FAILURE_LOGGED.make_true
+
+        logger.warn(
+          name: labels[:rate_limiter],
+          metric: metric.to_s,
+          Labkit::Fields::ERROR_TYPE => "rate_limit_metrics_error",
+          Labkit::Fields::CLASS_NAME => error.class.to_s,
+          Labkit::Fields::ERROR_MESSAGE => error.message
+        )
+      rescue StandardError
+        nil
+      end
+
+      # Emitted exactly once per #check call, including calls that fail open;
+      # summing by rate_limiter gives the request rate through the limiter.
+      def checks_total
+        Labkit::Metrics::Client.counter(
+          :gitlab_labkit_rate_limiter_checks_total,
+          'Total number of rate limit checks',
+          { rate_limiter: nil, action: nil, matched: nil, error: nil }
+        )
+      end
+
+      # Deprecated: superseded by checks_total and rule_evaluations_total.
+      # Emitted unchanged (once per matched rule, rule="unmatched" when none
+      # matched) until consumers migrate; will get removed in a follow-up release.
       def calls_total
         Labkit::Metrics::Client.counter(
           :gitlab_labkit_rate_limiter_calls_total,
@@ -16,6 +63,18 @@ module Labkit
         )
       end
 
+      # Emitted once per *evaluated* rule. Only matched rules are evaluated:
+      # Unmatched traffic is checks_total{matched="false"}
+      def rule_evaluations_total
+        Labkit::Metrics::Client.counter(
+          :gitlab_labkit_rate_limiter_rule_evaluations_total,
+          'Total number of rate limit rule evaluations',
+          { rate_limiter: nil, rule: nil, action: nil, result: nil }
+        )
+      end
+
+      # Deprecated: superseded by checks_total{error="true"}. Still emitted
+      # because it remains the only error metric for #peek.
       def errors_total
         Labkit::Metrics::Client.counter(
           :gitlab_labkit_rate_limiter_errors_total,

data/lib/labkit/rate_limit/result.rb

@@ -3,34 +3,114 @@
 module Labkit
   module RateLimit
     # Result is the return value of Limiter#check.
-    # matched?  - true if a rule's match conditions were satisfied
-    # exceeded? - true if the matched rule's counter exceeded its limit
-    # action    - the outcome: what the caller should do
-    #             :block = rule matched, exceeded, rule configured to block
-    #             :log   = rule matched, exceeded, rule configured to log only
-    #             :allow = rule matched but count within limit, rule configured to allow,
-    #                      rule configured to skip (bypass, nothing counted),
-    #                      no rule matched, or error (fail-open)
-    #             The rule's configured action is available via rule.action.
-    # rule      - the matched Rule object (nil when matched? is false)
-    # error?    - true if Redis was unavailable; result fails open (exceeded? is false)
-    # info      - Result::Info with per-window counters; nil when matched? is false,
-    #             error?, or the matched rule is :skip (no counter exists)
-    Result = Data.define(:matched, :exceeded, :action, :rule, :error, :info) do
-      def initialize(matched:, action: nil, exceeded: false, rule: nil, error: false, info: nil)
-        super
+    #
+    # It accumulates one Result::Evaluation per matched-and-counted rule; the
+    # reader methods report the single most-constraining evaluation (see
+    # #most_constraining). A matched :skip rule or a fail-open error replaces
+    # the evaluations as the source of the reported outcome.
+    #
+    # Result is a mutable accumulator, not a value object: two Results built
+    # from the same evaluations are not #==.
+    #
+    # matched?    - true if at least one rule's match conditions were satisfied
+    # exceeded?   - true if the reported evaluation's counter exceeded its limit
+    # action      - the outcome: what the caller should do
+    #               :block = a :limit rule matched and is over its limit
+    #               :allow = everything else, including an exceeded :log rule
+    #               (visible via exceeded?), a matched :skip rule, no rule
+    #               matched, and error (fail-open).
+    #               The rule's configured action is available via rule.action.
+    # rule        - the reported Rule: the :skip rule when one matched, otherwise
+    #               the most-constraining evaluated Rule (nil when matched? is
+    #               false). Other rules may also have matched and been counted;
+    #               see #evaluations.
+    # error?      - true if Redis was unavailable; result fails open (exceeded? is false)
+    # degraded?   - true if a matched count_distinct rule was skipped by the
+    #               missing-key fail-open path; the check still completed
+    # info        - Result::Info with per-window counters for the reported rule;
+    #               nil when matched? is false, error?, or the matched rule is
+    #               :skip (no counter exists)
+    # evaluations - every counted evaluation, in rule declaration order
+    class Result
+      attr_reader :evaluations
+
+      def self.error
+        new(error: true)
+      end
+
+      def initialize(error: false)
+        @evaluations = []
+        @skip_rule = nil
+        @error = error
+        @degraded = false
+        @most_constraining = nil
+      end
+
+      def add_evaluation(evaluation)
+        @most_constraining = nil
+        @evaluations << evaluation
+        self
+      end
+
+      # Records a matched :skip rule: the request is allowed and the skip rule
+      # is the one reported, regardless of any evaluations already collected.
+      def skip!(rule)
+        @skip_rule = rule
+        self
       end
 
       def matched?
-        matched
+        skipped? || @evaluations.any?
+      end
+
+      def skipped?
+        !!@skip_rule
+      end
+
+      def block?
+        !skipped? && @evaluations.any?(&:block?)
       end
 
       def exceeded?
-        exceeded
+        return false if skipped?
+
+        !most_constraining.nil? && most_constraining.exceeded?
       end
 
       def error?
-        error
+        @error
+      end
+
+      # Unlike error?, the check keeps going: the flag marks the outcome as
+      # degraded without changing it.
+      def degraded!
+        @degraded = true
+        self
+      end
+
+      def degraded?
+        @degraded
+      end
+
+      def action
+        block? ? :block : :allow
+      end
+
+      def rule
+        @skip_rule || most_constraining&.rule
+      end
+
+      def info
+        return nil if skipped?
+
+        most_constraining&.info
+      end
+
+      # The evaluation with the strongest claim on the outcome; ties keep the
+      # earliest-declared rule (min returns the first of tied elements).
+      # Memoized; add_evaluation invalidates.
+      def most_constraining
+        @most_constraining ||= @evaluations.min
       end
 
       # Returns RFC-compliant rate limit response headers, or {} when no rule matched or an error occurred.
@@ -48,6 +128,40 @@ module Labkit
       end
     end
 
+    # The outcome of counting one matched rule.
+    # rule     - the evaluated Rule
+    # exceeded - whether the post-increment count exceeded the resolved limit
+    # info     - Result::Info with the per-window counters
+    #
+    # Evaluations order by constraint: a blocking evaluation (a :limit rule over
+    # its limit, or one whose ban is still in force) ranks strictly first - it
+    # decides the request no matter what any other rule reports - then exceeded
+    # ones, then fewest remaining.
+    # remaining floors at 0, so an exceeded :log rule ties with one sitting
+    # exactly on its limit; ranking exceeded ahead keeps a breach from being
+    # hidden by a rule that merely reached its limit.
+    Result::Evaluation = Data.define(:rule, :exceeded, :info) do
+      include Comparable
+
+      def exceeded?
+        exceeded
+      end
+
+      def block?
+        exceeded? && rule.action == :limit
+      end
+
+      def <=>(other)
+        constraint_rank <=> other.constraint_rank
+      end
+
+      protected
+
+      def constraint_rank
+        [block? ? 0 : 1, exceeded? ? 0 : 1, info.remaining]
+      end
+    end
+
     # Per-window counter data attached to a matched Result.
     # resolved_limit  - the evaluated limit Integer for this rule
     # resolved_period - the evaluated period Integer (seconds) for this rule

data/lib/labkit/rate_limit/rule.rb

@@ -2,7 +2,7 @@
 
 module Labkit
   module RateLimit
-    KNOWN_ACTIONS = %i[block log allow skip].freeze
+    KNOWN_ACTIONS = %i[limit log skip].freeze
     RULE_NAME_PATTERN = /\A[a-z0-9_]+\z/
     RULE_NAME_MAX_LENGTH = 64
 
@@ -12,13 +12,19 @@ module Labkit
     #                   the rule to apply; empty hash matches any identifier
     # limit           - request threshold; may be a callable (resolved per check)
     # period          - window in seconds; may be a callable (resolved per check)
-    # action          - :block (enforce), :log (count and log only, do not block,
-    #                   evaluation continues to subsequent rules), :allow
-    #                   (count but always permit; terminates evaluation on match
-    #                   regardless of whether the limit was exceeded), or :skip
-    #                   (bypass: permit and terminate evaluation on match without
-    #                   counting; performs no Redis operation, so limit, period,
-    #                   characteristics, and count_distinct are inert)
+    # action          - :limit (enforce; the result blocks when the rule is over
+    #                   its limit, which also terminates evaluation), :log (count
+    #                   and log only, never blocks and never terminates),
+    #                   or :skip (bypass: permit and terminate evaluation on
+    #                   match without counting; performs no Redis operation, so
+    #                   limit, period, characteristics, and count_distinct are
+    #                   inert)
+    # ban_for         - optional ban duration in seconds. On crossing the limit
+    #                   the rule writes a separate ban key that outlives the
+    #                   counter window, and stops counting while it holds.
+    #                   +action+ still decides who is blocked: :limit enforces
+    #                   the ban, :log records what it would have done. Rejected
+    #                   on :skip. May be a callable, resolved per check.
     # characteristics - identifier keys used to build the compound Redis counter key
     # count_distinct  - optional Symbol naming an identifier key. When set, the rule
     #                   counts the number of distinct values seen for that key within
@@ -29,7 +35,27 @@ module Labkit
     # +name+ must be a lowercase alphanumeric-and-underscore string of at most 64
     # characters. It is used as the middle segment of every Redis counter key for
     # this rule, so changing a rule's name mid-window abandons its in-flight counters.
-    Rule = Data.define(:name, :match, :limit, :period, :action, :characteristics, :count_distinct) do
+    Rule = Data.define(:name, :match, :limit, :period, :action, :characteristics, :count_distinct, :ban_for) do
+      # ban_for changes the accounting; action still decides who gets blocked.
+      # A :skip rule counts nothing, so it has no limit to cross and no ban to
+      # write.
+      def self.normalize_ban_for(value, action_sym, count_distinct_sym)
+        return nil if value.nil?
+
+        raise ArgumentError, "ban_for is not valid on a :skip rule" if action_sym == :skip
+
+        # The ban path counts with INCRBYFLOAT, so a SET-backed rule would
+        # silently count calls instead of distinct values.
+        raise ArgumentError, "ban_for cannot be combined with count_distinct #{count_distinct_sym.inspect}" if count_distinct_sym
+
+        return value if value.respond_to?(:call)
+        # Whole seconds: the ban is written with SET EX, and anything under a
+        # second truncates to 0, which Redis rejects.
+        return value if value.is_a?(Numeric) && value >= 1
+
+        raise ArgumentError, "ban_for must be a Numeric of at least 1 second or a callable, got #{value.inspect}"
+      end
+
       def self.normalize_count_distinct(value, characteristics_arr)
         sym =
           case value
@@ -45,7 +71,10 @@ module Labkit
         sym
       end
 
-      def initialize(name:, limit:, period:, characteristics:, match: {}, action: :block, count_distinct: nil)
+      def initialize(
+        name:, limit:, period:, characteristics:,
+        match: {}, action: :limit, count_distinct: nil, ban_for: nil
+      )
         raise ArgumentError, "name must be a String or Symbol, got #{name.class}" unless name.is_a?(String) || name.is_a?(Symbol)
 
         name_str = name.to_s
@@ -60,6 +89,7 @@ module Labkit
         end
 
         characteristics_arr = Array(characteristics).map(&:to_sym).freeze
+        count_distinct_sym = self.class.normalize_count_distinct(count_distinct, characteristics_arr)
 
         super(
           name: name_str.freeze,
@@ -68,7 +98,8 @@ module Labkit
           period: period,
           action: action_sym,
           characteristics: characteristics_arr,
-          count_distinct: self.class.normalize_count_distinct(count_distinct, characteristics_arr)
+          count_distinct: count_distinct_sym,
+          ban_for: self.class.normalize_ban_for(ban_for, action_sym, count_distinct_sym)
         )
       end
     end

data/lib/labkit/rate_limit.rb

@@ -40,7 +40,8 @@ module Labkit
       #
       # @param name [String] call site name
       # @param identifier [Identifier, Hash] caller attributes
-      # @param rules [Array<Rule>] ordered list of rules (first match wins)
+      # @param rules [Array<Rule>] ordered list of rules; every matching rule is
+      #   counted, and the Result reports the most constraining one
       # @param redis [Object, nil] Redis client; falls back to config.redis
       # @param logger [Logger, nil] logger; falls back to config.logger
       # @param cost [Numeric] amount to add to the counter; see Limiter#check

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 2.8.1
+  version: 4.6.0
 platform: ruby
 authors:
 - Andrew Newdigate
@@ -716,7 +716,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Instrumentation for GitLab
 test_files: []