gitlab-labkit 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c6ab89c2e34ad08721cee1e920601a8d6d039bfe8a6552777021714f17d0c74
-  data.tar.gz: 6b9e36ac9ce69866890d6df28132d8abd280cf7d3b4ab7b748470864dbf8a70e
+  metadata.gz: 1457da9fa7cec587b47834870a4eb60f5a21c9966111d1d9edd0e7e0a49b4ff3
+  data.tar.gz: 631e56481d2be50e976a16aefa6fd6a250759089b56dde0414af059fd55c4ba7
 SHA512:
-  metadata.gz: af23293cd5d9afe2dd3ba93ca4cd98a6ae5178c04079069e60defff46652266158fc7d74fe335d9345391ee11e89602e0ebf09d1d536fc0f0140426676a23a16
-  data.tar.gz: 3ae857f6940bf2b871b123d88c3caf3e12589efe3b2a3733cb723b73cdca347c3bfa8506ecf3c931d0d83d7dd99882bf25a9abe7efa3e85f76e4958b111be312
+  metadata.gz: 5d6aba7952cc495dd4b74ea0aa452f473e01dc6a6bda2a94d1238d9102f5b448de2b4ae1765f87e2020d2fb460b1613044fe9403f0019b54b050b3a48261b425
+  data.tar.gz: 21ddb1d302b53049a605052d67bbdf2d51bee9a4e80f82c6c10f689d5b292397e0d913dae4b05e66c93ee4c93d831e14e885c448a77b049dab48ac1bda3b84b5

data/.copier-answers.yml

@@ -3,7 +3,7 @@
 # See the project for instructions on how to update the project
 #
 # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY
-_commit: v1.48.0
+_commit: v1.50.0
 _src_path: https://gitlab.com/gitlab-com/gl-infra/common-template-copier.git
 ee_licensed: false
 gitlab_namespace: gitlab-org/ruby/gems
@@ -11,6 +11,7 @@ golang: false
 helm: false
 initial_codeowners: '@reprazent @andrewn @mkaeppler @ayufan'
 jsonnet: false
+measure_performance: false
 project_name: labkit-ruby
 release_platform: false
 ruby: true

data/.gitlab-ci-other-versions.yml

@@ -0,0 +1,5 @@
+# This file should contain dependency versions not managed directly by mise
+# Use Renovate annotation comments to manage dependencies. Renovate is configured
+# to read this file automatically.
+
+variables: {}  # None yet...

data/.gitlab-ci.yml

@@ -32,7 +32,7 @@ include:
 # so this augments the base job (and is inherited by anything that extends it).
 rspec:
   services:
-    - name: redis:7-alpine
+    - name: redis:8-alpine
       alias: redis
   variables:
     LABKIT_TEST_REDIS_URL: redis://redis

data/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   redis:
-    image: redis:7-alpine
+    image: redis:8-alpine
     ports:
       - "6390:6379"
     healthcheck:

data/gitlab-labkit.gemspec

@@ -22,17 +22,17 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 3.3", "< 5"
 
   # Please maintain alphabetical order for dependencies
-  spec.add_runtime_dependency "actionpack", ">= 5.0.0", "< 8.1.0"
-  spec.add_runtime_dependency "activesupport", ">= 5.0.0", "< 8.1.0"
+  spec.add_runtime_dependency "actionpack", ">= 5.0.0", "< 8.2.0"
+  spec.add_runtime_dependency "activesupport", ">= 5.0.0", "< 8.2.0"
   spec.add_runtime_dependency "grpc", ">= 1.75" # Be sure to update the "grpc-tools" dev_dependency too
   spec.add_runtime_dependency "google-protobuf", ">= 3.25", "< 5.0"
   spec.add_runtime_dependency "jaeger-client", "~> 1.1.0"
   spec.add_runtime_dependency "json_schemer", ">= 2.3.0", "< 3.0"
   spec.add_runtime_dependency "openssl", "~> 3.3.2"
-  spec.add_runtime_dependency "opentelemetry-sdk", "~> 1.10"
-  spec.add_runtime_dependency "opentelemetry-instrumentation-all", "~> 0.89.1"
-  spec.add_runtime_dependency "opentelemetry-exporter-otlp", "~> 0.31.1"
-  spec.add_runtime_dependency "opentracing", "~> 0.4"
+  spec.add_runtime_dependency "opentelemetry-sdk", ">= 1.10", "< 2"
+  spec.add_runtime_dependency "opentelemetry-instrumentation-all", ">= 0.89", "< 1"
+  spec.add_runtime_dependency "opentelemetry-exporter-otlp", ">= 0.31", "< 1"
+  spec.add_runtime_dependency "opentracing", ">= 0.4", "< 1"
   spec.add_runtime_dependency "pg_query", ">= 6.1.0", "< 7.0"
   spec.add_runtime_dependency "prometheus-client-mmap", ">= 1.2", "< 2.0"
   spec.add_runtime_dependency "redis", "> 3.0.0", "< 6.0.0"
@@ -49,7 +49,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry", "~> 0.12"
   spec.add_development_dependency "pry-byebug", "~> 3.11"
   spec.add_development_dependency "rack", "~> 2.0"
-  spec.add_development_dependency "railties", ">= 5.0.0", "< 8.1.0"
+  spec.add_development_dependency "railties", ">= 5.0.0", "< 8.2.0"
   spec.add_development_dependency "rake", "~> 13.2"
   spec.add_development_dependency "rest-client", "~> 2.1.0"
   spec.add_development_dependency "rspec", "~> 3.12.0"

data/lib/labkit/rate_limit/evaluator.rb

@@ -38,6 +38,26 @@ module Labkit
         return {count, redis.call('TTL', KEYS[1])}
       LUA
 
+      # Atomic SADD + SCARD + conditional EXPIRE. SET-cardinality counterpart
+      # of INCR_SCRIPT; same shape (read TTL, mutate, set TTL when missing,
+      # return post-state {count, TTL}). count is SCARD, not the SADD return.
+      #
+      # ttl_before < 0 covers TTL=-2 (key missing) and TTL=-1 (no expiry),
+      # so this also self-heals orphan keys left without TTL.
+      SADD_SCRIPT = Labkit::Redis::Script.new(<<~LUA)
+        local ttl = ARGV[1]
+        local member = ARGV[2]
+        local ttl_before = redis.call('TTL', KEYS[1])
+
+        redis.call('SADD', KEYS[1], member)
+        local count = redis.call('SCARD', KEYS[1])
+        if ttl_before < 0 then
+          redis.call('EXPIRE', KEYS[1], ttl)
+        end
+
+        return {count, redis.call('TTL', KEYS[1])}
+      LUA
+
       def initialize(name:, rules:, redis:, logger:)
         @name   = name
         @rules  = rules
@@ -70,10 +90,21 @@ module Labkit
 
       # :log rules are non-terminating: they emit metrics and continue,
       # so a shadow :log rule cannot disable a following :block rule.
+      #
+      # SET-mode rules (rule.count_distinct set) that match but whose identifier
+      # is missing the count_distinct key fail open + log + bump errors_total, and
+      # the loop continues to the next rule (the rule is treated as not applicable
+      # rather than aborting the whole evaluation).
       def check_rules(identifier, cost)
         @rules.each do |rule|
           next unless rule_matches?(rule, identifier)
 
+          if rule.count_distinct && missing_count_distinct_value?(rule, identifier)
+            log_missing_count_distinct(rule, identifier)
+            report_error_metrics
+            next
+          end
+
           result = evaluate_rule(rule, identifier, cost)
           report_matched_metrics(result)
           return result unless rule.action == :log
@@ -85,6 +116,10 @@ module Labkit
 
       # Mirror of check_rules without metrics: peek skips :log rules (their state
       # is unobservable through peek).
+      #
+      # peek does not need the count_distinct identifier key - it reads SCARD on
+      # the rule-keyed compound key, which contains the cardinality across all
+      # members. So missing-key fail-open does not apply here.
       def peek_rules(identifier)
         @rules.each do |rule|
           next if rule.action == :log
@@ -100,12 +135,25 @@ module Labkit
         rule.match.all? { |key, matcher| matcher.match?(identifier[key]) }
       end
 
+      def missing_count_distinct_value?(rule, identifier)
+        value = identifier[rule.count_distinct]
+        value.nil? || value.to_s.empty?
+      end
+
       def evaluate_rule(rule, identifier, cost)
         redis_key = build_redis_key(rule, identifier)
         resolved_limit = Integer(resolve_value(rule.limit))
         resolved_period = Integer(resolve_value(rule.period))
 
-        count, ttl = incr_with_ttl(redis_key, resolved_period, cost)
+        # cost is ignored for count_distinct rules: SADD is binary (a member is
+        # either added or not), and the post-add count is SCARD regardless.
+        count, ttl =
+          if rule.count_distinct
+            sadd_with_ttl(redis_key, identifier[rule.count_distinct], resolved_period)
+          else
+            incr_with_ttl(redis_key, resolved_period, cost)
+          end
+
         build_result(rule, resolved_limit, resolved_period, count, ttl)
       end
 
@@ -114,7 +162,7 @@ module Labkit
         resolved_limit = Integer(resolve_value(rule.limit))
         resolved_period = Integer(resolve_value(rule.period))
 
-        count, ttl = read_with_ttl(redis_key)
+        count, ttl = rule.count_distinct ? scard_with_ttl(redis_key) : read_with_ttl(redis_key)
         build_result(rule, resolved_limit, resolved_period, count, ttl)
       end
 
@@ -168,7 +216,7 @@ module Labkit
       # otherwise.
       def incr_with_ttl(redis_key, period, cost)
         @redis.with do |conn|
-          raw_count, ttl = eval_incr_script(conn, redis_key, period, cost)
+          raw_count, ttl = INCR_SCRIPT.eval(conn, keys: [redis_key], argv: [period, cost])
           [Float(raw_count), ttl]
         end
       end
@@ -191,8 +239,29 @@ module Labkit
         end
       end
 
-      def eval_incr_script(conn, redis_key, period, cost)
-        INCR_SCRIPT.eval(conn, keys: [redis_key], argv: [period, cost])
+      # Atomic SADD + SCARD + conditional EXPIRE in one Redis operation via Lua.
+      # See SADD_SCRIPT for the body. Mirrors incr_with_ttl's shape, including
+      # the Float-typed count for uniformity with the INCR path.
+      def sadd_with_ttl(redis_key, member, period)
+        member_str = encode_char_value(member.to_s)
+        @redis.with do |conn|
+          raw_count, ttl = SADD_SCRIPT.eval(conn, keys: [redis_key], argv: [period, member_str])
+          [Float(raw_count), ttl]
+        end
+      end
+
+      # Pipelined SCARD + TTL. SCARD on a missing key returns 0, so no
+      # explicit nil handling is needed (unlike GET in read_with_ttl).
+      # SCARD is integer-valued; coerced to Float for type uniformity with
+      # the INCR path so callers see a consistent count type.
+      def scard_with_ttl(redis_key)
+        @redis.with do |conn|
+          scard, ttl = conn.pipelined do |pipe|
+            pipe.scard(redis_key)
+            pipe.ttl(redis_key)
+          end
+          [Float(scard), ttl]
+        end
       end
 
       def log_error(error, identifier)
@@ -204,6 +273,16 @@ module Labkit
         )
       end
 
+      def log_missing_count_distinct(rule, identifier)
+        @logger.warn(
+          message: "rate_limit_missing_count_distinct",
+          name: @name,
+          rule: rule.name,
+          count_distinct: rule.count_distinct.to_s,
+          identifier: identifier&.to_h
+        )
+      end
+
       def report_matched_metrics(result)
         Metrics.calls_total.increment(
           rate_limiter: @name,

data/lib/labkit/rate_limit/rule.rb

@@ -17,12 +17,32 @@ module Labkit
     #                   (count but always permit; terminates evaluation on match
     #                   regardless of whether the limit was exceeded)
     # characteristics - identifier keys used to build the compound Redis counter key
+    # count_distinct  - optional Symbol naming an identifier key. When set, the rule
+    #                   counts the number of distinct values seen for that key within
+    #                   the (characteristics-bucketed) period, backed by a Redis SET.
+    #                   When nil (default), the rule counts the number of calls,
+    #                   backed by INCR. The named key must not overlap +characteristics+.
     #
     # +name+ must be a lowercase alphanumeric-and-underscore string of at most 64
     # characters. It is used as the middle segment of every Redis counter key for
     # this rule, so changing a rule's name mid-window abandons its in-flight counters.
-    Rule = Data.define(:name, :match, :limit, :period, :action, :characteristics) do
-      def initialize(name:, limit:, period:, characteristics:, match: {}, action: :block)
+    Rule = Data.define(:name, :match, :limit, :period, :action, :characteristics, :count_distinct) do
+      def self.normalize_count_distinct(value, characteristics_arr)
+        sym =
+          case value
+          when nil    then nil
+          when Symbol then value
+          when String then value.to_sym
+          else
+            raise ArgumentError, "count_distinct must be a Symbol, String, or nil, got #{value.class}"
+          end
+
+        raise ArgumentError, "count_distinct #{sym.inspect} must not overlap characteristics #{characteristics_arr.inspect}" if sym && characteristics_arr.include?(sym)
+
+        sym
+      end
+
+      def initialize(name:, limit:, period:, characteristics:, match: {}, action: :block, count_distinct: nil)
         raise ArgumentError, "name must be a String or Symbol, got #{name.class}" unless name.is_a?(String) || name.is_a?(Symbol)
 
         name_str = name.to_s
@@ -36,13 +56,16 @@ module Labkit
           raise ArgumentError, "Rule name too long: #{name.inspect}. Maximum 64 characters" if name_str.length > RULE_NAME_MAX_LENGTH
         end
 
+        characteristics_arr = Array(characteristics).map(&:to_sym).freeze
+
         super(
           name: name_str.freeze,
           match: match.transform_keys(&:to_sym).transform_values { |v| Matcher.build(v) }.freeze,
           limit: limit,
           period: period,
           action: action_sym,
-          characteristics: Array(characteristics).map(&:to_sym).freeze
+          characteristics: characteristics_arr,
+          count_distinct: self.class.normalize_count_distinct(count_distinct, characteristics_arr)
         )
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Andrew Newdigate
@@ -18,7 +18,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 8.1.0
+        version: 8.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,7 +28,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 8.1.0
+        version: 8.2.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 8.1.0
+        version: 8.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -48,7 +48,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 8.1.0
+        version: 8.2.0
 - !ruby/object:Gem::Dependency
   name: grpc
   requirement: !ruby/object:Gem::Requirement
@@ -135,58 +135,82 @@ dependencies:
   name: opentelemetry-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.10'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.10'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: opentelemetry-instrumentation-all
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.89'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.89.1
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.89'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.89.1
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: opentelemetry-exporter-otlp
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.31'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.31.1
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.31'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.31.1
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: opentracing
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: pg_query
   requirement: !ruby/object:Gem::Requirement
@@ -410,7 +434,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 8.1.0
+        version: 8.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -420,7 +444,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 8.1.0
+        version: 8.2.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -523,6 +547,7 @@ files:
 - ".env.example.sh"
 - ".gitignore"
 - ".gitlab-ci-asdf-versions.yml"
+- ".gitlab-ci-other-versions.yml"
 - ".gitlab-ci.yml"
 - ".gitlab/CODEOWNERS"
 - ".gitlab/merge_request_templates/default.md"