gitlab-labkit 5.7.0 → 5.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2d9d587afe49d99a2ada5a8757230d69024a1085009f1e65c29ac97aa005e98
-  data.tar.gz: 35e55c1e1f9aef6836736e6917cffe4813aa37655eebf5534893f9b7b232fb89
+  metadata.gz: bbffb0883d80c949bfb28ffec2943b9cc89ffb88c2c02656afe99ad777e0f152
+  data.tar.gz: 93610eb46b3dbbfebe40c1b97d31d07b69af3f5e5ee0f152be29a66eca8da03e
 SHA512:
-  metadata.gz: 83a9378e749aa4c04cf05cca872fffabe8945802b169c996aba579b43b73e582d98dca6c37b31c82098fe3edda41932ddc81813010ab278e2df7569812c7926e
-  data.tar.gz: 5b00ebd19184fbfaec05c1cdf40de93b74a70c0a7aa2b167213425debada2ea6708b61874d4ad23a9de40c950724b47da599d37daf0116d323ed8b4db76a6d23
+  metadata.gz: c01235e9277e28adbde6cb2e14cedebbabe78b46a8f7e771664875da94cd59ea3c31e2033b152ea8e23dce3cd8ce84c0c5fd310c9691d1662006df0e87f66b67
+  data.tar.gz: 87c05f1a0543e7969b35023d227d32cabc87455caa99aeca54a83832a5e40a2968895afb3982c4ecfcadc31d891558ce4abccb249f0758f95903367fd56b82df

data/.copier-answers.yml

@@ -3,7 +3,7 @@
 # See the project for instructions on how to update the project
 #
 # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY
-_commit: v1.87.1
+_commit: v1.89.0
 _src_path: https://gitlab.com/gitlab-com/gl-infra/common-template-copier.git
 ee_licensed: false
 gitlab_namespace: gitlab-org/ruby/gems

data/AGENTS.md

@@ -1,5 +1,37 @@
-<<<<<<< before updating
-# AGENTS.md
+# Agent guidelines
+
+This project uses [Conventional Commits](https://www.conventionalcommits.org/).
+Commit messages drive automated [semantic releases](https://semantic-release.gitbook.io/)
+(see `.releaserc.json`), so use `feat:`, `fix:`, etc. prefixes to ensure the
+correct version bump and changelog entry.
+
+## AI-authored GitLab comments
+
+Wrap the full body of any comment, note, or reply you post to GitLab (issues,
+merge requests, epics — via `glab`/`glab api`) in `<:robot:>` / `</:robot:>`
+tags (GitLab emoji shortcode, renders as 🤖), so readers can tell at a glance
+that it's AI-generated:
+
+```
+<:robot:>
+
+...full comment body...
+
+</:robot:>
+```
+
+This applies to newly posted comments. It does not apply to code, commit
+messages, or MR/issue descriptions unless asked.
+
+<!--
+####################################################################
+#   🌊✨ PLEASE ADD PROJECT SPECIFIC RULES BELOW THIS LINE ✨🌊   #
+#   The section above is managed by the common-template-copier     #
+#   template and is updated automatically on `copier update`.      #
+#   Add your project-specific instructions below this line so      #
+#   they are preserved when the template is updated.               #
+####################################################################
+-->
 
 Instructions for AI agents and automated tooling working in this repository.
 
@@ -135,39 +167,3 @@ When a change does not fit one layer clearly, say so in the MR instead of pickin
   Mention the sync agent on the review thread to have it draft the scenario MR; if it cannot, open a labkit-spec issue and link it from the thread.
 - Never edit Lua scripts, key construction, header names, metric names, or log field names to fix a Ruby-only bug.
   If a fix seems to need that, it is a contract change.
-=======
-# Agent guidelines
-
-This project uses [Conventional Commits](https://www.conventionalcommits.org/).
-Commit messages drive automated [semantic releases](https://semantic-release.gitbook.io/)
-(see `.releaserc.json`), so use `feat:`, `fix:`, etc. prefixes to ensure the
-correct version bump and changelog entry.
-
-## AI-authored GitLab comments
-
-Wrap the full body of any comment, note, or reply you post to GitLab (issues,
-merge requests, epics — via `glab`/`glab api`) in `<:robot:>` / `</:robot:>`
-tags (GitLab emoji shortcode, renders as 🤖), so readers can tell at a glance
-that it's AI-generated:
-
-```
-<:robot:>
-
-...full comment body...
-
-</:robot:>
-```
-
-This applies to newly posted comments. It does not apply to code, commit
-messages, or MR/issue descriptions unless asked.
-
-<!--
-####################################################################
-#   🌊✨ PLEASE ADD PROJECT SPECIFIC RULES BELOW THIS LINE ✨🌊   #
-#   The section above is managed by the common-template-copier     #
-#   template and is updated automatically on `copier update`.      #
-#   Add your project-specific instructions below this line so      #
-#   they are preserved when the template is updated.               #
-####################################################################
--->
->>>>>>> after updating

data/lib/labkit/rate_limit/README.md

@@ -601,6 +601,56 @@ the request rate through a limiter — no exclusions or dedup needed. A single
 `check` call emits **one** `checks_total` increment and as many
 `rule_evaluations_total` increments as rules it evaluated (possibly zero).
 
+## Block summary
+
+`Labkit::RateLimit::BlockSummary` counts blocks per `(endpoint_id, rule)` in
+one process and logs the counts every interval (300 seconds by default). The
+number of lines grows with the number of distinct pairs the process blocked,
+not with request rate, so an attack does not multiply log volume.
+
+Each process tracks at most `max_pairs` pairs (10 by default) per interval.
+Blocks for any further pair are counted on one `endpoint_id: "other"`,
+`rate_limit_rule: "other"` line, with no `feature_category`, so a process
+writes at most `max_pairs + 1` lines per flush and the tally's memory stays
+bounded.
+
+```ruby
+BLOCK_SUMMARY = Labkit::RateLimit::BlockSummary.new(logger: logger)
+
+# On every blocked request. feature_category is optional:
+BLOCK_SUMMARY.add(endpoint_id: "GET /api/:version/projects", rule: "unauthenticated_api", feature_category: "source_code_management")
+
+# The caller owns the thread. Start it in each worker process, after any fork:
+thread = Thread.new { BLOCK_SUMMARY.call }
+
+# On shutdown, flush what is left and let the thread return:
+BLOCK_SUMMARY.stop
+thread.join
+```
+
+Each flush writes one INFO line per pair and then empties the counts:
+
+```json
+{"message": "rate_limit_block_summary", "endpoint_id": "GET /api/:version/projects", "rate_limit_rule": "unauthenticated_api", "feature_category": "source_code_management", "block_count": 412}
+```
+
+`feature_category` is left out when the caller passed none. It does not split
+a pair: the first non-nil one passed for a pair is the one logged. The first
+flush waits a random part of the interval, so processes started together do
+not all flush at once. A flush with nothing counted writes nothing. The caller
+decides what `endpoint_id` means. LabKit does not resolve it. Counts in a
+process that exits without `stop` are lost, up to one interval's worth. `stop`
+only signals the thread in `call`, which runs the final flush, so it flushes
+nothing if no thread is running `call`.
+
+`interval` must be a positive number and `max_pairs` a positive Integer, or
+`new` raises `ArgumentError`.
+
+A logging failure during a flush is logged at WARN with
+`message: "rate_limit_error"`, `error_type: "rate_limit_error"` and
+`dropped_pairs`, the number of lines that flush lost. It never raises, so the
+flush thread keeps running.
+
 ## Dev/test vs production guards
 
 `Limiter.new` and `Rule.new` validate names and configuration. In

data/lib/labkit/rate_limit/block_summary.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Labkit
+  module RateLimit
+    # Counts blocks per (endpoint_id, rule) in this process and logs the counts
+    # every interval. A line per block grows with an attack, so lines here grow
+    # only with the number of distinct pairs this process blocked, and never past
+    # max_pairs + 1.
+    #
+    # One line per pair rather than one line holding a map: map keys become
+    # index fields, one per endpoint name. The fields are rate_limit_rule and
+    # block_count, not rule and count, so they cannot clash with another line's
+    # type for a common name in a shared index.
+    #
+    # The caller owns the thread: run #call on one, and #stop it on shutdown so
+    # the last counts are flushed.
+    class BlockSummary
+      MESSAGE = "rate_limit_block_summary"
+      OTHER = "other"
+      OTHER_KEY = [OTHER, OTHER].freeze
+      DEFAULT_INTERVAL_SECONDS = 300
+      DEFAULT_MAX_PAIRS = 10
+
+      def initialize(logger: nil, interval: DEFAULT_INTERVAL_SECONDS, max_pairs: DEFAULT_MAX_PAIRS)
+        # A zero interval would make #call spin without waiting.
+        raise ArgumentError, "interval must be a positive Numeric, got #{interval.inspect}" unless interval.is_a?(Numeric) && interval.positive?
+
+        raise ArgumentError, "max_pairs must be a positive Integer, got #{max_pairs.inspect}" unless max_pairs.is_a?(Integer) && max_pairs.positive?
+
+        @logger = logger || RateLimit.config.logger || Labkit::Logging::JsonLogger.new($stdout)
+        @interval = interval
+        @max_pairs = max_pairs
+        @counts = {}
+        @mutex = Mutex.new
+        @stop_signal = Thread::Queue.new
+      end
+
+      # Keyed on the pair alone, so feature_category never adds a line. The first
+      # non-nil one passed for a pair is kept. Capped here rather than at flush,
+      # so the hash cannot grow during an attack.
+      def add(endpoint_id:, rule:, feature_category: nil)
+        key = [endpoint_id, rule]
+
+        @mutex.synchronize do
+          key = OTHER_KEY if !@counts.key?(key) && pairs_full?
+          entry = (@counts[key] ||= [0, nil])
+          entry[0] += 1
+          entry[1] ||= feature_category unless key == OTHER_KEY
+        end
+      end
+
+      # Waits on a queue rather than sleeping, so a stop that lands mid-flush is
+      # still seen and the final flush is never skipped. The first wait is
+      # random, so workers booted together do not all flush at once.
+      def call
+        timeout = rand * @interval
+
+        loop do
+          stopping = @stop_signal.pop(timeout: timeout)
+          flush
+          break if stopping
+
+          timeout = @interval
+        end
+      end
+
+      # Only signals #call, which runs the final flush. Without a thread in #call,
+      # nothing is flushed.
+      def stop
+        @stop_signal << true
+      end
+
+      def flush
+        drained = {}
+        logged = 0
+
+        drained = @mutex.synchronize do
+          counts = @counts
+          @counts = {}
+          counts
+        end
+
+        drained.each do |(endpoint_id, rule), (count, feature_category)|
+          @logger.info(line(endpoint_id, rule, feature_category, count))
+          logged += 1
+        end
+      rescue StandardError => e
+        log_failure(e, drained.size - logged)
+      end
+
+      private
+
+      def pairs_full?
+        @counts.size - (@counts.key?(OTHER_KEY) ? 1 : 0) >= @max_pairs
+      end
+
+      def line(endpoint_id, rule, feature_category, count)
+        payload = { message: MESSAGE, Labkit::Fields::ENDPOINT_ID => endpoint_id, rate_limit_rule: rule, block_count: count }
+        payload[:feature_category] = feature_category if feature_category
+        payload
+      end
+
+      # Never raises: a raise here would kill the caller's flush thread.
+      # The counts are already drained, so dropped_pairs says how many lines
+      # this flush lost.
+      def log_failure(error, dropped_pairs)
+        @logger.warn(
+          message: "rate_limit_error",
+          Labkit::Fields::ERROR_TYPE => "rate_limit_error",
+          Labkit::Fields::CLASS_NAME => error.class.to_s,
+          Labkit::Fields::ERROR_MESSAGE => error.message,
+          dropped_pairs: dropped_pairs
+        )
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/labkit/rate_limit.rb

@@ -25,6 +25,7 @@ module Labkit
     autoload :Evaluator, "labkit/rate_limit/evaluator"
     autoload :Limiter, "labkit/rate_limit/limiter"
     autoload :Metrics, "labkit/rate_limit/metrics"
+    autoload :BlockSummary, "labkit/rate_limit/block_summary"
 
     class << self
       def configure

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 5.7.0
+  version: 5.8.0
 platform: ruby
 authors:
 - Andrew Newdigate
@@ -651,6 +651,7 @@ files:
 - lib/labkit/net_http_publisher.rb
 - lib/labkit/rate_limit.rb
 - lib/labkit/rate_limit/README.md
+- lib/labkit/rate_limit/block_summary.rb
 - lib/labkit/rate_limit/configuration.rb
 - lib/labkit/rate_limit/evaluator.rb
 - lib/labkit/rate_limit/identifier.rb