gitlab-labkit 5.2.1 → 5.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a73e80c74e6f2578a9a86a5ad9c76c3c3c9ff4afc8c1deea262408763e84588
-  data.tar.gz: e7e1d625f3c634a04fd9a6aea0036270966c3b3a6e924edfebfd56713ede00ef
+  metadata.gz: e333c26dffb8c22bb8a131e820b01920887c2792dcffcee8f4ee012663b70831
+  data.tar.gz: 1cd8ac6e3b87b8bf1e214234e0d44dfc2b593798554fb1a1206d6e94cc8a2863
 SHA512:
-  metadata.gz: 77d32a2075415b452d10b1b01a4ef3679d46bbaacd1bef798ce2b17b0b0109a9ec26565f8065600df41f1090752ae311b5a90fadfd434b4e39025d57dcf2f46f
-  data.tar.gz: c8e099df23ea1312476d3d1301044f5de159b6ed25098d755b47df2b7653b8958a9f30c778d5742620c4d804255ac31250f7834e090981803e54d26f4ab6f67b
+  metadata.gz: 4f602e792b4e8573f1ce8e29a9654221f274a3789e262af412f3b52a937970f5605748bbf9f4580698f6457436c561c905c7e1021e964f545bc511508892f8bc
+  data.tar.gz: b131bf8715cecbd30484b89299607b9781e6cda5c690da7dc24f851b1558fdd5eb78a417cbd9baaaf2d93393d3694de8c98aa180b0103e27f5e61d598aa110a8

data/.gitlab-ci.yml

@@ -79,6 +79,13 @@ conformance-spec:
   variables:
     LABKIT_RUBY_SHA: $CI_COMMIT_SHA
   rules:
+    # Renovate's project-token bot cannot create pipelines in labkit-spec
+    # (GitLab forbids project bots from joining other projects), so its MRs
+    # would hard-fail the bridge. Attempt the trigger but do not block; a
+    # human re-running the pipeline gates the MR properly (it runs as them).
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $GITLAB_USER_LOGIN =~ /^project_\d+_bot/'
+      changes: !reference [.conformance-trigger-paths, changes]
+      allow_failure: true
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes: !reference [.conformance-trigger-paths, changes]
   trigger:

data/lib/labkit/rate_limit/README.md

@@ -207,17 +207,63 @@ A `match` hash gates whether a rule applies. Each value is normalised through
 | plain value       | `eq`     | `match: { user: nil }`, `match: { method: "POST" }`    |
 | `Regexp`          | `re`     | `match: { endpoint: %r{\A/api/} }`                     |
 | `{ eq: <value> }` | `eq`     | `match: { method: { eq: "POST" } }` (YAML-friendly)    |
+| `{ ne: <value> }` | `ne`     | `match: { method: { ne: "OPTIONS" } }`                 |
 | `{ re: <source> }`| `re`     | `match: { endpoint: { re: '\A/api/' } }` (YAML-friendly) |
+| `{ nre: <source> }`| `nre`   | `match: { endpoint: { nre: '\A/api/' } }`              |
 | `{ oneOf: <Array> }` | `oneOf` | `match: { user: { oneOf: ["7", "42"] } }` (YAML-friendly) |
+| `{ noneOf: <Array> }` | `noneOf` | `match: { plan: { noneOf: ["premium"] } }`         |
+| `Array` of matchers | `all` | `match: { endpoint: [%r{\A/api/}, { nre: 'internal' }] }` |
 
-`re` coerces the identifier value via `#to_s` before matching, so it can be
-used against non-String values (e.g. matching a 503 status against `{ re: '^5' }`).
+`re` and `nre` coerce the identifier value via `#to_s` before matching, so they
+can be used against non-String values (e.g. matching a 503 status against
+`{ re: '^5' }`).
 
 `oneOf` is set membership via `Set#include?` (`eql?`/`hash` equality, no
 coercion): it agrees with `eq` for Strings, Symbols, booleans, nil, and
 same-class numerics, but cross-type numerics differ (`1 == 1.0`, yet
-`{ oneOf: [1] }` does not match `1.0`). A bare Array is rejected; membership must
-be spelled `{ oneOf: [...] }` explicitly.
+`{ oneOf: [1] }` does not match `1.0`). `noneOf` is its complement.
+
+An Array means **AND**, so one identifier key can carry several predicates:
+
+```yaml
+endpoint:
+  - nre: '/oauth/'
+  - nre: '\A/api/'
+  - nre: '\A/-/(?:health|liveness|readiness|metrics)'
+```
+
+Its entries must be matcher Hashes or bare `Regexp`s, the two shapes that
+already mean "a matcher" on their own. An Array of plain values is rejected.
+A value meant as one matcher, arriving as a list, therefore fails loudly rather
+than becoming a membership test. Membership is spelled `{ oneOf: [...] }`.
+
+An Array holds matchers, not other Arrays, so a match value nests three deep at
+most: an Array, a matcher Hash inside it, and that Hash's `oneOf` list. A value
+nesting deeper than `Matcher::MAX_NESTING_DEPTH` is rejected when the rule is
+built.
+
+#### Negation and absent keys
+
+`ne`, `nre` and `noneOf` are plain complements of their positive forms, with no
+special case for a missing key. The `Evaluator` looks an identifier key up
+rather than requiring it, so an absent key reaches the matcher as `nil`, and a
+complement is therefore **true** for one: a request with no `endpoint` is not a
+request to an excluded endpoint.
+
+That widens a rule rather than narrowing it. A rule that should apply only when
+the key is present says so, with a presence matcher alongside the exclusions:
+
+```ruby
+match: { endpoint: [/./, { nre: '\A/api/' }] }
+```
+
+Since this follows from inversion, a matcher that does accept `nil` inverts the
+other way: `{ ne: nil }`, `{ noneOf: [nil] }` and `{ nre: '^$' }` are all false
+for an absent key, the last because `nil.to_s` is `""`.
+
+There is no cap on regex source length. Rules are written by operators, and the
+only bound on a pattern is `Matcher::MATCH_TIMEOUT_SECONDS`, a 50ms wall-clock
+budget applied per compiled `Regexp`.
 
 Glob and prefix matchers are intentionally out of scope.
 

data/lib/labkit/rate_limit/matcher.rb

@@ -12,27 +12,49 @@ module Labkit
     #   - any plain value (String, Symbol, Integer, ...) -> :eq matcher
     #   - a Regexp instance                              -> :re matcher (Ruby convenience)
     #   - { eq: <value> }                                -> :eq matcher (canonical, YAML-compatible)
+    #   - { ne: <value> }                                -> :ne matcher (complement of :eq)
     #   - { re: <String|Regexp> }                        -> :re matcher (canonical, YAML-compatible)
+    #   - { nre: <String|Regexp> }                       -> :nre matcher (complement of :re)
     #   - { oneOf: <Array> }                             -> :oneOf matcher (set membership, YAML-compatible)
+    #   - { noneOf: <Array> }                            -> :noneOf matcher (complement of :oneOf)
+    #   - [ <matcher>, <matcher>, ... ]                  -> :all matcher (every entry must match)
     #
-    # A bare Array is rejected rather than treated as :oneOf, so a value that
-    # was meant as a single matcher hash but arrived as an Array fails loudly
-    # instead of silently becoming a membership test.
+    # An Array means AND, so several predicates can apply to one identifier key.
+    # Its entries must be matcher Hashes or bare Regexps, the two shapes that
+    # already mean "a matcher" on their own. An Array of plain values is
+    # rejected, so a value that was meant as a single matcher but arrived as a
+    # list fails loudly instead of silently becoming a membership test. An Array
+    # holds matchers rather than other Arrays, so a match value nests at most
+    # MAX_NESTING_DEPTH deep; see too_deeply_nested?.
     #
-    # :oneOf membership is Set#include? (eql?/hash equality), not :eq's ==, so
-    # cross-type numerics differ (1 == 1.0, but {oneOf: [1]} does not match 1.0).
+    # :oneOf and :noneOf membership is Set#include? (eql?/hash equality), not
+    # :eq's ==, so cross-type numerics differ (1 == 1.0, but {oneOf: [1]} does
+    # not match 1.0).
     #
     # Hash-key naming follows the metrics-catalog selector pattern. Glob,
     # prefix, and other matcher kinds are intentionally out of scope here; see
     # gitlab-com/gl-infra/production-engineering#28853 for that follow-up.
     #
-    # An :re matcher coerces the identifier value via #to_s before applying
-    # the regex, so callers can match non-String identifier values such as
-    # Integer status codes (e.g. {status: { re: "^5" }} against status: 503).
+    # The :re and :nre matchers coerce the identifier value via #to_s before
+    # applying the regex, so callers can match non-String identifier values such
+    # as Integer status codes (e.g. {status: { re: "^5" }} against status: 503).
+    #
+    # Each negation matcher is the plain complement of its positive form, with
+    # no special case for a missing key. Evaluator passes identifier[key], so an
+    # absent key arrives as nil, and the complement is therefore usually true
+    # for one: a request with no path is not a request to an excluded path. A
+    # rule that should apply only when the key is present says so, by adding a
+    # presence matcher alongside the exclusions:
+    #
+    #   path: [/./, { nre: '\A/api/' }]
     class Matcher < Data.define(:type, :value)
-      KNOWN_HASH_KEYS = %i[eq re oneOf].freeze
-      MAX_REGEX_SOURCE_LENGTH = 200
+      KNOWN_HASH_KEYS = %i[eq ne re nre oneOf noneOf].freeze
       ERROR_INSPECT_LIMIT = 80
+      # No Hash or Array may sit this many levels below the match value, which
+      # is level 0. The deepest valid shape is an Array (0) holding a matcher
+      # Hash (1) whose :oneOf list (2) holds plain values; see
+      # too_deeply_nested?.
+      MAX_NESTING_DEPTH = 3
 
       # Wall-clock budget for a single #match? call. Without it a match is
       # bounded only by whatever global the host sets (40s in GitLab Rails,
@@ -45,16 +67,23 @@ module Labkit
       # timeouts were wall-clock stalls, since Ruby counts real time and GC
       # pauses on GitLab's git fleet have a p50 of ~72ms. Under 4x CPU
       # oversubscription 50ms still fires on 0.16% of those matches, 5ms 1.33%.
+      #
+      # This timeout is the only bound on a pattern. There is no cap on regex
+      # source length: rules are written by operators rather than end users, and
+      # a cap that applied to {re: "..."} but not to a bare Regexp bounded
+      # nothing in practice, since callers pass Regexp constants.
       MATCH_TIMEOUT_SECONDS = 0.05
 
       def self.build(input)
+        raise ArgumentError, nesting_error(input) if too_deeply_nested?(input)
+
         case input
         when Regexp
-          new(type: :re, value: with_match_timeout(input))
+          new(type: :re, value: with_match_timeout(:re, input))
         when Hash
           from_hash(input)
         when Array
-          raise ArgumentError, invalid_shape_error(input)
+          compose(input)
         else
           new(type: :eq, value: input)
         end
@@ -64,6 +93,15 @@ module Labkit
         raise ArgumentError, invalid_shape_error(input) if input.size != 1
 
         type, source = input.first
+
+        # Identifier rejects keys the same way. Without this, a key such as an
+        # Integer or nil leaves NoMethodError for the caller instead of the
+        # ArgumentError every other bad shape raises.
+        unless type.respond_to?(:to_sym)
+          raise ArgumentError,
+            "rate-limit match value type key #{truncate_for_error(type)} must be a String or Symbol"
+        end
+
         type_sym = type.to_sym
 
         unless KNOWN_HASH_KEYS.include?(type_sym)
@@ -77,31 +115,45 @@ module Labkit
 
       def self.compile(type_sym, source)
         case type_sym
-        when :eq
-          new(type: :eq, value: source)
-        when :oneOf
+        when :eq, :ne
+          new(type: type_sym, value: source)
+        when :oneOf, :noneOf
           unless source.is_a?(Array)
             raise ArgumentError,
-              "rate-limit match value {oneOf: ...} must be an Array, got #{truncate_for_error(source)}"
+              "rate-limit match value {#{type_sym}: ...} must be an Array, got #{truncate_for_error(source)}"
           end
 
-          new(type: :oneOf, value: Set.new(source).freeze)
-        when :re
-          if source.to_s.length > MAX_REGEX_SOURCE_LENGTH
+          # {oneOf: []} matches nothing, so the rule never applies and no harm
+          # follows. Its complement matches everything, which is an exclusion
+          # that excludes nothing: rejected for the same reason an empty Array
+          # of matchers is.
+          if type_sym == :noneOf && source.empty?
             raise ArgumentError,
-              "rate-limit match value {re: ...} source exceeds #{MAX_REGEX_SOURCE_LENGTH} characters"
+              "rate-limit match value {noneOf: []} must not be empty, it matches everything"
           end
 
-          begin
-            new(type: :re, value: with_match_timeout(source))
-          rescue RegexpError, TypeError => e
-            raise ArgumentError,
-              "rate-limit match value {re: #{truncate_for_error(source)}} failed to compile: #{e.message}"
-          end
+          new(type: type_sym, value: Set.new(source).freeze)
+        when :re, :nre
+          new(type: type_sym, value: with_match_timeout(type_sym, source))
         end
       end
       private_class_method :compile
 
+      # Builds the :all matcher behind the Array form. Entries go back through
+      # .build, which is what keeps a nested regex bounded by the match timeout.
+      def self.compose(input)
+        raise ArgumentError, "rate-limit match value [] must not be empty" if input.empty?
+
+        unless input.all? { |entry| entry.is_a?(Hash) || entry.is_a?(Regexp) }
+          raise ArgumentError,
+            "rate-limit match value Array entries must each be a matcher Hash or Regexp, " \
+              "got #{truncate_for_error(input)}"
+        end
+
+        new(type: :all, value: input.map { |entry| build(entry) }.freeze)
+      end
+      private_class_method :compose
+
       # Compiles +source+ (a String pattern or an existing Regexp) into a
       # Regexp bounded by MATCH_TIMEOUT_SECONDS.
       #
@@ -116,7 +168,7 @@ module Labkit
       # Recompiling an existing Regexp preserves its source and options
       # (including the fixed-encoding flags) and therefore +#==+; only object
       # identity changes.
-      def self.with_match_timeout(source)
+      def self.with_match_timeout(key, source)
         return source if source.is_a?(Regexp) && source.timeout
 
         if source.is_a?(Regexp)
@@ -124,12 +176,19 @@ module Labkit
         else
           Regexp.new(source, timeout: MATCH_TIMEOUT_SECONDS)
         end
+      rescue RegexpError, TypeError => e
+        raise ArgumentError,
+          "rate-limit match value {#{key}: #{truncate_for_error(source)}} failed to compile: #{e.message}"
       end
 
       private_class_method :with_match_timeout
 
       def self.invalid_shape_error(input)
-        "rate-limit match value must be a single-key Hash like {re: \"...\"}, {eq: ...} or {oneOf: [...]}, " \
+        # No key list here: this fires on the shape, and from_hash names the
+        # accepted keys when the shape is right but the key is not. Keeping the
+        # fixed part short is also what lets truncate_for_error bound the whole
+        # message.
+        "rate-limit match value must be a single-key Hash, or an Array of matchers, " \
           "got #{truncate_for_error(input)}"
       end
       private_class_method :invalid_shape_error
@@ -140,14 +199,65 @@ module Labkit
       end
       private_class_method :truncate_for_error
 
+      def self.nesting_error(input)
+        # Does not inspect the input: it is too deep to render, which is the
+        # reason this fires. The class is the only hint available cheaply, and
+        # the nesting may be in a key or a value.
+        "rate-limit match value nests deeper than #{MAX_NESTING_DEPTH} levels (#{input.class})"
+      end
+      private_class_method :nesting_error
+
+      # A valid match value nests three deep at most: an Array of matcher
+      # Hashes whose :oneOf value is an Array of plain values. Anything past
+      # MAX_NESTING_DEPTH is rejected rather than built.
+      #
+      # The check is iterative because the alternative is finding out through
+      # #inspect, which recurses once per level while rendering an error
+      # message. Rules are built at boot and SystemStackError is not a
+      # StandardError, so that would take the process down rather than reject
+      # one rule, and rescuing it is not an option either: the VM aborts on a
+      # second overflow in the same process.
+      def self.too_deeply_nested?(input)
+        # A flat stack of node, depth pairs, so a visit allocates no Array. The
+        # pops must mirror the pushes: depth comes off first, then its node.
+        stack = [input, 0]
+
+        until stack.empty?
+          depth = stack.pop
+          node = stack.pop
+          next unless node.is_a?(Hash) || node.is_a?(Array)
+          return true if depth >= MAX_NESTING_DEPTH
+
+          # Keys as well as values: #inspect renders both, so a deeply nested
+          # key overflows the same way a deeply nested value does.
+          if node.is_a?(Hash)
+            node.each { |key, value| stack.push(key, depth + 1, value, depth + 1) }
+          else
+            node.each { |child| stack.push(child, depth + 1) }
+          end
+        end
+
+        false
+      end
+      private_class_method :too_deeply_nested?
+
       def match?(identifier_value)
         case type
         when :eq
           value == identifier_value
+        when :ne
+          value != identifier_value
         when :oneOf
           value.include?(identifier_value)
+        when :noneOf
+          # exclude? is ActiveSupport, and nothing in rate_limit requires it.
+          !value.include?(identifier_value) # rubocop:disable Rails/NegateInclude
         when :re
           value.match?(identifier_value.to_s)
+        when :nre
+          !value.match?(identifier_value.to_s)
+        when :all
+          value.all? { |matcher| matcher.match?(identifier_value) }
         else
           raise ArgumentError, "unknown matcher type: #{type.inspect}"
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 5.3.0
 platform: ruby
 authors:
 - Andrew Newdigate