gitlab-labkit 5.10.0 → 5.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/gitlab-labkit.gemspec +3 -1
- data/lib/labkit/rate_limit/README.md +39 -4
- data/lib/labkit/rate_limit/config/document.rb +64 -0
- data/lib/labkit/rate_limit/config/loader.rb +246 -0
- data/lib/labkit/rate_limit/config/parser.rb +1 -2
- data/lib/labkit/rate_limit/config/schema.rb +324 -0
- data/lib/labkit/rate_limit/config.rb +3 -0
- data/lib/labkit/rate_limit/evaluator.rb +21 -1
- data/lib/labkit/rate_limit/matcher.rb +44 -10
- data/lib/labkit/rate_limit/rule.rb +28 -12
- data/scripts/vendor-ratelimit-proto.sh +21 -10
- metadata +24 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5f286c7029701485334c9e4f78b26e799a0d74a016e64d7fd7bb881dfb82a853
|
|
4
|
+
data.tar.gz: 8ee6361b4da8dda630297bed4526623f6b53ca813feb2ce2ebc2a5ea4522b1cd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cd0a8fd90cee39271ac936e277ff72dde660db5b9d6615860142572a53fd2a7fbda90019b9fad4e0dae62c2c9d20656ae7ef905d588e9bea481b7f206bf890bd
|
|
7
|
+
data.tar.gz: 79e35c32ea2e8e01e0b3df319fe563cbbe43986e2dc2446dd02220c60545d75b8fc2c4a393b1a9eea852b18af8bfb800b3aba3e641c43b2e401cf649cb1d7fea
|
data/gitlab-labkit.gemspec
CHANGED
|
@@ -15,7 +15,8 @@ Gem::Specification.new do |spec|
|
|
|
15
15
|
spec.metadata = { "source_code_uri" => "https://gitlab.com/gitlab-org/ruby/gems/labkit-ruby" }
|
|
16
16
|
spec.license = "MIT"
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
# The vendored .proto sources are read by the specs only; the gem ships the generated Ruby.
|
|
19
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|tools)/}) || f.end_with?(".proto") }
|
|
19
20
|
spec.bindir = "exe"
|
|
20
21
|
spec.executables = %w[labkit-logging]
|
|
21
22
|
spec.require_paths = ["lib"]
|
|
@@ -35,6 +36,7 @@ Gem::Specification.new do |spec|
|
|
|
35
36
|
spec.add_runtime_dependency "opentracing", ">= 0.4", "< 1"
|
|
36
37
|
spec.add_runtime_dependency "pg_query", ">= 6.1.0", "< 7.0"
|
|
37
38
|
spec.add_runtime_dependency "prometheus-client-mmap", ">= 1.2", "< 2.0"
|
|
39
|
+
spec.add_runtime_dependency "re2", ">= 2.15", "< 3"
|
|
38
40
|
spec.add_runtime_dependency "redis", "> 3.0.0", "< 6.0.0"
|
|
39
41
|
spec.add_runtime_dependency "thrift", "< 0.25" # Pinned for jaeger-client 1.1.0 compatibility; thrift 0.25 requires transports to implement message_boundaries?
|
|
40
42
|
|
|
@@ -724,19 +724,54 @@ A gem schema that lacks or changes a vendored field raises `Labkit::RateLimit::C
|
|
|
724
724
|
So upgrade the `gitlab-ratelimit-config-v1` gem in the application first, then the `gitlab-labkit` release that updates the vendored copy.
|
|
725
725
|
|
|
726
726
|
Loading runs in four steps, `parse`, `schema`, `contract` and `rule`, and a failure raises `Labkit::RateLimit::Config::Error` whose `#step` names the step.
|
|
727
|
-
|
|
728
|
-
Until the schema step lands, no mode validates a document: strict mode lets an enum number that no `Action` defines through, such as `action: 7`, and lenient mode leaves an unknown `action` name as `ACTION_UNSPECIFIED`.
|
|
729
|
-
The schema step rejects both.
|
|
727
|
+
All four steps exist; the registry that applies a document to a `Limiter` follows in [Part 3 of the plan](https://gitlab.com/gitlab-com/gl-infra/production-engineering/-/work_items/30263).
|
|
730
728
|
|
|
731
729
|
```ruby
|
|
732
730
|
doc = Labkit::RateLimit::Config::Parser.parse(
|
|
733
731
|
"config/rate_limits/rack_request.yaml",
|
|
734
732
|
Labkit::RateLimit::Config::Proto::RateLimits,
|
|
735
|
-
strict: true # CI rejects unknown fields. Production loads with strict: false
|
|
733
|
+
strict: true # CI rejects unknown fields. Production loads with strict: false
|
|
736
734
|
)
|
|
735
|
+
Labkit::RateLimit::Config::Schema.check!(doc)
|
|
737
736
|
doc.rate_limits["rack_request"].rules.first.name
|
|
738
737
|
```
|
|
739
738
|
|
|
739
|
+
`Schema.check!` mirrors every protovalidate rule of the `gitlab.ratelimit.config.v1` schema: name format, the period presets, the `limit` and `ban_for_s` ranges, every list cap, and the message rules such as `rule.limit_requires_threshold`.
|
|
740
|
+
It accepts `AvailableLimiters`, `RateLimits` and `PublishRulesRequest`.
|
|
741
|
+
A failure lists every violation with the field path, message and rule id that protovalidate reports, such as `rate_limits["rack_request"].rules[0].period_s: must be in list [1, 15, 30, 60, 180, 300, 600, 900, 1800, 3600, 28800, 86400] (uint32.in)`, so a failure reads the same in Ruby and Go.
|
|
742
|
+
`Schema.violations(doc)` returns them without raising.
|
|
743
|
+
Inside one message the order is protovalidate's: message rules, then the oneof, then fields in declaration order.
|
|
744
|
+
A protobuf map keeps no document order and Go iterates it in an unspecified order, so Ruby reports map entries by sorted key and a cross-SDK comparison matches violations per entry, not by position.
|
|
745
|
+
Map keys in a path are quoted as Go's `strconv.Quote` prints them.
|
|
746
|
+
The check is the strict reading of the schema: a matcher whose operator this binary does not know fails `required`.
|
|
747
|
+
Lenient loading, through the `Loader` below, drops such a rule with a warning before the schema step, so a newer operator does not reject the file.
|
|
748
|
+
`Labkit::RateLimit::Config::Loader` runs all four steps and returns a `Document`:
|
|
749
|
+
|
|
750
|
+
```ruby
|
|
751
|
+
loader = Labkit::RateLimit::Config::Loader.new(strict: Rails.env.test?, logger: Gitlab::AppLogger)
|
|
752
|
+
doc = loader.load(contracts: Dir["config/rate_limits/*.yaml"], rate_limits: "config/rate_limits.yaml")
|
|
753
|
+
doc.limiter?("rack_request") # => true
|
|
754
|
+
doc.available_properties("rack_request") # => [:plan, :path, :requester_id]
|
|
755
|
+
doc.default_rules("rack_request") # => [Rule, ...] from the contract, document order
|
|
756
|
+
doc.override_rules("rack_request") # => [Rule, ...] from the rate limits file
|
|
757
|
+
doc.warnings # => [{ step:, limiter:, rule:, message: }, ...]
|
|
758
|
+
```
|
|
759
|
+
|
|
760
|
+
The contract step merges the contract files: a limiter is declared, with its description, in exactly one file, a later file may add properties and append default rules, and every rule may use only the properties its limiter lists.
|
|
761
|
+
It rejects an override that takes a protected name or carries a `position`, and warns when an override drops a match key of the default it replaces.
|
|
762
|
+
A contract mistake rejects in both modes, because the contract is the application's own.
|
|
763
|
+
In lenient mode a rule for an unknown limiter, with an unknown property, with a protected name, or with a matcher whose only operator this version does not know is dropped with a warning, and a position is ignored; any other failure rejects the whole rate limits file.
|
|
764
|
+
Every warning is also logged as `rate_limit_config_warning` with `step`, `limiter`, `rule` and `detail`.
|
|
765
|
+
|
|
766
|
+
The rule step builds each proto rule as a `Rule` with `source: :file`.
|
|
767
|
+
A file rule compares identifier values as strings, as every SDK does for a rule from a document: `{ eq: "false" }` matches a `false` fact, a missing key reads as `""`, an integer reads as its decimal digits and a symbol as its name; other value types fall back to `to_s` and have no fixed cross-SDK form, so a host keeps them out of identifiers that file rules match on.
|
|
768
|
+
A rule built in Ruby keeps Ruby equality.
|
|
769
|
+
A file rule checks its name in every environment and compiles `re` and `nre` with the `re2` gem, so a pattern that only Ruby accepts, such as a lookbehind, is rejected at step `rule`, and `^` anchors the whole text as it does in Go and Rust.
|
|
770
|
+
`Document.empty` is a document with no limiters, for a host that has no contract yet.
|
|
771
|
+
|
|
772
|
+
`scripts/vendor-ratelimit-proto.sh` also copies `config.proto` and `constraint.proto` next to the generated code, and `schema_spec.rb` parses them and compares every field rule (caps, bounds, presets, flags), every message rule id and message, and the name pattern with `Schema`, so a `PROTOS_REF` bump that changes any of those fails the suite until `Schema` follows.
|
|
773
|
+
The CEL expressions themselves are mirrored by hand and pinned by the examples.
|
|
774
|
+
|
|
740
775
|
The parse step converts each YAML scalar by the type of the field it fills, so `eq: off`, `eq: True` and `eq: 9970` stay strings, `limit: "5"` becomes `5`, `action: "1"` becomes `ACTION_LIMIT`, and a YAML null leaves the field unset.
|
|
741
776
|
A repeated key, also one repeated under its camelCase `json_name`, a merge key (`<<`), an empty file, more than one document, or a non-scalar key fails at step `parse`, in both modes.
|
|
742
777
|
So does a value of the wrong kind for its field, such as a mapping where a scalar belongs, an integer out of range for its field, two fields of one `oneof`, such as `{ eq: a, re: b }`, a null list item or map value, and a document nested deeper than 64 levels.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Labkit
|
|
4
|
+
module RateLimit
|
|
5
|
+
module Config
|
|
6
|
+
# Document is what Loader#load returns: the merged contract and the rules
|
|
7
|
+
# of the rate limits file, per limiter, as Rule objects a Limiter can use.
|
|
8
|
+
# It is immutable; a host builds a new one to reload.
|
|
9
|
+
class Document
|
|
10
|
+
# One limiter's slice of the document.
|
|
11
|
+
Limiter = Data.define(:properties, :default_rules, :override_rules, :protected_rules)
|
|
12
|
+
|
|
13
|
+
attr_reader :warnings
|
|
14
|
+
|
|
15
|
+
# A document with no limiters: every host default applies unchanged.
|
|
16
|
+
def self.empty
|
|
17
|
+
new({}, [])
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# @param limiters [Hash{String => Limiter}]
|
|
21
|
+
# @param warnings [Array<Hash>] with :step, :limiter, :rule, :message
|
|
22
|
+
def initialize(limiters, warnings)
|
|
23
|
+
@limiters = limiters.freeze
|
|
24
|
+
@warnings = warnings.freeze
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def limiter?(name)
|
|
28
|
+
@limiters.key?(name.to_s)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @return [Array<String>] every limiter the contract declares
|
|
32
|
+
def limiters
|
|
33
|
+
@limiters.keys
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @return [Array<Symbol>] the properties a rule of this limiter may use
|
|
37
|
+
def available_properties(name)
|
|
38
|
+
slice(name)&.properties || []
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# @return [Array<Rule>] the file rules for this limiter, document order
|
|
42
|
+
def override_rules(name)
|
|
43
|
+
slice(name)&.override_rules || []
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @return [Array<Rule>] the contract's default rules, document order
|
|
47
|
+
def default_rules(name)
|
|
48
|
+
slice(name)&.default_rules || []
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# @return [Array<String>] default rule names no override may replace
|
|
52
|
+
def protected_rules(name)
|
|
53
|
+
slice(name)&.protected_rules || []
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def slice(name)
|
|
59
|
+
@limiters[name.to_s]
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Labkit
|
|
4
|
+
module RateLimit
|
|
5
|
+
module Config
|
|
6
|
+
# Loader runs the four steps of the config loader over a set of contract
|
|
7
|
+
# files and an optional rate limits file, and returns a Document.
|
|
8
|
+
#
|
|
9
|
+
# parse and schema are Parser and Schema. The contract step lives here: it
|
|
10
|
+
# merges the contract files, checks that every rule uses only properties
|
|
11
|
+
# its limiter lists, that no override takes a protected name, and that a
|
|
12
|
+
# file rule carries no position. The rule step converts each proto Rule
|
|
13
|
+
# into a Rule with source: :file, which compiles patterns with RE2.
|
|
14
|
+
#
|
|
15
|
+
# Strict mode, for CI, rejects on any failure. Lenient mode, for a running
|
|
16
|
+
# process, drops with a warning a rule for an unknown limiter, with an
|
|
17
|
+
# unknown property, with a protected name, or with a matcher whose only
|
|
18
|
+
# operator this binary does not know, and ignores a position. Any other
|
|
19
|
+
# failure rejects the whole file: the host then logs and runs on its
|
|
20
|
+
# defaults. Contract files are the application's own, so a contract
|
|
21
|
+
# mistake rejects in both modes.
|
|
22
|
+
class Loader
|
|
23
|
+
ACTIONS = { ACTION_LIMIT: :limit, ACTION_LOG: :log, ACTION_SKIP: :skip }.freeze
|
|
24
|
+
# The operators this version converts. A newer protos gem can decode
|
|
25
|
+
# an operator or an action outside these sets.
|
|
26
|
+
OPERATORS = %i[eq ne re nre one_of none_of all].freeze
|
|
27
|
+
|
|
28
|
+
# Merged view of one limiter across the contract files.
|
|
29
|
+
Spec = Struct.new(:properties, :default_rules, :protected_rules, :described)
|
|
30
|
+
|
|
31
|
+
# Default rules and properties per limiter, across every contract
|
|
32
|
+
# file. Schema caps one file's list; the merge must hold the same line.
|
|
33
|
+
MAX_DEFAULT_RULES = Schema::FIELD_RULES.fetch("LimiterSpec.default_rules").fetch(:max_items)
|
|
34
|
+
MAX_PROPERTIES = Schema::FIELD_RULES.fetch("LimiterSpec.available_properties").fetch(:max_items)
|
|
35
|
+
|
|
36
|
+
# @param strict [Boolean] reject on any failure instead of dropping rules
|
|
37
|
+
# @param logger [#warn, nil] receives each warning as it is recorded
|
|
38
|
+
def initialize(strict: false, logger: nil)
|
|
39
|
+
@strict = strict
|
|
40
|
+
@logger = logger || RateLimit.config.logger || Labkit::Logging::JsonLogger.new($stdout)
|
|
41
|
+
# load keeps the run's warnings on the instance, so two threads
|
|
42
|
+
# sharing one Loader would mix them.
|
|
43
|
+
@mutex = Mutex.new
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @param contracts [Array<String>] paths of available_limiters files, in order
|
|
47
|
+
# @param rate_limits [String, nil] path of the rate_limits file
|
|
48
|
+
# @return [Document]
|
|
49
|
+
# @raise [Error] with the step that failed
|
|
50
|
+
def load(contracts:, rate_limits: nil)
|
|
51
|
+
@mutex.synchronize { load_locked(contracts, rate_limits) }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
attr_reader :strict
|
|
57
|
+
|
|
58
|
+
def load_locked(contracts, rate_limits)
|
|
59
|
+
@warnings = []
|
|
60
|
+
specs = merge_contracts(contracts)
|
|
61
|
+
overrides = rate_limits ? load_rate_limits(rate_limits, specs) : {}
|
|
62
|
+
|
|
63
|
+
limiters = specs.to_h do |name, spec|
|
|
64
|
+
defaults = spec.default_rules.map { |rule| convert(name, rule) }
|
|
65
|
+
[name, Document::Limiter.new(spec.properties.freeze, defaults.freeze, overrides.fetch(name, []).freeze, spec.protected_rules.freeze)]
|
|
66
|
+
end
|
|
67
|
+
Document.new(limiters, @warnings)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def merge_contracts(paths)
|
|
71
|
+
specs = {}
|
|
72
|
+
paths.each do |path|
|
|
73
|
+
doc = Schema.check!(Parser.parse(path, Proto::AvailableLimiters, strict: true))
|
|
74
|
+
doc.available_limiters.each { |name, proto_spec| merge_limiter(specs, name, proto_spec, path) }
|
|
75
|
+
end
|
|
76
|
+
specs.each do |name, spec|
|
|
77
|
+
contract_error("limiter #{name} has no description in any contract file") unless spec.described
|
|
78
|
+
check_caps!(name, spec)
|
|
79
|
+
spec.default_rules.each { |rule| check_properties!(name, rule, spec) }
|
|
80
|
+
end
|
|
81
|
+
specs
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# A limiter is declared, with its description, in exactly one file. A
|
|
85
|
+
# later file may add properties and append default rules.
|
|
86
|
+
def merge_limiter(specs, name, proto_spec, path)
|
|
87
|
+
spec = specs[name] ||= Spec.new([], [], [], false)
|
|
88
|
+
unless proto_spec.description.empty?
|
|
89
|
+
contract_error("limiter #{name} is declared twice, again in #{path}") if spec.described
|
|
90
|
+
spec.described = true
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
proto_spec.available_properties.each { |p| spec.properties << p.to_sym unless spec.properties.include?(p.to_sym) }
|
|
94
|
+
proto_spec.default_rules.each do |rule|
|
|
95
|
+
contract_error("limiter #{name}: default rule #{rule.name} is defined twice, again in #{path}") if spec.default_rules.any? { |r| r.name == rule.name }
|
|
96
|
+
spec.default_rules << rule
|
|
97
|
+
end
|
|
98
|
+
spec.protected_rules.concat(proto_spec.protected_rules.to_a - spec.protected_rules)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def load_rate_limits(path, specs)
|
|
102
|
+
doc = Parser.parse(path, Proto::RateLimits, strict: strict)
|
|
103
|
+
drop_unknown_operators(doc) unless strict
|
|
104
|
+
Schema.check!(doc)
|
|
105
|
+
|
|
106
|
+
# Map#to_h ignores a block, so the overrides are collected by hand.
|
|
107
|
+
overrides = {}
|
|
108
|
+
doc.rate_limits.each do |name, list|
|
|
109
|
+
spec = specs[name]
|
|
110
|
+
if spec
|
|
111
|
+
overrides[name] = list.rules.filter_map { |rule| override_for(name, rule, spec) }
|
|
112
|
+
else
|
|
113
|
+
contract_error("rate_limits names limiter #{name}, which no contract declares") if strict
|
|
114
|
+
warn(:contract, name, "", "limiter #{name} is not in the contract; its rules are dropped")
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
overrides
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# An unknown operator is an empty oneof after the parse step, which the
|
|
121
|
+
# schema step would reject as a missing operator for the whole file.
|
|
122
|
+
def drop_unknown_operators(doc)
|
|
123
|
+
doc.rate_limits.each do |name, list|
|
|
124
|
+
kept = list.rules.reject do |rule|
|
|
125
|
+
dropped = rule.match.any? { |_, matcher| unknown_operator?(matcher) }
|
|
126
|
+
warn(:parse, name, rule.name, "a matcher uses an operator this version does not know; rule dropped") if dropped
|
|
127
|
+
dropped
|
|
128
|
+
end
|
|
129
|
+
list.rules.replace(kept)
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def unknown_operator?(matcher)
|
|
134
|
+
return true unless OPERATORS.include?(matcher.op)
|
|
135
|
+
|
|
136
|
+
matcher.op == :all && matcher.all.matchers.any? { |m| unknown_operator?(m) }
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# The contract checks on one file rule. Returns the Rule, or nil when
|
|
140
|
+
# lenient mode drops it.
|
|
141
|
+
def override_for(name, rule, spec)
|
|
142
|
+
return reject(:contract, name, rule.name, "#{rule.name} is a protected rule of #{name}; the contract alone sets its limit") if spec.protected_rules.include?(rule.name)
|
|
143
|
+
|
|
144
|
+
unknown = unknown_property(rule, spec)
|
|
145
|
+
return reject(:contract, name, rule.name, "property #{unknown} is not listed for limiter #{name}") if unknown
|
|
146
|
+
|
|
147
|
+
if rule.has_position?
|
|
148
|
+
contract_error("limiter #{name}: rule #{rule.name} has a position; only a published rule can") if strict
|
|
149
|
+
warn(:contract, name, rule.name, "position is ignored on a file rule")
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
warn_dropped_match_keys(name, rule, spec)
|
|
153
|
+
convert(name, rule)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# An override that matches less than the default it replaces widens the
|
|
157
|
+
# rule. Allowed, since every matching rule counts, but worth a line in
|
|
158
|
+
# the log.
|
|
159
|
+
def warn_dropped_match_keys(name, rule, spec)
|
|
160
|
+
default = spec.default_rules.find { |d| d.name == rule.name }
|
|
161
|
+
return unless default
|
|
162
|
+
|
|
163
|
+
dropped = default.match.keys - rule.match.keys
|
|
164
|
+
warn(:contract, name, rule.name, "dropped match keys: #{dropped.sort.join(', ')}") unless dropped.empty?
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def check_caps!(name, spec)
|
|
168
|
+
rules = spec.default_rules.size
|
|
169
|
+
contract_error("limiter #{name} has #{rules} default rules across the contract files; at most #{MAX_DEFAULT_RULES} are allowed") if rules > MAX_DEFAULT_RULES
|
|
170
|
+
|
|
171
|
+
properties = spec.properties.size
|
|
172
|
+
contract_error("limiter #{name} lists #{properties} properties across the contract files; at most #{MAX_PROPERTIES} are allowed") if properties > MAX_PROPERTIES
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def check_properties!(name, rule, spec)
|
|
176
|
+
unknown = unknown_property(rule, spec)
|
|
177
|
+
contract_error("limiter #{name}: default rule #{rule.name} uses property #{unknown}, which the limiter does not list") if unknown
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def unknown_property(rule, spec)
|
|
181
|
+
(used_properties(rule) - spec.properties).first
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def used_properties(rule)
|
|
185
|
+
used = rule.characteristics.map(&:to_sym) + rule.match.keys.map(&:to_sym)
|
|
186
|
+
used << rule.count_distinct.to_sym unless rule.count_distinct.empty?
|
|
187
|
+
used
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def convert(limiter, rule)
|
|
191
|
+
Rule.new(
|
|
192
|
+
name: rule.name,
|
|
193
|
+
action: ACTIONS.fetch(rule.action) { raise ArgumentError, "unknown action #{rule.action.inspect}" },
|
|
194
|
+
limit: rule.has_limit? ? rule.limit : nil,
|
|
195
|
+
period: rule.has_period_s? ? rule.period_s : nil,
|
|
196
|
+
ban_for: rule.has_ban_for_s? ? rule.ban_for_s : nil,
|
|
197
|
+
characteristics: rule.characteristics.to_a,
|
|
198
|
+
count_distinct: rule.count_distinct.empty? ? nil : rule.count_distinct,
|
|
199
|
+
match: match_input(rule.match),
|
|
200
|
+
source: :file
|
|
201
|
+
)
|
|
202
|
+
rescue ArgumentError => e
|
|
203
|
+
raise Error.new(:rule, "#{limiter}/#{rule.name}: #{e.message}")
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Map#to_h would turn the nested messages into plain Hashes, so the
|
|
207
|
+
# entries are walked as messages.
|
|
208
|
+
def match_input(map)
|
|
209
|
+
input = {}
|
|
210
|
+
map.each { |key, matcher| input[key] = matcher_input(matcher) }
|
|
211
|
+
input
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# The proto Matcher in the shape Rule.new accepts.
|
|
215
|
+
def matcher_input(matcher)
|
|
216
|
+
case matcher.op
|
|
217
|
+
when :eq, :ne, :re, :nre then { matcher.op => matcher[matcher.op.to_s] }
|
|
218
|
+
when :one_of then { oneOf: matcher.one_of.values.to_a }
|
|
219
|
+
when :none_of then { noneOf: matcher.none_of.values.to_a }
|
|
220
|
+
when :all then matcher.all.matchers.map { |m| matcher_input(m) }
|
|
221
|
+
else raise ArgumentError, "unknown matcher operator #{matcher.op.inspect}"
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def reject(step, limiter, rule, message)
|
|
226
|
+
contract_error("limiter #{limiter}: rule #{rule}: #{message}") if strict
|
|
227
|
+
|
|
228
|
+
warn(step, limiter, rule, message)
|
|
229
|
+
nil
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def warn(step, limiter, rule, message)
|
|
233
|
+
warning = { step: step, limiter: limiter, rule: rule, message: message }
|
|
234
|
+
@warnings << warning
|
|
235
|
+
# The warning's own :message is the detail; the log line keeps the
|
|
236
|
+
# fixed message every rate_limit log line has.
|
|
237
|
+
@logger.warn(message: "rate_limit_config_warning", step: step, limiter: limiter, rule: rule, detail: message)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def contract_error(message)
|
|
241
|
+
raise Error.new(:contract, message)
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
end
|
|
@@ -22,8 +22,7 @@ module Labkit
|
|
|
22
22
|
# number that no Action defines, such as `action: 7`, passes in both
|
|
23
23
|
# modes, and in lenient mode protobuf drops an unknown enum name together
|
|
24
24
|
# with the unknown fields, so a rule with a misspelt action reaches the
|
|
25
|
-
# schema step as ACTION_UNSPECIFIED.
|
|
26
|
-
# both.
|
|
25
|
+
# schema step as ACTION_UNSPECIFIED. Schema.check! rejects both.
|
|
27
26
|
module Parser
|
|
28
27
|
INT32 = (-(2**31))...(2**31)
|
|
29
28
|
UINT32 = 0...(2**32)
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Labkit
|
|
4
|
+
module RateLimit
|
|
5
|
+
module Config
|
|
6
|
+
# Schema is the schema step: the checks that gitlab/ratelimit/config/v1
|
|
7
|
+
# declares with protovalidate, mirrored in Ruby because there is no
|
|
8
|
+
# protovalidate runtime for Ruby. Each check keeps the protovalidate rule
|
|
9
|
+
# id, message and field path. Inside one message the order is
|
|
10
|
+
# protovalidate's: message rules, then the oneof, then fields in
|
|
11
|
+
# declaration order.
|
|
12
|
+
# A protobuf map keeps no document order and Go iterates it in an
|
|
13
|
+
# unspecified order, so Ruby reports map entries by sorted key and the
|
|
14
|
+
# corpus compares per entry.
|
|
15
|
+
#
|
|
16
|
+
# FIELD_RULES and MESSAGE_RULES are copied from the proto by hand, and
|
|
17
|
+
# schema_spec.rb compares both with the vendored config.proto and
|
|
18
|
+
# constraint.proto field by field, so a PROTOS_REF bump that changes a
|
|
19
|
+
# cap, a flag, a rule id or a message fails the suite until this file
|
|
20
|
+
# follows. The CEL expressions have no such check; the examples pin them.
|
|
21
|
+
#
|
|
22
|
+
# The check is the strict reading of the schema. Lenient loading (Loader)
|
|
23
|
+
# drops a rule whose matcher has no known operator before this step,
|
|
24
|
+
# because such a matcher fails `required` here.
|
|
25
|
+
module Schema
|
|
26
|
+
# for_key marks a violation on a map key; its path is the entry's path.
|
|
27
|
+
Violation = Data.define(:path, :rule_id, :message, :for_key) do
|
|
28
|
+
def initialize(path:, rule_id:, message:, for_key: false)
|
|
29
|
+
super
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def to_s
|
|
33
|
+
path.empty? ? "#{message} (#{rule_id})" : "#{path}: #{message} (#{rule_id})"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# CEL matches() is RE2, where ^ and $ anchor the whole text; Ruby's $
|
|
38
|
+
# matches before a newline, so the mirror anchors with \A and \z.
|
|
39
|
+
NAME = /\A[a-z0-9_]{1,64}\z/
|
|
40
|
+
NAME_MESSAGE = "invalid name; must match ^[a-z0-9_]{1,64}$"
|
|
41
|
+
PERIODS = [1, 15, 30, 60, 180, 300, 600, 900, 1800, 3600, 28800, 86400].freeze
|
|
42
|
+
SKIP = :ACTION_SKIP
|
|
43
|
+
|
|
44
|
+
# The field rules of config.proto, keyed "Message.field". The keys of a
|
|
45
|
+
# map field and the items of a repeated field carry their rule under
|
|
46
|
+
# keys_* and items_*.
|
|
47
|
+
FIELD_RULES = {
|
|
48
|
+
"AvailableLimiters.available_limiters" => { min_pairs: 1, keys_is_name: true },
|
|
49
|
+
"LimiterSpec.description" => { max_len: 1024 },
|
|
50
|
+
"LimiterSpec.available_properties" => { unique: true, max_items: 100, items_is_name: true },
|
|
51
|
+
"LimiterSpec.default_rules" => { max_items: 200 },
|
|
52
|
+
"LimiterSpec.protected_rules" => { unique: true, max_items: 200, items_is_name: true },
|
|
53
|
+
"RateLimits.rate_limits" => { keys_is_name: true },
|
|
54
|
+
"RuleList.rules" => { max_items: 200 },
|
|
55
|
+
"Rule.name" => { is_name: true },
|
|
56
|
+
"Rule.description" => { max_len: 1024 },
|
|
57
|
+
"Rule.action" => { required: true, defined_only: true },
|
|
58
|
+
"Rule.characteristics" => { unique: true, max_items: 10, items_is_name: true },
|
|
59
|
+
"Rule.match" => { max_pairs: 20, keys_is_name: true },
|
|
60
|
+
"Rule.limit" => { gte: 1, lte: 1_000_000 },
|
|
61
|
+
"Rule.period_s" => { in: PERIODS },
|
|
62
|
+
"Rule.count_distinct" => { is_name: true, ignore_empty: true },
|
|
63
|
+
"Rule.ban_for_s" => { gte: 1, lte: 86_400 },
|
|
64
|
+
"PublishRulesRequest.rate_limits" => { keys_is_name: true },
|
|
65
|
+
"PublishRulesRequest.remove" => { keys_is_name: true },
|
|
66
|
+
"RuleNames.names" => { min_items: 1, max_items: 200, unique: true, items_is_name: true },
|
|
67
|
+
"Matcher.eq" => { max_len: 1024 },
|
|
68
|
+
"Matcher.ne" => { max_len: 1024 },
|
|
69
|
+
"Matcher.re" => { max_len: 1024 },
|
|
70
|
+
"Matcher.nre" => { max_len: 1024 },
|
|
71
|
+
"StringList.values" => { min_items: 1, max_items: 100, unique: true, items_max_len: 1024 },
|
|
72
|
+
"MatcherList.matchers" => { min_items: 1, max_items: 20 }
|
|
73
|
+
}.freeze
|
|
74
|
+
|
|
75
|
+
# The message-level CEL rules of config.proto, by id.
|
|
76
|
+
MESSAGE_RULES = {
|
|
77
|
+
"limiter_spec.protected_rules_declared" => "every protected rule must be a default rule of this limiter",
|
|
78
|
+
"limiter_spec.unique_default_names" => "default rule names must be unique within a limiter",
|
|
79
|
+
"limiter_spec.no_position_in_defaults" => "position is not allowed on a default rule",
|
|
80
|
+
"rule_list.unique_names" => "rule names must be unique within a limiter",
|
|
81
|
+
"rule.limit_requires_threshold" => "limit and period_s are required unless action is skip",
|
|
82
|
+
"rule.ban_excludes_count_distinct" => "ban_for_s cannot be combined with count_distinct",
|
|
83
|
+
"rule.ban_not_on_skip" => "ban_for_s is not allowed on a skip rule",
|
|
84
|
+
"rule.no_counter_on_skip" => "limit, period_s, characteristics and count_distinct are not allowed on a skip rule",
|
|
85
|
+
"rule.count_distinct_not_characteristic" => "count_distinct cannot also be a characteristic",
|
|
86
|
+
"publish.name_in_both" => "a rule name cannot be in both rate_limits and remove for one limiter",
|
|
87
|
+
"matcher_list.no_nested_all" => "all cannot contain another all"
|
|
88
|
+
}.freeze
|
|
89
|
+
|
|
90
|
+
# The ASCII characters that strconv.Quote prints with a backslash escape.
|
|
91
|
+
ESCAPES = {
|
|
92
|
+
'"' => '\\"', "\\" => "\\\\", "\a" => "\\a", "\b" => "\\b", "\f" => "\\f",
|
|
93
|
+
"\n" => "\\n", "\r" => "\\r", "\t" => "\\t", "\v" => "\\v"
|
|
94
|
+
}.freeze
|
|
95
|
+
|
|
96
|
+
class << self
|
|
97
|
+
# @param message [Proto::AvailableLimiters, Proto::RateLimits, Proto::PublishRulesRequest]
|
|
98
|
+
# @return the message
|
|
99
|
+
# @raise [Error] step :schema, every violation joined with "; "
|
|
100
|
+
def check!(message)
|
|
101
|
+
found = violations(message)
|
|
102
|
+
raise Error.new(:schema, found.join("; ")) unless found.empty?
|
|
103
|
+
|
|
104
|
+
message
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# @return [Array<Violation>] every violation, in the order described above
|
|
108
|
+
def violations(message)
|
|
109
|
+
checker = Checker.new
|
|
110
|
+
case message
|
|
111
|
+
when Proto::AvailableLimiters then checker.available_limiters(message)
|
|
112
|
+
when Proto::RateLimits then checker.rate_limits(message)
|
|
113
|
+
when Proto::PublishRulesRequest then checker.publish_rules_request(message)
|
|
114
|
+
else
|
|
115
|
+
raise ArgumentError, "no schema for #{message.class}; expected AvailableLimiters, RateLimits or PublishRulesRequest"
|
|
116
|
+
end
|
|
117
|
+
checker.violations
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Quotes a map key for a path the way Go's strconv.Quote does, so the
|
|
121
|
+
# path reads the same in every SDK: backslash escapes for the ASCII
|
|
122
|
+
# controls that have one, \xNN for the other ASCII controls and DEL,
|
|
123
|
+
# \uNNNN or \UNNNNNNNN for a non-printable rune, everything else as is.
|
|
124
|
+
def quote(key)
|
|
125
|
+
out = +'"'
|
|
126
|
+
key.each_char { |c| out << (ESCAPES[c] || escape_rune(c)) }
|
|
127
|
+
out << '"'
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
private
|
|
131
|
+
|
|
132
|
+
# Go's unicode.IsPrint: letters, marks, numbers, punctuation, symbols
|
|
133
|
+
# and the ASCII space.
|
|
134
|
+
def escape_rune(char)
|
|
135
|
+
code = char.ord
|
|
136
|
+
return format("\\x%02x", code) if code < 0x20 || code == 0x7f
|
|
137
|
+
return char if char == " " || char.match?(/\A[\p{L}\p{M}\p{N}\p{P}\p{S}]\z/)
|
|
138
|
+
|
|
139
|
+
code > 0xffff ? format("\\U%08x", code) : format("\\u%04x", code)
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Walks one message and collects violations. Paths follow protovalidate:
|
|
144
|
+
# `rate_limits["rack_request"].rules[0].match["plan"].eq`.
|
|
145
|
+
class Checker
|
|
146
|
+
attr_reader :violations
|
|
147
|
+
|
|
148
|
+
def initialize
|
|
149
|
+
@violations = []
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def available_limiters(doc)
|
|
153
|
+
map_field("available_limiters", doc.available_limiters, FIELD_RULES["AvailableLimiters.available_limiters"]) { |path, spec| limiter_spec(path, spec) }
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def rate_limits(doc)
|
|
157
|
+
map_field("rate_limits", doc.rate_limits, FIELD_RULES["RateLimits.rate_limits"]) { |path, list| rule_list(path, list) }
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def publish_rules_request(req)
|
|
161
|
+
# The outcome of one name in both fields would depend on the order
|
|
162
|
+
# the store applies them.
|
|
163
|
+
both = req.rate_limits.any? do |key, list|
|
|
164
|
+
req.remove.has_key?(key) && list.rules.any? { |r| req.remove[key].names.include?(r.name) }
|
|
165
|
+
end
|
|
166
|
+
message_rule("", "publish.name_in_both") if both
|
|
167
|
+
map_field("rate_limits", req.rate_limits, FIELD_RULES["PublishRulesRequest.rate_limits"]) { |path, list| rule_list(path, list) }
|
|
168
|
+
map_field("remove", req.remove, FIELD_RULES["PublishRulesRequest.remove"]) do |path, names|
|
|
169
|
+
list("#{path}.names", names.names, FIELD_RULES["RuleNames.names"])
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
private
|
|
174
|
+
|
|
175
|
+
def limiter_spec(path, spec)
|
|
176
|
+
default_names = spec.default_rules.map(&:name)
|
|
177
|
+
message_rule(path, "limiter_spec.protected_rules_declared") unless spec.protected_rules.all? { |n| default_names.include?(n) }
|
|
178
|
+
message_rule(path, "limiter_spec.unique_default_names") unless default_names.uniq.size == default_names.size
|
|
179
|
+
# Default rules keep document order, so a position has no meaning there.
|
|
180
|
+
message_rule(path, "limiter_spec.no_position_in_defaults") if spec.default_rules.any?(&:has_position?)
|
|
181
|
+
|
|
182
|
+
string("#{path}.description", spec.description, FIELD_RULES["LimiterSpec.description"])
|
|
183
|
+
list("#{path}.available_properties", spec.available_properties, FIELD_RULES["LimiterSpec.available_properties"])
|
|
184
|
+
list("#{path}.default_rules", spec.default_rules, FIELD_RULES["LimiterSpec.default_rules"])
|
|
185
|
+
spec.default_rules.each_with_index { |r, i| rule("#{path}.default_rules[#{i}]", r) }
|
|
186
|
+
list("#{path}.protected_rules", spec.protected_rules, FIELD_RULES["LimiterSpec.protected_rules"])
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def rule_list(path, list)
|
|
190
|
+
names = list.rules.map(&:name)
|
|
191
|
+
message_rule(path, "rule_list.unique_names") unless names.uniq.size == names.size
|
|
192
|
+
list("#{path}.rules", list.rules, FIELD_RULES["RuleList.rules"])
|
|
193
|
+
list.rules.each_with_index { |r, i| rule("#{path}.rules[#{i}]", r) }
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def rule(path, rule)
|
|
197
|
+
rule_message_rules(path, rule)
|
|
198
|
+
string("#{path}.name", rule.name, FIELD_RULES["Rule.name"])
|
|
199
|
+
string("#{path}.description", rule.description, FIELD_RULES["Rule.description"])
|
|
200
|
+
action("#{path}.action", rule.action, FIELD_RULES["Rule.action"])
|
|
201
|
+
list("#{path}.characteristics", rule.characteristics, FIELD_RULES["Rule.characteristics"])
|
|
202
|
+
map_field("#{path}.match", rule.match, FIELD_RULES["Rule.match"]) { |entry, matcher| matcher(entry, matcher) }
|
|
203
|
+
integer("#{path}.limit", rule.limit, FIELD_RULES["Rule.limit"]) if rule.has_limit?
|
|
204
|
+
integer("#{path}.period_s", rule.period_s, FIELD_RULES["Rule.period_s"]) if rule.has_period_s?
|
|
205
|
+
string("#{path}.count_distinct", rule.count_distinct, FIELD_RULES["Rule.count_distinct"])
|
|
206
|
+
integer("#{path}.ban_for_s", rule.ban_for_s, FIELD_RULES["Rule.ban_for_s"]) if rule.has_ban_for_s?
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# In the order config.proto declares them.
|
|
210
|
+
def rule_message_rules(path, rule)
|
|
211
|
+
counts_distinct = !rule.count_distinct.empty?
|
|
212
|
+
message_rule(path, "rule.limit_requires_threshold") unless rule.action == SKIP || (rule.has_limit? && rule.has_period_s?)
|
|
213
|
+
message_rule(path, "rule.ban_excludes_count_distinct") if rule.has_ban_for_s? && counts_distinct
|
|
214
|
+
skip_rules(path, rule) if rule.action == SKIP
|
|
215
|
+
message_rule(path, "rule.count_distinct_not_characteristic") if counts_distinct && rule.characteristics.include?(rule.count_distinct)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# A skip rule has no counter and no limit to cross.
|
|
219
|
+
def skip_rules(path, rule)
|
|
220
|
+
counter = rule.has_limit? || rule.has_period_s? || !rule.characteristics.empty? || !rule.count_distinct.empty?
|
|
221
|
+
message_rule(path, "rule.ban_not_on_skip") if rule.has_ban_for_s?
|
|
222
|
+
message_rule(path, "rule.no_counter_on_skip") if counter
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# An unknown enum number decodes as an Integer in Ruby.
|
|
226
|
+
def action(path, value, rules)
|
|
227
|
+
if rules[:required] && value == :ACTION_UNSPECIFIED
|
|
228
|
+
add(path, "required", "value is required")
|
|
229
|
+
elsif rules[:defined_only] && value.is_a?(Integer)
|
|
230
|
+
add(path, "enum.defined_only", "value must be one of the defined enum values")
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def matcher(path, matcher)
|
|
235
|
+
op = matcher.op
|
|
236
|
+
return add("#{path}.op", "required", "exactly one field is required in oneof") if op.nil?
|
|
237
|
+
|
|
238
|
+
case op
|
|
239
|
+
when :eq, :ne, :re, :nre then string("#{path}.#{op}", matcher[op.to_s], FIELD_RULES["Matcher.#{op}"])
|
|
240
|
+
when :one_of, :none_of then list("#{path}.#{op}.values", matcher[op.to_s].values, FIELD_RULES["StringList.values"])
|
|
241
|
+
when :all then matcher_list("#{path}.all", matcher.all)
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def matcher_list(path, list)
|
|
246
|
+
# There is no `any`, so an `all` inside an `all` is one flat list.
|
|
247
|
+
message_rule(path, "matcher_list.no_nested_all") if list.matchers.any? { |m| m.op == :all }
|
|
248
|
+
list("#{path}.matchers", list.matchers, FIELD_RULES["MatcherList.matchers"])
|
|
249
|
+
list.matchers.each_with_index { |m, i| matcher("#{path}.matchers[#{i}]", m) }
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Rules of RepeatedRules in field order: min_items, max_items, unique,
|
|
253
|
+
# then the items.
|
|
254
|
+
def list(path, values, rules)
|
|
255
|
+
add(path, "repeated.min_items", "must contain at least #{rules[:min_items]} item(s)") if rules[:min_items] && values.size < rules[:min_items]
|
|
256
|
+
add(path, "repeated.max_items", "must contain no more than #{rules[:max_items]} item(s)") if rules[:max_items] && values.size > rules[:max_items]
|
|
257
|
+
add(path, "repeated.unique", "repeated value must contain unique items") if rules[:unique] && values.uniq.size != values.size
|
|
258
|
+
values.each_with_index { |v, i| name("#{path}[#{i}]", v) } if rules[:items_is_name]
|
|
259
|
+
values.each_with_index { |v, i| max_len("#{path}[#{i}]", v, rules[:items_max_len]) } if rules[:items_max_len]
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# Rules of MapRules in field order: min_pairs, max_pairs, then the
|
|
263
|
+
# keys. Yields each entry's path and value.
|
|
264
|
+
def map_field(path, map, rules)
|
|
265
|
+
add(path, "map.min_pairs", "map must be at least #{rules[:min_pairs]} entries") if rules[:min_pairs] && map.length < rules[:min_pairs]
|
|
266
|
+
add(path, "map.max_pairs", "map must be at most #{rules[:max_pairs]} entries") if rules[:max_pairs] && map.length > rules[:max_pairs]
|
|
267
|
+
map.keys.sort.each do |key|
|
|
268
|
+
value = map[key]
|
|
269
|
+
entry = "#{path}[#{Schema.quote(key)}]"
|
|
270
|
+
add(entry, "string.is_name", NAME_MESSAGE, for_key: true) if rules[:keys_is_name] && !NAME.match?(key)
|
|
271
|
+
yield entry, value
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def string(path, value, rules)
|
|
276
|
+
return if rules[:ignore_empty] && value.empty?
|
|
277
|
+
|
|
278
|
+
max_len(path, value, rules[:max_len]) if rules[:max_len]
|
|
279
|
+
name(path, value) if rules[:is_name]
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# protovalidate reports one of three ids for bounds, by which bounds
|
|
283
|
+
# the field declares.
|
|
284
|
+
def integer(path, value, rules)
|
|
285
|
+
if rules[:in]
|
|
286
|
+
add(path, "uint32.in", "must be in list #{rules[:in]}") unless rules[:in].include?(value)
|
|
287
|
+
return
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
gte, lte = rules.values_at(:gte, :lte)
|
|
291
|
+
below = gte && value < gte
|
|
292
|
+
above = lte && value > lte
|
|
293
|
+
return unless below || above
|
|
294
|
+
|
|
295
|
+
if gte && lte
|
|
296
|
+
add(path, "uint32.gte_lte", "must be greater than or equal to #{gte} and less than or equal to #{lte}")
|
|
297
|
+
elsif below
|
|
298
|
+
add(path, "uint32.gte", "must be greater than or equal to #{gte}")
|
|
299
|
+
else
|
|
300
|
+
add(path, "uint32.lte", "must be less than or equal to #{lte}")
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def message_rule(path, rule_id)
|
|
305
|
+
add(path, rule_id, MESSAGE_RULES.fetch(rule_id))
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
def name(path, value)
|
|
309
|
+
add(path, "string.is_name", NAME_MESSAGE) unless NAME.match?(value)
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def max_len(path, value, max)
|
|
313
|
+
add(path, "string.max_len", "must be at most #{max} characters") if value.length > max
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def add(path, rule_id, message, for_key: false)
|
|
317
|
+
@violations << Violation.new(path: path, rule_id: rule_id, message: message, for_key: for_key)
|
|
318
|
+
end
|
|
319
|
+
end
|
|
320
|
+
private_constant :Checker
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
end
|
|
@@ -16,7 +16,10 @@ module Labkit
|
|
|
16
16
|
Proto = Gitlab::Ratelimit::Config::V1
|
|
17
17
|
|
|
18
18
|
autoload :Error, "labkit/rate_limit/config/error"
|
|
19
|
+
autoload :Document, "labkit/rate_limit/config/document"
|
|
20
|
+
autoload :Loader, "labkit/rate_limit/config/loader"
|
|
19
21
|
autoload :Parser, "labkit/rate_limit/config/parser"
|
|
22
|
+
autoload :Schema, "labkit/rate_limit/config/schema"
|
|
20
23
|
end
|
|
21
24
|
end
|
|
22
25
|
end
|
|
@@ -321,8 +321,28 @@ module Labkit
|
|
|
321
321
|
cursor.rule = nil unless e
|
|
322
322
|
end
|
|
323
323
|
|
|
324
|
+
# A file rule compares as strings, the way every SDK does for a rule
|
|
325
|
+
# from a document. A code rule keeps Ruby equality, so {eq: true}
|
|
326
|
+
# matches true only.
|
|
324
327
|
def rule_matches?(rule, identifier)
|
|
325
|
-
rule.
|
|
328
|
+
if rule.source == :file
|
|
329
|
+
rule.match.all? { |key, matcher| matcher.match?(file_value(identifier[key])) }
|
|
330
|
+
else
|
|
331
|
+
rule.match.all? { |key, matcher| matcher.match?(identifier[key]) }
|
|
332
|
+
end
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
# The string form of an identifier value for a file rule. A missing key
|
|
336
|
+
# reads as "". Strings, symbols, booleans and integers have one form in
|
|
337
|
+
# every SDK; anything else falls back to to_s and is the host's to keep
|
|
338
|
+
# out of identifiers.
|
|
339
|
+
def file_value(value)
|
|
340
|
+
case value
|
|
341
|
+
when nil then ""
|
|
342
|
+
when String then value
|
|
343
|
+
when Symbol then value.name
|
|
344
|
+
else value.to_s
|
|
345
|
+
end
|
|
326
346
|
end
|
|
327
347
|
|
|
328
348
|
def missing_count_distinct_value?(rule, identifier)
|
|
@@ -47,6 +47,12 @@ module Labkit
|
|
|
47
47
|
# presence matcher alongside the exclusions:
|
|
48
48
|
#
|
|
49
49
|
# path: [/./, { nre: '\A/api/' }]
|
|
50
|
+
#
|
|
51
|
+
# Rules from a config file are built with engine: :re2. Their :re and :nre
|
|
52
|
+
# patterns compile with RE2, the syntax every SDK shares, so a pattern that
|
|
53
|
+
# only Ruby accepts (lookbehind, a possessive quantifier) is rejected and a
|
|
54
|
+
# `^` anchors the whole text, not each line. RE2 has no backtracking, so
|
|
55
|
+
# those matchers need no timeout.
|
|
50
56
|
class Matcher < Data.define(:type, :value)
|
|
51
57
|
KNOWN_HASH_KEYS = %i[eq ne re nre oneOf noneOf].freeze
|
|
52
58
|
ERROR_INSPECT_LIMIT = 80
|
|
@@ -74,22 +80,27 @@ module Labkit
|
|
|
74
80
|
# nothing in practice, since callers pass Regexp constants.
|
|
75
81
|
MATCH_TIMEOUT_SECONDS = 0.05
|
|
76
82
|
|
|
77
|
-
|
|
83
|
+
ENGINES = %i[ruby re2].freeze
|
|
84
|
+
|
|
85
|
+
# engine is positional, not a keyword: callers write build(eq: x) and a
|
|
86
|
+
# keyword here would swallow that Hash.
|
|
87
|
+
def self.build(input, engine = :ruby)
|
|
88
|
+
raise ArgumentError, "engine must be one of #{ENGINES.inspect}, got #{engine.inspect}" unless ENGINES.include?(engine)
|
|
78
89
|
raise ArgumentError, nesting_error(input) if too_deeply_nested?(input)
|
|
79
90
|
|
|
80
91
|
case input
|
|
81
92
|
when Regexp
|
|
82
|
-
new(type: :re, value:
|
|
93
|
+
new(type: :re, value: regexp(:re, input, engine))
|
|
83
94
|
when Hash
|
|
84
|
-
from_hash(input)
|
|
95
|
+
from_hash(input, engine)
|
|
85
96
|
when Array
|
|
86
|
-
compose(input)
|
|
97
|
+
compose(input, engine)
|
|
87
98
|
else
|
|
88
99
|
new(type: :eq, value: input)
|
|
89
100
|
end
|
|
90
101
|
end
|
|
91
102
|
|
|
92
|
-
def self.from_hash(input)
|
|
103
|
+
def self.from_hash(input, engine)
|
|
93
104
|
raise ArgumentError, invalid_shape_error(input) if input.size != 1
|
|
94
105
|
|
|
95
106
|
type, source = input.first
|
|
@@ -109,11 +120,11 @@ module Labkit
|
|
|
109
120
|
"rate-limit match value has unknown type key #{truncate_for_error(type)}; accepted: #{KNOWN_HASH_KEYS.inspect}"
|
|
110
121
|
end
|
|
111
122
|
|
|
112
|
-
compile(type_sym, source)
|
|
123
|
+
compile(type_sym, source, engine)
|
|
113
124
|
end
|
|
114
125
|
private_class_method :from_hash
|
|
115
126
|
|
|
116
|
-
def self.compile(type_sym, source)
|
|
127
|
+
def self.compile(type_sym, source, engine)
|
|
117
128
|
case type_sym
|
|
118
129
|
when :eq, :ne
|
|
119
130
|
new(type: type_sym, value: source)
|
|
@@ -134,14 +145,19 @@ module Labkit
|
|
|
134
145
|
|
|
135
146
|
new(type: type_sym, value: Set.new(source).freeze)
|
|
136
147
|
when :re, :nre
|
|
137
|
-
new(type: type_sym, value:
|
|
148
|
+
new(type: type_sym, value: regexp(type_sym, source, engine))
|
|
138
149
|
end
|
|
139
150
|
end
|
|
140
151
|
private_class_method :compile
|
|
141
152
|
|
|
153
|
+
def self.regexp(key, source, engine)
|
|
154
|
+
engine == :re2 ? re2_regexp(key, source) : with_match_timeout(key, source)
|
|
155
|
+
end
|
|
156
|
+
private_class_method :regexp
|
|
157
|
+
|
|
142
158
|
# Builds the :all matcher behind the Array form. Entries go back through
|
|
143
159
|
# .build, which is what keeps a nested regex bounded by the match timeout.
|
|
144
|
-
def self.compose(input)
|
|
160
|
+
def self.compose(input, engine)
|
|
145
161
|
raise ArgumentError, "rate-limit match value [] must not be empty" if input.empty?
|
|
146
162
|
|
|
147
163
|
unless input.all? { |entry| entry.is_a?(Hash) || entry.is_a?(Regexp) }
|
|
@@ -150,10 +166,28 @@ module Labkit
|
|
|
150
166
|
"got #{truncate_for_error(input)}"
|
|
151
167
|
end
|
|
152
168
|
|
|
153
|
-
new(type: :all, value: input.map { |entry| build(entry) }.freeze)
|
|
169
|
+
new(type: :all, value: input.map { |entry| build(entry, engine) }.freeze)
|
|
154
170
|
end
|
|
155
171
|
private_class_method :compose
|
|
156
172
|
|
|
173
|
+
# The re2 gem is loaded here, not at require time, so a host that builds
|
|
174
|
+
# no file rules never pays for it.
|
|
175
|
+
# A Ruby Regexp with options cannot be handed to RE2 as its source alone:
|
|
176
|
+
# /api/i would silently turn case-sensitive.
|
|
177
|
+
def self.re2_regexp(key, source)
|
|
178
|
+
require "re2"
|
|
179
|
+
flagged = source.is_a?(Regexp) && source.options.anybits?(Regexp::IGNORECASE | Regexp::MULTILINE | Regexp::EXTENDED)
|
|
180
|
+
raise ArgumentError, "rate-limit match value {#{key}: #{truncate_for_error(source)}} has Regexp options, which RE2 does not take; write them into the pattern" if flagged
|
|
181
|
+
|
|
182
|
+
pattern = source.is_a?(Regexp) ? source.source : source.to_s
|
|
183
|
+
compiled = RE2::Regexp.new(pattern, log_errors: false)
|
|
184
|
+
return compiled if compiled.ok?
|
|
185
|
+
|
|
186
|
+
raise ArgumentError,
|
|
187
|
+
"rate-limit match value {#{key}: #{truncate_for_error(source)}} failed to compile: #{compiled.error}"
|
|
188
|
+
end
|
|
189
|
+
private_class_method :re2_regexp
|
|
190
|
+
|
|
157
191
|
# Compiles +source+ (a String pattern or an existing Regexp) into a
|
|
158
192
|
# Regexp bounded by MATCH_TIMEOUT_SECONDS.
|
|
159
193
|
#
|
|
@@ -41,11 +41,16 @@ module Labkit
|
|
|
41
41
|
# admitting rules after counter rules. Default false; nil
|
|
42
42
|
# means not set.
|
|
43
43
|
#
|
|
44
|
+
# source - :code for a rule built in Ruby (default), :file for a rule
|
|
45
|
+
# loaded from a rate limit config document. A :file rule
|
|
46
|
+
# compiles its patterns with RE2 and checks its name in
|
|
47
|
+
# every environment, because an operator wrote it.
|
|
48
|
+
#
|
|
44
49
|
# +name+ must be a lowercase alphanumeric-and-underscore string of at most 64
|
|
45
50
|
# characters. It is used as the middle segment of every Redis counter key for
|
|
46
51
|
# this rule, so changing a rule's name mid-window abandons its in-flight counters.
|
|
47
52
|
Rule = Data.define(
|
|
48
|
-
:name, :match, :limit, :period, :action, :characteristics, :count_distinct, :ban_for, :admit_counted_values
|
|
53
|
+
:name, :match, :limit, :period, :action, :characteristics, :count_distinct, :ban_for, :admit_counted_values, :source
|
|
49
54
|
) do
|
|
50
55
|
# ban_for changes the accounting; action still decides who gets blocked.
|
|
51
56
|
# A :skip rule counts nothing, so it has no limit to cross and no ban to
|
|
@@ -106,36 +111,47 @@ module Labkit
|
|
|
106
111
|
sym
|
|
107
112
|
end
|
|
108
113
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
)
|
|
114
|
+
# A :file rule is checked in every environment: an operator wrote it, and
|
|
115
|
+
# Limiter's production fallback of sanitising the name would silently
|
|
116
|
+
# change the counter key of a rule the operator can see in the file.
|
|
117
|
+
def self.validate_name!(name, source)
|
|
113
118
|
raise ArgumentError, "name must be a String or Symbol, got #{name.class}" unless name.is_a?(String) || name.is_a?(Symbol)
|
|
114
119
|
|
|
115
120
|
name_str = name.to_s
|
|
116
121
|
raise ArgumentError, "name must not be empty" if name_str.empty?
|
|
122
|
+
return name_str unless source == :file || Labkit.dev_or_test?
|
|
123
|
+
|
|
124
|
+
raise ArgumentError, "Invalid rule name: #{name.inspect}. Must match /\\A[a-z0-9_]+\\z/" unless RULE_NAME_PATTERN.match?(name_str)
|
|
125
|
+
raise ArgumentError, "Rule name too long: #{name.inspect}. Maximum 64 characters" if name_str.length > RULE_NAME_MAX_LENGTH
|
|
126
|
+
|
|
127
|
+
name_str
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def initialize( # rubocop:disable Metrics/ParameterLists -- one keyword per Rule field; the public API grows by keywords with defaults
|
|
131
|
+
name:, limit:, period:, characteristics:,
|
|
132
|
+
match: {}, action: :limit, count_distinct: nil, ban_for: nil, admit_counted_values: false, source: :code
|
|
133
|
+
)
|
|
134
|
+
raise ArgumentError, "source must be :code or :file, got #{source.inspect}" unless %i[code file].include?(source)
|
|
135
|
+
|
|
136
|
+
name_str = self.class.validate_name!(name, source)
|
|
117
137
|
|
|
118
138
|
action_sym = action.to_sym
|
|
119
139
|
raise ArgumentError, "Invalid action: #{action.inspect}. Must be one of: #{KNOWN_ACTIONS.inspect}" unless KNOWN_ACTIONS.include?(action_sym)
|
|
120
140
|
|
|
121
|
-
if Labkit.dev_or_test?
|
|
122
|
-
raise ArgumentError, "Invalid rule name: #{name.inspect}. Must match /\\A[a-z0-9_]+\\z/" unless RULE_NAME_PATTERN.match?(name_str)
|
|
123
|
-
raise ArgumentError, "Rule name too long: #{name.inspect}. Maximum 64 characters" if name_str.length > RULE_NAME_MAX_LENGTH
|
|
124
|
-
end
|
|
125
|
-
|
|
126
141
|
characteristics_arr = Array(characteristics).map(&:to_sym).freeze
|
|
127
142
|
count_distinct_sym = self.class.normalize_count_distinct(count_distinct, characteristics_arr)
|
|
128
143
|
|
|
129
144
|
super(
|
|
130
145
|
name: name_str.freeze,
|
|
131
|
-
match: match.transform_keys(&:to_sym).transform_values { |v| Matcher.build(v) }.freeze,
|
|
146
|
+
match: match.transform_keys(&:to_sym).transform_values { |v| Matcher.build(v, source == :file ? :re2 : :ruby) }.freeze,
|
|
132
147
|
limit: self.class.normalize_numeric(:limit, limit, action_sym),
|
|
133
148
|
period: self.class.normalize_numeric(:period, period, action_sym),
|
|
134
149
|
action: action_sym,
|
|
135
150
|
characteristics: characteristics_arr,
|
|
136
151
|
count_distinct: count_distinct_sym,
|
|
137
152
|
ban_for: self.class.normalize_ban_for(ban_for, action_sym, count_distinct_sym),
|
|
138
|
-
admit_counted_values: self.class.normalize_admit_counted_values(admit_counted_values, action_sym, count_distinct_sym)
|
|
153
|
+
admit_counted_values: self.class.normalize_admit_counted_values(admit_counted_values, action_sym, count_distinct_sym),
|
|
154
|
+
source: source
|
|
139
155
|
)
|
|
140
156
|
end
|
|
141
157
|
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
|
-
# Copies the generated Ruby code for gitlab.ratelimit.config.v1
|
|
2
|
+
# Copies the generated Ruby code and the proto sources for gitlab.ratelimit.config.v1
|
|
3
|
+
# from gitlab-org/protos. The sources are what schema_spec.rb checks Schema against.
|
|
3
4
|
# A published gem cannot depend on the protos Bundler git gem, so the file is
|
|
4
5
|
# vendored and CI checks that it matches PROTOS_REF (job vendored-proto-check).
|
|
5
6
|
set -euo pipefail
|
|
@@ -7,12 +8,20 @@ set -euo pipefail
|
|
|
7
8
|
# Pinned protos module tag. Bump it when a newer schema release is needed.
|
|
8
9
|
PROTOS_REF="${PROTOS_REF:-gen/ruby/gitlab/ratelimit/config/v1/v0.1.0}"
|
|
9
10
|
SOURCE_PATH="gen/ruby/gitlab/ratelimit/config/v1/lib/gitlab/ratelimit/config/v1/config_pb.rb"
|
|
10
|
-
|
|
11
|
+
PROTO_DIR="proto/ratelimit/gitlab/ratelimit/config/v1"
|
|
12
|
+
TARGET_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/lib/labkit/rate_limit/config/proto"
|
|
13
|
+
TARGET="${TARGET_DIR}/config_pb.rb"
|
|
11
14
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
15
|
+
raw="https://gitlab.com/gitlab-org/protos/-/raw/${PROTOS_REF}"
|
|
16
|
+
# Everything lands in a temp dir first, so a failed download or check
|
|
17
|
+
# leaves the vendored files as they were.
|
|
18
|
+
work="$(mktemp -d)"
|
|
19
|
+
trap 'rm -rf "$work"' EXIT
|
|
20
|
+
tmp="${work}/config_pb.rb"
|
|
21
|
+
curl --fail --silent --show-error --location "${raw}/${SOURCE_PATH}" -o "$tmp"
|
|
22
|
+
for proto in config.proto constraint.proto; do
|
|
23
|
+
curl --fail --silent --show-error --location "${raw}/${PROTO_DIR}/${proto}" -o "${work}/${proto}"
|
|
24
|
+
done
|
|
16
25
|
|
|
17
26
|
# The generated file registers gitlab/ratelimit/config/v1/config.proto in the
|
|
18
27
|
# global descriptor pool. The protos gem registers the same file name, so the
|
|
@@ -27,10 +36,12 @@ guard='return unless Labkit::RateLimit::Config::VendoredProto.register(pool, des
|
|
|
27
36
|
grep -v '^# frozen_string_literal: true$' "$tmp" |
|
|
28
37
|
sed -e "s|^require 'google/protobuf'\$|require 'google/protobuf'\nrequire 'labkit/rate_limit/config/vendored_proto'|" \
|
|
29
38
|
-e "s|^pool.add_serialized_file(descriptor_data)\$|${guard}|"
|
|
30
|
-
} >"$
|
|
31
|
-
if ! grep -q 'VendoredProto.register' "$
|
|
32
|
-
! grep -qx "require 'labkit/rate_limit/config/vendored_proto'" "$
|
|
39
|
+
} >"${work}/vendored.rb"
|
|
40
|
+
if ! grep -q 'VendoredProto.register' "${work}/vendored.rb" ||
|
|
41
|
+
! grep -qx "require 'labkit/rate_limit/config/vendored_proto'" "${work}/vendored.rb"; then
|
|
33
42
|
echo "vendor: the generated file no longer requires VendoredProto or registers the descriptor as expected; update the sed in this script" >&2
|
|
34
43
|
exit 1
|
|
35
44
|
fi
|
|
36
|
-
|
|
45
|
+
mv "${work}/vendored.rb" "$TARGET"
|
|
46
|
+
mv "${work}/config.proto" "${work}/constraint.proto" "$TARGET_DIR/"
|
|
47
|
+
echo "vendored ${SOURCE_PATH}, config.proto and constraint.proto @${PROTOS_REF} -> ${TARGET_DIR#"$PWD"/}"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gitlab-labkit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.
|
|
4
|
+
version: 5.12.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andrew Newdigate
|
|
@@ -257,6 +257,26 @@ dependencies:
|
|
|
257
257
|
- - "<"
|
|
258
258
|
- !ruby/object:Gem::Version
|
|
259
259
|
version: '2.0'
|
|
260
|
+
- !ruby/object:Gem::Dependency
|
|
261
|
+
name: re2
|
|
262
|
+
requirement: !ruby/object:Gem::Requirement
|
|
263
|
+
requirements:
|
|
264
|
+
- - ">="
|
|
265
|
+
- !ruby/object:Gem::Version
|
|
266
|
+
version: '2.15'
|
|
267
|
+
- - "<"
|
|
268
|
+
- !ruby/object:Gem::Version
|
|
269
|
+
version: '3'
|
|
270
|
+
type: :runtime
|
|
271
|
+
prerelease: false
|
|
272
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
273
|
+
requirements:
|
|
274
|
+
- - ">="
|
|
275
|
+
- !ruby/object:Gem::Version
|
|
276
|
+
version: '2.15'
|
|
277
|
+
- - "<"
|
|
278
|
+
- !ruby/object:Gem::Version
|
|
279
|
+
version: '3'
|
|
260
280
|
- !ruby/object:Gem::Dependency
|
|
261
281
|
name: redis
|
|
262
282
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -652,9 +672,12 @@ files:
|
|
|
652
672
|
- lib/labkit/rate_limit/README.md
|
|
653
673
|
- lib/labkit/rate_limit/block_summary.rb
|
|
654
674
|
- lib/labkit/rate_limit/config.rb
|
|
675
|
+
- lib/labkit/rate_limit/config/document.rb
|
|
655
676
|
- lib/labkit/rate_limit/config/error.rb
|
|
677
|
+
- lib/labkit/rate_limit/config/loader.rb
|
|
656
678
|
- lib/labkit/rate_limit/config/parser.rb
|
|
657
679
|
- lib/labkit/rate_limit/config/proto/config_pb.rb
|
|
680
|
+
- lib/labkit/rate_limit/config/schema.rb
|
|
658
681
|
- lib/labkit/rate_limit/config/vendored_proto.rb
|
|
659
682
|
- lib/labkit/rate_limit/configuration.rb
|
|
660
683
|
- lib/labkit/rate_limit/evaluator.rb
|