gitlab-labkit 5.9.0 → 5.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +13 -0
- data/.rubocop.yml +8 -0
- data/Gemfile +6 -0
- data/gitlab-labkit.gemspec +2 -1
- data/lib/labkit/rate_limit/README.md +44 -0
- data/lib/labkit/rate_limit/config/error.rb +18 -0
- data/lib/labkit/rate_limit/config/parser.rb +242 -0
- data/lib/labkit/rate_limit/config/proto/config_pb.rb +34 -0
- data/lib/labkit/rate_limit/config/schema.rb +324 -0
- data/lib/labkit/rate_limit/config/vendored_proto.rb +105 -0
- data/lib/labkit/rate_limit/config.rb +23 -0
- data/lib/labkit/rate_limit.rb +1 -0
- data/scripts/vendor-ratelimit-proto.sh +47 -0
- metadata +8 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: adf3a44ad840a0b124ef408e68697340e1c62ca66e0965da67ed3e21aa608a92
|
|
4
|
+
data.tar.gz: d3ccd488458bc4c365c476217e6b1234bb7abec41efee94413d81b69497ff460
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7bbbd90bd5eb0e65f794845225fd8707817bea9d8a80f47da64063b4e8e9a7c988177dae2ad0f44e0fb7ca22dca6dd2846e0fc2f2dd9db038c4554d2738d4364
|
|
7
|
+
data.tar.gz: c15ff6de4430a52692e33dd685ca88e4e080a12c6026f4c59f6520b91171113db5e9d13e319d1ac783e9dfea2b5648637ab4c0c8224feedc927ef2f70bc704ea
|
data/.gitlab-ci.yml
CHANGED
|
@@ -81,6 +81,19 @@ deploy:
|
|
|
81
81
|
- Gemfile
|
|
82
82
|
- .gitlab-ci.yml
|
|
83
83
|
|
|
84
|
+
# The generated protobuf code for gitlab.ratelimit.config.v1 is vendored from
|
|
85
|
+
# gitlab-org/protos, because a published gem cannot depend on the protos git gem.
|
|
86
|
+
vendored-proto-check:
|
|
87
|
+
stage: validate
|
|
88
|
+
needs: []
|
|
89
|
+
image: ruby:3.4
|
|
90
|
+
rules:
|
|
91
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
92
|
+
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
93
|
+
script:
|
|
94
|
+
- ./scripts/vendor-ratelimit-proto.sh
|
|
95
|
+
- git diff --exit-code -- lib/labkit/rate_limit/config/proto/
|
|
96
|
+
|
|
84
97
|
# Gate SDK changes on the labkit-spec conformance suite, run remotely: trigger
|
|
85
98
|
# a downstream pipeline in labkit-spec that checks out this pipeline's commit
|
|
86
99
|
# (LABKIT_RUBY_SHA) and runs the Ruby conformance suite against it, verifying
|
data/.rubocop.yml
CHANGED
|
@@ -4,5 +4,13 @@ inherit_gem:
|
|
|
4
4
|
gitlab-styles:
|
|
5
5
|
- rubocop-default.yml
|
|
6
6
|
|
|
7
|
+
inherit_mode:
|
|
8
|
+
merge:
|
|
9
|
+
- Exclude
|
|
10
|
+
|
|
7
11
|
AllCops:
|
|
8
12
|
NewCops: enable
|
|
13
|
+
Exclude:
|
|
14
|
+
# Generated by protoc and vendored from gitlab-org/protos; regenerate with
|
|
15
|
+
# scripts/vendor-ratelimit-proto.sh instead of editing.
|
|
16
|
+
- lib/labkit/rate_limit/config/proto/**/*
|
data/Gemfile
CHANGED
|
@@ -8,3 +8,9 @@ gemspec
|
|
|
8
8
|
# Exporter source: https://gitlab.com/gitlab-org/quality/analytics/test-metrics-exporters
|
|
9
9
|
# Owned by: @gitlab-org/quality/analytics (group::development analytics)
|
|
10
10
|
gem 'gitlab-rspec-metrics-exporter', '~> 0.2', groups: [:development, :test]
|
|
11
|
+
|
|
12
|
+
# regexp_parser 2.13.2 made Regexp::Syntax::Base.implements private, and
|
|
13
|
+
# json_schemer 2.5.0 calls it at load time, so every fresh bundle fails on
|
|
14
|
+
# `require "json_schemer"`. Drop the pin once json_schemer or regexp_parser
|
|
15
|
+
# ships a fix: https://github.com/davishmcclurg/json_schemer/issues/238
|
|
16
|
+
gem "regexp_parser", "!= 2.13.2", groups: [:development, :test]
|
data/gitlab-labkit.gemspec
CHANGED
|
@@ -15,7 +15,8 @@ Gem::Specification.new do |spec|
|
|
|
15
15
|
spec.metadata = { "source_code_uri" => "https://gitlab.com/gitlab-org/ruby/gems/labkit-ruby" }
|
|
16
16
|
spec.license = "MIT"
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
# The vendored .proto sources are read by the specs only; the gem ships the generated Ruby.
|
|
19
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|tools)/}) || f.end_with?(".proto") }
|
|
19
20
|
spec.bindir = "exe"
|
|
20
21
|
spec.executables = %w[labkit-logging]
|
|
21
22
|
spec.require_paths = ["lib"]
|
|
@@ -711,6 +711,50 @@ A logging failure during a flush is logged at WARN with
|
|
|
711
711
|
`dropped_pairs`, the number of lines that flush lost. It never raises, so the
|
|
712
712
|
flush thread keeps running.
|
|
713
713
|
|
|
714
|
+
## Config files
|
|
715
|
+
|
|
716
|
+
`Labkit::RateLimit::Config` loads rules from YAML files that follow the `gitlab.ratelimit.config.v1` schema from [gitlab-org/protos](https://gitlab.com/gitlab-org/protos): the contract an application ships (`AvailableLimiters`) and the rate limits an operator writes (`RateLimits`).
|
|
717
|
+
The generated protobuf classes are vendored under `lib/labkit/rate_limit/config/proto/` and reachable as `Labkit::RateLimit::Config::Proto`.
|
|
718
|
+
Run `scripts/vendor-ratelimit-proto.sh` to update them; CI fails on drift.
|
|
719
|
+
|
|
720
|
+
An application can also install the `gitlab-ratelimit-config-v1` gem from gitlab-org/protos.
|
|
721
|
+
Then labkit uses the gem's schema and compares it with the vendored copy when `Labkit::RateLimit::Config` loads.
|
|
722
|
+
A gem schema that is newer gives a warning.
|
|
723
|
+
A gem schema that lacks or changes a vendored field raises `Labkit::RateLimit::Config::VendoredProto::Mismatch`.
|
|
724
|
+
So upgrade the `gitlab-ratelimit-config-v1` gem in the application first, then the `gitlab-labkit` release that updates the vendored copy.
|
|
725
|
+
|
|
726
|
+
Loading runs in four steps, `parse`, `schema`, `contract` and `rule`, and a failure raises `Labkit::RateLimit::Config::Error` whose `#step` names the step.
|
|
727
|
+
The parse and schema steps exist; the contract and rule steps follow in [Part 3 of the plan](https://gitlab.com/gitlab-com/gl-infra/production-engineering/-/work_items/30263).
|
|
728
|
+
|
|
729
|
+
```ruby
|
|
730
|
+
doc = Labkit::RateLimit::Config::Parser.parse(
|
|
731
|
+
"config/rate_limits/rack_request.yaml",
|
|
732
|
+
Labkit::RateLimit::Config::Proto::RateLimits,
|
|
733
|
+
strict: true # CI rejects unknown fields. Production loads with strict: false
|
|
734
|
+
)
|
|
735
|
+
Labkit::RateLimit::Config::Schema.check!(doc)
|
|
736
|
+
doc.rate_limits["rack_request"].rules.first.name
|
|
737
|
+
```
|
|
738
|
+
|
|
739
|
+
`Schema.check!` mirrors every protovalidate rule of the `gitlab.ratelimit.config.v1` schema: name format, the period presets, the `limit` and `ban_for_s` ranges, every list cap, and the message rules such as `rule.limit_requires_threshold`.
|
|
740
|
+
It accepts `AvailableLimiters`, `RateLimits` and `PublishRulesRequest`.
|
|
741
|
+
A failure lists every violation with the field path, message and rule id that protovalidate reports, such as `rate_limits["rack_request"].rules[0].period_s: must be in list [1, 15, 30, 60, 180, 300, 600, 900, 1800, 3600, 28800, 86400] (uint32.in)`, so a failure reads the same in Ruby and Go.
|
|
742
|
+
`Schema.violations(doc)` returns them without raising.
|
|
743
|
+
Inside one message the order is protovalidate's: message rules, then the oneof, then fields in declaration order.
|
|
744
|
+
A protobuf map keeps no document order and Go iterates it in an unspecified order, so Ruby reports map entries by sorted key and a cross-SDK comparison matches violations per entry, not by position.
|
|
745
|
+
Map keys in a path are quoted as Go's `strconv.Quote` prints them.
|
|
746
|
+
The check is the strict reading of the schema: a matcher whose operator this binary does not know fails `required`.
|
|
747
|
+
Lenient loading, through the loader that follows in Part 3, drops such a rule with a warning before the schema step, so a newer operator does not reject the file.
|
|
748
|
+
`scripts/vendor-ratelimit-proto.sh` also copies `config.proto` and `constraint.proto` next to the generated code, and `schema_spec.rb` parses them and compares every field rule (caps, bounds, presets, flags), every message rule id and message, and the name pattern with `Schema`, so a `PROTOS_REF` bump that changes any of those fails the suite until `Schema` follows.
|
|
749
|
+
The CEL expressions themselves are mirrored by hand and pinned by the examples.
|
|
750
|
+
|
|
751
|
+
The parse step converts each YAML scalar by the type of the field it fills, so `eq: off`, `eq: True` and `eq: 9970` stay strings, `limit: "5"` becomes `5`, `action: "1"` becomes `ACTION_LIMIT`, and a YAML null leaves the field unset.
|
|
752
|
+
A repeated key, also one repeated under its camelCase `json_name`, a merge key (`<<`), an empty file, more than one document, or a non-scalar key fails at step `parse`, in both modes.
|
|
753
|
+
So does a value of the wrong kind for its field, such as a mapping where a scalar belongs, an integer out of range for its field, two fields of one `oneof`, such as `{ eq: a, re: b }`, a null list item or map value, and a document nested deeper than 64 levels.
|
|
754
|
+
Each of these errors names the YAML line.
|
|
755
|
+
An unknown field or an unknown enum name, such as `action: ACTION_LIMT`, fails in strict mode with its YAML line.
|
|
756
|
+
Lenient mode ignores an unknown field and leaves an unknown enum name unset.
|
|
757
|
+
|
|
714
758
|
## Dev/test vs production guards
|
|
715
759
|
|
|
716
760
|
`Limiter.new` and `Rule.new` validate names and configuration. In
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Labkit
|
|
4
|
+
module RateLimit
|
|
5
|
+
module Config
|
|
6
|
+
# Error is raised when a document fails one of the load steps.
|
|
7
|
+
# step is :parse, :schema, :contract or :rule.
|
|
8
|
+
class Error < StandardError
|
|
9
|
+
attr_reader :step
|
|
10
|
+
|
|
11
|
+
def initialize(step, message)
|
|
12
|
+
@step = step
|
|
13
|
+
super("#{step}: #{message}")
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "psych"
|
|
5
|
+
require "google/protobuf/descriptor_pb"
|
|
6
|
+
|
|
7
|
+
module Labkit
|
|
8
|
+
module RateLimit
|
|
9
|
+
module Config
|
|
10
|
+
# Parser is the parse step: a YAML file into a protobuf message.
|
|
11
|
+
#
|
|
12
|
+
# YAML types do not follow the schema, so each scalar is converted by the
|
|
13
|
+
# type of the field it fills, the way protoyaml does in Go: a string field
|
|
14
|
+
# keeps the scalar text (`off`, `True` and `9970` stay strings), an integer
|
|
15
|
+
# or enum field gets an Integer when the text is one, quoted or not, and
|
|
16
|
+
# the YAML null forms mean the field is absent. Psych keeps the last value
|
|
17
|
+
# of a repeated key, so repeated keys are rejected here, also when one
|
|
18
|
+
# spelling is the snake_case field name and the other its camelCase
|
|
19
|
+
# json_name.
|
|
20
|
+
#
|
|
21
|
+
# This step checks the shape of a document, not its values. An enum
|
|
22
|
+
# number that no Action defines, such as `action: 7`, passes in both
|
|
23
|
+
# modes, and in lenient mode protobuf drops an unknown enum name together
|
|
24
|
+
# with the unknown fields, so a rule with a misspelt action reaches the
|
|
25
|
+
# schema step as ACTION_UNSPECIFIED. Schema.check! rejects both.
|
|
26
|
+
module Parser
|
|
27
|
+
INT32 = (-(2**31))...(2**31)
|
|
28
|
+
UINT32 = 0...(2**32)
|
|
29
|
+
INT64 = (-(2**63))...(2**63)
|
|
30
|
+
UINT64 = 0...(2**64)
|
|
31
|
+
INTEGER_RANGES = {
|
|
32
|
+
int32: INT32, sint32: INT32, sfixed32: INT32, enum: INT32,
|
|
33
|
+
uint32: UINT32, fixed32: UINT32,
|
|
34
|
+
int64: INT64, sint64: INT64, sfixed64: INT64,
|
|
35
|
+
uint64: UINT64, fixed64: UINT64
|
|
36
|
+
}.freeze
|
|
37
|
+
INTEGER = /\A[-+]?\d+\z/
|
|
38
|
+
NULLS = ["", "~", "null", "Null", "NULL"].freeze
|
|
39
|
+
# The protobuf JSON decoder stops at 64 levels of objects and arrays.
|
|
40
|
+
# Mappings and lists are counted here so the error names the YAML line
|
|
41
|
+
# and a deeper document cannot exhaust the stack.
|
|
42
|
+
MAX_DEPTH = 64
|
|
43
|
+
|
|
44
|
+
class << self
|
|
45
|
+
# @param path [String] YAML file
|
|
46
|
+
# @param klass [Class] a Proto message class, AvailableLimiters or RateLimits
|
|
47
|
+
# @param strict [Boolean] when true an unknown field is an error
|
|
48
|
+
# @return [Google::Protobuf::AbstractMessage]
|
|
49
|
+
# @raise [Error] with step :parse
|
|
50
|
+
def parse(path, klass, strict:)
|
|
51
|
+
tree = tree_for(document_root(path), klass.descriptor, strict: strict, depth: 0)
|
|
52
|
+
klass.decode_json(JSON.generate(tree), ignore_unknown_fields: !strict)
|
|
53
|
+
rescue Psych::SyntaxError, SystemCallError => e
|
|
54
|
+
# Both messages already name the file.
|
|
55
|
+
raise Error.new(:parse, e.message)
|
|
56
|
+
rescue Google::Protobuf::ParseError, Psych::Exception, JSON::JSONError => e
|
|
57
|
+
raise Error.new(:parse, "#{path}: #{e.message}")
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
|
|
62
|
+
# One YAML document per file. Psych would silently keep only the first
|
|
63
|
+
# of several, and an empty file has no root to walk.
|
|
64
|
+
def document_root(path)
|
|
65
|
+
documents = Psych.parse_stream(File.read(path), filename: path).children
|
|
66
|
+
raise Psych::Exception, "the file holds no YAML document" if documents.empty?
|
|
67
|
+
raise Psych::Exception, "the file holds #{documents.size} YAML documents, expected 1" if documents.size > 1
|
|
68
|
+
|
|
69
|
+
documents.first.root
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# One value for one slot. type is a field type Symbol, a message
|
|
73
|
+
# Descriptor or an EnumDescriptor. A node of the wrong kind is
|
|
74
|
+
# rejected here so the error names the YAML line, not an offset into
|
|
75
|
+
# the generated JSON. depth counts the mappings and lists above node.
|
|
76
|
+
def tree_for(node, type, strict:, depth:)
|
|
77
|
+
expects_message = type.is_a?(Google::Protobuf::Descriptor)
|
|
78
|
+
|
|
79
|
+
case node
|
|
80
|
+
when Psych::Nodes::Mapping
|
|
81
|
+
fail_at(node, "expected a scalar, found a mapping") unless expects_message
|
|
82
|
+
mapping_for(node, type, strict: strict, depth: depth)
|
|
83
|
+
when Psych::Nodes::Sequence
|
|
84
|
+
fail_at(node, "expected a single value, found a list")
|
|
85
|
+
when Psych::Nodes::Scalar
|
|
86
|
+
return nil if null?(node)
|
|
87
|
+
|
|
88
|
+
fail_at(node, "expected a mapping, found a scalar") if expects_message
|
|
89
|
+
scalar_for(node, type, strict: strict)
|
|
90
|
+
when Psych::Nodes::Alias
|
|
91
|
+
fail_at(node, "aliases are not supported")
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def mapping_for(node, descriptor, strict:, depth:)
|
|
96
|
+
check_depth!(node, depth)
|
|
97
|
+
entries = node.children.each_slice(2).map do |key, value|
|
|
98
|
+
name = scalar_key(key)
|
|
99
|
+
field = field_for(descriptor, name)
|
|
100
|
+
fail_at(key, "unknown field #{name.inspect}") if strict && field.nil?
|
|
101
|
+
[name, key, field, value]
|
|
102
|
+
end
|
|
103
|
+
reject_repeated_keys!(entries.map { |name, key, field, _| [field ? field.name : name, key] })
|
|
104
|
+
reject_oneof_conflicts!(descriptor, entries)
|
|
105
|
+
|
|
106
|
+
maps = map_fields(descriptor)
|
|
107
|
+
# An unknown field is not walked: strict mode rejected its name
|
|
108
|
+
# above and lenient mode ignores it, so a newer schema may put
|
|
109
|
+
# anything there.
|
|
110
|
+
entries.to_h do |name, _, field, value|
|
|
111
|
+
next [name, nil] unless field
|
|
112
|
+
|
|
113
|
+
[name, field_value_for(value, field, map: maps.include?(field.name), strict: strict, depth: depth + 1)]
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Descriptor#options, which carries the map_entry flag, arrived after
|
|
118
|
+
# google-protobuf 3.25.0, so a blank message tells a map from a list.
|
|
119
|
+
# A message type keeps one Descriptor object, so the cache is bounded
|
|
120
|
+
# by the number of message types.
|
|
121
|
+
def map_fields(descriptor)
|
|
122
|
+
(@map_fields ||= {})[descriptor] ||= begin
|
|
123
|
+
blank = descriptor.msgclass.new
|
|
124
|
+
descriptor.select { |field| blank[field.name].is_a?(Google::Protobuf::Map) }.map(&:name).freeze
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# A mapping key must be a plain scalar; a sequence, mapping or alias
|
|
129
|
+
# as a key has no field to fill. A merge key is rejected in both
|
|
130
|
+
# modes: lenient mode would ignore it as an unknown field and drop
|
|
131
|
+
# the merged fields with no error.
|
|
132
|
+
def scalar_key(node)
|
|
133
|
+
fail_at(node, "a mapping key must be a scalar") unless node.is_a?(Psych::Nodes::Scalar)
|
|
134
|
+
fail_at(node, "merge keys (<<) are not supported") if node.plain && node.value == "<<"
|
|
135
|
+
|
|
136
|
+
node.value
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# The error points at the repeated key, not at the start of the mapping.
|
|
140
|
+
def reject_repeated_keys!(named_keys)
|
|
141
|
+
seen = {}
|
|
142
|
+
named_keys.each do |name, key|
|
|
143
|
+
fail_at(key, "duplicate key #{name.inspect}") if seen[name]
|
|
144
|
+
seen[name] = true
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Two fields of one oneof, such as two Matcher operators, would
|
|
149
|
+
# otherwise fail in the JSON decoder with no YAML line.
|
|
150
|
+
def reject_oneof_conflicts!(descriptor, entries)
|
|
151
|
+
descriptor.each_oneof do |oneof|
|
|
152
|
+
names = oneof.map(&:name)
|
|
153
|
+
set = entries.select { |_, _, field, value| field && names.include?(field.name) && !null?(value) }
|
|
154
|
+
next if set.size < 2
|
|
155
|
+
|
|
156
|
+
fail_at(set[1][1], "#{set[0][2].name} and #{set[1][2].name} are both set; set only one field of #{oneof.name}")
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def check_depth!(node, depth)
|
|
161
|
+
fail_at(node, "nesting deeper than #{MAX_DEPTH} levels") if depth >= MAX_DEPTH
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def fail_at(node, message)
|
|
165
|
+
raise Psych::Exception, "line #{node.start_line + 1}: #{message}"
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Files are snake_case; protojson also accepts the camelCase json_name.
|
|
169
|
+
def field_for(descriptor, name)
|
|
170
|
+
descriptor.lookup(name) || descriptor.find { |field| field.json_name == name }
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def field_value_for(node, field, map:, strict:, depth:)
|
|
174
|
+
return map_for(node, field, strict: strict, depth: depth) if map
|
|
175
|
+
return tree_for(node, element_type(field), strict: strict, depth: depth) unless field.label == :repeated
|
|
176
|
+
return nil if null?(node)
|
|
177
|
+
|
|
178
|
+
fail_at(node, "expected a list") unless node.is_a?(Psych::Nodes::Sequence)
|
|
179
|
+
check_depth!(node, depth)
|
|
180
|
+
node.children.map do |child|
|
|
181
|
+
fail_at(child, "a list item cannot be null") if null?(child)
|
|
182
|
+
|
|
183
|
+
tree_for(child, element_type(field), strict: strict, depth: depth + 1)
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def map_for(node, field, strict:, depth:)
|
|
188
|
+
return nil if null?(node)
|
|
189
|
+
|
|
190
|
+
fail_at(node, "expected a mapping") unless node.is_a?(Psych::Nodes::Mapping)
|
|
191
|
+
check_depth!(node, depth)
|
|
192
|
+
pairs = node.children.each_slice(2)
|
|
193
|
+
reject_repeated_keys!(pairs.map { |key, _| [scalar_key(key), key] })
|
|
194
|
+
value_type = element_type(field.subtype.lookup("value"))
|
|
195
|
+
pairs.to_h do |key, value|
|
|
196
|
+
fail_at(value, "a map value cannot be null") if null?(value)
|
|
197
|
+
|
|
198
|
+
[scalar_key(key), tree_for(value, value_type, strict: strict, depth: depth + 1)]
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# A message or enum field passes its descriptor down, so a scalar
|
|
203
|
+
# can be checked against the enum's names.
|
|
204
|
+
def element_type(field)
|
|
205
|
+
%i[message enum].include?(field.type) ? field.subtype : field.type
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def null?(node)
|
|
209
|
+
node.is_a?(Psych::Nodes::Scalar) && node.plain && NULLS.include?(node.value)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Integer text is converted quoted or not: protojson takes a quoted
|
|
213
|
+
# number for an integer field but not for an enum. The range is
|
|
214
|
+
# checked here so the error names the YAML line.
|
|
215
|
+
def scalar_for(node, type, strict:)
|
|
216
|
+
enum = type if type.is_a?(Google::Protobuf::EnumDescriptor)
|
|
217
|
+
kind = enum ? :enum : type
|
|
218
|
+
range = INTEGER_RANGES[kind]
|
|
219
|
+
if range && INTEGER.match?(node.value)
|
|
220
|
+
value = Integer(node.value, 10)
|
|
221
|
+
fail_at(node, "#{node.value} is out of range for #{kind}") unless range.cover?(value)
|
|
222
|
+
return value
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# Lenient mode passes an unknown name on, and protobuf drops it
|
|
226
|
+
# with the unknown fields, so a newer schema's value still loads.
|
|
227
|
+
fail_at(node, "unknown value #{node.value.inspect} for enum #{enum.name}") if enum && strict && enum.lookup_name(node.value.to_sym).nil?
|
|
228
|
+
|
|
229
|
+
return node.value unless node.plain
|
|
230
|
+
|
|
231
|
+
case kind
|
|
232
|
+
when :bool
|
|
233
|
+
{ "true" => true, "false" => false }.fetch(node.value, node.value)
|
|
234
|
+
else
|
|
235
|
+
node.value
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Vendored from gitlab-org/protos at gen/ruby/gitlab/ratelimit/config/v1/v0.1.0, gen/ruby/gitlab/ratelimit/config/v1/lib/gitlab/ratelimit/config/v1/config_pb.rb.
|
|
3
|
+
# Do not edit. Run scripts/vendor-ratelimit-proto.sh to update.
|
|
4
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
5
|
+
# source: gitlab/ratelimit/config/v1/config.proto
|
|
6
|
+
|
|
7
|
+
require 'google/protobuf'
|
|
8
|
+
require 'labkit/rate_limit/config/vendored_proto'
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
descriptor_data = "\n\'gitlab/ratelimit/config/v1/config.proto\x12\x1agitlab.ratelimit.config.v1\"\xf7\x01\n\x11\x41vailableLimiters\x12s\n\x12\x61vailable_limiters\x18\x01 \x03(\x0b\x32\x44.gitlab.ratelimit.config.v1.AvailableLimiters.AvailableLimitersEntryR\x11\x61vailableLimiters\x1am\n\x16\x41vailableLimitersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12=\n\x05value\x18\x02 \x01(\x0b\x32\'.gitlab.ratelimit.config.v1.LimiterSpecR\x05value:\x02\x38\x01\"\xd2\x01\n\x0bLimiterSpec\x12 \n\x0b\x64\x65scription\x18\x01 \x01(\tR\x0b\x64\x65scription\x12\x31\n\x14\x61vailable_properties\x18\x02 \x03(\tR\x13\x61vailableProperties\x12\x45\n\rdefault_rules\x18\x03 \x03(\x0b\x32 .gitlab.ratelimit.config.v1.RuleR\x0c\x64\x65\x66\x61ultRules\x12\'\n\x0fprotected_rules\x18\x04 \x03(\tR\x0eprotectedRules\"\xca\x01\n\nRateLimits\x12W\n\x0brate_limits\x18\x01 \x03(\x0b\x32\x36.gitlab.ratelimit.config.v1.RateLimits.RateLimitsEntryR\nrateLimits\x1a\x63\n\x0fRateLimitsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12:\n\x05value\x18\x02 \x01(\x0b\x32$.gitlab.ratelimit.config.v1.RuleListR\x05value:\x02\x38\x01\"B\n\x08RuleList\x12\x36\n\x05rules\x18\x01 \x03(\x0b\x32 .gitlab.ratelimit.config.v1.RuleR\x05rules\"\x9a\x04\n\x04Rule\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\x12:\n\x06\x61\x63tion\x18\x03 \x01(\x0e\x32\".gitlab.ratelimit.config.v1.ActionR\x06\x61\x63tion\x12(\n\x0f\x63haracteristics\x18\x04 \x03(\tR\x0f\x63haracteristics\x12\x41\n\x05match\x18\x05 \x03(\x0b\x32+.gitlab.ratelimit.config.v1.Rule.MatchEntryR\x05match\x12\x19\n\x05limit\x18\x06 \x01(\rH\x00R\x05limit\x88\x01\x01\x12\x1e\n\x08period_s\x18\x07 \x01(\rH\x01R\x07periodS\x88\x01\x01\x12%\n\x0e\x63ount_distinct\x18\x08 \x01(\tR\rcountDistinct\x12\x1f\n\tban_for_s\x18\t \x01(\rH\x02R\x07\x62\x61nForS\x88\x01\x01\x12\x1f\n\x08position\x18\n \x01(\rH\x03R\x08position\x88\x01\x01\x1a]\n\nMatchEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32#.gitlab.ratelimit.config.v1.MatcherR\x05value:\x02\x38\x01\x42\x08\n\x06_limitB\x0b\n\t_period_sB\x0c\n\n_ban_for_sB\x0b\n\t_position\"\x93\x03\n\x13PublishRulesRequest\x12`\n\x0brate_limits\x18\x01 \x03(\x0b\x32?.gitlab.ratelimit.config.v1.PublishRulesRequest.RateLimitsEntryR\nrateLimits\x12S\n\x06remove\x18\x02 \x03(\x0b\x32;.gitlab.ratelimit.config.v1.PublishRulesRequest.RemoveEntryR\x06remove\x1a\x63\n\x0fRateLimitsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12:\n\x05value\x18\x02 \x01(\x0b\x32$.gitlab.ratelimit.config.v1.RuleListR\x05value:\x02\x38\x01\x1a`\n\x0bRemoveEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12;\n\x05value\x18\x02 \x01(\x0b\x32%.gitlab.ratelimit.config.v1.RuleNamesR\x05value:\x02\x38\x01\"!\n\tRuleNames\x12\x14\n\x05names\x18\x01 \x03(\tR\x05names\"\x9a\x02\n\x07Matcher\x12\x10\n\x02\x65q\x18\x01 \x01(\tH\x00R\x02\x65q\x12\x10\n\x02ne\x18\x02 \x01(\tH\x00R\x02ne\x12\x10\n\x02re\x18\x03 \x01(\tH\x00R\x02re\x12\x12\n\x03nre\x18\x04 \x01(\tH\x00R\x03nre\x12?\n\x06one_of\x18\x05 \x01(\x0b\x32&.gitlab.ratelimit.config.v1.StringListH\x00R\x05oneOf\x12\x41\n\x07none_of\x18\x06 \x01(\x0b\x32&.gitlab.ratelimit.config.v1.StringListH\x00R\x06noneOf\x12;\n\x03\x61ll\x18\x07 \x01(\x0b\x32\'.gitlab.ratelimit.config.v1.MatcherListH\x00R\x03\x61llB\x04\n\x02op\"$\n\nStringList\x12\x16\n\x06values\x18\x01 \x03(\tR\x06values\"N\n\x0bMatcherList\x12?\n\x08matchers\x18\x01 \x03(\x0b\x32#.gitlab.ratelimit.config.v1.MatcherR\x08matchers*S\n\x06\x41\x63tion\x12\x16\n\x12\x41\x43TION_UNSPECIFIED\x10\x00\x12\x10\n\x0c\x41\x43TION_LIMIT\x10\x01\x12\x0e\n\nACTION_LOG\x10\x02\x12\x0f\n\x0b\x41\x43TION_SKIP\x10\x03\x62\x06proto3"
|
|
12
|
+
|
|
13
|
+
pool = ::Google::Protobuf::DescriptorPool.generated_pool
|
|
14
|
+
return unless Labkit::RateLimit::Config::VendoredProto.register(pool, descriptor_data)
|
|
15
|
+
|
|
16
|
+
module Gitlab
|
|
17
|
+
module Ratelimit
|
|
18
|
+
module Config
|
|
19
|
+
module V1
|
|
20
|
+
AvailableLimiters = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.AvailableLimiters").msgclass
|
|
21
|
+
LimiterSpec = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.LimiterSpec").msgclass
|
|
22
|
+
RateLimits = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RateLimits").msgclass
|
|
23
|
+
RuleList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RuleList").msgclass
|
|
24
|
+
Rule = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Rule").msgclass
|
|
25
|
+
PublishRulesRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.PublishRulesRequest").msgclass
|
|
26
|
+
RuleNames = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RuleNames").msgclass
|
|
27
|
+
Matcher = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Matcher").msgclass
|
|
28
|
+
StringList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.StringList").msgclass
|
|
29
|
+
MatcherList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.MatcherList").msgclass
|
|
30
|
+
Action = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Action").enummodule
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Labkit
|
|
4
|
+
module RateLimit
|
|
5
|
+
module Config
|
|
6
|
+
# Schema is the schema step: the checks that gitlab/ratelimit/config/v1
|
|
7
|
+
# declares with protovalidate, mirrored in Ruby because there is no
|
|
8
|
+
# protovalidate runtime for Ruby. Each check keeps the protovalidate rule
|
|
9
|
+
# id, message and field path. Inside one message the order is
|
|
10
|
+
# protovalidate's: message rules, then the oneof, then fields in
|
|
11
|
+
# declaration order.
|
|
12
|
+
# A protobuf map keeps no document order and Go iterates it in an
|
|
13
|
+
# unspecified order, so Ruby reports map entries by sorted key and the
|
|
14
|
+
# corpus compares per entry.
|
|
15
|
+
#
|
|
16
|
+
# FIELD_RULES and MESSAGE_RULES are copied from the proto by hand, and
|
|
17
|
+
# schema_spec.rb compares both with the vendored config.proto and
|
|
18
|
+
# constraint.proto field by field, so a PROTOS_REF bump that changes a
|
|
19
|
+
# cap, a flag, a rule id or a message fails the suite until this file
|
|
20
|
+
# follows. The CEL expressions have no such check; the examples pin them.
|
|
21
|
+
#
|
|
22
|
+
# The check is the strict reading of the schema. Lenient loading drops a
|
|
23
|
+
# rule whose matcher has no known operator before this step (the loader,
|
|
24
|
+
# Task 3.3), because such a matcher fails `required` here.
|
|
25
|
+
module Schema
|
|
26
|
+
# for_key marks a violation on a map key; its path is the entry's path.
|
|
27
|
+
Violation = Data.define(:path, :rule_id, :message, :for_key) do
|
|
28
|
+
def initialize(path:, rule_id:, message:, for_key: false)
|
|
29
|
+
super
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def to_s
|
|
33
|
+
path.empty? ? "#{message} (#{rule_id})" : "#{path}: #{message} (#{rule_id})"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# CEL matches() is RE2, where ^ and $ anchor the whole text; Ruby's $
|
|
38
|
+
# matches before a newline, so the mirror anchors with \A and \z.
|
|
39
|
+
NAME = /\A[a-z0-9_]{1,64}\z/
|
|
40
|
+
NAME_MESSAGE = "invalid name; must match ^[a-z0-9_]{1,64}$"
|
|
41
|
+
PERIODS = [1, 15, 30, 60, 180, 300, 600, 900, 1800, 3600, 28800, 86400].freeze
|
|
42
|
+
SKIP = :ACTION_SKIP
|
|
43
|
+
|
|
44
|
+
# The field rules of config.proto, keyed "Message.field". The keys of a
|
|
45
|
+
# map field and the items of a repeated field carry their rule under
|
|
46
|
+
# keys_* and items_*.
|
|
47
|
+
FIELD_RULES = {
|
|
48
|
+
"AvailableLimiters.available_limiters" => { min_pairs: 1, keys_is_name: true },
|
|
49
|
+
"LimiterSpec.description" => { max_len: 1024 },
|
|
50
|
+
"LimiterSpec.available_properties" => { unique: true, max_items: 100, items_is_name: true },
|
|
51
|
+
"LimiterSpec.default_rules" => { max_items: 200 },
|
|
52
|
+
"LimiterSpec.protected_rules" => { unique: true, max_items: 200, items_is_name: true },
|
|
53
|
+
"RateLimits.rate_limits" => { keys_is_name: true },
|
|
54
|
+
"RuleList.rules" => { max_items: 200 },
|
|
55
|
+
"Rule.name" => { is_name: true },
|
|
56
|
+
"Rule.description" => { max_len: 1024 },
|
|
57
|
+
"Rule.action" => { required: true, defined_only: true },
|
|
58
|
+
"Rule.characteristics" => { unique: true, max_items: 10, items_is_name: true },
|
|
59
|
+
"Rule.match" => { max_pairs: 20, keys_is_name: true },
|
|
60
|
+
"Rule.limit" => { gte: 1, lte: 1_000_000 },
|
|
61
|
+
"Rule.period_s" => { in: PERIODS },
|
|
62
|
+
"Rule.count_distinct" => { is_name: true, ignore_empty: true },
|
|
63
|
+
"Rule.ban_for_s" => { gte: 1, lte: 86_400 },
|
|
64
|
+
"PublishRulesRequest.rate_limits" => { keys_is_name: true },
|
|
65
|
+
"PublishRulesRequest.remove" => { keys_is_name: true },
|
|
66
|
+
"RuleNames.names" => { min_items: 1, max_items: 200, unique: true, items_is_name: true },
|
|
67
|
+
"Matcher.eq" => { max_len: 1024 },
|
|
68
|
+
"Matcher.ne" => { max_len: 1024 },
|
|
69
|
+
"Matcher.re" => { max_len: 1024 },
|
|
70
|
+
"Matcher.nre" => { max_len: 1024 },
|
|
71
|
+
"StringList.values" => { min_items: 1, max_items: 100, unique: true, items_max_len: 1024 },
|
|
72
|
+
"MatcherList.matchers" => { min_items: 1, max_items: 20 }
|
|
73
|
+
}.freeze
|
|
74
|
+
|
|
75
|
+
# The message-level CEL rules of config.proto, by id.
|
|
76
|
+
MESSAGE_RULES = {
|
|
77
|
+
"limiter_spec.protected_rules_declared" => "every protected rule must be a default rule of this limiter",
|
|
78
|
+
"limiter_spec.unique_default_names" => "default rule names must be unique within a limiter",
|
|
79
|
+
"limiter_spec.no_position_in_defaults" => "position is not allowed on a default rule",
|
|
80
|
+
"rule_list.unique_names" => "rule names must be unique within a limiter",
|
|
81
|
+
"rule.limit_requires_threshold" => "limit and period_s are required unless action is skip",
|
|
82
|
+
"rule.ban_excludes_count_distinct" => "ban_for_s cannot be combined with count_distinct",
|
|
83
|
+
"rule.ban_not_on_skip" => "ban_for_s is not allowed on a skip rule",
|
|
84
|
+
"rule.no_counter_on_skip" => "limit, period_s, characteristics and count_distinct are not allowed on a skip rule",
|
|
85
|
+
"rule.count_distinct_not_characteristic" => "count_distinct cannot also be a characteristic",
|
|
86
|
+
"publish.name_in_both" => "a rule name cannot be in both rate_limits and remove for one limiter",
|
|
87
|
+
"matcher_list.no_nested_all" => "all cannot contain another all"
|
|
88
|
+
}.freeze
|
|
89
|
+
|
|
90
|
+
# The ASCII characters that strconv.Quote prints with a backslash escape.
|
|
91
|
+
ESCAPES = {
|
|
92
|
+
'"' => '\\"', "\\" => "\\\\", "\a" => "\\a", "\b" => "\\b", "\f" => "\\f",
|
|
93
|
+
"\n" => "\\n", "\r" => "\\r", "\t" => "\\t", "\v" => "\\v"
|
|
94
|
+
}.freeze
|
|
95
|
+
|
|
96
|
+
class << self
|
|
97
|
+
# @param message [Proto::AvailableLimiters, Proto::RateLimits, Proto::PublishRulesRequest]
|
|
98
|
+
# @return the message
|
|
99
|
+
# @raise [Error] step :schema, every violation joined with "; "
|
|
100
|
+
def check!(message)
|
|
101
|
+
found = violations(message)
|
|
102
|
+
raise Error.new(:schema, found.join("; ")) unless found.empty?
|
|
103
|
+
|
|
104
|
+
message
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# @return [Array<Violation>] every violation, in the order described above
|
|
108
|
+
def violations(message)
|
|
109
|
+
checker = Checker.new
|
|
110
|
+
case message
|
|
111
|
+
when Proto::AvailableLimiters then checker.available_limiters(message)
|
|
112
|
+
when Proto::RateLimits then checker.rate_limits(message)
|
|
113
|
+
when Proto::PublishRulesRequest then checker.publish_rules_request(message)
|
|
114
|
+
else
|
|
115
|
+
raise ArgumentError, "no schema for #{message.class}; expected AvailableLimiters, RateLimits or PublishRulesRequest"
|
|
116
|
+
end
|
|
117
|
+
checker.violations
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Quotes a map key for a path the way Go's strconv.Quote does, so the
|
|
121
|
+
# path reads the same in every SDK: backslash escapes for the ASCII
|
|
122
|
+
# controls that have one, \xNN for the other ASCII controls and DEL,
|
|
123
|
+
# \uNNNN or \UNNNNNNNN for a non-printable rune, everything else as is.
|
|
124
|
+
def quote(key)
|
|
125
|
+
out = +'"'
|
|
126
|
+
key.each_char { |c| out << (ESCAPES[c] || escape_rune(c)) }
|
|
127
|
+
out << '"'
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
private
|
|
131
|
+
|
|
132
|
+
# Go's unicode.IsPrint: letters, marks, numbers, punctuation, symbols
|
|
133
|
+
# and the ASCII space.
|
|
134
|
+
def escape_rune(char)
|
|
135
|
+
code = char.ord
|
|
136
|
+
return format("\\x%02x", code) if code < 0x20 || code == 0x7f
|
|
137
|
+
return char if char == " " || char.match?(/\A[\p{L}\p{M}\p{N}\p{P}\p{S}]\z/)
|
|
138
|
+
|
|
139
|
+
code > 0xffff ? format("\\U%08x", code) : format("\\u%04x", code)
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Walks one message and collects violations. Paths follow protovalidate:
|
|
144
|
+
# `rate_limits["rack_request"].rules[0].match["plan"].eq`.
|
|
145
|
+
class Checker
|
|
146
|
+
attr_reader :violations
|
|
147
|
+
|
|
148
|
+
def initialize
|
|
149
|
+
@violations = []
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def available_limiters(doc)
|
|
153
|
+
map_field("available_limiters", doc.available_limiters, FIELD_RULES["AvailableLimiters.available_limiters"]) { |path, spec| limiter_spec(path, spec) }
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def rate_limits(doc)
|
|
157
|
+
map_field("rate_limits", doc.rate_limits, FIELD_RULES["RateLimits.rate_limits"]) { |path, list| rule_list(path, list) }
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def publish_rules_request(req)
|
|
161
|
+
# The outcome of one name in both fields would depend on the order
|
|
162
|
+
# the store applies them.
|
|
163
|
+
both = req.rate_limits.any? do |key, list|
|
|
164
|
+
req.remove.has_key?(key) && list.rules.any? { |r| req.remove[key].names.include?(r.name) }
|
|
165
|
+
end
|
|
166
|
+
message_rule("", "publish.name_in_both") if both
|
|
167
|
+
map_field("rate_limits", req.rate_limits, FIELD_RULES["PublishRulesRequest.rate_limits"]) { |path, list| rule_list(path, list) }
|
|
168
|
+
map_field("remove", req.remove, FIELD_RULES["PublishRulesRequest.remove"]) do |path, names|
|
|
169
|
+
list("#{path}.names", names.names, FIELD_RULES["RuleNames.names"])
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
private
|
|
174
|
+
|
|
175
|
+
def limiter_spec(path, spec)
|
|
176
|
+
default_names = spec.default_rules.map(&:name)
|
|
177
|
+
message_rule(path, "limiter_spec.protected_rules_declared") unless spec.protected_rules.all? { |n| default_names.include?(n) }
|
|
178
|
+
message_rule(path, "limiter_spec.unique_default_names") unless default_names.uniq.size == default_names.size
|
|
179
|
+
# Default rules keep document order, so a position has no meaning there.
|
|
180
|
+
message_rule(path, "limiter_spec.no_position_in_defaults") if spec.default_rules.any?(&:has_position?)
|
|
181
|
+
|
|
182
|
+
string("#{path}.description", spec.description, FIELD_RULES["LimiterSpec.description"])
|
|
183
|
+
list("#{path}.available_properties", spec.available_properties, FIELD_RULES["LimiterSpec.available_properties"])
|
|
184
|
+
list("#{path}.default_rules", spec.default_rules, FIELD_RULES["LimiterSpec.default_rules"])
|
|
185
|
+
spec.default_rules.each_with_index { |r, i| rule("#{path}.default_rules[#{i}]", r) }
|
|
186
|
+
list("#{path}.protected_rules", spec.protected_rules, FIELD_RULES["LimiterSpec.protected_rules"])
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def rule_list(path, list)
|
|
190
|
+
names = list.rules.map(&:name)
|
|
191
|
+
message_rule(path, "rule_list.unique_names") unless names.uniq.size == names.size
|
|
192
|
+
list("#{path}.rules", list.rules, FIELD_RULES["RuleList.rules"])
|
|
193
|
+
list.rules.each_with_index { |r, i| rule("#{path}.rules[#{i}]", r) }
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def rule(path, rule)
|
|
197
|
+
rule_message_rules(path, rule)
|
|
198
|
+
string("#{path}.name", rule.name, FIELD_RULES["Rule.name"])
|
|
199
|
+
string("#{path}.description", rule.description, FIELD_RULES["Rule.description"])
|
|
200
|
+
action("#{path}.action", rule.action, FIELD_RULES["Rule.action"])
|
|
201
|
+
list("#{path}.characteristics", rule.characteristics, FIELD_RULES["Rule.characteristics"])
|
|
202
|
+
map_field("#{path}.match", rule.match, FIELD_RULES["Rule.match"]) { |entry, matcher| matcher(entry, matcher) }
|
|
203
|
+
integer("#{path}.limit", rule.limit, FIELD_RULES["Rule.limit"]) if rule.has_limit?
|
|
204
|
+
integer("#{path}.period_s", rule.period_s, FIELD_RULES["Rule.period_s"]) if rule.has_period_s?
|
|
205
|
+
string("#{path}.count_distinct", rule.count_distinct, FIELD_RULES["Rule.count_distinct"])
|
|
206
|
+
integer("#{path}.ban_for_s", rule.ban_for_s, FIELD_RULES["Rule.ban_for_s"]) if rule.has_ban_for_s?
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# In the order config.proto declares them.
|
|
210
|
+
def rule_message_rules(path, rule)
|
|
211
|
+
counts_distinct = !rule.count_distinct.empty?
|
|
212
|
+
message_rule(path, "rule.limit_requires_threshold") unless rule.action == SKIP || (rule.has_limit? && rule.has_period_s?)
|
|
213
|
+
message_rule(path, "rule.ban_excludes_count_distinct") if rule.has_ban_for_s? && counts_distinct
|
|
214
|
+
skip_rules(path, rule) if rule.action == SKIP
|
|
215
|
+
message_rule(path, "rule.count_distinct_not_characteristic") if counts_distinct && rule.characteristics.include?(rule.count_distinct)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# A skip rule has no counter and no limit to cross.
|
|
219
|
+
def skip_rules(path, rule)
|
|
220
|
+
counter = rule.has_limit? || rule.has_period_s? || !rule.characteristics.empty? || !rule.count_distinct.empty?
|
|
221
|
+
message_rule(path, "rule.ban_not_on_skip") if rule.has_ban_for_s?
|
|
222
|
+
message_rule(path, "rule.no_counter_on_skip") if counter
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# An unknown enum number decodes as an Integer in Ruby.
|
|
226
|
+
def action(path, value, rules)
|
|
227
|
+
if rules[:required] && value == :ACTION_UNSPECIFIED
|
|
228
|
+
add(path, "required", "value is required")
|
|
229
|
+
elsif rules[:defined_only] && value.is_a?(Integer)
|
|
230
|
+
add(path, "enum.defined_only", "value must be one of the defined enum values")
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def matcher(path, matcher)
|
|
235
|
+
op = matcher.op
|
|
236
|
+
return add("#{path}.op", "required", "exactly one field is required in oneof") if op.nil?
|
|
237
|
+
|
|
238
|
+
case op
|
|
239
|
+
when :eq, :ne, :re, :nre then string("#{path}.#{op}", matcher[op.to_s], FIELD_RULES["Matcher.#{op}"])
|
|
240
|
+
when :one_of, :none_of then list("#{path}.#{op}.values", matcher[op.to_s].values, FIELD_RULES["StringList.values"])
|
|
241
|
+
when :all then matcher_list("#{path}.all", matcher.all)
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def matcher_list(path, list)
|
|
246
|
+
# There is no `any`, so an `all` inside an `all` is one flat list.
|
|
247
|
+
message_rule(path, "matcher_list.no_nested_all") if list.matchers.any? { |m| m.op == :all }
|
|
248
|
+
list("#{path}.matchers", list.matchers, FIELD_RULES["MatcherList.matchers"])
|
|
249
|
+
list.matchers.each_with_index { |m, i| matcher("#{path}.matchers[#{i}]", m) }
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Rules of RepeatedRules in field order: min_items, max_items, unique,
|
|
253
|
+
# then the items.
|
|
254
|
+
def list(path, values, rules)
|
|
255
|
+
add(path, "repeated.min_items", "must contain at least #{rules[:min_items]} item(s)") if rules[:min_items] && values.size < rules[:min_items]
|
|
256
|
+
add(path, "repeated.max_items", "must contain no more than #{rules[:max_items]} item(s)") if rules[:max_items] && values.size > rules[:max_items]
|
|
257
|
+
add(path, "repeated.unique", "repeated value must contain unique items") if rules[:unique] && values.uniq.size != values.size
|
|
258
|
+
values.each_with_index { |v, i| name("#{path}[#{i}]", v) } if rules[:items_is_name]
|
|
259
|
+
values.each_with_index { |v, i| max_len("#{path}[#{i}]", v, rules[:items_max_len]) } if rules[:items_max_len]
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# Rules of MapRules in field order: min_pairs, max_pairs, then the
|
|
263
|
+
# keys. Yields each entry's path and value.
|
|
264
|
+
def map_field(path, map, rules)
|
|
265
|
+
add(path, "map.min_pairs", "map must be at least #{rules[:min_pairs]} entries") if rules[:min_pairs] && map.length < rules[:min_pairs]
|
|
266
|
+
add(path, "map.max_pairs", "map must be at most #{rules[:max_pairs]} entries") if rules[:max_pairs] && map.length > rules[:max_pairs]
|
|
267
|
+
map.keys.sort.each do |key|
|
|
268
|
+
value = map[key]
|
|
269
|
+
entry = "#{path}[#{Schema.quote(key)}]"
|
|
270
|
+
add(entry, "string.is_name", NAME_MESSAGE, for_key: true) if rules[:keys_is_name] && !NAME.match?(key)
|
|
271
|
+
yield entry, value
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def string(path, value, rules)
|
|
276
|
+
return if rules[:ignore_empty] && value.empty?
|
|
277
|
+
|
|
278
|
+
max_len(path, value, rules[:max_len]) if rules[:max_len]
|
|
279
|
+
name(path, value) if rules[:is_name]
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# protovalidate reports one of three ids for bounds, by which bounds
|
|
283
|
+
# the field declares.
|
|
284
|
+
def integer(path, value, rules)
|
|
285
|
+
if rules[:in]
|
|
286
|
+
add(path, "uint32.in", "must be in list #{rules[:in]}") unless rules[:in].include?(value)
|
|
287
|
+
return
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
gte, lte = rules.values_at(:gte, :lte)
|
|
291
|
+
below = gte && value < gte
|
|
292
|
+
above = lte && value > lte
|
|
293
|
+
return unless below || above
|
|
294
|
+
|
|
295
|
+
if gte && lte
|
|
296
|
+
add(path, "uint32.gte_lte", "must be greater than or equal to #{gte} and less than or equal to #{lte}")
|
|
297
|
+
elsif below
|
|
298
|
+
add(path, "uint32.gte", "must be greater than or equal to #{gte}")
|
|
299
|
+
else
|
|
300
|
+
add(path, "uint32.lte", "must be less than or equal to #{lte}")
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def message_rule(path, rule_id)
|
|
305
|
+
add(path, rule_id, MESSAGE_RULES.fetch(rule_id))
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
def name(path, value)
|
|
309
|
+
add(path, "string.is_name", NAME_MESSAGE) unless NAME.match?(value)
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def max_len(path, value, max)
|
|
313
|
+
add(path, "string.max_len", "must be at most #{max} characters") if value.length > max
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def add(path, rule_id, message, for_key: false)
|
|
317
|
+
@violations << Violation.new(path: path, rule_id: rule_id, message: message, for_key: for_key)
|
|
318
|
+
end
|
|
319
|
+
end
|
|
320
|
+
private_constant :Checker
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
end
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "google/protobuf/descriptor_pb"
|
|
4
|
+
|
|
5
|
+
module Labkit
|
|
6
|
+
module RateLimit
|
|
7
|
+
module Config
|
|
8
|
+
# VendoredProto registers the vendored gitlab.ratelimit.config.v1 file in
|
|
9
|
+
# the global descriptor pool. The gitlab-ratelimit-config-v1 gem from
|
|
10
|
+
# gitlab-org/protos registers the same file name, unguarded, and a second
|
|
11
|
+
# registration raises. So the gem is loaded first whenever it is
|
|
12
|
+
# installed, and the vendored copy is then compared with it: the same
|
|
13
|
+
# schema means the gem's registration is used, a different one is a boot
|
|
14
|
+
# error, because two versions of one schema in one process would make the
|
|
15
|
+
# loader read fields the other copy does not know.
|
|
16
|
+
module VendoredProto
|
|
17
|
+
# The gem registered a schema that differs from the vendored copy, or
|
|
18
|
+
# one this google-protobuf cannot compare.
|
|
19
|
+
Mismatch = Class.new(StandardError)
|
|
20
|
+
|
|
21
|
+
PACKAGE = "gitlab.ratelimit.config.v1"
|
|
22
|
+
PROBE = "#{PACKAGE}.RateLimits".freeze
|
|
23
|
+
GEM_FILE = "gitlab/ratelimit/config/v1/config_pb"
|
|
24
|
+
|
|
25
|
+
class << self
|
|
26
|
+
# @param pool [Google::Protobuf::DescriptorPool]
|
|
27
|
+
# @param serialized [String] a serialized FileDescriptorProto
|
|
28
|
+
# @return [Boolean] true when this call registered the file and the
|
|
29
|
+
# vendored file must define its constants; false when the gem's
|
|
30
|
+
# registration is in use and the vendored file must stop loading
|
|
31
|
+
# @raise [Mismatch] when the gem registered a different schema
|
|
32
|
+
def register(pool, serialized)
|
|
33
|
+
load_gem_first
|
|
34
|
+
|
|
35
|
+
existing = pool.lookup(PROBE)
|
|
36
|
+
if existing.nil?
|
|
37
|
+
pool.add_serialized_file(serialized)
|
|
38
|
+
return true
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
check_same_schema!(existing.file_descriptor, serialized)
|
|
42
|
+
false
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
# Ruby's require loads the gem's file once, so whichever side comes
|
|
48
|
+
# first, the gem registers before the vendored copy is compared. Only
|
|
49
|
+
# an absent gem is expected; a LoadError from inside the gem's own
|
|
50
|
+
# file is a real failure and propagates.
|
|
51
|
+
def load_gem_first
|
|
52
|
+
require GEM_FILE
|
|
53
|
+
rescue ::LoadError => e
|
|
54
|
+
raise unless e.path == GEM_FILE
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The gem may be ahead of the vendored copy: an additive release adds
|
|
58
|
+
# fields the loader does not read, and that is fine. The gem may not
|
|
59
|
+
# be behind it: the loader would then read a field the gem's schema
|
|
60
|
+
# does not have.
|
|
61
|
+
def check_same_schema!(file, serialized)
|
|
62
|
+
# FileDescriptor#to_proto arrived in google-protobuf 4.30. The gem
|
|
63
|
+
# keeps its 3.25 floor for consumers that never reach this path, so
|
|
64
|
+
# the requirement is raised here, where it applies.
|
|
65
|
+
raise Mismatch, "#{PACKAGE} is registered by another gem and this google-protobuf cannot compare schemas; rate limit config needs google-protobuf >= 4.30" unless file.respond_to?(:to_proto)
|
|
66
|
+
|
|
67
|
+
registered = signature(file.to_proto)
|
|
68
|
+
vendored = signature(Google::Protobuf::FileDescriptorProto.decode(serialized))
|
|
69
|
+
missing = vendored.reject { |path, sig| registered[path] == sig }.keys
|
|
70
|
+
detail = "lacks or changes #{missing.join(', ')}; update the gitlab-labkit vendored copy or the gem"
|
|
71
|
+
raise Mismatch, "#{PACKAGE} is registered by another gem with a schema that #{detail}" unless missing.empty?
|
|
72
|
+
|
|
73
|
+
extra = registered.keys - vendored.keys
|
|
74
|
+
warn "gitlab-labkit: #{PACKAGE} from another gem is a newer schema (#{extra.join(', ')}); the vendored copy is older" unless extra.empty?
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# "Message.field" => its wire and API shape, "Enum.VALUE" => number,
|
|
78
|
+
# for every message, nested message and enum. proto3 optional decides
|
|
79
|
+
# whether has_x? exists, the oneof name which fields exclude each
|
|
80
|
+
# other, and json_name what the parser accepts as a key.
|
|
81
|
+
def signature(file)
|
|
82
|
+
out = {}
|
|
83
|
+
walk = lambda do |messages, enums, prefix|
|
|
84
|
+
enums.each { |e| e.value.each { |val| out["#{prefix}#{e.name}.#{val.name}"] = val.number } }
|
|
85
|
+
messages.each do |m|
|
|
86
|
+
m.field.each { |f| out["#{prefix}#{m.name}.#{f.name}"] = field_signature(f, m) }
|
|
87
|
+
walk.call(m.nested_type, m.enum_type, "#{prefix}#{m.name}.")
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
walk.call(file.message_type, file.enum_type, "")
|
|
91
|
+
out
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# protoc numbers oneofs in declaration order, and every proto3
|
|
95
|
+
# optional field adds a synthetic one, so an additive field shifts
|
|
96
|
+
# the index of every oneof after it. The name does not move.
|
|
97
|
+
def field_signature(field, message)
|
|
98
|
+
oneof = field.has_oneof_index? ? message.oneof_decl[field.oneof_index].name : nil
|
|
99
|
+
[field.number, field.type, field.label, field.type_name, field.json_name, field.proto3_optional, oneof]
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "labkit/rate_limit/config/vendored_proto"
|
|
4
|
+
require "labkit/rate_limit/config/proto/config_pb"
|
|
5
|
+
|
|
6
|
+
module Labkit
|
|
7
|
+
module RateLimit
|
|
8
|
+
# Config loads rate limit rules from YAML files that follow the
|
|
9
|
+
# gitlab.ratelimit.config.v1 schema: the contract an application ships
|
|
10
|
+
# (AvailableLimiters) and the rate limits an operator writes (RateLimits).
|
|
11
|
+
# Loading runs in four steps, parse, schema, contract and rule, and a
|
|
12
|
+
# failure names its step so CI output and the conformance corpus agree
|
|
13
|
+
# across SDKs.
|
|
14
|
+
module Config
|
|
15
|
+
# The generated protobuf classes, vendored from gitlab-org/protos.
|
|
16
|
+
Proto = Gitlab::Ratelimit::Config::V1
|
|
17
|
+
|
|
18
|
+
autoload :Error, "labkit/rate_limit/config/error"
|
|
19
|
+
autoload :Parser, "labkit/rate_limit/config/parser"
|
|
20
|
+
autoload :Schema, "labkit/rate_limit/config/schema"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
data/lib/labkit/rate_limit.rb
CHANGED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Copies the generated Ruby code and the proto sources for gitlab.ratelimit.config.v1
|
|
3
|
+
# from gitlab-org/protos. The sources are what schema_spec.rb checks Schema against.
|
|
4
|
+
# A published gem cannot depend on the protos Bundler git gem, so the file is
|
|
5
|
+
# vendored and CI checks that it matches PROTOS_REF (job vendored-proto-check).
|
|
6
|
+
set -euo pipefail
|
|
7
|
+
|
|
8
|
+
# Pinned protos module tag. Bump it when a newer schema release is needed.
|
|
9
|
+
PROTOS_REF="${PROTOS_REF:-gen/ruby/gitlab/ratelimit/config/v1/v0.1.0}"
|
|
10
|
+
SOURCE_PATH="gen/ruby/gitlab/ratelimit/config/v1/lib/gitlab/ratelimit/config/v1/config_pb.rb"
|
|
11
|
+
PROTO_DIR="proto/ratelimit/gitlab/ratelimit/config/v1"
|
|
12
|
+
TARGET_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/lib/labkit/rate_limit/config/proto"
|
|
13
|
+
TARGET="${TARGET_DIR}/config_pb.rb"
|
|
14
|
+
|
|
15
|
+
raw="https://gitlab.com/gitlab-org/protos/-/raw/${PROTOS_REF}"
|
|
16
|
+
# Everything lands in a temp dir first, so a failed download or check
|
|
17
|
+
# leaves the vendored files as they were.
|
|
18
|
+
work="$(mktemp -d)"
|
|
19
|
+
trap 'rm -rf "$work"' EXIT
|
|
20
|
+
tmp="${work}/config_pb.rb"
|
|
21
|
+
curl --fail --silent --show-error --location "${raw}/${SOURCE_PATH}" -o "$tmp"
|
|
22
|
+
for proto in config.proto constraint.proto; do
|
|
23
|
+
curl --fail --silent --show-error --location "${raw}/${PROTO_DIR}/${proto}" -o "${work}/${proto}"
|
|
24
|
+
done
|
|
25
|
+
|
|
26
|
+
# The generated file registers gitlab/ratelimit/config/v1/config.proto in the
|
|
27
|
+
# global descriptor pool. The protos gem registers the same file name, so the
|
|
28
|
+
# vendored copy goes through VendoredProto.register, which loads the gem first
|
|
29
|
+
# when it is installed and then compares schemas. When the gem's registration
|
|
30
|
+
# is in use the file returns early, so its constants are not defined twice.
|
|
31
|
+
guard='return unless Labkit::RateLimit::Config::VendoredProto.register(pool, descriptor_data)'
|
|
32
|
+
{
|
|
33
|
+
echo "# frozen_string_literal: true"
|
|
34
|
+
echo "# Vendored from gitlab-org/protos at ${PROTOS_REF}, ${SOURCE_PATH}."
|
|
35
|
+
echo "# Do not edit. Run scripts/vendor-ratelimit-proto.sh to update."
|
|
36
|
+
grep -v '^# frozen_string_literal: true$' "$tmp" |
|
|
37
|
+
sed -e "s|^require 'google/protobuf'\$|require 'google/protobuf'\nrequire 'labkit/rate_limit/config/vendored_proto'|" \
|
|
38
|
+
-e "s|^pool.add_serialized_file(descriptor_data)\$|${guard}|"
|
|
39
|
+
} >"${work}/vendored.rb"
|
|
40
|
+
if ! grep -q 'VendoredProto.register' "${work}/vendored.rb" ||
|
|
41
|
+
! grep -qx "require 'labkit/rate_limit/config/vendored_proto'" "${work}/vendored.rb"; then
|
|
42
|
+
echo "vendor: the generated file no longer requires VendoredProto or registers the descriptor as expected; update the sed in this script" >&2
|
|
43
|
+
exit 1
|
|
44
|
+
fi
|
|
45
|
+
mv "${work}/vendored.rb" "$TARGET"
|
|
46
|
+
mv "${work}/config.proto" "${work}/constraint.proto" "$TARGET_DIR/"
|
|
47
|
+
echo "vendored ${SOURCE_PATH}, config.proto and constraint.proto @${PROTOS_REF} -> ${TARGET_DIR#"$PWD"/}"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gitlab-labkit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.
|
|
4
|
+
version: 5.11.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andrew Newdigate
|
|
@@ -651,6 +651,12 @@ files:
|
|
|
651
651
|
- lib/labkit/rate_limit.rb
|
|
652
652
|
- lib/labkit/rate_limit/README.md
|
|
653
653
|
- lib/labkit/rate_limit/block_summary.rb
|
|
654
|
+
- lib/labkit/rate_limit/config.rb
|
|
655
|
+
- lib/labkit/rate_limit/config/error.rb
|
|
656
|
+
- lib/labkit/rate_limit/config/parser.rb
|
|
657
|
+
- lib/labkit/rate_limit/config/proto/config_pb.rb
|
|
658
|
+
- lib/labkit/rate_limit/config/schema.rb
|
|
659
|
+
- lib/labkit/rate_limit/config/vendored_proto.rb
|
|
654
660
|
- lib/labkit/rate_limit/configuration.rb
|
|
655
661
|
- lib/labkit/rate_limit/evaluator.rb
|
|
656
662
|
- lib/labkit/rate_limit/identifier.rb
|
|
@@ -719,6 +725,7 @@ files:
|
|
|
719
725
|
- renovate.json
|
|
720
726
|
- scripts/prepare-dev-env.sh
|
|
721
727
|
- scripts/update-asdf-version-variables.sh
|
|
728
|
+
- scripts/vendor-ratelimit-proto.sh
|
|
722
729
|
homepage: https://gitlab.com/gitlab-org/labkit-ruby
|
|
723
730
|
licenses:
|
|
724
731
|
- MIT
|