gitlab-labkit 5.9.0 → 5.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +13 -0
- data/.rubocop.yml +8 -0
- data/Gemfile +6 -0
- data/lib/labkit/rate_limit/README.md +33 -0
- data/lib/labkit/rate_limit/config/error.rb +18 -0
- data/lib/labkit/rate_limit/config/parser.rb +243 -0
- data/lib/labkit/rate_limit/config/proto/config_pb.rb +34 -0
- data/lib/labkit/rate_limit/config/vendored_proto.rb +105 -0
- data/lib/labkit/rate_limit/config.rb +22 -0
- data/lib/labkit/rate_limit.rb +1 -0
- data/scripts/vendor-ratelimit-proto.sh +36 -0
- metadata +7 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3cf9de3ab607ca2b1f4e24b04a55efc3081556acaf61c8fd4c451fd786aa632a
|
|
4
|
+
data.tar.gz: 6a7d1d5fd0434e9298c41bfa6057f96ae08e37c3ea100f37db6d4879ecea688e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 931e421f2c52d3d98602f3657088701bb5a48b2f98bd9289becffad20df9d991c3fc1d2f9b03d5ba6cbc26593ca38176227827afc66b973393302e1f789f89cd
|
|
7
|
+
data.tar.gz: b971f78dfcf2529cda540c44e950651876cfd89e5d68af517c10f5013ead19d50c7f603d7ecc582787e790ab6ae1deb46dbc342b010676a48aff8964fb89b109
|
data/.gitlab-ci.yml
CHANGED
|
@@ -81,6 +81,19 @@ deploy:
|
|
|
81
81
|
- Gemfile
|
|
82
82
|
- .gitlab-ci.yml
|
|
83
83
|
|
|
84
|
+
# The generated protobuf code for gitlab.ratelimit.config.v1 is vendored from
|
|
85
|
+
# gitlab-org/protos, because a published gem cannot depend on the protos git gem.
|
|
86
|
+
vendored-proto-check:
|
|
87
|
+
stage: validate
|
|
88
|
+
needs: []
|
|
89
|
+
image: ruby:3.4
|
|
90
|
+
rules:
|
|
91
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
92
|
+
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
93
|
+
script:
|
|
94
|
+
- ./scripts/vendor-ratelimit-proto.sh
|
|
95
|
+
- git diff --exit-code -- lib/labkit/rate_limit/config/proto/
|
|
96
|
+
|
|
84
97
|
# Gate SDK changes on the labkit-spec conformance suite, run remotely: trigger
|
|
85
98
|
# a downstream pipeline in labkit-spec that checks out this pipeline's commit
|
|
86
99
|
# (LABKIT_RUBY_SHA) and runs the Ruby conformance suite against it, verifying
|
data/.rubocop.yml
CHANGED
|
@@ -4,5 +4,13 @@ inherit_gem:
|
|
|
4
4
|
gitlab-styles:
|
|
5
5
|
- rubocop-default.yml
|
|
6
6
|
|
|
7
|
+
inherit_mode:
|
|
8
|
+
merge:
|
|
9
|
+
- Exclude
|
|
10
|
+
|
|
7
11
|
AllCops:
|
|
8
12
|
NewCops: enable
|
|
13
|
+
Exclude:
|
|
14
|
+
# Generated by protoc and vendored from gitlab-org/protos; regenerate with
|
|
15
|
+
# scripts/vendor-ratelimit-proto.sh instead of editing.
|
|
16
|
+
- lib/labkit/rate_limit/config/proto/**/*
|
data/Gemfile
CHANGED
|
@@ -8,3 +8,9 @@ gemspec
|
|
|
8
8
|
# Exporter source: https://gitlab.com/gitlab-org/quality/analytics/test-metrics-exporters
|
|
9
9
|
# Owned by: @gitlab-org/quality/analytics (group::development analytics)
|
|
10
10
|
gem 'gitlab-rspec-metrics-exporter', '~> 0.2', groups: [:development, :test]
|
|
11
|
+
|
|
12
|
+
# regexp_parser 2.13.2 made Regexp::Syntax::Base.implements private, and
|
|
13
|
+
# json_schemer 2.5.0 calls it at load time, so every fresh bundle fails on
|
|
14
|
+
# `require "json_schemer"`. Drop the pin once json_schemer or regexp_parser
|
|
15
|
+
# ships a fix: https://github.com/davishmcclurg/json_schemer/issues/238
|
|
16
|
+
gem "regexp_parser", "!= 2.13.2", groups: [:development, :test]
|
|
@@ -711,6 +711,39 @@ A logging failure during a flush is logged at WARN with
|
|
|
711
711
|
`dropped_pairs`, the number of lines that flush lost. It never raises, so the
|
|
712
712
|
flush thread keeps running.
|
|
713
713
|
|
|
714
|
+
## Config files
|
|
715
|
+
|
|
716
|
+
`Labkit::RateLimit::Config` loads rules from YAML files that follow the `gitlab.ratelimit.config.v1` schema from [gitlab-org/protos](https://gitlab.com/gitlab-org/protos): the contract an application ships (`AvailableLimiters`) and the rate limits an operator writes (`RateLimits`).
|
|
717
|
+
The generated protobuf classes are vendored under `lib/labkit/rate_limit/config/proto/` and reachable as `Labkit::RateLimit::Config::Proto`.
|
|
718
|
+
Run `scripts/vendor-ratelimit-proto.sh` to update them; CI fails on drift.
|
|
719
|
+
|
|
720
|
+
An application can also install the `gitlab-ratelimit-config-v1` gem from gitlab-org/protos.
|
|
721
|
+
Then labkit uses the gem's schema and compares it with the vendored copy when `Labkit::RateLimit::Config` loads.
|
|
722
|
+
A gem schema that is newer gives a warning.
|
|
723
|
+
A gem schema that lacks or changes a vendored field raises `Labkit::RateLimit::Config::VendoredProto::Mismatch`.
|
|
724
|
+
So upgrade the `gitlab-ratelimit-config-v1` gem in the application first, then the `gitlab-labkit` release that updates the vendored copy.
|
|
725
|
+
|
|
726
|
+
Loading runs in four steps, `parse`, `schema`, `contract` and `rule`, and a failure raises `Labkit::RateLimit::Config::Error` whose `#step` names the step.
|
|
727
|
+
Only the parse step exists so far; the schema, contract and rule steps follow.
|
|
728
|
+
Until the schema step lands, no mode validates a document: strict mode lets an enum number that no `Action` defines through, such as `action: 7`, and lenient mode leaves an unknown `action` name as `ACTION_UNSPECIFIED`.
|
|
729
|
+
The schema step rejects both.
|
|
730
|
+
|
|
731
|
+
```ruby
|
|
732
|
+
doc = Labkit::RateLimit::Config::Parser.parse(
|
|
733
|
+
"config/rate_limits/rack_request.yaml",
|
|
734
|
+
Labkit::RateLimit::Config::Proto::RateLimits,
|
|
735
|
+
strict: true # CI rejects unknown fields. Production loads with strict: false once the schema step exists
|
|
736
|
+
)
|
|
737
|
+
doc.rate_limits["rack_request"].rules.first.name
|
|
738
|
+
```
|
|
739
|
+
|
|
740
|
+
The parse step converts each YAML scalar by the type of the field it fills, so `eq: off`, `eq: True` and `eq: 9970` stay strings, `limit: "5"` becomes `5`, `action: "1"` becomes `ACTION_LIMIT`, and a YAML null leaves the field unset.
|
|
741
|
+
A repeated key, also one repeated under its camelCase `json_name`, a merge key (`<<`), an empty file, more than one document, or a non-scalar key fails at step `parse`, in both modes.
|
|
742
|
+
So does a value of the wrong kind for its field, such as a mapping where a scalar belongs, an integer out of range for its field, two fields of one `oneof`, such as `{ eq: a, re: b }`, a null list item or map value, and a document nested deeper than 64 levels.
|
|
743
|
+
Each of these errors names the YAML line.
|
|
744
|
+
An unknown field or an unknown enum name, such as `action: ACTION_LIMT`, fails in strict mode with its YAML line.
|
|
745
|
+
Lenient mode ignores an unknown field and leaves an unknown enum name unset.
|
|
746
|
+
|
|
714
747
|
## Dev/test vs production guards
|
|
715
748
|
|
|
716
749
|
`Limiter.new` and `Rule.new` validate names and configuration. In
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Labkit
|
|
4
|
+
module RateLimit
|
|
5
|
+
module Config
|
|
6
|
+
# Error is raised when a document fails one of the load steps.
|
|
7
|
+
# step is :parse, :schema, :contract or :rule.
|
|
8
|
+
class Error < StandardError
|
|
9
|
+
attr_reader :step
|
|
10
|
+
|
|
11
|
+
def initialize(step, message)
|
|
12
|
+
@step = step
|
|
13
|
+
super("#{step}: #{message}")
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "psych"
|
|
5
|
+
require "google/protobuf/descriptor_pb"
|
|
6
|
+
|
|
7
|
+
module Labkit
|
|
8
|
+
module RateLimit
|
|
9
|
+
module Config
|
|
10
|
+
# Parser is the parse step: a YAML file into a protobuf message.
|
|
11
|
+
#
|
|
12
|
+
# YAML types do not follow the schema, so each scalar is converted by the
|
|
13
|
+
# type of the field it fills, the way protoyaml does in Go: a string field
|
|
14
|
+
# keeps the scalar text (`off`, `True` and `9970` stay strings), an integer
|
|
15
|
+
# or enum field gets an Integer when the text is one, quoted or not, and
|
|
16
|
+
# the YAML null forms mean the field is absent. Psych keeps the last value
|
|
17
|
+
# of a repeated key, so repeated keys are rejected here, also when one
|
|
18
|
+
# spelling is the snake_case field name and the other its camelCase
|
|
19
|
+
# json_name.
|
|
20
|
+
#
|
|
21
|
+
# This step checks the shape of a document, not its values. An enum
|
|
22
|
+
# number that no Action defines, such as `action: 7`, passes in both
|
|
23
|
+
# modes, and in lenient mode protobuf drops an unknown enum name together
|
|
24
|
+
# with the unknown fields, so a rule with a misspelt action reaches the
|
|
25
|
+
# schema step as ACTION_UNSPECIFIED. The schema step rejects or drops
|
|
26
|
+
# both.
|
|
27
|
+
module Parser
|
|
28
|
+
INT32 = (-(2**31))...(2**31)
|
|
29
|
+
UINT32 = 0...(2**32)
|
|
30
|
+
INT64 = (-(2**63))...(2**63)
|
|
31
|
+
UINT64 = 0...(2**64)
|
|
32
|
+
INTEGER_RANGES = {
|
|
33
|
+
int32: INT32, sint32: INT32, sfixed32: INT32, enum: INT32,
|
|
34
|
+
uint32: UINT32, fixed32: UINT32,
|
|
35
|
+
int64: INT64, sint64: INT64, sfixed64: INT64,
|
|
36
|
+
uint64: UINT64, fixed64: UINT64
|
|
37
|
+
}.freeze
|
|
38
|
+
INTEGER = /\A[-+]?\d+\z/
|
|
39
|
+
NULLS = ["", "~", "null", "Null", "NULL"].freeze
|
|
40
|
+
# The protobuf JSON decoder stops at 64 levels of objects and arrays.
|
|
41
|
+
# Mappings and lists are counted here so the error names the YAML line
|
|
42
|
+
# and a deeper document cannot exhaust the stack.
|
|
43
|
+
MAX_DEPTH = 64
|
|
44
|
+
|
|
45
|
+
class << self
|
|
46
|
+
# @param path [String] YAML file
|
|
47
|
+
# @param klass [Class] a Proto message class, AvailableLimiters or RateLimits
|
|
48
|
+
# @param strict [Boolean] when true an unknown field is an error
|
|
49
|
+
# @return [Google::Protobuf::AbstractMessage]
|
|
50
|
+
# @raise [Error] with step :parse
|
|
51
|
+
def parse(path, klass, strict:)
|
|
52
|
+
tree = tree_for(document_root(path), klass.descriptor, strict: strict, depth: 0)
|
|
53
|
+
klass.decode_json(JSON.generate(tree), ignore_unknown_fields: !strict)
|
|
54
|
+
rescue Psych::SyntaxError, SystemCallError => e
|
|
55
|
+
# Both messages already name the file.
|
|
56
|
+
raise Error.new(:parse, e.message)
|
|
57
|
+
rescue Google::Protobuf::ParseError, Psych::Exception, JSON::JSONError => e
|
|
58
|
+
raise Error.new(:parse, "#{path}: #{e.message}")
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
# One YAML document per file. Psych would silently keep only the first
|
|
64
|
+
# of several, and an empty file has no root to walk.
|
|
65
|
+
def document_root(path)
|
|
66
|
+
documents = Psych.parse_stream(File.read(path), filename: path).children
|
|
67
|
+
raise Psych::Exception, "the file holds no YAML document" if documents.empty?
|
|
68
|
+
raise Psych::Exception, "the file holds #{documents.size} YAML documents, expected 1" if documents.size > 1
|
|
69
|
+
|
|
70
|
+
documents.first.root
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# One value for one slot. type is a field type Symbol, a message
|
|
74
|
+
# Descriptor or an EnumDescriptor. A node of the wrong kind is
|
|
75
|
+
# rejected here so the error names the YAML line, not an offset into
|
|
76
|
+
# the generated JSON. depth counts the mappings and lists above node.
|
|
77
|
+
def tree_for(node, type, strict:, depth:)
|
|
78
|
+
expects_message = type.is_a?(Google::Protobuf::Descriptor)
|
|
79
|
+
|
|
80
|
+
case node
|
|
81
|
+
when Psych::Nodes::Mapping
|
|
82
|
+
fail_at(node, "expected a scalar, found a mapping") unless expects_message
|
|
83
|
+
mapping_for(node, type, strict: strict, depth: depth)
|
|
84
|
+
when Psych::Nodes::Sequence
|
|
85
|
+
fail_at(node, "expected a single value, found a list")
|
|
86
|
+
when Psych::Nodes::Scalar
|
|
87
|
+
return nil if null?(node)
|
|
88
|
+
|
|
89
|
+
fail_at(node, "expected a mapping, found a scalar") if expects_message
|
|
90
|
+
scalar_for(node, type, strict: strict)
|
|
91
|
+
when Psych::Nodes::Alias
|
|
92
|
+
fail_at(node, "aliases are not supported")
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def mapping_for(node, descriptor, strict:, depth:)
|
|
97
|
+
check_depth!(node, depth)
|
|
98
|
+
entries = node.children.each_slice(2).map do |key, value|
|
|
99
|
+
name = scalar_key(key)
|
|
100
|
+
field = field_for(descriptor, name)
|
|
101
|
+
fail_at(key, "unknown field #{name.inspect}") if strict && field.nil?
|
|
102
|
+
[name, key, field, value]
|
|
103
|
+
end
|
|
104
|
+
reject_repeated_keys!(entries.map { |name, key, field, _| [field ? field.name : name, key] })
|
|
105
|
+
reject_oneof_conflicts!(descriptor, entries)
|
|
106
|
+
|
|
107
|
+
maps = map_fields(descriptor)
|
|
108
|
+
# An unknown field is not walked: strict mode rejected its name
|
|
109
|
+
# above and lenient mode ignores it, so a newer schema may put
|
|
110
|
+
# anything there.
|
|
111
|
+
entries.to_h do |name, _, field, value|
|
|
112
|
+
next [name, nil] unless field
|
|
113
|
+
|
|
114
|
+
[name, field_value_for(value, field, map: maps.include?(field.name), strict: strict, depth: depth + 1)]
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Descriptor#options, which carries the map_entry flag, arrived after
|
|
119
|
+
# google-protobuf 3.25.0, so a blank message tells a map from a list.
|
|
120
|
+
# A message type keeps one Descriptor object, so the cache is bounded
|
|
121
|
+
# by the number of message types.
|
|
122
|
+
def map_fields(descriptor)
|
|
123
|
+
(@map_fields ||= {})[descriptor] ||= begin
|
|
124
|
+
blank = descriptor.msgclass.new
|
|
125
|
+
descriptor.select { |field| blank[field.name].is_a?(Google::Protobuf::Map) }.map(&:name).freeze
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# A mapping key must be a plain scalar; a sequence, mapping or alias
|
|
130
|
+
# as a key has no field to fill. A merge key is rejected in both
|
|
131
|
+
# modes: lenient mode would ignore it as an unknown field and drop
|
|
132
|
+
# the merged fields with no error.
|
|
133
|
+
def scalar_key(node)
|
|
134
|
+
fail_at(node, "a mapping key must be a scalar") unless node.is_a?(Psych::Nodes::Scalar)
|
|
135
|
+
fail_at(node, "merge keys (<<) are not supported") if node.plain && node.value == "<<"
|
|
136
|
+
|
|
137
|
+
node.value
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# The error points at the repeated key, not at the start of the mapping.
|
|
141
|
+
def reject_repeated_keys!(named_keys)
|
|
142
|
+
seen = {}
|
|
143
|
+
named_keys.each do |name, key|
|
|
144
|
+
fail_at(key, "duplicate key #{name.inspect}") if seen[name]
|
|
145
|
+
seen[name] = true
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Two fields of one oneof, such as two Matcher operators, would
|
|
150
|
+
# otherwise fail in the JSON decoder with no YAML line.
|
|
151
|
+
def reject_oneof_conflicts!(descriptor, entries)
|
|
152
|
+
descriptor.each_oneof do |oneof|
|
|
153
|
+
names = oneof.map(&:name)
|
|
154
|
+
set = entries.select { |_, _, field, value| field && names.include?(field.name) && !null?(value) }
|
|
155
|
+
next if set.size < 2
|
|
156
|
+
|
|
157
|
+
fail_at(set[1][1], "#{set[0][2].name} and #{set[1][2].name} are both set; set only one field of #{oneof.name}")
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def check_depth!(node, depth)
|
|
162
|
+
fail_at(node, "nesting deeper than #{MAX_DEPTH} levels") if depth >= MAX_DEPTH
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def fail_at(node, message)
|
|
166
|
+
raise Psych::Exception, "line #{node.start_line + 1}: #{message}"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Files are snake_case; protojson also accepts the camelCase json_name.
|
|
170
|
+
def field_for(descriptor, name)
|
|
171
|
+
descriptor.lookup(name) || descriptor.find { |field| field.json_name == name }
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def field_value_for(node, field, map:, strict:, depth:)
|
|
175
|
+
return map_for(node, field, strict: strict, depth: depth) if map
|
|
176
|
+
return tree_for(node, element_type(field), strict: strict, depth: depth) unless field.label == :repeated
|
|
177
|
+
return nil if null?(node)
|
|
178
|
+
|
|
179
|
+
fail_at(node, "expected a list") unless node.is_a?(Psych::Nodes::Sequence)
|
|
180
|
+
check_depth!(node, depth)
|
|
181
|
+
node.children.map do |child|
|
|
182
|
+
fail_at(child, "a list item cannot be null") if null?(child)
|
|
183
|
+
|
|
184
|
+
tree_for(child, element_type(field), strict: strict, depth: depth + 1)
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def map_for(node, field, strict:, depth:)
|
|
189
|
+
return nil if null?(node)
|
|
190
|
+
|
|
191
|
+
fail_at(node, "expected a mapping") unless node.is_a?(Psych::Nodes::Mapping)
|
|
192
|
+
check_depth!(node, depth)
|
|
193
|
+
pairs = node.children.each_slice(2)
|
|
194
|
+
reject_repeated_keys!(pairs.map { |key, _| [scalar_key(key), key] })
|
|
195
|
+
value_type = element_type(field.subtype.lookup("value"))
|
|
196
|
+
pairs.to_h do |key, value|
|
|
197
|
+
fail_at(value, "a map value cannot be null") if null?(value)
|
|
198
|
+
|
|
199
|
+
[scalar_key(key), tree_for(value, value_type, strict: strict, depth: depth + 1)]
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# A message or enum field passes its descriptor down, so a scalar
|
|
204
|
+
# can be checked against the enum's names.
|
|
205
|
+
def element_type(field)
|
|
206
|
+
%i[message enum].include?(field.type) ? field.subtype : field.type
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def null?(node)
|
|
210
|
+
node.is_a?(Psych::Nodes::Scalar) && node.plain && NULLS.include?(node.value)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Integer text is converted quoted or not: protojson takes a quoted
|
|
214
|
+
# number for an integer field but not for an enum. The range is
|
|
215
|
+
# checked here so the error names the YAML line.
|
|
216
|
+
def scalar_for(node, type, strict:)
|
|
217
|
+
enum = type if type.is_a?(Google::Protobuf::EnumDescriptor)
|
|
218
|
+
kind = enum ? :enum : type
|
|
219
|
+
range = INTEGER_RANGES[kind]
|
|
220
|
+
if range && INTEGER.match?(node.value)
|
|
221
|
+
value = Integer(node.value, 10)
|
|
222
|
+
fail_at(node, "#{node.value} is out of range for #{kind}") unless range.cover?(value)
|
|
223
|
+
return value
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# Lenient mode passes an unknown name on, and protobuf drops it
|
|
227
|
+
# with the unknown fields, so a newer schema's value still loads.
|
|
228
|
+
fail_at(node, "unknown value #{node.value.inspect} for enum #{enum.name}") if enum && strict && enum.lookup_name(node.value.to_sym).nil?
|
|
229
|
+
|
|
230
|
+
return node.value unless node.plain
|
|
231
|
+
|
|
232
|
+
case kind
|
|
233
|
+
when :bool
|
|
234
|
+
{ "true" => true, "false" => false }.fetch(node.value, node.value)
|
|
235
|
+
else
|
|
236
|
+
node.value
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Vendored from gitlab-org/protos at gen/ruby/gitlab/ratelimit/config/v1/v0.1.0, gen/ruby/gitlab/ratelimit/config/v1/lib/gitlab/ratelimit/config/v1/config_pb.rb.
|
|
3
|
+
# Do not edit. Run scripts/vendor-ratelimit-proto.sh to update.
|
|
4
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
5
|
+
# source: gitlab/ratelimit/config/v1/config.proto
|
|
6
|
+
|
|
7
|
+
require 'google/protobuf'
|
|
8
|
+
require 'labkit/rate_limit/config/vendored_proto'
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
descriptor_data = "\n\'gitlab/ratelimit/config/v1/config.proto\x12\x1agitlab.ratelimit.config.v1\"\xf7\x01\n\x11\x41vailableLimiters\x12s\n\x12\x61vailable_limiters\x18\x01 \x03(\x0b\x32\x44.gitlab.ratelimit.config.v1.AvailableLimiters.AvailableLimitersEntryR\x11\x61vailableLimiters\x1am\n\x16\x41vailableLimitersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12=\n\x05value\x18\x02 \x01(\x0b\x32\'.gitlab.ratelimit.config.v1.LimiterSpecR\x05value:\x02\x38\x01\"\xd2\x01\n\x0bLimiterSpec\x12 \n\x0b\x64\x65scription\x18\x01 \x01(\tR\x0b\x64\x65scription\x12\x31\n\x14\x61vailable_properties\x18\x02 \x03(\tR\x13\x61vailableProperties\x12\x45\n\rdefault_rules\x18\x03 \x03(\x0b\x32 .gitlab.ratelimit.config.v1.RuleR\x0c\x64\x65\x66\x61ultRules\x12\'\n\x0fprotected_rules\x18\x04 \x03(\tR\x0eprotectedRules\"\xca\x01\n\nRateLimits\x12W\n\x0brate_limits\x18\x01 \x03(\x0b\x32\x36.gitlab.ratelimit.config.v1.RateLimits.RateLimitsEntryR\nrateLimits\x1a\x63\n\x0fRateLimitsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12:\n\x05value\x18\x02 \x01(\x0b\x32$.gitlab.ratelimit.config.v1.RuleListR\x05value:\x02\x38\x01\"B\n\x08RuleList\x12\x36\n\x05rules\x18\x01 \x03(\x0b\x32 .gitlab.ratelimit.config.v1.RuleR\x05rules\"\x9a\x04\n\x04Rule\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\x12:\n\x06\x61\x63tion\x18\x03 \x01(\x0e\x32\".gitlab.ratelimit.config.v1.ActionR\x06\x61\x63tion\x12(\n\x0f\x63haracteristics\x18\x04 \x03(\tR\x0f\x63haracteristics\x12\x41\n\x05match\x18\x05 \x03(\x0b\x32+.gitlab.ratelimit.config.v1.Rule.MatchEntryR\x05match\x12\x19\n\x05limit\x18\x06 \x01(\rH\x00R\x05limit\x88\x01\x01\x12\x1e\n\x08period_s\x18\x07 \x01(\rH\x01R\x07periodS\x88\x01\x01\x12%\n\x0e\x63ount_distinct\x18\x08 \x01(\tR\rcountDistinct\x12\x1f\n\tban_for_s\x18\t \x01(\rH\x02R\x07\x62\x61nForS\x88\x01\x01\x12\x1f\n\x08position\x18\n \x01(\rH\x03R\x08position\x88\x01\x01\x1a]\n\nMatchEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32#.gitlab.ratelimit.config.v1.MatcherR\x05value:\x02\x38\x01\x42\x08\n\x06_limitB\x0b\n\t_period_sB\x0c\n\n_ban_for_sB\x0b\n\t_position\"\x93\x03\n\x13PublishRulesRequest\x12`\n\x0brate_limits\x18\x01 \x03(\x0b\x32?.gitlab.ratelimit.config.v1.PublishRulesRequest.RateLimitsEntryR\nrateLimits\x12S\n\x06remove\x18\x02 \x03(\x0b\x32;.gitlab.ratelimit.config.v1.PublishRulesRequest.RemoveEntryR\x06remove\x1a\x63\n\x0fRateLimitsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12:\n\x05value\x18\x02 \x01(\x0b\x32$.gitlab.ratelimit.config.v1.RuleListR\x05value:\x02\x38\x01\x1a`\n\x0bRemoveEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12;\n\x05value\x18\x02 \x01(\x0b\x32%.gitlab.ratelimit.config.v1.RuleNamesR\x05value:\x02\x38\x01\"!\n\tRuleNames\x12\x14\n\x05names\x18\x01 \x03(\tR\x05names\"\x9a\x02\n\x07Matcher\x12\x10\n\x02\x65q\x18\x01 \x01(\tH\x00R\x02\x65q\x12\x10\n\x02ne\x18\x02 \x01(\tH\x00R\x02ne\x12\x10\n\x02re\x18\x03 \x01(\tH\x00R\x02re\x12\x12\n\x03nre\x18\x04 \x01(\tH\x00R\x03nre\x12?\n\x06one_of\x18\x05 \x01(\x0b\x32&.gitlab.ratelimit.config.v1.StringListH\x00R\x05oneOf\x12\x41\n\x07none_of\x18\x06 \x01(\x0b\x32&.gitlab.ratelimit.config.v1.StringListH\x00R\x06noneOf\x12;\n\x03\x61ll\x18\x07 \x01(\x0b\x32\'.gitlab.ratelimit.config.v1.MatcherListH\x00R\x03\x61llB\x04\n\x02op\"$\n\nStringList\x12\x16\n\x06values\x18\x01 \x03(\tR\x06values\"N\n\x0bMatcherList\x12?\n\x08matchers\x18\x01 \x03(\x0b\x32#.gitlab.ratelimit.config.v1.MatcherR\x08matchers*S\n\x06\x41\x63tion\x12\x16\n\x12\x41\x43TION_UNSPECIFIED\x10\x00\x12\x10\n\x0c\x41\x43TION_LIMIT\x10\x01\x12\x0e\n\nACTION_LOG\x10\x02\x12\x0f\n\x0b\x41\x43TION_SKIP\x10\x03\x62\x06proto3"
|
|
12
|
+
|
|
13
|
+
pool = ::Google::Protobuf::DescriptorPool.generated_pool
|
|
14
|
+
return unless Labkit::RateLimit::Config::VendoredProto.register(pool, descriptor_data)
|
|
15
|
+
|
|
16
|
+
module Gitlab
|
|
17
|
+
module Ratelimit
|
|
18
|
+
module Config
|
|
19
|
+
module V1
|
|
20
|
+
AvailableLimiters = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.AvailableLimiters").msgclass
|
|
21
|
+
LimiterSpec = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.LimiterSpec").msgclass
|
|
22
|
+
RateLimits = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RateLimits").msgclass
|
|
23
|
+
RuleList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RuleList").msgclass
|
|
24
|
+
Rule = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Rule").msgclass
|
|
25
|
+
PublishRulesRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.PublishRulesRequest").msgclass
|
|
26
|
+
RuleNames = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RuleNames").msgclass
|
|
27
|
+
Matcher = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Matcher").msgclass
|
|
28
|
+
StringList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.StringList").msgclass
|
|
29
|
+
MatcherList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.MatcherList").msgclass
|
|
30
|
+
Action = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Action").enummodule
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "google/protobuf/descriptor_pb"
|
|
4
|
+
|
|
5
|
+
module Labkit
|
|
6
|
+
module RateLimit
|
|
7
|
+
module Config
|
|
8
|
+
# VendoredProto registers the vendored gitlab.ratelimit.config.v1 file in
|
|
9
|
+
# the global descriptor pool. The gitlab-ratelimit-config-v1 gem from
|
|
10
|
+
# gitlab-org/protos registers the same file name, unguarded, and a second
|
|
11
|
+
# registration raises. So the gem is loaded first whenever it is
|
|
12
|
+
# installed, and the vendored copy is then compared with it: the same
|
|
13
|
+
# schema means the gem's registration is used, a different one is a boot
|
|
14
|
+
# error, because two versions of one schema in one process would make the
|
|
15
|
+
# loader read fields the other copy does not know.
|
|
16
|
+
module VendoredProto
|
|
17
|
+
# The gem registered a schema that differs from the vendored copy, or
|
|
18
|
+
# one this google-protobuf cannot compare.
|
|
19
|
+
Mismatch = Class.new(StandardError)
|
|
20
|
+
|
|
21
|
+
PACKAGE = "gitlab.ratelimit.config.v1"
|
|
22
|
+
PROBE = "#{PACKAGE}.RateLimits".freeze
|
|
23
|
+
GEM_FILE = "gitlab/ratelimit/config/v1/config_pb"
|
|
24
|
+
|
|
25
|
+
class << self
|
|
26
|
+
# @param pool [Google::Protobuf::DescriptorPool]
|
|
27
|
+
# @param serialized [String] a serialized FileDescriptorProto
|
|
28
|
+
# @return [Boolean] true when this call registered the file and the
|
|
29
|
+
# vendored file must define its constants; false when the gem's
|
|
30
|
+
# registration is in use and the vendored file must stop loading
|
|
31
|
+
# @raise [Mismatch] when the gem registered a different schema
|
|
32
|
+
def register(pool, serialized)
|
|
33
|
+
load_gem_first
|
|
34
|
+
|
|
35
|
+
existing = pool.lookup(PROBE)
|
|
36
|
+
if existing.nil?
|
|
37
|
+
pool.add_serialized_file(serialized)
|
|
38
|
+
return true
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
check_same_schema!(existing.file_descriptor, serialized)
|
|
42
|
+
false
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
# Ruby's require loads the gem's file once, so whichever side comes
|
|
48
|
+
# first, the gem registers before the vendored copy is compared. Only
|
|
49
|
+
# an absent gem is expected; a LoadError from inside the gem's own
|
|
50
|
+
# file is a real failure and propagates.
|
|
51
|
+
def load_gem_first
|
|
52
|
+
require GEM_FILE
|
|
53
|
+
rescue ::LoadError => e
|
|
54
|
+
raise unless e.path == GEM_FILE
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The gem may be ahead of the vendored copy: an additive release adds
|
|
58
|
+
# fields the loader does not read, and that is fine. The gem may not
|
|
59
|
+
# be behind it: the loader would then read a field the gem's schema
|
|
60
|
+
# does not have.
|
|
61
|
+
def check_same_schema!(file, serialized)
|
|
62
|
+
# FileDescriptor#to_proto arrived in google-protobuf 4.30. The gem
|
|
63
|
+
# keeps its 3.25 floor for consumers that never reach this path, so
|
|
64
|
+
# the requirement is raised here, where it applies.
|
|
65
|
+
raise Mismatch, "#{PACKAGE} is registered by another gem and this google-protobuf cannot compare schemas; rate limit config needs google-protobuf >= 4.30" unless file.respond_to?(:to_proto)
|
|
66
|
+
|
|
67
|
+
registered = signature(file.to_proto)
|
|
68
|
+
vendored = signature(Google::Protobuf::FileDescriptorProto.decode(serialized))
|
|
69
|
+
missing = vendored.reject { |path, sig| registered[path] == sig }.keys
|
|
70
|
+
detail = "lacks or changes #{missing.join(', ')}; update the gitlab-labkit vendored copy or the gem"
|
|
71
|
+
raise Mismatch, "#{PACKAGE} is registered by another gem with a schema that #{detail}" unless missing.empty?
|
|
72
|
+
|
|
73
|
+
extra = registered.keys - vendored.keys
|
|
74
|
+
warn "gitlab-labkit: #{PACKAGE} from another gem is a newer schema (#{extra.join(', ')}); the vendored copy is older" unless extra.empty?
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# "Message.field" => its wire and API shape, "Enum.VALUE" => number,
|
|
78
|
+
# for every message, nested message and enum. proto3 optional decides
|
|
79
|
+
# whether has_x? exists, the oneof name which fields exclude each
|
|
80
|
+
# other, and json_name what the parser accepts as a key.
|
|
81
|
+
def signature(file)
|
|
82
|
+
out = {}
|
|
83
|
+
walk = lambda do |messages, enums, prefix|
|
|
84
|
+
enums.each { |e| e.value.each { |val| out["#{prefix}#{e.name}.#{val.name}"] = val.number } }
|
|
85
|
+
messages.each do |m|
|
|
86
|
+
m.field.each { |f| out["#{prefix}#{m.name}.#{f.name}"] = field_signature(f, m) }
|
|
87
|
+
walk.call(m.nested_type, m.enum_type, "#{prefix}#{m.name}.")
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
walk.call(file.message_type, file.enum_type, "")
|
|
91
|
+
out
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# protoc numbers oneofs in declaration order, and every proto3
|
|
95
|
+
# optional field adds a synthetic one, so an additive field shifts
|
|
96
|
+
# the index of every oneof after it. The name does not move.
|
|
97
|
+
def field_signature(field, message)
|
|
98
|
+
oneof = field.has_oneof_index? ? message.oneof_decl[field.oneof_index].name : nil
|
|
99
|
+
[field.number, field.type, field.label, field.type_name, field.json_name, field.proto3_optional, oneof]
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "labkit/rate_limit/config/vendored_proto"
|
|
4
|
+
require "labkit/rate_limit/config/proto/config_pb"
|
|
5
|
+
|
|
6
|
+
module Labkit
|
|
7
|
+
module RateLimit
|
|
8
|
+
# Config loads rate limit rules from YAML files that follow the
|
|
9
|
+
# gitlab.ratelimit.config.v1 schema: the contract an application ships
|
|
10
|
+
# (AvailableLimiters) and the rate limits an operator writes (RateLimits).
|
|
11
|
+
# Loading runs in four steps, parse, schema, contract and rule, and a
|
|
12
|
+
# failure names its step so CI output and the conformance corpus agree
|
|
13
|
+
# across SDKs.
|
|
14
|
+
module Config
|
|
15
|
+
# The generated protobuf classes, vendored from gitlab-org/protos.
|
|
16
|
+
Proto = Gitlab::Ratelimit::Config::V1
|
|
17
|
+
|
|
18
|
+
autoload :Error, "labkit/rate_limit/config/error"
|
|
19
|
+
autoload :Parser, "labkit/rate_limit/config/parser"
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
data/lib/labkit/rate_limit.rb
CHANGED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Copies the generated Ruby code for gitlab.ratelimit.config.v1 from gitlab-org/protos.
|
|
3
|
+
# A published gem cannot depend on the protos Bundler git gem, so the file is
|
|
4
|
+
# vendored and CI checks that it matches PROTOS_REF (job vendored-proto-check).
|
|
5
|
+
set -euo pipefail
|
|
6
|
+
|
|
7
|
+
# Pinned protos module tag. Bump it when a newer schema release is needed.
|
|
8
|
+
PROTOS_REF="${PROTOS_REF:-gen/ruby/gitlab/ratelimit/config/v1/v0.1.0}"
|
|
9
|
+
SOURCE_PATH="gen/ruby/gitlab/ratelimit/config/v1/lib/gitlab/ratelimit/config/v1/config_pb.rb"
|
|
10
|
+
TARGET="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/lib/labkit/rate_limit/config/proto/config_pb.rb"
|
|
11
|
+
|
|
12
|
+
url="https://gitlab.com/gitlab-org/protos/-/raw/${PROTOS_REF}/${SOURCE_PATH}"
|
|
13
|
+
tmp="$(mktemp)"
|
|
14
|
+
trap 'rm -f "$tmp"' EXIT
|
|
15
|
+
curl --fail --silent --show-error --location "$url" -o "$tmp"
|
|
16
|
+
|
|
17
|
+
# The generated file registers gitlab/ratelimit/config/v1/config.proto in the
|
|
18
|
+
# global descriptor pool. The protos gem registers the same file name, so the
|
|
19
|
+
# vendored copy goes through VendoredProto.register, which loads the gem first
|
|
20
|
+
# when it is installed and then compares schemas. When the gem's registration
|
|
21
|
+
# is in use the file returns early, so its constants are not defined twice.
|
|
22
|
+
guard='return unless Labkit::RateLimit::Config::VendoredProto.register(pool, descriptor_data)'
|
|
23
|
+
{
|
|
24
|
+
echo "# frozen_string_literal: true"
|
|
25
|
+
echo "# Vendored from gitlab-org/protos at ${PROTOS_REF}, ${SOURCE_PATH}."
|
|
26
|
+
echo "# Do not edit. Run scripts/vendor-ratelimit-proto.sh to update."
|
|
27
|
+
grep -v '^# frozen_string_literal: true$' "$tmp" |
|
|
28
|
+
sed -e "s|^require 'google/protobuf'\$|require 'google/protobuf'\nrequire 'labkit/rate_limit/config/vendored_proto'|" \
|
|
29
|
+
-e "s|^pool.add_serialized_file(descriptor_data)\$|${guard}|"
|
|
30
|
+
} >"$TARGET"
|
|
31
|
+
if ! grep -q 'VendoredProto.register' "$TARGET" ||
|
|
32
|
+
! grep -qx "require 'labkit/rate_limit/config/vendored_proto'" "$TARGET"; then
|
|
33
|
+
echo "vendor: the generated file no longer requires VendoredProto or registers the descriptor as expected; update the sed in this script" >&2
|
|
34
|
+
exit 1
|
|
35
|
+
fi
|
|
36
|
+
echo "vendored ${SOURCE_PATH}@${PROTOS_REF} -> ${TARGET#"$PWD"/}"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gitlab-labkit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.
|
|
4
|
+
version: 5.10.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andrew Newdigate
|
|
@@ -651,6 +651,11 @@ files:
|
|
|
651
651
|
- lib/labkit/rate_limit.rb
|
|
652
652
|
- lib/labkit/rate_limit/README.md
|
|
653
653
|
- lib/labkit/rate_limit/block_summary.rb
|
|
654
|
+
- lib/labkit/rate_limit/config.rb
|
|
655
|
+
- lib/labkit/rate_limit/config/error.rb
|
|
656
|
+
- lib/labkit/rate_limit/config/parser.rb
|
|
657
|
+
- lib/labkit/rate_limit/config/proto/config_pb.rb
|
|
658
|
+
- lib/labkit/rate_limit/config/vendored_proto.rb
|
|
654
659
|
- lib/labkit/rate_limit/configuration.rb
|
|
655
660
|
- lib/labkit/rate_limit/evaluator.rb
|
|
656
661
|
- lib/labkit/rate_limit/identifier.rb
|
|
@@ -719,6 +724,7 @@ files:
|
|
|
719
724
|
- renovate.json
|
|
720
725
|
- scripts/prepare-dev-env.sh
|
|
721
726
|
- scripts/update-asdf-version-variables.sh
|
|
727
|
+
- scripts/vendor-ratelimit-proto.sh
|
|
722
728
|
homepage: https://gitlab.com/gitlab-org/labkit-ruby
|
|
723
729
|
licenses:
|
|
724
730
|
- MIT
|