gitlab-labkit 5.9.0 → 5.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 23080f09f7295046f43c9519b76d3a71bef133c8178f60fb9d33bc13311c0947
4
- data.tar.gz: 35ca7c5427293cad362e700551c1a6a33ba439866f8acd60e065944ab8085b6f
3
+ metadata.gz: 3cf9de3ab607ca2b1f4e24b04a55efc3081556acaf61c8fd4c451fd786aa632a
4
+ data.tar.gz: 6a7d1d5fd0434e9298c41bfa6057f96ae08e37c3ea100f37db6d4879ecea688e
5
5
  SHA512:
6
- metadata.gz: e83c05dc9f364acadcf7fa4c5343441476e8c4ab1fdb7b1ed36122499649caef0f6a6607d01e0c511042eb3f940fe4e799e0b422749ed28b8f8857ddc5c5fc6f
7
- data.tar.gz: 0d9a4dccbb22b559ba33e2403151b4284b4e8240970384757ab1ec455d478bf8ad9865f7573883b7c11a819f68718c8111781e39856f6b4ad28f99dbf117e0a7
6
+ metadata.gz: 931e421f2c52d3d98602f3657088701bb5a48b2f98bd9289becffad20df9d991c3fc1d2f9b03d5ba6cbc26593ca38176227827afc66b973393302e1f789f89cd
7
+ data.tar.gz: b971f78dfcf2529cda540c44e950651876cfd89e5d68af517c10f5013ead19d50c7f603d7ecc582787e790ab6ae1deb46dbc342b010676a48aff8964fb89b109
data/.gitlab-ci.yml CHANGED
@@ -81,6 +81,19 @@ deploy:
81
81
  - Gemfile
82
82
  - .gitlab-ci.yml
83
83
 
84
+ # The generated protobuf code for gitlab.ratelimit.config.v1 is vendored from
85
+ # gitlab-org/protos, because a published gem cannot depend on the protos git gem.
86
+ vendored-proto-check:
87
+ stage: validate
88
+ needs: []
89
+ image: ruby:3.4
90
+ rules:
91
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
92
+ - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
93
+ script:
94
+ - ./scripts/vendor-ratelimit-proto.sh
95
+ - git diff --exit-code -- lib/labkit/rate_limit/config/proto/
96
+
84
97
  # Gate SDK changes on the labkit-spec conformance suite, run remotely: trigger
85
98
  # a downstream pipeline in labkit-spec that checks out this pipeline's commit
86
99
  # (LABKIT_RUBY_SHA) and runs the Ruby conformance suite against it, verifying
data/.rubocop.yml CHANGED
@@ -4,5 +4,13 @@ inherit_gem:
4
4
  gitlab-styles:
5
5
  - rubocop-default.yml
6
6
 
7
+ inherit_mode:
8
+ merge:
9
+ - Exclude
10
+
7
11
  AllCops:
8
12
  NewCops: enable
13
+ Exclude:
14
+ # Generated by protoc and vendored from gitlab-org/protos; regenerate with
15
+ # scripts/vendor-ratelimit-proto.sh instead of editing.
16
+ - lib/labkit/rate_limit/config/proto/**/*
data/Gemfile CHANGED
@@ -8,3 +8,9 @@ gemspec
8
8
  # Exporter source: https://gitlab.com/gitlab-org/quality/analytics/test-metrics-exporters
9
9
  # Owned by: @gitlab-org/quality/analytics (group::development analytics)
10
10
  gem 'gitlab-rspec-metrics-exporter', '~> 0.2', groups: [:development, :test]
11
+
12
+ # regexp_parser 2.13.2 made Regexp::Syntax::Base.implements private, and
13
+ # json_schemer 2.5.0 calls it at load time, so every fresh bundle fails on
14
+ # `require "json_schemer"`. Drop the pin once json_schemer or regexp_parser
15
+ # ships a fix: https://github.com/davishmcclurg/json_schemer/issues/238
16
+ gem "regexp_parser", "!= 2.13.2", groups: [:development, :test]
@@ -711,6 +711,39 @@ A logging failure during a flush is logged at WARN with
711
711
  `dropped_pairs`, the number of lines that flush lost. It never raises, so the
712
712
  flush thread keeps running.
713
713
 
714
+ ## Config files
715
+
716
+ `Labkit::RateLimit::Config` loads rules from YAML files that follow the `gitlab.ratelimit.config.v1` schema from [gitlab-org/protos](https://gitlab.com/gitlab-org/protos): the contract an application ships (`AvailableLimiters`) and the rate limits an operator writes (`RateLimits`).
717
+ The generated protobuf classes are vendored under `lib/labkit/rate_limit/config/proto/` and reachable as `Labkit::RateLimit::Config::Proto`.
718
+ Run `scripts/vendor-ratelimit-proto.sh` to update them; CI fails on drift.
719
+
720
+ An application can also install the `gitlab-ratelimit-config-v1` gem from gitlab-org/protos.
721
+ Then labkit uses the gem's schema and compares it with the vendored copy when `Labkit::RateLimit::Config` loads.
722
+ A gem schema that is newer gives a warning.
723
+ A gem schema that lacks or changes a vendored field raises `Labkit::RateLimit::Config::VendoredProto::Mismatch`.
724
+ So upgrade the `gitlab-ratelimit-config-v1` gem in the application first, then the `gitlab-labkit` release that updates the vendored copy.
725
+
726
+ Loading runs in four steps, `parse`, `schema`, `contract` and `rule`, and a failure raises `Labkit::RateLimit::Config::Error` whose `#step` names the step.
727
+ Only the parse step exists so far; the schema, contract and rule steps follow.
728
+ Until the schema step lands, no mode validates a document: strict mode lets an enum number that no `Action` defines through, such as `action: 7`, and lenient mode leaves an unknown `action` name as `ACTION_UNSPECIFIED`.
729
+ The schema step rejects both.
730
+
731
+ ```ruby
732
+ doc = Labkit::RateLimit::Config::Parser.parse(
733
+ "config/rate_limits/rack_request.yaml",
734
+ Labkit::RateLimit::Config::Proto::RateLimits,
735
+ strict: true # CI rejects unknown fields. Production loads with strict: false once the schema step exists
736
+ )
737
+ doc.rate_limits["rack_request"].rules.first.name
738
+ ```
739
+
740
+ The parse step converts each YAML scalar by the type of the field it fills, so `eq: off`, `eq: True` and `eq: 9970` stay strings, `limit: "5"` becomes `5`, `action: "1"` becomes `ACTION_LIMIT`, and a YAML null leaves the field unset.
741
+ A repeated key, also one repeated under its camelCase `json_name`, a merge key (`<<`), an empty file, more than one document, or a non-scalar key fails at step `parse`, in both modes.
742
+ So does a value of the wrong kind for its field, such as a mapping where a scalar belongs, an integer out of range for its field, two fields of one `oneof`, such as `{ eq: a, re: b }`, a null list item or map value, and a document nested deeper than 64 levels.
743
+ Each of these errors names the YAML line.
744
+ An unknown field or an unknown enum name, such as `action: ACTION_LIMT`, fails in strict mode with its YAML line.
745
+ Lenient mode ignores an unknown field and leaves an unknown enum name unset.
746
+
714
747
  ## Dev/test vs production guards
715
748
 
716
749
  `Limiter.new` and `Rule.new` validate names and configuration. In
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Labkit
4
+ module RateLimit
5
+ module Config
6
+ # Error is raised when a document fails one of the load steps.
7
+ # step is :parse, :schema, :contract or :rule.
8
+ class Error < StandardError
9
+ attr_reader :step
10
+
11
+ def initialize(step, message)
12
+ @step = step
13
+ super("#{step}: #{message}")
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,243 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "psych"
5
+ require "google/protobuf/descriptor_pb"
6
+
7
+ module Labkit
8
+ module RateLimit
9
+ module Config
10
+ # Parser is the parse step: a YAML file into a protobuf message.
11
+ #
12
+ # YAML types do not follow the schema, so each scalar is converted by the
13
+ # type of the field it fills, the way protoyaml does in Go: a string field
14
+ # keeps the scalar text (`off`, `True` and `9970` stay strings), an integer
15
+ # or enum field gets an Integer when the text is one, quoted or not, and
16
+ # the YAML null forms mean the field is absent. Psych keeps the last value
17
+ # of a repeated key, so repeated keys are rejected here, also when one
18
+ # spelling is the snake_case field name and the other its camelCase
19
+ # json_name.
20
+ #
21
+ # This step checks the shape of a document, not its values. An enum
22
+ # number that no Action defines, such as `action: 7`, passes in both
23
+ # modes, and in lenient mode protobuf drops an unknown enum name together
24
+ # with the unknown fields, so a rule with a misspelt action reaches the
25
+ # schema step as ACTION_UNSPECIFIED. The schema step rejects or drops
26
+ # both.
27
+ module Parser
28
+ INT32 = (-(2**31))...(2**31)
29
+ UINT32 = 0...(2**32)
30
+ INT64 = (-(2**63))...(2**63)
31
+ UINT64 = 0...(2**64)
32
+ INTEGER_RANGES = {
33
+ int32: INT32, sint32: INT32, sfixed32: INT32, enum: INT32,
34
+ uint32: UINT32, fixed32: UINT32,
35
+ int64: INT64, sint64: INT64, sfixed64: INT64,
36
+ uint64: UINT64, fixed64: UINT64
37
+ }.freeze
38
+ INTEGER = /\A[-+]?\d+\z/
39
+ NULLS = ["", "~", "null", "Null", "NULL"].freeze
40
+ # The protobuf JSON decoder stops at 64 levels of objects and arrays.
41
+ # Mappings and lists are counted here so the error names the YAML line
42
+ # and a deeper document cannot exhaust the stack.
43
+ MAX_DEPTH = 64
44
+
45
+ class << self
46
+ # @param path [String] YAML file
47
+ # @param klass [Class] a Proto message class, AvailableLimiters or RateLimits
48
+ # @param strict [Boolean] when true an unknown field is an error
49
+ # @return [Google::Protobuf::AbstractMessage]
50
+ # @raise [Error] with step :parse
51
+ def parse(path, klass, strict:)
52
+ tree = tree_for(document_root(path), klass.descriptor, strict: strict, depth: 0)
53
+ klass.decode_json(JSON.generate(tree), ignore_unknown_fields: !strict)
54
+ rescue Psych::SyntaxError, SystemCallError => e
55
+ # Both messages already name the file.
56
+ raise Error.new(:parse, e.message)
57
+ rescue Google::Protobuf::ParseError, Psych::Exception, JSON::JSONError => e
58
+ raise Error.new(:parse, "#{path}: #{e.message}")
59
+ end
60
+
61
+ private
62
+
63
+ # One YAML document per file. Psych would silently keep only the first
64
+ # of several, and an empty file has no root to walk.
65
+ def document_root(path)
66
+ documents = Psych.parse_stream(File.read(path), filename: path).children
67
+ raise Psych::Exception, "the file holds no YAML document" if documents.empty?
68
+ raise Psych::Exception, "the file holds #{documents.size} YAML documents, expected 1" if documents.size > 1
69
+
70
+ documents.first.root
71
+ end
72
+
73
+ # One value for one slot. type is a field type Symbol, a message
74
+ # Descriptor or an EnumDescriptor. A node of the wrong kind is
75
+ # rejected here so the error names the YAML line, not an offset into
76
+ # the generated JSON. depth counts the mappings and lists above node.
77
+ def tree_for(node, type, strict:, depth:)
78
+ expects_message = type.is_a?(Google::Protobuf::Descriptor)
79
+
80
+ case node
81
+ when Psych::Nodes::Mapping
82
+ fail_at(node, "expected a scalar, found a mapping") unless expects_message
83
+ mapping_for(node, type, strict: strict, depth: depth)
84
+ when Psych::Nodes::Sequence
85
+ fail_at(node, "expected a single value, found a list")
86
+ when Psych::Nodes::Scalar
87
+ return nil if null?(node)
88
+
89
+ fail_at(node, "expected a mapping, found a scalar") if expects_message
90
+ scalar_for(node, type, strict: strict)
91
+ when Psych::Nodes::Alias
92
+ fail_at(node, "aliases are not supported")
93
+ end
94
+ end
95
+
96
+ def mapping_for(node, descriptor, strict:, depth:)
97
+ check_depth!(node, depth)
98
+ entries = node.children.each_slice(2).map do |key, value|
99
+ name = scalar_key(key)
100
+ field = field_for(descriptor, name)
101
+ fail_at(key, "unknown field #{name.inspect}") if strict && field.nil?
102
+ [name, key, field, value]
103
+ end
104
+ reject_repeated_keys!(entries.map { |name, key, field, _| [field ? field.name : name, key] })
105
+ reject_oneof_conflicts!(descriptor, entries)
106
+
107
+ maps = map_fields(descriptor)
108
+ # An unknown field is not walked: strict mode rejected its name
109
+ # above and lenient mode ignores it, so a newer schema may put
110
+ # anything there.
111
+ entries.to_h do |name, _, field, value|
112
+ next [name, nil] unless field
113
+
114
+ [name, field_value_for(value, field, map: maps.include?(field.name), strict: strict, depth: depth + 1)]
115
+ end
116
+ end
117
+
118
+ # Descriptor#options, which carries the map_entry flag, arrived after
119
+ # google-protobuf 3.25.0, so a blank message tells a map from a list.
120
+ # A message type keeps one Descriptor object, so the cache is bounded
121
+ # by the number of message types.
122
+ def map_fields(descriptor)
123
+ (@map_fields ||= {})[descriptor] ||= begin
124
+ blank = descriptor.msgclass.new
125
+ descriptor.select { |field| blank[field.name].is_a?(Google::Protobuf::Map) }.map(&:name).freeze
126
+ end
127
+ end
128
+
129
+ # A mapping key must be a plain scalar; a sequence, mapping or alias
130
+ # as a key has no field to fill. A merge key is rejected in both
131
+ # modes: lenient mode would ignore it as an unknown field and drop
132
+ # the merged fields with no error.
133
+ def scalar_key(node)
134
+ fail_at(node, "a mapping key must be a scalar") unless node.is_a?(Psych::Nodes::Scalar)
135
+ fail_at(node, "merge keys (<<) are not supported") if node.plain && node.value == "<<"
136
+
137
+ node.value
138
+ end
139
+
140
+ # The error points at the repeated key, not at the start of the mapping.
141
+ def reject_repeated_keys!(named_keys)
142
+ seen = {}
143
+ named_keys.each do |name, key|
144
+ fail_at(key, "duplicate key #{name.inspect}") if seen[name]
145
+ seen[name] = true
146
+ end
147
+ end
148
+
149
+ # Two fields of one oneof, such as two Matcher operators, would
150
+ # otherwise fail in the JSON decoder with no YAML line.
151
+ def reject_oneof_conflicts!(descriptor, entries)
152
+ descriptor.each_oneof do |oneof|
153
+ names = oneof.map(&:name)
154
+ set = entries.select { |_, _, field, value| field && names.include?(field.name) && !null?(value) }
155
+ next if set.size < 2
156
+
157
+ fail_at(set[1][1], "#{set[0][2].name} and #{set[1][2].name} are both set; set only one field of #{oneof.name}")
158
+ end
159
+ end
160
+
161
+ def check_depth!(node, depth)
162
+ fail_at(node, "nesting deeper than #{MAX_DEPTH} levels") if depth >= MAX_DEPTH
163
+ end
164
+
165
+ def fail_at(node, message)
166
+ raise Psych::Exception, "line #{node.start_line + 1}: #{message}"
167
+ end
168
+
169
+ # Files are snake_case; protojson also accepts the camelCase json_name.
170
+ def field_for(descriptor, name)
171
+ descriptor.lookup(name) || descriptor.find { |field| field.json_name == name }
172
+ end
173
+
174
+ def field_value_for(node, field, map:, strict:, depth:)
175
+ return map_for(node, field, strict: strict, depth: depth) if map
176
+ return tree_for(node, element_type(field), strict: strict, depth: depth) unless field.label == :repeated
177
+ return nil if null?(node)
178
+
179
+ fail_at(node, "expected a list") unless node.is_a?(Psych::Nodes::Sequence)
180
+ check_depth!(node, depth)
181
+ node.children.map do |child|
182
+ fail_at(child, "a list item cannot be null") if null?(child)
183
+
184
+ tree_for(child, element_type(field), strict: strict, depth: depth + 1)
185
+ end
186
+ end
187
+
188
+ def map_for(node, field, strict:, depth:)
189
+ return nil if null?(node)
190
+
191
+ fail_at(node, "expected a mapping") unless node.is_a?(Psych::Nodes::Mapping)
192
+ check_depth!(node, depth)
193
+ pairs = node.children.each_slice(2)
194
+ reject_repeated_keys!(pairs.map { |key, _| [scalar_key(key), key] })
195
+ value_type = element_type(field.subtype.lookup("value"))
196
+ pairs.to_h do |key, value|
197
+ fail_at(value, "a map value cannot be null") if null?(value)
198
+
199
+ [scalar_key(key), tree_for(value, value_type, strict: strict, depth: depth + 1)]
200
+ end
201
+ end
202
+
203
+ # A message or enum field passes its descriptor down, so a scalar
204
+ # can be checked against the enum's names.
205
+ def element_type(field)
206
+ %i[message enum].include?(field.type) ? field.subtype : field.type
207
+ end
208
+
209
+ def null?(node)
210
+ node.is_a?(Psych::Nodes::Scalar) && node.plain && NULLS.include?(node.value)
211
+ end
212
+
213
+ # Integer text is converted quoted or not: protojson takes a quoted
214
+ # number for an integer field but not for an enum. The range is
215
+ # checked here so the error names the YAML line.
216
+ def scalar_for(node, type, strict:)
217
+ enum = type if type.is_a?(Google::Protobuf::EnumDescriptor)
218
+ kind = enum ? :enum : type
219
+ range = INTEGER_RANGES[kind]
220
+ if range && INTEGER.match?(node.value)
221
+ value = Integer(node.value, 10)
222
+ fail_at(node, "#{node.value} is out of range for #{kind}") unless range.cover?(value)
223
+ return value
224
+ end
225
+
226
+ # Lenient mode passes an unknown name on, and protobuf drops it
227
+ # with the unknown fields, so a newer schema's value still loads.
228
+ fail_at(node, "unknown value #{node.value.inspect} for enum #{enum.name}") if enum && strict && enum.lookup_name(node.value.to_sym).nil?
229
+
230
+ return node.value unless node.plain
231
+
232
+ case kind
233
+ when :bool
234
+ { "true" => true, "false" => false }.fetch(node.value, node.value)
235
+ else
236
+ node.value
237
+ end
238
+ end
239
+ end
240
+ end
241
+ end
242
+ end
243
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+ # Vendored from gitlab-org/protos at gen/ruby/gitlab/ratelimit/config/v1/v0.1.0, gen/ruby/gitlab/ratelimit/config/v1/lib/gitlab/ratelimit/config/v1/config_pb.rb.
3
+ # Do not edit. Run scripts/vendor-ratelimit-proto.sh to update.
4
+ # Generated by the protocol buffer compiler. DO NOT EDIT!
5
+ # source: gitlab/ratelimit/config/v1/config.proto
6
+
7
+ require 'google/protobuf'
8
+ require 'labkit/rate_limit/config/vendored_proto'
9
+
10
+
11
+ descriptor_data = "\n\'gitlab/ratelimit/config/v1/config.proto\x12\x1agitlab.ratelimit.config.v1\"\xf7\x01\n\x11\x41vailableLimiters\x12s\n\x12\x61vailable_limiters\x18\x01 \x03(\x0b\x32\x44.gitlab.ratelimit.config.v1.AvailableLimiters.AvailableLimitersEntryR\x11\x61vailableLimiters\x1am\n\x16\x41vailableLimitersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12=\n\x05value\x18\x02 \x01(\x0b\x32\'.gitlab.ratelimit.config.v1.LimiterSpecR\x05value:\x02\x38\x01\"\xd2\x01\n\x0bLimiterSpec\x12 \n\x0b\x64\x65scription\x18\x01 \x01(\tR\x0b\x64\x65scription\x12\x31\n\x14\x61vailable_properties\x18\x02 \x03(\tR\x13\x61vailableProperties\x12\x45\n\rdefault_rules\x18\x03 \x03(\x0b\x32 .gitlab.ratelimit.config.v1.RuleR\x0c\x64\x65\x66\x61ultRules\x12\'\n\x0fprotected_rules\x18\x04 \x03(\tR\x0eprotectedRules\"\xca\x01\n\nRateLimits\x12W\n\x0brate_limits\x18\x01 \x03(\x0b\x32\x36.gitlab.ratelimit.config.v1.RateLimits.RateLimitsEntryR\nrateLimits\x1a\x63\n\x0fRateLimitsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12:\n\x05value\x18\x02 \x01(\x0b\x32$.gitlab.ratelimit.config.v1.RuleListR\x05value:\x02\x38\x01\"B\n\x08RuleList\x12\x36\n\x05rules\x18\x01 \x03(\x0b\x32 .gitlab.ratelimit.config.v1.RuleR\x05rules\"\x9a\x04\n\x04Rule\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\x12:\n\x06\x61\x63tion\x18\x03 \x01(\x0e\x32\".gitlab.ratelimit.config.v1.ActionR\x06\x61\x63tion\x12(\n\x0f\x63haracteristics\x18\x04 \x03(\tR\x0f\x63haracteristics\x12\x41\n\x05match\x18\x05 \x03(\x0b\x32+.gitlab.ratelimit.config.v1.Rule.MatchEntryR\x05match\x12\x19\n\x05limit\x18\x06 \x01(\rH\x00R\x05limit\x88\x01\x01\x12\x1e\n\x08period_s\x18\x07 \x01(\rH\x01R\x07periodS\x88\x01\x01\x12%\n\x0e\x63ount_distinct\x18\x08 \x01(\tR\rcountDistinct\x12\x1f\n\tban_for_s\x18\t \x01(\rH\x02R\x07\x62\x61nForS\x88\x01\x01\x12\x1f\n\x08position\x18\n \x01(\rH\x03R\x08position\x88\x01\x01\x1a]\n\nMatchEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32#.gitlab.ratelimit.config.v1.MatcherR\x05value:\x02\x38\x01\x42\x08\n\x06_limitB\x0b\n\t_period_sB\x0c\n\n_ban_for_sB\x0b\n\t_position\"\x93\x03\n\x13PublishRulesRequest\x12`\n\x0brate_limits\x18\x01 \x03(\x0b\x32?.gitlab.ratelimit.config.v1.PublishRulesRequest.RateLimitsEntryR\nrateLimits\x12S\n\x06remove\x18\x02 \x03(\x0b\x32;.gitlab.ratelimit.config.v1.PublishRulesRequest.RemoveEntryR\x06remove\x1a\x63\n\x0fRateLimitsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12:\n\x05value\x18\x02 \x01(\x0b\x32$.gitlab.ratelimit.config.v1.RuleListR\x05value:\x02\x38\x01\x1a`\n\x0bRemoveEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12;\n\x05value\x18\x02 \x01(\x0b\x32%.gitlab.ratelimit.config.v1.RuleNamesR\x05value:\x02\x38\x01\"!\n\tRuleNames\x12\x14\n\x05names\x18\x01 \x03(\tR\x05names\"\x9a\x02\n\x07Matcher\x12\x10\n\x02\x65q\x18\x01 \x01(\tH\x00R\x02\x65q\x12\x10\n\x02ne\x18\x02 \x01(\tH\x00R\x02ne\x12\x10\n\x02re\x18\x03 \x01(\tH\x00R\x02re\x12\x12\n\x03nre\x18\x04 \x01(\tH\x00R\x03nre\x12?\n\x06one_of\x18\x05 \x01(\x0b\x32&.gitlab.ratelimit.config.v1.StringListH\x00R\x05oneOf\x12\x41\n\x07none_of\x18\x06 \x01(\x0b\x32&.gitlab.ratelimit.config.v1.StringListH\x00R\x06noneOf\x12;\n\x03\x61ll\x18\x07 \x01(\x0b\x32\'.gitlab.ratelimit.config.v1.MatcherListH\x00R\x03\x61llB\x04\n\x02op\"$\n\nStringList\x12\x16\n\x06values\x18\x01 \x03(\tR\x06values\"N\n\x0bMatcherList\x12?\n\x08matchers\x18\x01 \x03(\x0b\x32#.gitlab.ratelimit.config.v1.MatcherR\x08matchers*S\n\x06\x41\x63tion\x12\x16\n\x12\x41\x43TION_UNSPECIFIED\x10\x00\x12\x10\n\x0c\x41\x43TION_LIMIT\x10\x01\x12\x0e\n\nACTION_LOG\x10\x02\x12\x0f\n\x0b\x41\x43TION_SKIP\x10\x03\x62\x06proto3"
12
+
13
+ pool = ::Google::Protobuf::DescriptorPool.generated_pool
14
+ return unless Labkit::RateLimit::Config::VendoredProto.register(pool, descriptor_data)
15
+
16
+ module Gitlab
17
+ module Ratelimit
18
+ module Config
19
+ module V1
20
+ AvailableLimiters = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.AvailableLimiters").msgclass
21
+ LimiterSpec = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.LimiterSpec").msgclass
22
+ RateLimits = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RateLimits").msgclass
23
+ RuleList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RuleList").msgclass
24
+ Rule = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Rule").msgclass
25
+ PublishRulesRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.PublishRulesRequest").msgclass
26
+ RuleNames = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RuleNames").msgclass
27
+ Matcher = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Matcher").msgclass
28
+ StringList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.StringList").msgclass
29
+ MatcherList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.MatcherList").msgclass
30
+ Action = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Action").enummodule
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "google/protobuf/descriptor_pb"
4
+
5
+ module Labkit
6
+ module RateLimit
7
+ module Config
8
+ # VendoredProto registers the vendored gitlab.ratelimit.config.v1 file in
9
+ # the global descriptor pool. The gitlab-ratelimit-config-v1 gem from
10
+ # gitlab-org/protos registers the same file name, unguarded, and a second
11
+ # registration raises. So the gem is loaded first whenever it is
12
+ # installed, and the vendored copy is then compared with it: the same
13
+ # schema means the gem's registration is used, a different one is a boot
14
+ # error, because two versions of one schema in one process would make the
15
+ # loader read fields the other copy does not know.
16
+ module VendoredProto
17
+ # The gem registered a schema that differs from the vendored copy, or
18
+ # one this google-protobuf cannot compare.
19
+ Mismatch = Class.new(StandardError)
20
+
21
+ PACKAGE = "gitlab.ratelimit.config.v1"
22
+ PROBE = "#{PACKAGE}.RateLimits".freeze
23
+ GEM_FILE = "gitlab/ratelimit/config/v1/config_pb"
24
+
25
+ class << self
26
+ # @param pool [Google::Protobuf::DescriptorPool]
27
+ # @param serialized [String] a serialized FileDescriptorProto
28
+ # @return [Boolean] true when this call registered the file and the
29
+ # vendored file must define its constants; false when the gem's
30
+ # registration is in use and the vendored file must stop loading
31
+ # @raise [Mismatch] when the gem registered a different schema
32
+ def register(pool, serialized)
33
+ load_gem_first
34
+
35
+ existing = pool.lookup(PROBE)
36
+ if existing.nil?
37
+ pool.add_serialized_file(serialized)
38
+ return true
39
+ end
40
+
41
+ check_same_schema!(existing.file_descriptor, serialized)
42
+ false
43
+ end
44
+
45
+ private
46
+
47
+ # Ruby's require loads the gem's file once, so whichever side comes
48
+ # first, the gem registers before the vendored copy is compared. Only
49
+ # an absent gem is expected; a LoadError from inside the gem's own
50
+ # file is a real failure and propagates.
51
+ def load_gem_first
52
+ require GEM_FILE
53
+ rescue ::LoadError => e
54
+ raise unless e.path == GEM_FILE
55
+ end
56
+
57
+ # The gem may be ahead of the vendored copy: an additive release adds
58
+ # fields the loader does not read, and that is fine. The gem may not
59
+ # be behind it: the loader would then read a field the gem's schema
60
+ # does not have.
61
+ def check_same_schema!(file, serialized)
62
+ # FileDescriptor#to_proto arrived in google-protobuf 4.30. The gem
63
+ # keeps its 3.25 floor for consumers that never reach this path, so
64
+ # the requirement is raised here, where it applies.
65
+ raise Mismatch, "#{PACKAGE} is registered by another gem and this google-protobuf cannot compare schemas; rate limit config needs google-protobuf >= 4.30" unless file.respond_to?(:to_proto)
66
+
67
+ registered = signature(file.to_proto)
68
+ vendored = signature(Google::Protobuf::FileDescriptorProto.decode(serialized))
69
+ missing = vendored.reject { |path, sig| registered[path] == sig }.keys
70
+ detail = "lacks or changes #{missing.join(', ')}; update the gitlab-labkit vendored copy or the gem"
71
+ raise Mismatch, "#{PACKAGE} is registered by another gem with a schema that #{detail}" unless missing.empty?
72
+
73
+ extra = registered.keys - vendored.keys
74
+ warn "gitlab-labkit: #{PACKAGE} from another gem is a newer schema (#{extra.join(', ')}); the vendored copy is older" unless extra.empty?
75
+ end
76
+
77
+ # "Message.field" => its wire and API shape, "Enum.VALUE" => number,
78
+ # for every message, nested message and enum. proto3 optional decides
79
+ # whether has_x? exists, the oneof name which fields exclude each
80
+ # other, and json_name what the parser accepts as a key.
81
+ def signature(file)
82
+ out = {}
83
+ walk = lambda do |messages, enums, prefix|
84
+ enums.each { |e| e.value.each { |val| out["#{prefix}#{e.name}.#{val.name}"] = val.number } }
85
+ messages.each do |m|
86
+ m.field.each { |f| out["#{prefix}#{m.name}.#{f.name}"] = field_signature(f, m) }
87
+ walk.call(m.nested_type, m.enum_type, "#{prefix}#{m.name}.")
88
+ end
89
+ end
90
+ walk.call(file.message_type, file.enum_type, "")
91
+ out
92
+ end
93
+
94
+ # protoc numbers oneofs in declaration order, and every proto3
95
+ # optional field adds a synthetic one, so an additive field shifts
96
+ # the index of every oneof after it. The name does not move.
97
+ def field_signature(field, message)
98
+ oneof = field.has_oneof_index? ? message.oneof_decl[field.oneof_index].name : nil
99
+ [field.number, field.type, field.label, field.type_name, field.json_name, field.proto3_optional, oneof]
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "labkit/rate_limit/config/vendored_proto"
4
+ require "labkit/rate_limit/config/proto/config_pb"
5
+
6
+ module Labkit
7
+ module RateLimit
8
+ # Config loads rate limit rules from YAML files that follow the
9
+ # gitlab.ratelimit.config.v1 schema: the contract an application ships
10
+ # (AvailableLimiters) and the rate limits an operator writes (RateLimits).
11
+ # Loading runs in four steps, parse, schema, contract and rule, and a
12
+ # failure names its step so CI output and the conformance corpus agree
13
+ # across SDKs.
14
+ module Config
15
+ # The generated protobuf classes, vendored from gitlab-org/protos.
16
+ Proto = Gitlab::Ratelimit::Config::V1
17
+
18
+ autoload :Error, "labkit/rate_limit/config/error"
19
+ autoload :Parser, "labkit/rate_limit/config/parser"
20
+ end
21
+ end
22
+ end
@@ -26,6 +26,7 @@ module Labkit
26
26
  autoload :Limiter, "labkit/rate_limit/limiter"
27
27
  autoload :Metrics, "labkit/rate_limit/metrics"
28
28
  autoload :BlockSummary, "labkit/rate_limit/block_summary"
29
+ autoload :Config, "labkit/rate_limit/config"
29
30
 
30
31
  class << self
31
32
  def configure
@@ -0,0 +1,36 @@
1
+ #!/usr/bin/env bash
2
+ # Copies the generated Ruby code for gitlab.ratelimit.config.v1 from gitlab-org/protos.
3
+ # A published gem cannot depend on the protos Bundler git gem, so the file is
4
+ # vendored and CI checks that it matches PROTOS_REF (job vendored-proto-check).
5
+ set -euo pipefail
6
+
7
+ # Pinned protos module tag. Bump it when a newer schema release is needed.
8
+ PROTOS_REF="${PROTOS_REF:-gen/ruby/gitlab/ratelimit/config/v1/v0.1.0}"
9
+ SOURCE_PATH="gen/ruby/gitlab/ratelimit/config/v1/lib/gitlab/ratelimit/config/v1/config_pb.rb"
10
+ TARGET="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/lib/labkit/rate_limit/config/proto/config_pb.rb"
11
+
12
+ url="https://gitlab.com/gitlab-org/protos/-/raw/${PROTOS_REF}/${SOURCE_PATH}"
13
+ tmp="$(mktemp)"
14
+ trap 'rm -f "$tmp"' EXIT
15
+ curl --fail --silent --show-error --location "$url" -o "$tmp"
16
+
17
+ # The generated file registers gitlab/ratelimit/config/v1/config.proto in the
18
+ # global descriptor pool. The protos gem registers the same file name, so the
19
+ # vendored copy goes through VendoredProto.register, which loads the gem first
20
+ # when it is installed and then compares schemas. When the gem's registration
21
+ # is in use the file returns early, so its constants are not defined twice.
22
+ guard='return unless Labkit::RateLimit::Config::VendoredProto.register(pool, descriptor_data)'
23
+ {
24
+ echo "# frozen_string_literal: true"
25
+ echo "# Vendored from gitlab-org/protos at ${PROTOS_REF}, ${SOURCE_PATH}."
26
+ echo "# Do not edit. Run scripts/vendor-ratelimit-proto.sh to update."
27
+ grep -v '^# frozen_string_literal: true$' "$tmp" |
28
+ sed -e "s|^require 'google/protobuf'\$|require 'google/protobuf'\nrequire 'labkit/rate_limit/config/vendored_proto'|" \
29
+ -e "s|^pool.add_serialized_file(descriptor_data)\$|${guard}|"
30
+ } >"$TARGET"
31
+ if ! grep -q 'VendoredProto.register' "$TARGET" ||
32
+ ! grep -qx "require 'labkit/rate_limit/config/vendored_proto'" "$TARGET"; then
33
+ echo "vendor: the generated file no longer requires VendoredProto or registers the descriptor as expected; update the sed in this script" >&2
34
+ exit 1
35
+ fi
36
+ echo "vendored ${SOURCE_PATH}@${PROTOS_REF} -> ${TARGET#"$PWD"/}"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-labkit
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.9.0
4
+ version: 5.10.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Newdigate
@@ -651,6 +651,11 @@ files:
651
651
  - lib/labkit/rate_limit.rb
652
652
  - lib/labkit/rate_limit/README.md
653
653
  - lib/labkit/rate_limit/block_summary.rb
654
+ - lib/labkit/rate_limit/config.rb
655
+ - lib/labkit/rate_limit/config/error.rb
656
+ - lib/labkit/rate_limit/config/parser.rb
657
+ - lib/labkit/rate_limit/config/proto/config_pb.rb
658
+ - lib/labkit/rate_limit/config/vendored_proto.rb
654
659
  - lib/labkit/rate_limit/configuration.rb
655
660
  - lib/labkit/rate_limit/evaluator.rb
656
661
  - lib/labkit/rate_limit/identifier.rb
@@ -719,6 +724,7 @@ files:
719
724
  - renovate.json
720
725
  - scripts/prepare-dev-env.sh
721
726
  - scripts/update-asdf-version-variables.sh
727
+ - scripts/vendor-ratelimit-proto.sh
722
728
  homepage: https://gitlab.com/gitlab-org/labkit-ruby
723
729
  licenses:
724
730
  - MIT