gitlab-labkit 5.9.0 → 5.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 23080f09f7295046f43c9519b76d3a71bef133c8178f60fb9d33bc13311c0947
4
- data.tar.gz: 35ca7c5427293cad362e700551c1a6a33ba439866f8acd60e065944ab8085b6f
3
+ metadata.gz: adf3a44ad840a0b124ef408e68697340e1c62ca66e0965da67ed3e21aa608a92
4
+ data.tar.gz: d3ccd488458bc4c365c476217e6b1234bb7abec41efee94413d81b69497ff460
5
5
  SHA512:
6
- metadata.gz: e83c05dc9f364acadcf7fa4c5343441476e8c4ab1fdb7b1ed36122499649caef0f6a6607d01e0c511042eb3f940fe4e799e0b422749ed28b8f8857ddc5c5fc6f
7
- data.tar.gz: 0d9a4dccbb22b559ba33e2403151b4284b4e8240970384757ab1ec455d478bf8ad9865f7573883b7c11a819f68718c8111781e39856f6b4ad28f99dbf117e0a7
6
+ metadata.gz: 7bbbd90bd5eb0e65f794845225fd8707817bea9d8a80f47da64063b4e8e9a7c988177dae2ad0f44e0fb7ca22dca6dd2846e0fc2f2dd9db038c4554d2738d4364
7
+ data.tar.gz: c15ff6de4430a52692e33dd685ca88e4e080a12c6026f4c59f6520b91171113db5e9d13e319d1ac783e9dfea2b5648637ab4c0c8224feedc927ef2f70bc704ea
data/.gitlab-ci.yml CHANGED
@@ -81,6 +81,19 @@ deploy:
81
81
  - Gemfile
82
82
  - .gitlab-ci.yml
83
83
 
84
+ # The generated protobuf code for gitlab.ratelimit.config.v1 is vendored from
85
+ # gitlab-org/protos, because a published gem cannot depend on the protos git gem.
86
+ vendored-proto-check:
87
+ stage: validate
88
+ needs: []
89
+ image: ruby:3.4
90
+ rules:
91
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
92
+ - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
93
+ script:
94
+ - ./scripts/vendor-ratelimit-proto.sh
95
+ - git diff --exit-code -- lib/labkit/rate_limit/config/proto/
96
+
84
97
  # Gate SDK changes on the labkit-spec conformance suite, run remotely: trigger
85
98
  # a downstream pipeline in labkit-spec that checks out this pipeline's commit
86
99
  # (LABKIT_RUBY_SHA) and runs the Ruby conformance suite against it, verifying
data/.rubocop.yml CHANGED
@@ -4,5 +4,13 @@ inherit_gem:
4
4
  gitlab-styles:
5
5
  - rubocop-default.yml
6
6
 
7
+ inherit_mode:
8
+ merge:
9
+ - Exclude
10
+
7
11
  AllCops:
8
12
  NewCops: enable
13
+ Exclude:
14
+ # Generated by protoc and vendored from gitlab-org/protos; regenerate with
15
+ # scripts/vendor-ratelimit-proto.sh instead of editing.
16
+ - lib/labkit/rate_limit/config/proto/**/*
data/Gemfile CHANGED
@@ -8,3 +8,9 @@ gemspec
8
8
  # Exporter source: https://gitlab.com/gitlab-org/quality/analytics/test-metrics-exporters
9
9
  # Owned by: @gitlab-org/quality/analytics (group::development analytics)
10
10
  gem 'gitlab-rspec-metrics-exporter', '~> 0.2', groups: [:development, :test]
11
+
12
+ # regexp_parser 2.13.2 made Regexp::Syntax::Base.implements private, and
13
+ # json_schemer 2.5.0 calls it at load time, so every fresh bundle fails on
14
+ # `require "json_schemer"`. Drop the pin once json_schemer or regexp_parser
15
+ # ships a fix: https://github.com/davishmcclurg/json_schemer/issues/238
16
+ gem "regexp_parser", "!= 2.13.2", groups: [:development, :test]
@@ -15,7 +15,8 @@ Gem::Specification.new do |spec|
15
15
  spec.metadata = { "source_code_uri" => "https://gitlab.com/gitlab-org/ruby/gems/labkit-ruby" }
16
16
  spec.license = "MIT"
17
17
 
18
- spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|tools)/}) }
18
+ # The vendored .proto sources are read by the specs only; the gem ships the generated Ruby.
19
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|tools)/}) || f.end_with?(".proto") }
19
20
  spec.bindir = "exe"
20
21
  spec.executables = %w[labkit-logging]
21
22
  spec.require_paths = ["lib"]
@@ -711,6 +711,50 @@ A logging failure during a flush is logged at WARN with
711
711
  `dropped_pairs`, the number of lines that flush lost. It never raises, so the
712
712
  flush thread keeps running.
713
713
 
714
+ ## Config files
715
+
716
+ `Labkit::RateLimit::Config` loads rules from YAML files that follow the `gitlab.ratelimit.config.v1` schema from [gitlab-org/protos](https://gitlab.com/gitlab-org/protos): the contract an application ships (`AvailableLimiters`) and the rate limits an operator writes (`RateLimits`).
717
+ The generated protobuf classes are vendored under `lib/labkit/rate_limit/config/proto/` and reachable as `Labkit::RateLimit::Config::Proto`.
718
+ Run `scripts/vendor-ratelimit-proto.sh` to update them; CI fails on drift.
719
+
720
+ An application can also install the `gitlab-ratelimit-config-v1` gem from gitlab-org/protos.
721
+ Then labkit uses the gem's schema and compares it with the vendored copy when `Labkit::RateLimit::Config` loads.
722
+ A gem schema that is newer gives a warning.
723
+ A gem schema that lacks or changes a vendored field raises `Labkit::RateLimit::Config::VendoredProto::Mismatch`.
724
+ So upgrade the `gitlab-ratelimit-config-v1` gem in the application first, then the `gitlab-labkit` release that updates the vendored copy.
725
+
726
+ Loading runs in four steps, `parse`, `schema`, `contract` and `rule`, and a failure raises `Labkit::RateLimit::Config::Error` whose `#step` names the step.
727
+ The parse and schema steps exist; the contract and rule steps follow in [Part 3 of the plan](https://gitlab.com/gitlab-com/gl-infra/production-engineering/-/work_items/30263).
728
+
729
+ ```ruby
730
+ doc = Labkit::RateLimit::Config::Parser.parse(
731
+ "config/rate_limits/rack_request.yaml",
732
+ Labkit::RateLimit::Config::Proto::RateLimits,
733
+ strict: true # CI rejects unknown fields. Production loads with strict: false
734
+ )
735
+ Labkit::RateLimit::Config::Schema.check!(doc)
736
+ doc.rate_limits["rack_request"].rules.first.name
737
+ ```
738
+
739
+ `Schema.check!` mirrors every protovalidate rule of the `gitlab.ratelimit.config.v1` schema: name format, the period presets, the `limit` and `ban_for_s` ranges, every list cap, and the message rules such as `rule.limit_requires_threshold`.
740
+ It accepts `AvailableLimiters`, `RateLimits` and `PublishRulesRequest`.
741
+ A failure lists every violation with the field path, message and rule id that protovalidate reports, such as `rate_limits["rack_request"].rules[0].period_s: must be in list [1, 15, 30, 60, 180, 300, 600, 900, 1800, 3600, 28800, 86400] (uint32.in)`, so a failure reads the same in Ruby and Go.
742
+ `Schema.violations(doc)` returns them without raising.
743
+ Inside one message the order is protovalidate's: message rules, then the oneof, then fields in declaration order.
744
+ A protobuf map keeps no document order and Go iterates it in an unspecified order, so Ruby reports map entries by sorted key and a cross-SDK comparison matches violations per entry, not by position.
745
+ Map keys in a path are quoted as Go's `strconv.Quote` prints them.
746
+ The check is the strict reading of the schema: a matcher whose operator this binary does not know fails `required`.
747
+ Lenient loading, through the loader that follows in Part 3, drops such a rule with a warning before the schema step, so a newer operator does not reject the file.
748
+ `scripts/vendor-ratelimit-proto.sh` also copies `config.proto` and `constraint.proto` next to the generated code, and `schema_spec.rb` parses them and compares every field rule (caps, bounds, presets, flags), every message rule id and message, and the name pattern with `Schema`, so a `PROTOS_REF` bump that changes any of those fails the suite until `Schema` follows.
749
+ The CEL expressions themselves are mirrored by hand and pinned by the examples.
750
+
751
+ The parse step converts each YAML scalar by the type of the field it fills, so `eq: off`, `eq: True` and `eq: 9970` stay strings, `limit: "5"` becomes `5`, `action: "1"` becomes `ACTION_LIMIT`, and a YAML null leaves the field unset.
752
+ A repeated key, also one repeated under its camelCase `json_name`, a merge key (`<<`), an empty file, more than one document, or a non-scalar key fails at step `parse`, in both modes.
753
+ So does a value of the wrong kind for its field, such as a mapping where a scalar belongs, an integer out of range for its field, two fields of one `oneof`, such as `{ eq: a, re: b }`, a null list item or map value, and a document nested deeper than 64 levels.
754
+ Each of these errors names the YAML line.
755
+ An unknown field or an unknown enum name, such as `action: ACTION_LIMT`, fails in strict mode with its YAML line.
756
+ Lenient mode ignores an unknown field and leaves an unknown enum name unset.
757
+
714
758
  ## Dev/test vs production guards
715
759
 
716
760
  `Limiter.new` and `Rule.new` validate names and configuration. In
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Labkit
4
+ module RateLimit
5
+ module Config
6
+ # Error is raised when a document fails one of the load steps.
7
+ # step is :parse, :schema, :contract or :rule.
8
+ class Error < StandardError
9
+ attr_reader :step
10
+
11
+ def initialize(step, message)
12
+ @step = step
13
+ super("#{step}: #{message}")
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,242 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "psych"
5
+ require "google/protobuf/descriptor_pb"
6
+
7
+ module Labkit
8
+ module RateLimit
9
+ module Config
10
+ # Parser is the parse step: a YAML file into a protobuf message.
11
+ #
12
+ # YAML types do not follow the schema, so each scalar is converted by the
13
+ # type of the field it fills, the way protoyaml does in Go: a string field
14
+ # keeps the scalar text (`off`, `True` and `9970` stay strings), an integer
15
+ # or enum field gets an Integer when the text is one, quoted or not, and
16
+ # the YAML null forms mean the field is absent. Psych keeps the last value
17
+ # of a repeated key, so repeated keys are rejected here, also when one
18
+ # spelling is the snake_case field name and the other its camelCase
19
+ # json_name.
20
+ #
21
+ # This step checks the shape of a document, not its values. An enum
22
+ # number that no Action defines, such as `action: 7`, passes in both
23
+ # modes, and in lenient mode protobuf drops an unknown enum name together
24
+ # with the unknown fields, so a rule with a misspelt action reaches the
25
+ # schema step as ACTION_UNSPECIFIED. Schema.check! rejects both.
26
+ module Parser
27
+ INT32 = (-(2**31))...(2**31)
28
+ UINT32 = 0...(2**32)
29
+ INT64 = (-(2**63))...(2**63)
30
+ UINT64 = 0...(2**64)
31
+ INTEGER_RANGES = {
32
+ int32: INT32, sint32: INT32, sfixed32: INT32, enum: INT32,
33
+ uint32: UINT32, fixed32: UINT32,
34
+ int64: INT64, sint64: INT64, sfixed64: INT64,
35
+ uint64: UINT64, fixed64: UINT64
36
+ }.freeze
37
+ INTEGER = /\A[-+]?\d+\z/
38
+ NULLS = ["", "~", "null", "Null", "NULL"].freeze
39
+ # The protobuf JSON decoder stops at 64 levels of objects and arrays.
40
+ # Mappings and lists are counted here so the error names the YAML line
41
+ # and a deeper document cannot exhaust the stack.
42
+ MAX_DEPTH = 64
43
+
44
+ class << self
45
+ # @param path [String] YAML file
46
+ # @param klass [Class] a Proto message class, AvailableLimiters or RateLimits
47
+ # @param strict [Boolean] when true an unknown field is an error
48
+ # @return [Google::Protobuf::AbstractMessage]
49
+ # @raise [Error] with step :parse
50
+ def parse(path, klass, strict:)
51
+ tree = tree_for(document_root(path), klass.descriptor, strict: strict, depth: 0)
52
+ klass.decode_json(JSON.generate(tree), ignore_unknown_fields: !strict)
53
+ rescue Psych::SyntaxError, SystemCallError => e
54
+ # Both messages already name the file.
55
+ raise Error.new(:parse, e.message)
56
+ rescue Google::Protobuf::ParseError, Psych::Exception, JSON::JSONError => e
57
+ raise Error.new(:parse, "#{path}: #{e.message}")
58
+ end
59
+
60
+ private
61
+
62
+ # One YAML document per file. Psych would silently keep only the first
63
+ # of several, and an empty file has no root to walk.
64
+ def document_root(path)
65
+ documents = Psych.parse_stream(File.read(path), filename: path).children
66
+ raise Psych::Exception, "the file holds no YAML document" if documents.empty?
67
+ raise Psych::Exception, "the file holds #{documents.size} YAML documents, expected 1" if documents.size > 1
68
+
69
+ documents.first.root
70
+ end
71
+
72
+ # One value for one slot. type is a field type Symbol, a message
73
+ # Descriptor or an EnumDescriptor. A node of the wrong kind is
74
+ # rejected here so the error names the YAML line, not an offset into
75
+ # the generated JSON. depth counts the mappings and lists above node.
76
+ def tree_for(node, type, strict:, depth:)
77
+ expects_message = type.is_a?(Google::Protobuf::Descriptor)
78
+
79
+ case node
80
+ when Psych::Nodes::Mapping
81
+ fail_at(node, "expected a scalar, found a mapping") unless expects_message
82
+ mapping_for(node, type, strict: strict, depth: depth)
83
+ when Psych::Nodes::Sequence
84
+ fail_at(node, "expected a single value, found a list")
85
+ when Psych::Nodes::Scalar
86
+ return nil if null?(node)
87
+
88
+ fail_at(node, "expected a mapping, found a scalar") if expects_message
89
+ scalar_for(node, type, strict: strict)
90
+ when Psych::Nodes::Alias
91
+ fail_at(node, "aliases are not supported")
92
+ end
93
+ end
94
+
95
+ def mapping_for(node, descriptor, strict:, depth:)
96
+ check_depth!(node, depth)
97
+ entries = node.children.each_slice(2).map do |key, value|
98
+ name = scalar_key(key)
99
+ field = field_for(descriptor, name)
100
+ fail_at(key, "unknown field #{name.inspect}") if strict && field.nil?
101
+ [name, key, field, value]
102
+ end
103
+ reject_repeated_keys!(entries.map { |name, key, field, _| [field ? field.name : name, key] })
104
+ reject_oneof_conflicts!(descriptor, entries)
105
+
106
+ maps = map_fields(descriptor)
107
+ # An unknown field is not walked: strict mode rejected its name
108
+ # above and lenient mode ignores it, so a newer schema may put
109
+ # anything there.
110
+ entries.to_h do |name, _, field, value|
111
+ next [name, nil] unless field
112
+
113
+ [name, field_value_for(value, field, map: maps.include?(field.name), strict: strict, depth: depth + 1)]
114
+ end
115
+ end
116
+
117
+ # Descriptor#options, which carries the map_entry flag, arrived after
118
+ # google-protobuf 3.25.0, so a blank message tells a map from a list.
119
+ # A message type keeps one Descriptor object, so the cache is bounded
120
+ # by the number of message types.
121
+ def map_fields(descriptor)
122
+ (@map_fields ||= {})[descriptor] ||= begin
123
+ blank = descriptor.msgclass.new
124
+ descriptor.select { |field| blank[field.name].is_a?(Google::Protobuf::Map) }.map(&:name).freeze
125
+ end
126
+ end
127
+
128
+ # A mapping key must be a plain scalar; a sequence, mapping or alias
129
+ # as a key has no field to fill. A merge key is rejected in both
130
+ # modes: lenient mode would ignore it as an unknown field and drop
131
+ # the merged fields with no error.
132
+ def scalar_key(node)
133
+ fail_at(node, "a mapping key must be a scalar") unless node.is_a?(Psych::Nodes::Scalar)
134
+ fail_at(node, "merge keys (<<) are not supported") if node.plain && node.value == "<<"
135
+
136
+ node.value
137
+ end
138
+
139
+ # The error points at the repeated key, not at the start of the mapping.
140
+ def reject_repeated_keys!(named_keys)
141
+ seen = {}
142
+ named_keys.each do |name, key|
143
+ fail_at(key, "duplicate key #{name.inspect}") if seen[name]
144
+ seen[name] = true
145
+ end
146
+ end
147
+
148
+ # Two fields of one oneof, such as two Matcher operators, would
149
+ # otherwise fail in the JSON decoder with no YAML line.
150
+ def reject_oneof_conflicts!(descriptor, entries)
151
+ descriptor.each_oneof do |oneof|
152
+ names = oneof.map(&:name)
153
+ set = entries.select { |_, _, field, value| field && names.include?(field.name) && !null?(value) }
154
+ next if set.size < 2
155
+
156
+ fail_at(set[1][1], "#{set[0][2].name} and #{set[1][2].name} are both set; set only one field of #{oneof.name}")
157
+ end
158
+ end
159
+
160
+ def check_depth!(node, depth)
161
+ fail_at(node, "nesting deeper than #{MAX_DEPTH} levels") if depth >= MAX_DEPTH
162
+ end
163
+
164
+ def fail_at(node, message)
165
+ raise Psych::Exception, "line #{node.start_line + 1}: #{message}"
166
+ end
167
+
168
+ # Files are snake_case; protojson also accepts the camelCase json_name.
169
+ def field_for(descriptor, name)
170
+ descriptor.lookup(name) || descriptor.find { |field| field.json_name == name }
171
+ end
172
+
173
+ def field_value_for(node, field, map:, strict:, depth:)
174
+ return map_for(node, field, strict: strict, depth: depth) if map
175
+ return tree_for(node, element_type(field), strict: strict, depth: depth) unless field.label == :repeated
176
+ return nil if null?(node)
177
+
178
+ fail_at(node, "expected a list") unless node.is_a?(Psych::Nodes::Sequence)
179
+ check_depth!(node, depth)
180
+ node.children.map do |child|
181
+ fail_at(child, "a list item cannot be null") if null?(child)
182
+
183
+ tree_for(child, element_type(field), strict: strict, depth: depth + 1)
184
+ end
185
+ end
186
+
187
+ def map_for(node, field, strict:, depth:)
188
+ return nil if null?(node)
189
+
190
+ fail_at(node, "expected a mapping") unless node.is_a?(Psych::Nodes::Mapping)
191
+ check_depth!(node, depth)
192
+ pairs = node.children.each_slice(2)
193
+ reject_repeated_keys!(pairs.map { |key, _| [scalar_key(key), key] })
194
+ value_type = element_type(field.subtype.lookup("value"))
195
+ pairs.to_h do |key, value|
196
+ fail_at(value, "a map value cannot be null") if null?(value)
197
+
198
+ [scalar_key(key), tree_for(value, value_type, strict: strict, depth: depth + 1)]
199
+ end
200
+ end
201
+
202
+ # A message or enum field passes its descriptor down, so a scalar
203
+ # can be checked against the enum's names.
204
+ def element_type(field)
205
+ %i[message enum].include?(field.type) ? field.subtype : field.type
206
+ end
207
+
208
+ def null?(node)
209
+ node.is_a?(Psych::Nodes::Scalar) && node.plain && NULLS.include?(node.value)
210
+ end
211
+
212
+ # Integer text is converted quoted or not: protojson takes a quoted
213
+ # number for an integer field but not for an enum. The range is
214
+ # checked here so the error names the YAML line.
215
+ def scalar_for(node, type, strict:)
216
+ enum = type if type.is_a?(Google::Protobuf::EnumDescriptor)
217
+ kind = enum ? :enum : type
218
+ range = INTEGER_RANGES[kind]
219
+ if range && INTEGER.match?(node.value)
220
+ value = Integer(node.value, 10)
221
+ fail_at(node, "#{node.value} is out of range for #{kind}") unless range.cover?(value)
222
+ return value
223
+ end
224
+
225
+ # Lenient mode passes an unknown name on, and protobuf drops it
226
+ # with the unknown fields, so a newer schema's value still loads.
227
+ fail_at(node, "unknown value #{node.value.inspect} for enum #{enum.name}") if enum && strict && enum.lookup_name(node.value.to_sym).nil?
228
+
229
+ return node.value unless node.plain
230
+
231
+ case kind
232
+ when :bool
233
+ { "true" => true, "false" => false }.fetch(node.value, node.value)
234
+ else
235
+ node.value
236
+ end
237
+ end
238
+ end
239
+ end
240
+ end
241
+ end
242
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+ # Vendored from gitlab-org/protos at gen/ruby/gitlab/ratelimit/config/v1/v0.1.0, gen/ruby/gitlab/ratelimit/config/v1/lib/gitlab/ratelimit/config/v1/config_pb.rb.
3
+ # Do not edit. Run scripts/vendor-ratelimit-proto.sh to update.
4
+ # Generated by the protocol buffer compiler. DO NOT EDIT!
5
+ # source: gitlab/ratelimit/config/v1/config.proto
6
+
7
+ require 'google/protobuf'
8
+ require 'labkit/rate_limit/config/vendored_proto'
9
+
10
+
11
+ descriptor_data = "\n\'gitlab/ratelimit/config/v1/config.proto\x12\x1agitlab.ratelimit.config.v1\"\xf7\x01\n\x11\x41vailableLimiters\x12s\n\x12\x61vailable_limiters\x18\x01 \x03(\x0b\x32\x44.gitlab.ratelimit.config.v1.AvailableLimiters.AvailableLimitersEntryR\x11\x61vailableLimiters\x1am\n\x16\x41vailableLimitersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12=\n\x05value\x18\x02 \x01(\x0b\x32\'.gitlab.ratelimit.config.v1.LimiterSpecR\x05value:\x02\x38\x01\"\xd2\x01\n\x0bLimiterSpec\x12 \n\x0b\x64\x65scription\x18\x01 \x01(\tR\x0b\x64\x65scription\x12\x31\n\x14\x61vailable_properties\x18\x02 \x03(\tR\x13\x61vailableProperties\x12\x45\n\rdefault_rules\x18\x03 \x03(\x0b\x32 .gitlab.ratelimit.config.v1.RuleR\x0c\x64\x65\x66\x61ultRules\x12\'\n\x0fprotected_rules\x18\x04 \x03(\tR\x0eprotectedRules\"\xca\x01\n\nRateLimits\x12W\n\x0brate_limits\x18\x01 \x03(\x0b\x32\x36.gitlab.ratelimit.config.v1.RateLimits.RateLimitsEntryR\nrateLimits\x1a\x63\n\x0fRateLimitsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12:\n\x05value\x18\x02 \x01(\x0b\x32$.gitlab.ratelimit.config.v1.RuleListR\x05value:\x02\x38\x01\"B\n\x08RuleList\x12\x36\n\x05rules\x18\x01 \x03(\x0b\x32 .gitlab.ratelimit.config.v1.RuleR\x05rules\"\x9a\x04\n\x04Rule\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\x12:\n\x06\x61\x63tion\x18\x03 \x01(\x0e\x32\".gitlab.ratelimit.config.v1.ActionR\x06\x61\x63tion\x12(\n\x0f\x63haracteristics\x18\x04 \x03(\tR\x0f\x63haracteristics\x12\x41\n\x05match\x18\x05 \x03(\x0b\x32+.gitlab.ratelimit.config.v1.Rule.MatchEntryR\x05match\x12\x19\n\x05limit\x18\x06 \x01(\rH\x00R\x05limit\x88\x01\x01\x12\x1e\n\x08period_s\x18\x07 \x01(\rH\x01R\x07periodS\x88\x01\x01\x12%\n\x0e\x63ount_distinct\x18\x08 \x01(\tR\rcountDistinct\x12\x1f\n\tban_for_s\x18\t \x01(\rH\x02R\x07\x62\x61nForS\x88\x01\x01\x12\x1f\n\x08position\x18\n \x01(\rH\x03R\x08position\x88\x01\x01\x1a]\n\nMatchEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32#.gitlab.ratelimit.config.v1.MatcherR\x05value:\x02\x38\x01\x42\x08\n\x06_limitB\x0b\n\t_period_sB\x0c\n\n_ban_for_sB\x0b\n\t_position\"\x93\x03\n\x13PublishRulesRequest\x12`\n\x0brate_limits\x18\x01 \x03(\x0b\x32?.gitlab.ratelimit.config.v1.PublishRulesRequest.RateLimitsEntryR\nrateLimits\x12S\n\x06remove\x18\x02 \x03(\x0b\x32;.gitlab.ratelimit.config.v1.PublishRulesRequest.RemoveEntryR\x06remove\x1a\x63\n\x0fRateLimitsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12:\n\x05value\x18\x02 \x01(\x0b\x32$.gitlab.ratelimit.config.v1.RuleListR\x05value:\x02\x38\x01\x1a`\n\x0bRemoveEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12;\n\x05value\x18\x02 \x01(\x0b\x32%.gitlab.ratelimit.config.v1.RuleNamesR\x05value:\x02\x38\x01\"!\n\tRuleNames\x12\x14\n\x05names\x18\x01 \x03(\tR\x05names\"\x9a\x02\n\x07Matcher\x12\x10\n\x02\x65q\x18\x01 \x01(\tH\x00R\x02\x65q\x12\x10\n\x02ne\x18\x02 \x01(\tH\x00R\x02ne\x12\x10\n\x02re\x18\x03 \x01(\tH\x00R\x02re\x12\x12\n\x03nre\x18\x04 \x01(\tH\x00R\x03nre\x12?\n\x06one_of\x18\x05 \x01(\x0b\x32&.gitlab.ratelimit.config.v1.StringListH\x00R\x05oneOf\x12\x41\n\x07none_of\x18\x06 \x01(\x0b\x32&.gitlab.ratelimit.config.v1.StringListH\x00R\x06noneOf\x12;\n\x03\x61ll\x18\x07 \x01(\x0b\x32\'.gitlab.ratelimit.config.v1.MatcherListH\x00R\x03\x61llB\x04\n\x02op\"$\n\nStringList\x12\x16\n\x06values\x18\x01 \x03(\tR\x06values\"N\n\x0bMatcherList\x12?\n\x08matchers\x18\x01 \x03(\x0b\x32#.gitlab.ratelimit.config.v1.MatcherR\x08matchers*S\n\x06\x41\x63tion\x12\x16\n\x12\x41\x43TION_UNSPECIFIED\x10\x00\x12\x10\n\x0c\x41\x43TION_LIMIT\x10\x01\x12\x0e\n\nACTION_LOG\x10\x02\x12\x0f\n\x0b\x41\x43TION_SKIP\x10\x03\x62\x06proto3"
12
+
13
+ pool = ::Google::Protobuf::DescriptorPool.generated_pool
14
+ return unless Labkit::RateLimit::Config::VendoredProto.register(pool, descriptor_data)
15
+
16
+ module Gitlab
17
+ module Ratelimit
18
+ module Config
19
+ module V1
20
+ AvailableLimiters = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.AvailableLimiters").msgclass
21
+ LimiterSpec = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.LimiterSpec").msgclass
22
+ RateLimits = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RateLimits").msgclass
23
+ RuleList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RuleList").msgclass
24
+ Rule = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Rule").msgclass
25
+ PublishRulesRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.PublishRulesRequest").msgclass
26
+ RuleNames = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.RuleNames").msgclass
27
+ Matcher = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Matcher").msgclass
28
+ StringList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.StringList").msgclass
29
+ MatcherList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.MatcherList").msgclass
30
+ Action = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.ratelimit.config.v1.Action").enummodule
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,324 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Labkit
4
+ module RateLimit
5
+ module Config
6
+ # Schema is the schema step: the checks that gitlab/ratelimit/config/v1
7
+ # declares with protovalidate, mirrored in Ruby because there is no
8
+ # protovalidate runtime for Ruby. Each check keeps the protovalidate rule
9
+ # id, message and field path. Inside one message the order is
10
+ # protovalidate's: message rules, then the oneof, then fields in
11
+ # declaration order.
12
+ # A protobuf map keeps no document order and Go iterates it in an
13
+ # unspecified order, so Ruby reports map entries by sorted key and the
14
+ # corpus compares per entry.
15
+ #
16
+ # FIELD_RULES and MESSAGE_RULES are copied from the proto by hand, and
17
+ # schema_spec.rb compares both with the vendored config.proto and
18
+ # constraint.proto field by field, so a PROTOS_REF bump that changes a
19
+ # cap, a flag, a rule id or a message fails the suite until this file
20
+ # follows. The CEL expressions have no such check; the examples pin them.
21
+ #
22
+ # The check is the strict reading of the schema. Lenient loading drops a
23
+ # rule whose matcher has no known operator before this step (the loader,
24
+ # Task 3.3), because such a matcher fails `required` here.
25
+ module Schema
26
+ # for_key marks a violation on a map key; its path is the entry's path.
27
+ Violation = Data.define(:path, :rule_id, :message, :for_key) do
28
+ def initialize(path:, rule_id:, message:, for_key: false)
29
+ super
30
+ end
31
+
32
+ def to_s
33
+ path.empty? ? "#{message} (#{rule_id})" : "#{path}: #{message} (#{rule_id})"
34
+ end
35
+ end
36
+
37
+ # CEL matches() is RE2, where ^ and $ anchor the whole text; Ruby's $
38
+ # matches before a newline, so the mirror anchors with \A and \z.
39
+ NAME = /\A[a-z0-9_]{1,64}\z/
40
+ NAME_MESSAGE = "invalid name; must match ^[a-z0-9_]{1,64}$"
41
+ PERIODS = [1, 15, 30, 60, 180, 300, 600, 900, 1800, 3600, 28800, 86400].freeze
42
+ SKIP = :ACTION_SKIP
43
+
44
+ # The field rules of config.proto, keyed "Message.field". The keys of a
45
+ # map field and the items of a repeated field carry their rule under
46
+ # keys_* and items_*.
47
+ FIELD_RULES = {
48
+ "AvailableLimiters.available_limiters" => { min_pairs: 1, keys_is_name: true },
49
+ "LimiterSpec.description" => { max_len: 1024 },
50
+ "LimiterSpec.available_properties" => { unique: true, max_items: 100, items_is_name: true },
51
+ "LimiterSpec.default_rules" => { max_items: 200 },
52
+ "LimiterSpec.protected_rules" => { unique: true, max_items: 200, items_is_name: true },
53
+ "RateLimits.rate_limits" => { keys_is_name: true },
54
+ "RuleList.rules" => { max_items: 200 },
55
+ "Rule.name" => { is_name: true },
56
+ "Rule.description" => { max_len: 1024 },
57
+ "Rule.action" => { required: true, defined_only: true },
58
+ "Rule.characteristics" => { unique: true, max_items: 10, items_is_name: true },
59
+ "Rule.match" => { max_pairs: 20, keys_is_name: true },
60
+ "Rule.limit" => { gte: 1, lte: 1_000_000 },
61
+ "Rule.period_s" => { in: PERIODS },
62
+ "Rule.count_distinct" => { is_name: true, ignore_empty: true },
63
+ "Rule.ban_for_s" => { gte: 1, lte: 86_400 },
64
+ "PublishRulesRequest.rate_limits" => { keys_is_name: true },
65
+ "PublishRulesRequest.remove" => { keys_is_name: true },
66
+ "RuleNames.names" => { min_items: 1, max_items: 200, unique: true, items_is_name: true },
67
+ "Matcher.eq" => { max_len: 1024 },
68
+ "Matcher.ne" => { max_len: 1024 },
69
+ "Matcher.re" => { max_len: 1024 },
70
+ "Matcher.nre" => { max_len: 1024 },
71
+ "StringList.values" => { min_items: 1, max_items: 100, unique: true, items_max_len: 1024 },
72
+ "MatcherList.matchers" => { min_items: 1, max_items: 20 }
73
+ }.freeze
74
+
75
+ # The message-level CEL rules of config.proto, by id.
76
+ MESSAGE_RULES = {
77
+ "limiter_spec.protected_rules_declared" => "every protected rule must be a default rule of this limiter",
78
+ "limiter_spec.unique_default_names" => "default rule names must be unique within a limiter",
79
+ "limiter_spec.no_position_in_defaults" => "position is not allowed on a default rule",
80
+ "rule_list.unique_names" => "rule names must be unique within a limiter",
81
+ "rule.limit_requires_threshold" => "limit and period_s are required unless action is skip",
82
+ "rule.ban_excludes_count_distinct" => "ban_for_s cannot be combined with count_distinct",
83
+ "rule.ban_not_on_skip" => "ban_for_s is not allowed on a skip rule",
84
+ "rule.no_counter_on_skip" => "limit, period_s, characteristics and count_distinct are not allowed on a skip rule",
85
+ "rule.count_distinct_not_characteristic" => "count_distinct cannot also be a characteristic",
86
+ "publish.name_in_both" => "a rule name cannot be in both rate_limits and remove for one limiter",
87
+ "matcher_list.no_nested_all" => "all cannot contain another all"
88
+ }.freeze
89
+
90
+ # The ASCII characters that strconv.Quote prints with a backslash escape.
91
+ ESCAPES = {
92
+ '"' => '\\"', "\\" => "\\\\", "\a" => "\\a", "\b" => "\\b", "\f" => "\\f",
93
+ "\n" => "\\n", "\r" => "\\r", "\t" => "\\t", "\v" => "\\v"
94
+ }.freeze
95
+
96
+ class << self
97
+ # @param message [Proto::AvailableLimiters, Proto::RateLimits, Proto::PublishRulesRequest]
98
+ # @return the message
99
+ # @raise [Error] step :schema, every violation joined with "; "
100
+ def check!(message)
101
+ found = violations(message)
102
+ raise Error.new(:schema, found.join("; ")) unless found.empty?
103
+
104
+ message
105
+ end
106
+
107
+ # @return [Array<Violation>] every violation, in the order described above
108
+ def violations(message)
109
+ checker = Checker.new
110
+ case message
111
+ when Proto::AvailableLimiters then checker.available_limiters(message)
112
+ when Proto::RateLimits then checker.rate_limits(message)
113
+ when Proto::PublishRulesRequest then checker.publish_rules_request(message)
114
+ else
115
+ raise ArgumentError, "no schema for #{message.class}; expected AvailableLimiters, RateLimits or PublishRulesRequest"
116
+ end
117
+ checker.violations
118
+ end
119
+
120
+ # Quotes a map key for a path the way Go's strconv.Quote does, so the
121
+ # path reads the same in every SDK: backslash escapes for the ASCII
122
+ # controls that have one, \xNN for the other ASCII controls and DEL,
123
+ # \uNNNN or \UNNNNNNNN for a non-printable rune, everything else as is.
124
+ def quote(key)
125
+ out = +'"'
126
+ key.each_char { |c| out << (ESCAPES[c] || escape_rune(c)) }
127
+ out << '"'
128
+ end
129
+
130
+ private
131
+
132
+ # Go's unicode.IsPrint: letters, marks, numbers, punctuation, symbols
133
+ # and the ASCII space.
134
+ def escape_rune(char)
135
+ code = char.ord
136
+ return format("\\x%02x", code) if code < 0x20 || code == 0x7f
137
+ return char if char == " " || char.match?(/\A[\p{L}\p{M}\p{N}\p{P}\p{S}]\z/)
138
+
139
+ code > 0xffff ? format("\\U%08x", code) : format("\\u%04x", code)
140
+ end
141
+ end
142
+
143
+ # Walks one message and collects violations. Paths follow protovalidate:
144
+ # `rate_limits["rack_request"].rules[0].match["plan"].eq`.
145
+ class Checker
146
+ attr_reader :violations
147
+
148
+ def initialize
149
+ @violations = []
150
+ end
151
+
152
+ def available_limiters(doc)
153
+ map_field("available_limiters", doc.available_limiters, FIELD_RULES["AvailableLimiters.available_limiters"]) { |path, spec| limiter_spec(path, spec) }
154
+ end
155
+
156
+ def rate_limits(doc)
157
+ map_field("rate_limits", doc.rate_limits, FIELD_RULES["RateLimits.rate_limits"]) { |path, list| rule_list(path, list) }
158
+ end
159
+
160
+ def publish_rules_request(req)
161
+ # The outcome of one name in both fields would depend on the order
162
+ # the store applies them.
163
+ both = req.rate_limits.any? do |key, list|
164
+ req.remove.has_key?(key) && list.rules.any? { |r| req.remove[key].names.include?(r.name) }
165
+ end
166
+ message_rule("", "publish.name_in_both") if both
167
+ map_field("rate_limits", req.rate_limits, FIELD_RULES["PublishRulesRequest.rate_limits"]) { |path, list| rule_list(path, list) }
168
+ map_field("remove", req.remove, FIELD_RULES["PublishRulesRequest.remove"]) do |path, names|
169
+ list("#{path}.names", names.names, FIELD_RULES["RuleNames.names"])
170
+ end
171
+ end
172
+
173
+ private
174
+
175
+ def limiter_spec(path, spec)
176
+ default_names = spec.default_rules.map(&:name)
177
+ message_rule(path, "limiter_spec.protected_rules_declared") unless spec.protected_rules.all? { |n| default_names.include?(n) }
178
+ message_rule(path, "limiter_spec.unique_default_names") unless default_names.uniq.size == default_names.size
179
+ # Default rules keep document order, so a position has no meaning there.
180
+ message_rule(path, "limiter_spec.no_position_in_defaults") if spec.default_rules.any?(&:has_position?)
181
+
182
+ string("#{path}.description", spec.description, FIELD_RULES["LimiterSpec.description"])
183
+ list("#{path}.available_properties", spec.available_properties, FIELD_RULES["LimiterSpec.available_properties"])
184
+ list("#{path}.default_rules", spec.default_rules, FIELD_RULES["LimiterSpec.default_rules"])
185
+ spec.default_rules.each_with_index { |r, i| rule("#{path}.default_rules[#{i}]", r) }
186
+ list("#{path}.protected_rules", spec.protected_rules, FIELD_RULES["LimiterSpec.protected_rules"])
187
+ end
188
+
189
+ def rule_list(path, list)
190
+ names = list.rules.map(&:name)
191
+ message_rule(path, "rule_list.unique_names") unless names.uniq.size == names.size
192
+ list("#{path}.rules", list.rules, FIELD_RULES["RuleList.rules"])
193
+ list.rules.each_with_index { |r, i| rule("#{path}.rules[#{i}]", r) }
194
+ end
195
+
196
+ def rule(path, rule)
197
+ rule_message_rules(path, rule)
198
+ string("#{path}.name", rule.name, FIELD_RULES["Rule.name"])
199
+ string("#{path}.description", rule.description, FIELD_RULES["Rule.description"])
200
+ action("#{path}.action", rule.action, FIELD_RULES["Rule.action"])
201
+ list("#{path}.characteristics", rule.characteristics, FIELD_RULES["Rule.characteristics"])
202
+ map_field("#{path}.match", rule.match, FIELD_RULES["Rule.match"]) { |entry, matcher| matcher(entry, matcher) }
203
+ integer("#{path}.limit", rule.limit, FIELD_RULES["Rule.limit"]) if rule.has_limit?
204
+ integer("#{path}.period_s", rule.period_s, FIELD_RULES["Rule.period_s"]) if rule.has_period_s?
205
+ string("#{path}.count_distinct", rule.count_distinct, FIELD_RULES["Rule.count_distinct"])
206
+ integer("#{path}.ban_for_s", rule.ban_for_s, FIELD_RULES["Rule.ban_for_s"]) if rule.has_ban_for_s?
207
+ end
208
+
209
+ # In the order config.proto declares them.
210
+ def rule_message_rules(path, rule)
211
+ counts_distinct = !rule.count_distinct.empty?
212
+ message_rule(path, "rule.limit_requires_threshold") unless rule.action == SKIP || (rule.has_limit? && rule.has_period_s?)
213
+ message_rule(path, "rule.ban_excludes_count_distinct") if rule.has_ban_for_s? && counts_distinct
214
+ skip_rules(path, rule) if rule.action == SKIP
215
+ message_rule(path, "rule.count_distinct_not_characteristic") if counts_distinct && rule.characteristics.include?(rule.count_distinct)
216
+ end
217
+
218
+ # A skip rule has no counter and no limit to cross.
219
+ def skip_rules(path, rule)
220
+ counter = rule.has_limit? || rule.has_period_s? || !rule.characteristics.empty? || !rule.count_distinct.empty?
221
+ message_rule(path, "rule.ban_not_on_skip") if rule.has_ban_for_s?
222
+ message_rule(path, "rule.no_counter_on_skip") if counter
223
+ end
224
+
225
+ # An unknown enum number decodes as an Integer in Ruby.
226
+ def action(path, value, rules)
227
+ if rules[:required] && value == :ACTION_UNSPECIFIED
228
+ add(path, "required", "value is required")
229
+ elsif rules[:defined_only] && value.is_a?(Integer)
230
+ add(path, "enum.defined_only", "value must be one of the defined enum values")
231
+ end
232
+ end
233
+
234
+ def matcher(path, matcher)
235
+ op = matcher.op
236
+ return add("#{path}.op", "required", "exactly one field is required in oneof") if op.nil?
237
+
238
+ case op
239
+ when :eq, :ne, :re, :nre then string("#{path}.#{op}", matcher[op.to_s], FIELD_RULES["Matcher.#{op}"])
240
+ when :one_of, :none_of then list("#{path}.#{op}.values", matcher[op.to_s].values, FIELD_RULES["StringList.values"])
241
+ when :all then matcher_list("#{path}.all", matcher.all)
242
+ end
243
+ end
244
+
245
+ def matcher_list(path, list)
246
+ # There is no `any`, so an `all` inside an `all` is one flat list.
247
+ message_rule(path, "matcher_list.no_nested_all") if list.matchers.any? { |m| m.op == :all }
248
+ list("#{path}.matchers", list.matchers, FIELD_RULES["MatcherList.matchers"])
249
+ list.matchers.each_with_index { |m, i| matcher("#{path}.matchers[#{i}]", m) }
250
+ end
251
+
252
+ # Rules of RepeatedRules in field order: min_items, max_items, unique,
253
+ # then the items.
254
+ def list(path, values, rules)
255
+ add(path, "repeated.min_items", "must contain at least #{rules[:min_items]} item(s)") if rules[:min_items] && values.size < rules[:min_items]
256
+ add(path, "repeated.max_items", "must contain no more than #{rules[:max_items]} item(s)") if rules[:max_items] && values.size > rules[:max_items]
257
+ add(path, "repeated.unique", "repeated value must contain unique items") if rules[:unique] && values.uniq.size != values.size
258
+ values.each_with_index { |v, i| name("#{path}[#{i}]", v) } if rules[:items_is_name]
259
+ values.each_with_index { |v, i| max_len("#{path}[#{i}]", v, rules[:items_max_len]) } if rules[:items_max_len]
260
+ end
261
+
262
+ # Rules of MapRules in field order: min_pairs, max_pairs, then the
263
+ # keys. Yields each entry's path and value.
264
+ def map_field(path, map, rules)
265
+ add(path, "map.min_pairs", "map must be at least #{rules[:min_pairs]} entries") if rules[:min_pairs] && map.length < rules[:min_pairs]
266
+ add(path, "map.max_pairs", "map must be at most #{rules[:max_pairs]} entries") if rules[:max_pairs] && map.length > rules[:max_pairs]
267
+ map.keys.sort.each do |key|
268
+ value = map[key]
269
+ entry = "#{path}[#{Schema.quote(key)}]"
270
+ add(entry, "string.is_name", NAME_MESSAGE, for_key: true) if rules[:keys_is_name] && !NAME.match?(key)
271
+ yield entry, value
272
+ end
273
+ end
274
+
275
+ def string(path, value, rules)
276
+ return if rules[:ignore_empty] && value.empty?
277
+
278
+ max_len(path, value, rules[:max_len]) if rules[:max_len]
279
+ name(path, value) if rules[:is_name]
280
+ end
281
+
282
+ # protovalidate reports one of three ids for bounds, by which bounds
283
+ # the field declares.
284
+ def integer(path, value, rules)
285
+ if rules[:in]
286
+ add(path, "uint32.in", "must be in list #{rules[:in]}") unless rules[:in].include?(value)
287
+ return
288
+ end
289
+
290
+ gte, lte = rules.values_at(:gte, :lte)
291
+ below = gte && value < gte
292
+ above = lte && value > lte
293
+ return unless below || above
294
+
295
+ if gte && lte
296
+ add(path, "uint32.gte_lte", "must be greater than or equal to #{gte} and less than or equal to #{lte}")
297
+ elsif below
298
+ add(path, "uint32.gte", "must be greater than or equal to #{gte}")
299
+ else
300
+ add(path, "uint32.lte", "must be less than or equal to #{lte}")
301
+ end
302
+ end
303
+
304
+ def message_rule(path, rule_id)
305
+ add(path, rule_id, MESSAGE_RULES.fetch(rule_id))
306
+ end
307
+
308
+ def name(path, value)
309
+ add(path, "string.is_name", NAME_MESSAGE) unless NAME.match?(value)
310
+ end
311
+
312
+ def max_len(path, value, max)
313
+ add(path, "string.max_len", "must be at most #{max} characters") if value.length > max
314
+ end
315
+
316
+ def add(path, rule_id, message, for_key: false)
317
+ @violations << Violation.new(path: path, rule_id: rule_id, message: message, for_key: for_key)
318
+ end
319
+ end
320
+ private_constant :Checker
321
+ end
322
+ end
323
+ end
324
+ end
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "google/protobuf/descriptor_pb"
4
+
5
+ module Labkit
6
+ module RateLimit
7
+ module Config
8
+ # VendoredProto registers the vendored gitlab.ratelimit.config.v1 file in
9
+ # the global descriptor pool. The gitlab-ratelimit-config-v1 gem from
10
+ # gitlab-org/protos registers the same file name, unguarded, and a second
11
+ # registration raises. So the gem is loaded first whenever it is
12
+ # installed, and the vendored copy is then compared with it: the same
13
+ # schema means the gem's registration is used, a different one is a boot
14
+ # error, because two versions of one schema in one process would make the
15
+ # loader read fields the other copy does not know.
16
+ module VendoredProto
17
+ # The gem registered a schema that differs from the vendored copy, or
18
+ # one this google-protobuf cannot compare.
19
+ Mismatch = Class.new(StandardError)
20
+
21
+ PACKAGE = "gitlab.ratelimit.config.v1"
22
+ PROBE = "#{PACKAGE}.RateLimits".freeze
23
+ GEM_FILE = "gitlab/ratelimit/config/v1/config_pb"
24
+
25
+ class << self
26
+ # @param pool [Google::Protobuf::DescriptorPool]
27
+ # @param serialized [String] a serialized FileDescriptorProto
28
+ # @return [Boolean] true when this call registered the file and the
29
+ # vendored file must define its constants; false when the gem's
30
+ # registration is in use and the vendored file must stop loading
31
+ # @raise [Mismatch] when the gem registered a different schema
32
+ def register(pool, serialized)
33
+ load_gem_first
34
+
35
+ existing = pool.lookup(PROBE)
36
+ if existing.nil?
37
+ pool.add_serialized_file(serialized)
38
+ return true
39
+ end
40
+
41
+ check_same_schema!(existing.file_descriptor, serialized)
42
+ false
43
+ end
44
+
45
+ private
46
+
47
+ # Ruby's require loads the gem's file once, so whichever side comes
48
+ # first, the gem registers before the vendored copy is compared. Only
49
+ # an absent gem is expected; a LoadError from inside the gem's own
50
+ # file is a real failure and propagates.
51
+ def load_gem_first
52
+ require GEM_FILE
53
+ rescue ::LoadError => e
54
+ raise unless e.path == GEM_FILE
55
+ end
56
+
57
+ # The gem may be ahead of the vendored copy: an additive release adds
58
+ # fields the loader does not read, and that is fine. The gem may not
59
+ # be behind it: the loader would then read a field the gem's schema
60
+ # does not have.
61
+ def check_same_schema!(file, serialized)
62
+ # FileDescriptor#to_proto arrived in google-protobuf 4.30. The gem
63
+ # keeps its 3.25 floor for consumers that never reach this path, so
64
+ # the requirement is raised here, where it applies.
65
+ raise Mismatch, "#{PACKAGE} is registered by another gem and this google-protobuf cannot compare schemas; rate limit config needs google-protobuf >= 4.30" unless file.respond_to?(:to_proto)
66
+
67
+ registered = signature(file.to_proto)
68
+ vendored = signature(Google::Protobuf::FileDescriptorProto.decode(serialized))
69
+ missing = vendored.reject { |path, sig| registered[path] == sig }.keys
70
+ detail = "lacks or changes #{missing.join(', ')}; update the gitlab-labkit vendored copy or the gem"
71
+ raise Mismatch, "#{PACKAGE} is registered by another gem with a schema that #{detail}" unless missing.empty?
72
+
73
+ extra = registered.keys - vendored.keys
74
+ warn "gitlab-labkit: #{PACKAGE} from another gem is a newer schema (#{extra.join(', ')}); the vendored copy is older" unless extra.empty?
75
+ end
76
+
77
+ # "Message.field" => its wire and API shape, "Enum.VALUE" => number,
78
+ # for every message, nested message and enum. proto3 optional decides
79
+ # whether has_x? exists, the oneof name which fields exclude each
80
+ # other, and json_name what the parser accepts as a key.
81
+ def signature(file)
82
+ out = {}
83
+ walk = lambda do |messages, enums, prefix|
84
+ enums.each { |e| e.value.each { |val| out["#{prefix}#{e.name}.#{val.name}"] = val.number } }
85
+ messages.each do |m|
86
+ m.field.each { |f| out["#{prefix}#{m.name}.#{f.name}"] = field_signature(f, m) }
87
+ walk.call(m.nested_type, m.enum_type, "#{prefix}#{m.name}.")
88
+ end
89
+ end
90
+ walk.call(file.message_type, file.enum_type, "")
91
+ out
92
+ end
93
+
94
+ # protoc numbers oneofs in declaration order, and every proto3
95
+ # optional field adds a synthetic one, so an additive field shifts
96
+ # the index of every oneof after it. The name does not move.
97
+ def field_signature(field, message)
98
+ oneof = field.has_oneof_index? ? message.oneof_decl[field.oneof_index].name : nil
99
+ [field.number, field.type, field.label, field.type_name, field.json_name, field.proto3_optional, oneof]
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "labkit/rate_limit/config/vendored_proto"
4
+ require "labkit/rate_limit/config/proto/config_pb"
5
+
6
+ module Labkit
7
+ module RateLimit
8
+ # Config loads rate limit rules from YAML files that follow the
9
+ # gitlab.ratelimit.config.v1 schema: the contract an application ships
10
+ # (AvailableLimiters) and the rate limits an operator writes (RateLimits).
11
+ # Loading runs in four steps, parse, schema, contract and rule, and a
12
+ # failure names its step so CI output and the conformance corpus agree
13
+ # across SDKs.
14
+ module Config
15
+ # The generated protobuf classes, vendored from gitlab-org/protos.
16
+ Proto = Gitlab::Ratelimit::Config::V1
17
+
18
+ autoload :Error, "labkit/rate_limit/config/error"
19
+ autoload :Parser, "labkit/rate_limit/config/parser"
20
+ autoload :Schema, "labkit/rate_limit/config/schema"
21
+ end
22
+ end
23
+ end
@@ -26,6 +26,7 @@ module Labkit
26
26
  autoload :Limiter, "labkit/rate_limit/limiter"
27
27
  autoload :Metrics, "labkit/rate_limit/metrics"
28
28
  autoload :BlockSummary, "labkit/rate_limit/block_summary"
29
+ autoload :Config, "labkit/rate_limit/config"
29
30
 
30
31
  class << self
31
32
  def configure
@@ -0,0 +1,47 @@
1
+ #!/usr/bin/env bash
2
+ # Copies the generated Ruby code and the proto sources for gitlab.ratelimit.config.v1
3
+ # from gitlab-org/protos. The sources are what schema_spec.rb checks Schema against.
4
+ # A published gem cannot depend on the protos Bundler git gem, so the file is
5
+ # vendored and CI checks that it matches PROTOS_REF (job vendored-proto-check).
6
+ set -euo pipefail
7
+
8
+ # Pinned protos module tag. Bump it when a newer schema release is needed.
9
+ PROTOS_REF="${PROTOS_REF:-gen/ruby/gitlab/ratelimit/config/v1/v0.1.0}"
10
+ SOURCE_PATH="gen/ruby/gitlab/ratelimit/config/v1/lib/gitlab/ratelimit/config/v1/config_pb.rb"
11
+ PROTO_DIR="proto/ratelimit/gitlab/ratelimit/config/v1"
12
+ TARGET_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/lib/labkit/rate_limit/config/proto"
13
+ TARGET="${TARGET_DIR}/config_pb.rb"
14
+
15
+ raw="https://gitlab.com/gitlab-org/protos/-/raw/${PROTOS_REF}"
16
+ # Everything lands in a temp dir first, so a failed download or check
17
+ # leaves the vendored files as they were.
18
+ work="$(mktemp -d)"
19
+ trap 'rm -rf "$work"' EXIT
20
+ tmp="${work}/config_pb.rb"
21
+ curl --fail --silent --show-error --location "${raw}/${SOURCE_PATH}" -o "$tmp"
22
+ for proto in config.proto constraint.proto; do
23
+ curl --fail --silent --show-error --location "${raw}/${PROTO_DIR}/${proto}" -o "${work}/${proto}"
24
+ done
25
+
26
+ # The generated file registers gitlab/ratelimit/config/v1/config.proto in the
27
+ # global descriptor pool. The protos gem registers the same file name, so the
28
+ # vendored copy goes through VendoredProto.register, which loads the gem first
29
+ # when it is installed and then compares schemas. When the gem's registration
30
+ # is in use the file returns early, so its constants are not defined twice.
31
+ guard='return unless Labkit::RateLimit::Config::VendoredProto.register(pool, descriptor_data)'
32
+ {
33
+ echo "# frozen_string_literal: true"
34
+ echo "# Vendored from gitlab-org/protos at ${PROTOS_REF}, ${SOURCE_PATH}."
35
+ echo "# Do not edit. Run scripts/vendor-ratelimit-proto.sh to update."
36
+ grep -v '^# frozen_string_literal: true$' "$tmp" |
37
+ sed -e "s|^require 'google/protobuf'\$|require 'google/protobuf'\nrequire 'labkit/rate_limit/config/vendored_proto'|" \
38
+ -e "s|^pool.add_serialized_file(descriptor_data)\$|${guard}|"
39
+ } >"${work}/vendored.rb"
40
+ if ! grep -q 'VendoredProto.register' "${work}/vendored.rb" ||
41
+ ! grep -qx "require 'labkit/rate_limit/config/vendored_proto'" "${work}/vendored.rb"; then
42
+ echo "vendor: the generated file no longer requires VendoredProto or registers the descriptor as expected; update the sed in this script" >&2
43
+ exit 1
44
+ fi
45
+ mv "${work}/vendored.rb" "$TARGET"
46
+ mv "${work}/config.proto" "${work}/constraint.proto" "$TARGET_DIR/"
47
+ echo "vendored ${SOURCE_PATH}, config.proto and constraint.proto @${PROTOS_REF} -> ${TARGET_DIR#"$PWD"/}"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-labkit
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.9.0
4
+ version: 5.11.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Newdigate
@@ -651,6 +651,12 @@ files:
651
651
  - lib/labkit/rate_limit.rb
652
652
  - lib/labkit/rate_limit/README.md
653
653
  - lib/labkit/rate_limit/block_summary.rb
654
+ - lib/labkit/rate_limit/config.rb
655
+ - lib/labkit/rate_limit/config/error.rb
656
+ - lib/labkit/rate_limit/config/parser.rb
657
+ - lib/labkit/rate_limit/config/proto/config_pb.rb
658
+ - lib/labkit/rate_limit/config/schema.rb
659
+ - lib/labkit/rate_limit/config/vendored_proto.rb
654
660
  - lib/labkit/rate_limit/configuration.rb
655
661
  - lib/labkit/rate_limit/evaluator.rb
656
662
  - lib/labkit/rate_limit/identifier.rb
@@ -719,6 +725,7 @@ files:
719
725
  - renovate.json
720
726
  - scripts/prepare-dev-env.sh
721
727
  - scripts/update-asdf-version-variables.sh
728
+ - scripts/vendor-ratelimit-proto.sh
722
729
  homepage: https://gitlab.com/gitlab-org/labkit-ruby
723
730
  licenses:
724
731
  - MIT