active_sanction 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +538 -0
  3. data/CODE_OF_CONDUCT.md +74 -0
  4. data/CONTRIBUTING.md +312 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +616 -0
  7. data/SECURITY.md +97 -0
  8. data/active_sanction.gemspec +98 -0
  9. data/docs/adding_a_source.md +1083 -0
  10. data/docs/api_stability.md +357 -0
  11. data/docs/bundle_format.md +407 -0
  12. data/lib/active_sanction/address.rb +170 -0
  13. data/lib/active_sanction/client.rb +341 -0
  14. data/lib/active_sanction/configuration.rb +721 -0
  15. data/lib/active_sanction/countries.txt +292 -0
  16. data/lib/active_sanction/country.rb +164 -0
  17. data/lib/active_sanction/deprecation.rb +168 -0
  18. data/lib/active_sanction/diff/change.rb +208 -0
  19. data/lib/active_sanction/diff.rb +348 -0
  20. data/lib/active_sanction/doctor/checkup.rb +356 -0
  21. data/lib/active_sanction/doctor/diagnosis.rb +245 -0
  22. data/lib/active_sanction/doctor/finding.rb +183 -0
  23. data/lib/active_sanction/doctor/profile.rb +384 -0
  24. data/lib/active_sanction/doctor/report.rb +227 -0
  25. data/lib/active_sanction/doctor.rb +347 -0
  26. data/lib/active_sanction/entity.rb +293 -0
  27. data/lib/active_sanction/error.rb +332 -0
  28. data/lib/active_sanction/fetcher/result.rb +116 -0
  29. data/lib/active_sanction/fetcher.rb +242 -0
  30. data/lib/active_sanction/http_client/errors.rb +94 -0
  31. data/lib/active_sanction/http_client/response.rb +129 -0
  32. data/lib/active_sanction/http_client.rb +324 -0
  33. data/lib/active_sanction/identifier.rb +212 -0
  34. data/lib/active_sanction/index/builder.rb +89 -0
  35. data/lib/active_sanction/index/candidate.rb +63 -0
  36. data/lib/active_sanction/index/entry.rb +66 -0
  37. data/lib/active_sanction/index/features.rb +112 -0
  38. data/lib/active_sanction/index.rb +381 -0
  39. data/lib/active_sanction/match_result.rb +345 -0
  40. data/lib/active_sanction/matcher.rb +388 -0
  41. data/lib/active_sanction/name.rb +183 -0
  42. data/lib/active_sanction/normalizer/cache.rb +82 -0
  43. data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
  44. data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
  45. data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
  46. data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
  47. data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
  48. data/lib/active_sanction/normalizer/dictionary.rb +243 -0
  49. data/lib/active_sanction/normalizer/form.rb +279 -0
  50. data/lib/active_sanction/normalizer.rb +137 -0
  51. data/lib/active_sanction/parsers/column_shape.rb +274 -0
  52. data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
  53. data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
  54. data/lib/active_sanction/parsers/delimited_table.rb +151 -0
  55. data/lib/active_sanction/parsers/format.rb +107 -0
  56. data/lib/active_sanction/parsers/join.rb +149 -0
  57. data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
  58. data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
  59. data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
  60. data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
  61. data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
  62. data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
  63. data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
  64. data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
  65. data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
  66. data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
  67. data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
  68. data/lib/active_sanction/parsers/xml_records.rb +140 -0
  69. data/lib/active_sanction/parsers.rb +94 -0
  70. data/lib/active_sanction/partial_date/parser.rb +124 -0
  71. data/lib/active_sanction/partial_date.rb +303 -0
  72. data/lib/active_sanction/payload_cache/checksum.rb +66 -0
  73. data/lib/active_sanction/payload_cache/entry.rb +304 -0
  74. data/lib/active_sanction/payload_cache.rb +366 -0
  75. data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
  76. data/lib/active_sanction/phonetics.rb +40 -0
  77. data/lib/active_sanction/query.rb +290 -0
  78. data/lib/active_sanction/rescreen/alert.rb +314 -0
  79. data/lib/active_sanction/rescreen.rb +332 -0
  80. data/lib/active_sanction/scorer/adjustments.rb +241 -0
  81. data/lib/active_sanction/scorer/name_score.rb +279 -0
  82. data/lib/active_sanction/scorer/reason.rb +136 -0
  83. data/lib/active_sanction/scorer/result.rb +129 -0
  84. data/lib/active_sanction/scorer/subject.rb +235 -0
  85. data/lib/active_sanction/scorer/weights.rb +306 -0
  86. data/lib/active_sanction/scorer.rb +303 -0
  87. data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
  88. data/lib/active_sanction/similarity/levenshtein.rb +179 -0
  89. data/lib/active_sanction/similarity/token_set.rb +159 -0
  90. data/lib/active_sanction/similarity/token_sort.rb +120 -0
  91. data/lib/active_sanction/similarity.rb +183 -0
  92. data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
  93. data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
  94. data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
  95. data/lib/active_sanction/snapshot/bundle.rb +381 -0
  96. data/lib/active_sanction/snapshot.rb +305 -0
  97. data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
  98. data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
  99. data/lib/active_sanction/sources/australia_dfat.rb +291 -0
  100. data/lib/active_sanction/sources/base.rb +295 -0
  101. data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
  102. data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
  103. data/lib/active_sanction/sources/canada_sema.rb +153 -0
  104. data/lib/active_sanction/sources/definition.rb +360 -0
  105. data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
  106. data/lib/active_sanction/sources/eu_fsf.rb +203 -0
  107. data/lib/active_sanction/sources/ofac/record.rb +251 -0
  108. data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
  109. data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
  110. data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
  111. data/lib/active_sanction/sources/ofac.rb +248 -0
  112. data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
  113. data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
  114. data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
  115. data/lib/active_sanction/sources/remarks.rb +84 -0
  116. data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
  117. data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
  118. data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
  119. data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
  120. data/lib/active_sanction/sources/un_consolidated.rb +140 -0
  121. data/lib/active_sanction/sources.rb +196 -0
  122. data/lib/active_sanction/storage/active_record/reader.rb +124 -0
  123. data/lib/active_sanction/storage/active_record/row.rb +175 -0
  124. data/lib/active_sanction/storage/active_record/writer.rb +141 -0
  125. data/lib/active_sanction/storage/active_record.rb +277 -0
  126. data/lib/active_sanction/storage/base.rb +216 -0
  127. data/lib/active_sanction/storage/file_system.rb +373 -0
  128. data/lib/active_sanction/storage/memory.rb +67 -0
  129. data/lib/active_sanction/storage/meta.rb +188 -0
  130. data/lib/active_sanction/storage.rb +74 -0
  131. data/lib/active_sanction/subject.rb +271 -0
  132. data/lib/active_sanction/sync/report.rb +245 -0
  133. data/lib/active_sanction/sync/result.rb +296 -0
  134. data/lib/active_sanction/sync.rb +390 -0
  135. data/lib/active_sanction/validator_store/file_system.rb +103 -0
  136. data/lib/active_sanction/validator_store/memory.rb +41 -0
  137. data/lib/active_sanction/validator_store.rb +119 -0
  138. data/lib/active_sanction/validators.rb +195 -0
  139. data/lib/active_sanction/version.rb +23 -0
  140. data/lib/active_sanction.rb +331 -0
  141. data/lib/generators/active_sanction/install/install_generator.rb +60 -0
  142. data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
  143. metadata +236 -0
@@ -0,0 +1,302 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "json"
7
+ require "time"
8
+ require "active_sanction/error"
9
+ require "active_sanction/snapshot"
10
+ require "active_sanction/sources"
11
+ require "active_sanction/version"
12
+
13
+ module ActiveSanction
14
+ class Snapshot
15
+ module Bundle
16
+ # What a bundle says about itself, in one line, before any of its records
17
+ # are read.
18
+ #
19
+ # header = ActiveSanction::Snapshot::Bundle.header(io)
20
+ #
21
+ # header.source # => :ofac_sdn
22
+ # header.record_count # => 19015
23
+ # header.snapshot_checksum # => "sha256:9f86d081884c7d65..."
24
+ # header.generator # => "active_sanction/1.0.0"
25
+ # header.fetched_at # => 2026-08-28 09:30:00 UTC
26
+ #
27
+ # This is Storage::Meta's job for a file somebody sent you, and the two
28
+ # are deliberately the same shape: small, cheap, and answerable without
29
+ # inflating tens of megabytes. `active_sanction import` prints one, a
30
+ # mirror indexes them, and a deploy decides whether it already holds this
31
+ # list version by comparing `snapshot_checksum` against what it stored.
32
+ #
33
+ # ### It is the thing that gets signed
34
+ #
35
+ # A signature covers these bytes and nothing else -- see Bundle::Signature.
36
+ # It can, because `payload_digest` is a SHA-256 over every record in the
37
+ # file: signing ~300 bytes here transitively covers all 19,015 of them,
38
+ # and a verifier settles who published a bundle before it inflates a byte
39
+ # of what they sent.
40
+ #
41
+ # So every field that could change what the payload *means* is in here,
42
+ # and a field whose value is not reproducible from the snapshot is not:
43
+ # there is no written-at timestamp, because two writes of one snapshot
44
+ # have to produce identical files.
45
+ #
46
+ # Instances are frozen on construction and compare by value.
47
+ class Header
48
+ extend T::Sig
49
+
50
+ # Canonical member order. This is the order #to_h builds and #to_line
51
+ # serializes, and it is part of the format: a reader in another language
52
+ # that emits these keys in another order produces a different file.
53
+ #
54
+ # @api private
55
+ MEMBERS = T.let(
56
+ %i[format_version gem_version generator source schema_version snapshot_checksum record_count
57
+ fetched_at source_version payload_encoding payload_compression payload_digest payload_bytes].freeze,
58
+ T::Array[Symbol]
59
+ )
60
+
61
+ # The one field a publisher may leave out, because plenty of lists
62
+ # publish no version of their own. It still serializes, as null: the
63
+ # key order is the format, so nothing is omitted from the line.
64
+ #
65
+ # @api private
66
+ OPTIONAL_MEMBERS = T.let(%i[source_version].freeze, T::Array[Symbol])
67
+
68
+ # `sha256:` and 64 hex digits, the form this library quotes every digest
69
+ # in -- a snapshot's checksum, a cached payload's, and both of the ones
70
+ # here.
71
+ #
72
+ # @api private
73
+ DIGEST_PATTERN = T.let(/\A#{Snapshot::ALGORITHM}:\h{64}\z/, Regexp)
74
+
75
+ # Who wrote the file, defaulted to this gem and this version. A
76
+ # publisher that is not this gem -- a commercial mirror, a bank's
77
+ # internal pipeline -- overrides it, which is the point of the field:
78
+ # `gem_version` says what code serialized the records, and this says
79
+ # whose bundle it is.
80
+ sig { returns(String) }
81
+ def self.default_generator = -"active_sanction/#{VERSION}"
82
+
83
+ # The header for a snapshot about to be written, given what the payload
84
+ # came to. Both payload values are measurements of bytes that already
85
+ # exist rather than promises about bytes to come -- see Bundle.write.
86
+ sig do
87
+ params(snapshot: Snapshot, payload_digest: String, payload_bytes: Integer,
88
+ generator: T.untyped).returns(T.attached_class)
89
+ end
90
+ def self.from_snapshot(snapshot, payload_digest:, payload_bytes:, generator: nil)
91
+ new(format_version: FORMAT_VERSION, gem_version: VERSION, generator: generator || default_generator,
92
+ source: snapshot.source, schema_version: snapshot.schema_version,
93
+ snapshot_checksum: snapshot.checksum, record_count: snapshot.record_count,
94
+ fetched_at: snapshot.fetched_at, source_version: snapshot.source_version,
95
+ payload_digest: payload_digest, payload_bytes: payload_bytes)
96
+ end
97
+
98
+ # Reads one serialized header line. Everything that can be wrong with it
99
+ # raises -- an unknown key, a missing one, a digest that is not a
100
+ # digest -- because a header this code half understands is a header it
101
+ # cannot say the signature covers.
102
+ sig { params(line: T.untyped).returns(T.attached_class) }
103
+ def self.parse(line)
104
+ parsed = JSON.parse(line.to_s)
105
+ raise InvalidArgument, "a bundle header is a JSON object, got #{parsed.class}" unless parsed.is_a?(Hash)
106
+
107
+ from_h(parsed)
108
+ end
109
+
110
+ # Rebuilds from #to_h output, accepting string keys so a header
111
+ # survives the round-trip through JSON.
112
+ sig { params(hash: T.untyped).returns(T.attached_class) }
113
+ def self.from_h(hash)
114
+ attributes = hash.to_h.transform_keys(&:to_sym)
115
+ unknown = attributes.keys - MEMBERS
116
+ raise InvalidArgument, "unknown bundle header field(s): #{unknown.join(", ")}" if unknown.any?
117
+
118
+ missing = MEMBERS - attributes.keys - OPTIONAL_MEMBERS
119
+ raise InvalidArgument, "bundle header is missing #{missing.join(", ")}" if missing.any?
120
+
121
+ # `new(**hash)` past required keyword parameters is one of the few
122
+ # things Sorbet cannot check statically. #initialize validates what
123
+ # arrives, which is where a bad header is caught.
124
+ T.unsafe(self).new(**attributes)
125
+ end
126
+
127
+ sig { returns(Integer) }
128
+ attr_reader :format_version
129
+
130
+ # The active_sanction that serialized the records, which is what a
131
+ # `schema_version` this reader does not know is diagnosed against.
132
+ sig { returns(String) }
133
+ attr_reader :gem_version
134
+
135
+ sig { returns(String) }
136
+ attr_reader :generator
137
+
138
+ sig { returns(Symbol) }
139
+ attr_reader :source
140
+
141
+ # Snapshot::SCHEMA_VERSION the records were written under.
142
+ sig { returns(Integer) }
143
+ attr_reader :schema_version
144
+
145
+ # The checksum of the list itself: content only, and identical to what
146
+ # the snapshot had before it was ever written to a file. This is what a
147
+ # stored MatchResult cites, so a bundle can be matched to a screening
148
+ # decision made years earlier.
149
+ sig { returns(String) }
150
+ attr_reader :snapshot_checksum
151
+
152
+ sig { returns(Integer) }
153
+ attr_reader :record_count
154
+
155
+ # UTC, truncated to the second: when the publisher's file was fetched,
156
+ # not when this bundle was written.
157
+ sig { returns(Time) }
158
+ attr_reader :fetched_at
159
+
160
+ sig { returns(T.nilable(String)) }
161
+ attr_reader :source_version
162
+
163
+ # How the records are laid out once decompressed. `ndjson` in v1, and
164
+ # named so that a later version can add another without a reader having
165
+ # to guess which it is looking at.
166
+ sig { returns(String) }
167
+ attr_reader :payload_encoding
168
+
169
+ # `deflate` in v1: a raw zlib stream, RFC 1950.
170
+ sig { returns(String) }
171
+ attr_reader :payload_compression
172
+
173
+ # SHA-256 over the *uncompressed* payload. Over the uncompressed bytes
174
+ # because that is the half of the file two machines can be held to:
175
+ # zlib builds differ in what they emit for identical input, and the
176
+ # records do not.
177
+ sig { returns(String) }
178
+ attr_reader :payload_digest
179
+
180
+ # The length of the uncompressed payload. Checked as a reader inflates,
181
+ # so a bundle that decompresses to more than it declared is refused part
182
+ # way through rather than absorbed.
183
+ sig { returns(Integer) }
184
+ attr_reader :payload_bytes
185
+
186
+ sig do
187
+ params(format_version: T.untyped, gem_version: T.untyped, generator: T.untyped, source: T.untyped,
188
+ schema_version: T.untyped, snapshot_checksum: T.untyped, record_count: T.untyped,
189
+ fetched_at: T.untyped, payload_digest: T.untyped, payload_bytes: T.untyped,
190
+ source_version: T.untyped, payload_encoding: T.untyped, payload_compression: T.untyped).void
191
+ end
192
+ def initialize(format_version:, gem_version:, generator:, source:, schema_version:, snapshot_checksum:,
193
+ record_count:, fetched_at:, payload_digest:, payload_bytes:, source_version: nil,
194
+ payload_encoding: ENCODING, payload_compression: COMPRESSION)
195
+ @format_version = T.let(version!(:format_version, format_version), Integer)
196
+ @gem_version = T.let(string!(:gem_version, gem_version), String)
197
+ @generator = T.let(string!(:generator, generator), String)
198
+ @source = T.let(Sources::Definition.key!(source), Symbol)
199
+ @schema_version = T.let(version!(:schema_version, schema_version), Integer)
200
+ @snapshot_checksum = T.let(digest!(:snapshot_checksum, snapshot_checksum), String)
201
+ @record_count = T.let(count!(:record_count, record_count), Integer)
202
+ @fetched_at = T.let(time!(fetched_at), Time)
203
+ @source_version = T.let(string_or_nil(source_version), T.nilable(String))
204
+ @payload_encoding = T.let(string!(:payload_encoding, payload_encoding), String)
205
+ @payload_compression = T.let(string!(:payload_compression, payload_compression), String)
206
+ @payload_digest = T.let(digest!(:payload_digest, payload_digest), String)
207
+ @payload_bytes = T.let(count!(:payload_bytes, payload_bytes), Integer)
208
+ freeze
209
+ end
210
+
211
+ # The documented shape, in the documented order.
212
+ sig { returns(T::Hash[Symbol, T.untyped]) }
213
+ def to_h
214
+ {
215
+ format_version: format_version, gem_version: gem_version, generator: generator,
216
+ source: source.to_s, schema_version: schema_version, snapshot_checksum: snapshot_checksum,
217
+ record_count: record_count, fetched_at: fetched_at.iso8601, source_version: source_version,
218
+ payload_encoding: payload_encoding, payload_compression: payload_compression,
219
+ payload_digest: payload_digest, payload_bytes: payload_bytes
220
+ }
221
+ end
222
+
223
+ # The exact bytes that go into the file, and the exact bytes a
224
+ # signature is computed over. No trailing newline: the newline is the
225
+ # container's, not the header's, so a reader that strips line endings
226
+ # differently still verifies.
227
+ sig { returns(String) }
228
+ def to_line = JSON.generate(to_h)
229
+
230
+ sig { params(other: T.untyped).returns(T::Boolean) }
231
+ def ==(other)
232
+ return false unless other.instance_of?(self.class)
233
+
234
+ to_h == other.to_h
235
+ end
236
+ alias eql? ==
237
+
238
+ sig { returns(Integer) }
239
+ def hash = [self.class, to_h].hash
240
+
241
+ sig { returns(String) }
242
+ def inspect
243
+ "#<#{self.class} #{source} #{record_count} entities #{snapshot_checksum} by #{generator}>"
244
+ end
245
+
246
+ private
247
+
248
+ sig { params(member: Symbol, value: T.untyped).returns(Integer) }
249
+ def version!(member, value)
250
+ integer = Integer(value)
251
+ raise InvalidArgument, "#{member} must be positive, got #{integer}" unless integer.positive?
252
+
253
+ integer
254
+ end
255
+
256
+ sig { params(member: Symbol, value: T.untyped).returns(String) }
257
+ def string!(member, value)
258
+ string = value.to_s.strip
259
+ raise InvalidArgument, "#{member} is required" if string.empty?
260
+
261
+ -string
262
+ end
263
+
264
+ sig { params(member: Symbol, value: T.untyped).returns(String) }
265
+ def digest!(member, value)
266
+ string = value.to_s.strip
267
+ return -string if DIGEST_PATTERN.match?(string)
268
+
269
+ raise InvalidArgument, "#{member} is not a #{Snapshot::ALGORITHM} digest: #{value.inspect}"
270
+ end
271
+
272
+ sig { params(member: Symbol, value: T.untyped).returns(Integer) }
273
+ def count!(member, value)
274
+ integer = Integer(value)
275
+ raise InvalidArgument, "#{member} cannot be negative, got #{integer}" if integer.negative?
276
+
277
+ integer
278
+ end
279
+
280
+ # Truncated to the second, the precision #to_h serializes, so a header
281
+ # read back is equal to the one that was written.
282
+ sig { params(value: T.untyped).returns(Time) }
283
+ def time!(value)
284
+ time = case value
285
+ when Time then value
286
+ when String then Time.parse(value)
287
+ else raise InvalidArgument, "fetched_at is not a time: #{value.inspect}"
288
+ end
289
+ Time.at(time.to_i).utc
290
+ end
291
+
292
+ sig { params(value: T.untyped).returns(T.nilable(String)) }
293
+ def string_or_nil(value)
294
+ return nil if value.nil?
295
+
296
+ string = value.to_s.strip
297
+ string.empty? ? nil : -string
298
+ end
299
+ end
300
+ end
301
+ end
302
+ end
@@ -0,0 +1,185 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "digest"
7
+ require "zlib"
8
+ require "active_sanction/error"
9
+
10
+ module ActiveSanction
11
+ class Snapshot
12
+ module Bundle
13
+ # The records half of a bundle: newline-delimited JSON, deflated, with a
14
+ # SHA-256 over what it says before it was compressed.
15
+ #
16
+ # Two rules are worth stating outright, because between them they are why
17
+ # a bundle can be published once and trusted everywhere.
18
+ #
19
+ # ### The digest is over the uncompressed bytes
20
+ #
21
+ # zlib builds disagree. The same records fed to zlib 1.3 and to zlib-ng
22
+ # at the same level can come out as different bytes, so a digest over the
23
+ # compressed stream would make "the same snapshot produces the same
24
+ # bundle" a claim about which Linux distribution built somebody's Ruby.
25
+ # Over the records it is a claim about the records, which is the one worth
26
+ # signing.
27
+ #
28
+ # ### Reading never holds the whole list
29
+ #
30
+ # `.unpack` inflates in 64 KiB pieces and yields one record at a time, so
31
+ # a 25 MB EU bundle is read with a buffer of one chunk plus one line
32
+ # rather than with a decompressed copy beside the entities being built
33
+ # from it. The digest is accumulated on the way past.
34
+ #
35
+ # It also refuses to inflate more than the header declared. A file that
36
+ # says it holds 48 MB and keeps producing bytes at 49 MB is either damaged
37
+ # or built to exhaust whoever opens it, and either way there is nothing to
38
+ # gain by decompressing the rest of it.
39
+ #
40
+ # @api private
41
+ module Payload
42
+ extend T::Sig
43
+ extend T::Helpers
44
+
45
+ # Called as `Payload.pack` -- module functions on a module, which is an
46
+ # Object, which is where `raise` comes from.
47
+ requires_ancestor { Kernel }
48
+
49
+ # How much compressed input is inflated at a time. The same 64 KiB
50
+ # PayloadCache digests a download in, and for the same reason.
51
+ CHUNK_SIZE = T.let(64 * 1024, Integer)
52
+
53
+ # Named rather than Zlib::DEFAULT_COMPRESSION, which is -1 and means
54
+ # "whatever this build calls default". Two writers of one snapshot
55
+ # should not produce different files because one of them linked a
56
+ # different zlib, and a level in the specification is what a non-Ruby
57
+ # implementation has to be told anyway.
58
+ COMPRESSION_LEVEL = T.let(6, Integer)
59
+
60
+ module_function
61
+
62
+ # Compresses `records` -- JSON strings, one per entity, in the order
63
+ # they belong in -- and returns the compressed bytes, the digest of the
64
+ # uncompressed stream, and its length.
65
+ #
66
+ # Compressed output is buffered rather than streamed to the file,
67
+ # because the header that has to be written *before* it states its
68
+ # digest and its length, and neither is known until the last record has
69
+ # gone past. The snapshot is already wholly in memory by then; this adds
70
+ # the compressed copy of it, which for the largest list published is
71
+ # about 25 MB and is released as soon as it is written.
72
+ sig { params(records: T::Enumerable[String]).returns([String, String, Integer]) }
73
+ def pack(records)
74
+ digest = Digest::SHA256.new
75
+ deflate = Zlib::Deflate.new(COMPRESSION_LEVEL)
76
+ bytes = T.let(0, Integer)
77
+ compressed = String.new(encoding: Encoding::BINARY)
78
+ begin
79
+ records.each do |record|
80
+ line = "#{record}\n"
81
+ digest << line
82
+ bytes += line.bytesize
83
+ compressed << deflate.deflate(line)
84
+ end
85
+ compressed << deflate.finish
86
+ ensure
87
+ deflate.close
88
+ end
89
+ [compressed, -"#{Snapshot::ALGORITHM}:#{digest.hexdigest}", bytes]
90
+ end
91
+
92
+ # Inflates the rest of `io`, yielding each record as the JSON string it
93
+ # was written as, and returns the digest and length of what it read so
94
+ # a caller can hold them against what the header promised.
95
+ sig do
96
+ params(io: T.untyped, limit: Integer, block: T.proc.params(record: String).void).returns([String, Integer])
97
+ end
98
+ def unpack(io, limit:, &block)
99
+ digest = Digest::SHA256.new
100
+ bytes = T.let(0, Integer)
101
+ buffer = String.new(encoding: Encoding::BINARY)
102
+ inflate = Zlib::Inflate.new
103
+ fed = T.let(0, Integer)
104
+ begin
105
+ while (chunk = io.read(CHUNK_SIZE))
106
+ fed += chunk.bytesize
107
+ buffer << inflate!(inflate, chunk)
108
+ bytes = drain(buffer, bytes, limit, digest, &block)
109
+ end
110
+ bytes = finish(inflate, buffer, bytes, limit, digest, &block)
111
+ trailing!(fed, inflate.total_in)
112
+ ensure
113
+ inflate.close unless inflate.closed?
114
+ end
115
+ [-"#{Snapshot::ALGORITHM}:#{digest.hexdigest}", bytes]
116
+ end
117
+
118
+ # Every complete line the buffer now holds, leaving any partial one
119
+ # behind for the next chunk.
120
+ sig do
121
+ params(buffer: String, bytes: Integer, limit: Integer, digest: Digest::SHA256,
122
+ block: T.proc.params(record: String).void).returns(Integer)
123
+ end
124
+ def drain(buffer, bytes, limit, digest, &block)
125
+ total = bytes
126
+ while (index = buffer.index("\n"))
127
+ line = T.must(buffer.slice!(0, index + 1))
128
+ digest << line
129
+ total += line.bytesize
130
+ oversized!(total, limit)
131
+ block.call(line.chomp.force_encoding(Encoding::UTF_8))
132
+ end
133
+ total
134
+ end
135
+
136
+ # What is left when the compressed stream ends: nothing, if the writer
137
+ # terminated its last record. A trailing partial line is a truncated
138
+ # file and says so.
139
+ sig do
140
+ params(inflate: Zlib::Inflate, buffer: String, bytes: Integer, limit: Integer, digest: Digest::SHA256,
141
+ block: T.proc.params(record: String).void).returns(Integer)
142
+ end
143
+ def finish(inflate, buffer, bytes, limit, digest, &block)
144
+ buffer << inflate.finish unless inflate.finished?
145
+ total = drain(buffer, bytes, limit, digest, &block)
146
+ return total if buffer.empty?
147
+
148
+ raise Corrupt, "the last record in this bundle is unterminated, so the payload was cut short"
149
+ rescue Zlib::Error => e
150
+ raise Corrupt, "this bundle's payload ends mid-stream and cannot be decompressed (#{e.message})"
151
+ end
152
+
153
+ sig { params(inflate: Zlib::Inflate, chunk: String).returns(String) }
154
+ def inflate!(inflate, chunk)
155
+ inflate.inflate(chunk)
156
+ rescue Zlib::Error => e
157
+ raise Corrupt, "this bundle's payload is not readable as #{COMPRESSION} (#{e.message})"
158
+ end
159
+
160
+ # Bytes after the end of the compressed stream. zlib stops reading at
161
+ # the end of what it was given and would let them pass unmentioned,
162
+ # which would make a bundle somebody appended to indistinguishable from
163
+ # the one that was signed.
164
+ sig { params(fed: Integer, consumed: Integer).void }
165
+ def trailing!(fed, consumed)
166
+ return if fed <= consumed
167
+
168
+ raise Corrupt,
169
+ "this bundle carries #{fed - consumed} bytes after the end of its payload. Nothing was written " \
170
+ "there, so something else put them there"
171
+ end
172
+
173
+ sig { params(total: Integer, limit: Integer).void }
174
+ def oversized!(total, limit)
175
+ return if total <= limit
176
+
177
+ raise Corrupt,
178
+ "this bundle's payload decompresses to more than the #{limit} bytes its header declares. " \
179
+ "Nothing further is read: a file that lies about its own size is either damaged or built to " \
180
+ "exhaust whoever opens it"
181
+ end
182
+ end
183
+ end
184
+ end
185
+ end
@@ -0,0 +1,201 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "openssl"
7
+ require "active_sanction/error"
8
+
9
+ module ActiveSanction
10
+ class Snapshot
11
+ module Bundle
12
+ # The third line of a bundle: who says this is the file they published.
13
+ #
14
+ # key = OpenSSL::PKey::EC.generate("prime256v1")
15
+ # Bundle.write(snapshot, io: io, sign_with: key)
16
+ # Bundle.read(io, verify_with: key) # => Snapshot, trusted?
17
+ #
18
+ # ### Detached, and over the header only
19
+ #
20
+ # The signed bytes are the header line's, without its newline. That line
21
+ # carries `payload_digest`, a SHA-256 over every record in the file, so
22
+ # ~300 bytes of signature input stand for all 19,015 of them. Three things
23
+ # follow, and all three are the reason it is done this way:
24
+ #
25
+ # - **A verifier settles who published a bundle before inflating a byte
26
+ # of it.** Compressed data from a party you have not yet authenticated
27
+ # is exactly what you do not want to expand.
28
+ # - **Verification costs the same for OFAC and for the EU.** One
29
+ # signature over a fixed-size line, not over 25 MB.
30
+ # - **A bundle without a signature is still a bundle.** The line reads
31
+ # `-`, everything else about the file is unchanged, and a reader that
32
+ # was given no key never looks at it. Signing is a claim about
33
+ # provenance laid on top of a format that works without one.
34
+ #
35
+ # ### What it does not cover
36
+ #
37
+ # Nothing about the file that is not in the header. A bundle re-compressed
38
+ # at a different level still verifies, because the digest is over the
39
+ # uncompressed records -- which is correct: the claim is about the list,
40
+ # not about the packing. Tampering with the records themselves fails, but
41
+ # it fails as corruption (the digest) rather than as a bad signature, and
42
+ # Bundle raises two different errors to say which.
43
+ #
44
+ # ### Keys
45
+ #
46
+ # RSA and EC, both of which every OpenSSL this gem can run against
47
+ # supports. `ed25519` is reserved as an algorithm name and deliberately
48
+ # not implemented yet: support for it varies by OpenSSL build, and a
49
+ # format whose signatures verify on some machines is worse than one that
50
+ # signs with a key everybody has.
51
+ #
52
+ # Note that an ECDSA signature is randomized -- signing one snapshot twice
53
+ # with one key produces two different lines. Determinism is a property of
54
+ # the magic line, the header and the payload; see Bundle.
55
+ #
56
+ # @api private
57
+ module Signature
58
+ extend T::Sig
59
+ extend T::Helpers
60
+
61
+ # Called as `Signature.sign` -- module functions on a module, which is
62
+ # an Object, which is where `raise` comes from.
63
+ requires_ancestor { Kernel }
64
+
65
+ # What an unsigned bundle carries. A literal, rather than an empty
66
+ # line, so that a truncated file cannot read as an unsigned one.
67
+ NONE = T.let("-", String)
68
+
69
+ # The algorithm names the format defines, and the kind of OpenSSL key
70
+ # each one is produced by. `ed25519` is reserved and not in here: a name
71
+ # a v1 reader rejects is a name a v1 reader cannot get wrong.
72
+ ALGORITHMS = T.let({ "rsa-sha256" => OpenSSL::PKey::RSA, "ecdsa-sha256" => OpenSSL::PKey::EC }.freeze,
73
+ T::Hash[String, T.untyped])
74
+
75
+ # Reserved, and refused with a message that says so rather than with
76
+ # "unknown algorithm", so a bundle from a newer publisher diagnoses
77
+ # itself.
78
+ RESERVED_ALGORITHMS = T.let(%w[ed25519].freeze, T::Array[String])
79
+
80
+ DIGEST = T.let("SHA256", String)
81
+
82
+ module_function
83
+
84
+ # The signature line for a header, or NONE when there is no key.
85
+ sig { params(header_line: String, key: T.untyped).returns(String) }
86
+ def sign(header_line, key)
87
+ return NONE if key.nil?
88
+
89
+ pkey = key!(key)
90
+ algorithm = algorithm_for(pkey)
91
+ "#{algorithm} #{[pkey.sign(OpenSSL::Digest.new(DIGEST), header_line)].pack("m0")}"
92
+ end
93
+
94
+ # Whether a line carries a signature at all.
95
+ sig { params(line: T.untyped).returns(T::Boolean) }
96
+ def signed?(line) = !line.nil? && line.to_s.strip != NONE && !line.to_s.strip.empty?
97
+
98
+ # True when the signature on `line` was made over `header_line` by the
99
+ # holder of `key`. Raises rather than answering false: a caller that
100
+ # asked for verification and got a boolean it forgot to check would
101
+ # screen against unverified data believing otherwise.
102
+ sig { params(line: T.untyped, header_line: String, key: T.untyped).returns(T::Boolean) }
103
+ def verify!(line, header_line, key)
104
+ raise Unsigned, unsigned_message unless signed?(line)
105
+
106
+ algorithm, encoded = split!(line)
107
+ pkey = key!(key)
108
+ expected!(algorithm, pkey)
109
+ verified = begin
110
+ pkey.verify(OpenSSL::Digest.new(DIGEST), decode!(encoded), header_line)
111
+ rescue OpenSSL::OpenSSLError
112
+ false
113
+ end
114
+ return true if verified
115
+
116
+ raise UntrustedSignature, mismatch_message(algorithm)
117
+ end
118
+
119
+ # An OpenSSL key from what a caller supplied: a key, or a PEM.
120
+ sig { params(value: T.untyped).returns(OpenSSL::PKey::PKey) }
121
+ def key!(value)
122
+ return value if value.is_a?(OpenSSL::PKey::PKey)
123
+
124
+ if value.is_a?(String)
125
+ begin
126
+ return OpenSSL::PKey.read(value)
127
+ rescue OpenSSL::OpenSSLError => e
128
+ raise InvalidArgument, "the key given is not a readable PEM (#{e.message})"
129
+ end
130
+ end
131
+
132
+ raise InvalidArgument,
133
+ "a bundle is signed and verified with an OpenSSL::PKey or a PEM string, got #{value.class}. " \
134
+ "Read a key file yourself -- OpenSSL::PKey.read(File.read(\"public.pem\"))"
135
+ end
136
+
137
+ # The format's name for the key's algorithm.
138
+ sig { params(pkey: OpenSSL::PKey::PKey).returns(String) }
139
+ def algorithm_for(pkey)
140
+ name = ALGORITHMS.find { |_, kind| pkey.is_a?(kind) }&.first
141
+ return name if name
142
+
143
+ raise UnsupportedError,
144
+ "a bundle cannot be signed with a #{pkey.class} key. The format defines " \
145
+ "#{ALGORITHMS.keys.join(" and ")}, and reserves #{RESERVED_ALGORITHMS.join(", ")}"
146
+ end
147
+
148
+ # The two fields of a signature line.
149
+ sig { params(line: T.untyped).returns([String, String]) }
150
+ def split!(line)
151
+ algorithm, encoded = line.to_s.strip.split(" ", 2)
152
+ raise Corrupt, "#{line.to_s.strip.inspect} is not a signature line" if algorithm.nil? || encoded.nil?
153
+
154
+ [algorithm, encoded]
155
+ end
156
+
157
+ # A signature this reader could verify if it had the right key, or the
158
+ # exception that says why it never could.
159
+ sig { params(algorithm: String, pkey: OpenSSL::PKey::PKey).void }
160
+ def expected!(algorithm, pkey)
161
+ unless ALGORITHMS.key?(algorithm)
162
+ raise UnsupportedFormat, unsupported_message(algorithm) if RESERVED_ALGORITHMS.include?(algorithm)
163
+
164
+ raise Corrupt, "#{algorithm.inspect} is not a signature algorithm this format defines"
165
+ end
166
+ return if pkey.is_a?(ALGORITHMS.fetch(algorithm))
167
+
168
+ raise UntrustedSignature,
169
+ "this bundle is signed #{algorithm}, and the key supplied is a #{pkey.class}. It was signed " \
170
+ "by somebody else, or verification was pointed at the wrong key"
171
+ end
172
+
173
+ sig { params(encoded: String).returns(String) }
174
+ def decode!(encoded)
175
+ encoded.unpack1("m0")
176
+ rescue ArgumentError
177
+ raise Corrupt, "the signature is not base64"
178
+ end
179
+
180
+ sig { params(algorithm: String).returns(String) }
181
+ def unsupported_message(algorithm)
182
+ "this bundle is signed #{algorithm}, which active_sanction #{VERSION} reserves but does not verify. " \
183
+ "Upgrade the gem, or read the bundle without verify_with: and accept that it is unverified"
184
+ end
185
+
186
+ sig { params(algorithm: String).returns(String) }
187
+ def mismatch_message(algorithm)
188
+ "this bundle's #{algorithm} signature was not made by the key supplied. The file is intact -- its " \
189
+ "records still hash to what its header says -- but somebody other than the expected publisher " \
190
+ "signed it, so nothing here says where it came from"
191
+ end
192
+
193
+ sig { returns(String) }
194
+ def unsigned_message
195
+ "this bundle carries no signature, and a key was supplied to verify one. An unsigned bundle is " \
196
+ "readable without verify_with:, but then nothing attests to who published it"
197
+ end
198
+ end
199
+ end
200
+ end
201
+ end