active_sanction 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +538 -0
  3. data/CODE_OF_CONDUCT.md +74 -0
  4. data/CONTRIBUTING.md +312 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +616 -0
  7. data/SECURITY.md +97 -0
  8. data/active_sanction.gemspec +98 -0
  9. data/docs/adding_a_source.md +1083 -0
  10. data/docs/api_stability.md +357 -0
  11. data/docs/bundle_format.md +407 -0
  12. data/lib/active_sanction/address.rb +170 -0
  13. data/lib/active_sanction/client.rb +341 -0
  14. data/lib/active_sanction/configuration.rb +721 -0
  15. data/lib/active_sanction/countries.txt +292 -0
  16. data/lib/active_sanction/country.rb +164 -0
  17. data/lib/active_sanction/deprecation.rb +168 -0
  18. data/lib/active_sanction/diff/change.rb +208 -0
  19. data/lib/active_sanction/diff.rb +348 -0
  20. data/lib/active_sanction/doctor/checkup.rb +356 -0
  21. data/lib/active_sanction/doctor/diagnosis.rb +245 -0
  22. data/lib/active_sanction/doctor/finding.rb +183 -0
  23. data/lib/active_sanction/doctor/profile.rb +384 -0
  24. data/lib/active_sanction/doctor/report.rb +227 -0
  25. data/lib/active_sanction/doctor.rb +347 -0
  26. data/lib/active_sanction/entity.rb +293 -0
  27. data/lib/active_sanction/error.rb +332 -0
  28. data/lib/active_sanction/fetcher/result.rb +116 -0
  29. data/lib/active_sanction/fetcher.rb +242 -0
  30. data/lib/active_sanction/http_client/errors.rb +94 -0
  31. data/lib/active_sanction/http_client/response.rb +129 -0
  32. data/lib/active_sanction/http_client.rb +324 -0
  33. data/lib/active_sanction/identifier.rb +212 -0
  34. data/lib/active_sanction/index/builder.rb +89 -0
  35. data/lib/active_sanction/index/candidate.rb +63 -0
  36. data/lib/active_sanction/index/entry.rb +66 -0
  37. data/lib/active_sanction/index/features.rb +112 -0
  38. data/lib/active_sanction/index.rb +381 -0
  39. data/lib/active_sanction/match_result.rb +345 -0
  40. data/lib/active_sanction/matcher.rb +388 -0
  41. data/lib/active_sanction/name.rb +183 -0
  42. data/lib/active_sanction/normalizer/cache.rb +82 -0
  43. data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
  44. data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
  45. data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
  46. data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
  47. data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
  48. data/lib/active_sanction/normalizer/dictionary.rb +243 -0
  49. data/lib/active_sanction/normalizer/form.rb +279 -0
  50. data/lib/active_sanction/normalizer.rb +137 -0
  51. data/lib/active_sanction/parsers/column_shape.rb +274 -0
  52. data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
  53. data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
  54. data/lib/active_sanction/parsers/delimited_table.rb +151 -0
  55. data/lib/active_sanction/parsers/format.rb +107 -0
  56. data/lib/active_sanction/parsers/join.rb +149 -0
  57. data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
  58. data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
  59. data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
  60. data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
  61. data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
  62. data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
  63. data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
  64. data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
  65. data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
  66. data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
  67. data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
  68. data/lib/active_sanction/parsers/xml_records.rb +140 -0
  69. data/lib/active_sanction/parsers.rb +94 -0
  70. data/lib/active_sanction/partial_date/parser.rb +124 -0
  71. data/lib/active_sanction/partial_date.rb +303 -0
  72. data/lib/active_sanction/payload_cache/checksum.rb +66 -0
  73. data/lib/active_sanction/payload_cache/entry.rb +304 -0
  74. data/lib/active_sanction/payload_cache.rb +366 -0
  75. data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
  76. data/lib/active_sanction/phonetics.rb +40 -0
  77. data/lib/active_sanction/query.rb +290 -0
  78. data/lib/active_sanction/rescreen/alert.rb +314 -0
  79. data/lib/active_sanction/rescreen.rb +332 -0
  80. data/lib/active_sanction/scorer/adjustments.rb +241 -0
  81. data/lib/active_sanction/scorer/name_score.rb +279 -0
  82. data/lib/active_sanction/scorer/reason.rb +136 -0
  83. data/lib/active_sanction/scorer/result.rb +129 -0
  84. data/lib/active_sanction/scorer/subject.rb +235 -0
  85. data/lib/active_sanction/scorer/weights.rb +306 -0
  86. data/lib/active_sanction/scorer.rb +303 -0
  87. data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
  88. data/lib/active_sanction/similarity/levenshtein.rb +179 -0
  89. data/lib/active_sanction/similarity/token_set.rb +159 -0
  90. data/lib/active_sanction/similarity/token_sort.rb +120 -0
  91. data/lib/active_sanction/similarity.rb +183 -0
  92. data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
  93. data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
  94. data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
  95. data/lib/active_sanction/snapshot/bundle.rb +381 -0
  96. data/lib/active_sanction/snapshot.rb +305 -0
  97. data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
  98. data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
  99. data/lib/active_sanction/sources/australia_dfat.rb +291 -0
  100. data/lib/active_sanction/sources/base.rb +295 -0
  101. data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
  102. data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
  103. data/lib/active_sanction/sources/canada_sema.rb +153 -0
  104. data/lib/active_sanction/sources/definition.rb +360 -0
  105. data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
  106. data/lib/active_sanction/sources/eu_fsf.rb +203 -0
  107. data/lib/active_sanction/sources/ofac/record.rb +251 -0
  108. data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
  109. data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
  110. data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
  111. data/lib/active_sanction/sources/ofac.rb +248 -0
  112. data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
  113. data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
  114. data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
  115. data/lib/active_sanction/sources/remarks.rb +84 -0
  116. data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
  117. data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
  118. data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
  119. data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
  120. data/lib/active_sanction/sources/un_consolidated.rb +140 -0
  121. data/lib/active_sanction/sources.rb +196 -0
  122. data/lib/active_sanction/storage/active_record/reader.rb +124 -0
  123. data/lib/active_sanction/storage/active_record/row.rb +175 -0
  124. data/lib/active_sanction/storage/active_record/writer.rb +141 -0
  125. data/lib/active_sanction/storage/active_record.rb +277 -0
  126. data/lib/active_sanction/storage/base.rb +216 -0
  127. data/lib/active_sanction/storage/file_system.rb +373 -0
  128. data/lib/active_sanction/storage/memory.rb +67 -0
  129. data/lib/active_sanction/storage/meta.rb +188 -0
  130. data/lib/active_sanction/storage.rb +74 -0
  131. data/lib/active_sanction/subject.rb +271 -0
  132. data/lib/active_sanction/sync/report.rb +245 -0
  133. data/lib/active_sanction/sync/result.rb +296 -0
  134. data/lib/active_sanction/sync.rb +390 -0
  135. data/lib/active_sanction/validator_store/file_system.rb +103 -0
  136. data/lib/active_sanction/validator_store/memory.rb +41 -0
  137. data/lib/active_sanction/validator_store.rb +119 -0
  138. data/lib/active_sanction/validators.rb +195 -0
  139. data/lib/active_sanction/version.rb +23 -0
  140. data/lib/active_sanction.rb +331 -0
  141. data/lib/generators/active_sanction/install/install_generator.rb +60 -0
  142. data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
  143. metadata +236 -0
@@ -0,0 +1,293 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ module ActiveSanction
7
+ # The single source-agnostic record every adapter produces. Nothing
8
+ # downstream -- storage, index, matcher -- should ever need to know which
9
+ # government published a record.
10
+ #
11
+ # ActiveSanction::Entity.new(
12
+ # id: "ofac_sdn:2674",
13
+ # source: :ofac_sdn,
14
+ # source_ref: "2674",
15
+ # type: :individual,
16
+ # names: [Name, ...],
17
+ # addresses: [Address, ...],
18
+ # identifiers: [Identifier, ...],
19
+ # dates_of_birth: [PartialDate, ...],
20
+ # nationalities: ["EG"],
21
+ # programs: ["SDGT"],
22
+ # listed_on: PartialDate,
23
+ # remarks: "..."
24
+ # )
25
+ #
26
+ # Instances are frozen on construction and compare by value.
27
+ class Entity
28
+ extend T::Sig
29
+
30
+ # `vessel` and `aircraft` are first-class because they are ~10% of the OFAC
31
+ # SDN list (1,540 vessels, 342 aircraft) and carry name-like strings. Without
32
+ # a distinct type a search for a person can rank a ship.
33
+ TYPES = T.let(%i[individual organization vessel aircraft].freeze, T::Array[Symbol])
34
+
35
+ # Canonical member order. Snapshot (#8) checksums the serialized form, so
36
+ # #to_h must lay its keys out the same way every time.
37
+ #
38
+ # @api private
39
+ MEMBERS = T.let(%i[
40
+ id source source_ref type names addresses identifiers dates_of_birth
41
+ nationalities programs listed_on remarks
42
+ ].freeze, T::Array[Symbol])
43
+
44
+ # Plural, and it is not a hedge. The UN publishes more than one date of
45
+ # birth for 140 of its 736 individuals and as many as ten for one of them,
46
+ # because that is the honest state of the intelligence: several
47
+ # governments reported several dates and the Committee listed all of them.
48
+ # Collapsing that to one would mean choosing, on no evidence, which
49
+ # report to believe -- and a screening decision that clears someone whose
50
+ # DOB matched the discarded one is exactly the failure this library exists
51
+ # to prevent. The scorer (#32) reads them the way PartialDate#overlaps?
52
+ # already reads a single imprecise date: any of them matching is a match.
53
+ #
54
+ # @api private
55
+ DATE_MEMBERS = T.let(%i[dates_of_birth].freeze, T::Array[Symbol])
56
+
57
+ # Which class rebuilds each nested member from a hash. The names are
58
+ # strings, resolved lazily through `const_get`, so this file depends on
59
+ # none of those four classes at load time -- and Sorbet cannot see through
60
+ # that, deliberately. What the checker holds instead is the other end:
61
+ # #initialize declares all four member types, so a hash that rebuilds into
62
+ # the wrong thing is caught where the entity is built rather than here.
63
+ #
64
+ # @api private
65
+ COLLECTION_TYPES = T.let({
66
+ names: "ActiveSanction::Name",
67
+ addresses: "ActiveSanction::Address",
68
+ identifiers: "ActiveSanction::Identifier",
69
+ dates_of_birth: "ActiveSanction::PartialDate"
70
+ }.freeze, T::Hash[Symbol, String])
71
+
72
+ # @api private
73
+ SCALAR_TYPES = T.let({ listed_on: "ActiveSanction::PartialDate" }.freeze, T::Hash[Symbol, String])
74
+
75
+ # Namespaced, and never nil: #initialize derives one from the source and
76
+ # the publisher's own reference when the caller gives none.
77
+ sig { returns(String) }
78
+ attr_reader :id
79
+
80
+ sig { returns(Symbol) }
81
+ attr_reader :source
82
+
83
+ sig { returns(T.nilable(String)) }
84
+ attr_reader :source_ref
85
+
86
+ sig { returns(Symbol) }
87
+ attr_reader :type
88
+
89
+ sig { returns(T::Array[Name]) }
90
+ attr_reader :names
91
+
92
+ sig { returns(T::Array[Address]) }
93
+ attr_reader :addresses
94
+
95
+ sig { returns(T::Array[Identifier]) }
96
+ attr_reader :identifiers
97
+
98
+ sig { returns(T::Array[PartialDate]) }
99
+ attr_reader :dates_of_birth
100
+
101
+ sig { returns(T::Array[String]) }
102
+ attr_reader :nationalities
103
+
104
+ sig { returns(T::Array[String]) }
105
+ attr_reader :programs
106
+
107
+ sig { returns(T.nilable(PartialDate)) }
108
+ attr_reader :listed_on
109
+
110
+ sig { returns(T.nilable(String)) }
111
+ attr_reader :remarks
112
+
113
+ # Rebuilds an entity from #to_h output. Accepts string keys too, so a record
114
+ # that has been through JSON round-trips without a separate coercion step.
115
+ sig { params(hash: T.untyped).returns(T.attached_class) }
116
+ def self.from_h(hash)
117
+ attributes = hash.to_h.transform_keys(&:to_sym)
118
+ unknown = attributes.keys - MEMBERS
119
+ raise InvalidArgument, "unknown Entity attribute(s): #{unknown.join(", ")}" if unknown.any?
120
+
121
+ # `new(**hash)` past required keyword parameters is one of the few
122
+ # things Sorbet cannot check statically. The hash is validated on the two
123
+ # lines above and by #initialize below, so what is lost here is only the
124
+ # checker's ability to see it happen.
125
+ T.unsafe(self).new(**coerce_members(attributes))
126
+ end
127
+
128
+ sig { params(attributes: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
129
+ def self.coerce_members(attributes)
130
+ COLLECTION_TYPES.each do |member, class_name|
131
+ attributes[member] &&= attributes[member].map { |value| build(class_name, value) }
132
+ end
133
+ SCALAR_TYPES.each { |member, class_name| attributes[member] &&= build(class_name, attributes[member]) }
134
+ attributes
135
+ end
136
+ private_class_method :coerce_members
137
+
138
+ # Values that are already value objects pass through untouched, so from_h is
139
+ # safe to call on a half-deserialized hash.
140
+ sig { params(class_name: String, value: T.untyped).returns(T.untyped) }
141
+ def self.build(class_name, value)
142
+ return value unless value.is_a?(Hash)
143
+
144
+ Object.const_get(class_name).from_h(value)
145
+ end
146
+ private_class_method :build
147
+
148
+ # Each collection is nilable because nil is how "the publisher listed
149
+ # none" arrives from a store or a half-built hash; #list! turns it into the
150
+ # empty array the reader hands back.
151
+ #
152
+ # The four collection members and `listed_on` are declared, and the rest is
153
+ # `T.untyped` on purpose. The difference is who wrote the value: the nested
154
+ # members are canonical objects an adapter builds, and declaring them is
155
+ # what makes `srb tc` refuse an adapter that hands over the string a
156
+ # publisher wrote where a PartialDate belongs. The runtime check that comes
157
+ # with the signature is shallow -- it sees the Array and not what is in it
158
+ # -- so the adapter conformance group goes on asserting the element types
159
+ # per fixture, which is what covers an adapter written outside this repo.
160
+ #
161
+ # Everything else is the publisher's own text arriving as whatever the
162
+ # parser made of it, and the coercions below say what happens to it in
163
+ # messages written for whoever has to fix the record. A type error would
164
+ # say less.
165
+ sig do
166
+ params(
167
+ source: T.untyped,
168
+ type: T.untyped,
169
+ id: T.untyped,
170
+ source_ref: T.untyped,
171
+ names: T.nilable(T::Array[Name]),
172
+ addresses: T.nilable(T::Array[Address]),
173
+ identifiers: T.nilable(T::Array[Identifier]),
174
+ dates_of_birth: T.nilable(T::Array[PartialDate]),
175
+ nationalities: T.untyped,
176
+ programs: T.untyped,
177
+ listed_on: T.nilable(PartialDate),
178
+ remarks: T.untyped
179
+ ).void
180
+ end
181
+ def initialize(source:, type:, id: nil, source_ref: nil, names: [], addresses: [], identifiers: [],
182
+ dates_of_birth: [], nationalities: [], programs: [], listed_on: nil, remarks: nil)
183
+ @source = T.let(symbol!(:source, source), Symbol)
184
+ @type = T.let(type!(type), Symbol)
185
+ @source_ref = T.let(string_or_nil(source_ref), T.nilable(String))
186
+ @id = T.let(string_or_nil(id) || derived_id, String)
187
+ @names = T.let(list!(:names, names), T::Array[Name])
188
+ @addresses = T.let(list!(:addresses, addresses), T::Array[Address])
189
+ @identifiers = T.let(list!(:identifiers, identifiers), T::Array[Identifier])
190
+ @dates_of_birth = T.let(list!(:dates_of_birth, dates_of_birth), T::Array[PartialDate])
191
+ @nationalities = T.let(strings!(:nationalities, nationalities), T::Array[String])
192
+ @programs = T.let(strings!(:programs, programs), T::Array[String])
193
+ @listed_on = T.let(listed_on, T.nilable(PartialDate))
194
+ # original free text, always retained verbatim
195
+ @remarks = T.let(string_or_nil(remarks), T.nilable(String))
196
+ freeze
197
+ end
198
+
199
+ # True when the publisher gave no date at all, which is most of OFAC --
200
+ # its dates are prose in Remarks and stay there until #19 reads them.
201
+ sig { returns(T::Boolean) }
202
+ def dates_of_birth? = dates_of_birth.any?
203
+
204
+ # The name an adapter marked `:primary`, falling back to the first name for
205
+ # sources such as Canada that publish no alias kinds at all.
206
+ sig { returns(T.nilable(Name)) }
207
+ def primary_name
208
+ names.find(&:primary?) || names.first
209
+ end
210
+
211
+ sig { returns(T::Hash[Symbol, T.untyped]) }
212
+ def to_h
213
+ {
214
+ id: id,
215
+ source: source,
216
+ source_ref: source_ref,
217
+ type: type,
218
+ names: names.map(&:to_h),
219
+ addresses: addresses.map(&:to_h),
220
+ identifiers: identifiers.map(&:to_h),
221
+ dates_of_birth: dates_of_birth.map(&:to_h),
222
+ nationalities: nationalities,
223
+ programs: programs,
224
+ listed_on: listed_on&.to_h,
225
+ remarks: remarks
226
+ }
227
+ end
228
+
229
+ # Compared through #to_h so nested members only have to serialize, not
230
+ # implement value equality themselves. Class is part of the comparison to
231
+ # keep #== and #hash agreeing, which is what Hash and Set rely on.
232
+ sig { params(other: T.untyped).returns(T::Boolean) }
233
+ def ==(other)
234
+ return false unless other.instance_of?(self.class)
235
+
236
+ to_h == other.to_h
237
+ end
238
+ alias eql? ==
239
+
240
+ sig { returns(Integer) }
241
+ def hash
242
+ [self.class, to_h].hash
243
+ end
244
+
245
+ sig { returns(String) }
246
+ def inspect
247
+ "#<#{self.class} id=#{id.inspect} type=#{type.inspect} names=#{names.size}>"
248
+ end
249
+
250
+ private
251
+
252
+ sig { returns(String) }
253
+ def derived_id
254
+ raise InvalidArgument, "id is required when source_ref is nil" if source_ref.nil?
255
+
256
+ # Namespaced so ids stay unique and stable across sources.
257
+ -"#{source}:#{source_ref}"
258
+ end
259
+
260
+ sig { params(value: T.untyped).returns(Symbol) }
261
+ def type!(value)
262
+ type = symbol!(:type, value)
263
+ return type if TYPES.include?(type)
264
+
265
+ raise InvalidArgument, "unknown type #{type.inspect}, expected one of #{TYPES.join(", ")}"
266
+ end
267
+
268
+ sig { params(member: Symbol, value: T.untyped).returns(Symbol) }
269
+ def symbol!(member, value)
270
+ raise InvalidArgument, "#{member} is required" if value.nil? || value.to_s.empty?
271
+
272
+ value.to_sym
273
+ end
274
+
275
+ sig { params(value: T.untyped).returns(T.nilable(String)) }
276
+ def string_or_nil(value)
277
+ value.nil? ? nil : -value.to_s
278
+ end
279
+
280
+ sig { params(member: Symbol, value: T.untyped).returns(T.untyped) }
281
+ def list!(member, value)
282
+ return [].freeze if value.nil?
283
+ raise InvalidArgument, "#{member} must be an Array" unless value.is_a?(Array)
284
+
285
+ value.dup.freeze
286
+ end
287
+
288
+ sig { params(member: Symbol, value: T.untyped).returns(T::Array[String]) }
289
+ def strings!(member, value)
290
+ list!(member, value).map { |item| -item.to_s }.freeze
291
+ end
292
+ end
293
+ end
@@ -0,0 +1,332 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ module ActiveSanction
7
+ # The one rescue that covers this library.
8
+ #
9
+ # begin
10
+ # ActiveSanction.sync!
11
+ # rescue ActiveSanction::Error => e
12
+ # raise unless e.retryable?
13
+ #
14
+ # RetryLater.enqueue(e.source_id)
15
+ # end
16
+ #
17
+ # Every error raised out of a public method answers to this, and carries
18
+ # what a caller needs to decide between the only three responses there are
19
+ # to a screening failure: **retry this** (`retryable?`), **alert somebody**
20
+ # (anything else that is not the caller's fault), and **this is a bug in my
21
+ # call** (ConfigurationError, InvalidArgument, QueryError). None of those
22
+ # decisions should be made by matching on a message string, so none of them
23
+ # has to be.
24
+ #
25
+ # ### The hierarchy
26
+ #
27
+ # Error the marker; rescue this
28
+ # ConfigurationError this installation is set up wrong; never retry
29
+ # SourceError something went wrong with one list
30
+ # FetchError the bytes could not be obtained
31
+ # ParseError the bytes could not be read
32
+ # IntegrityError the bytes are not what they claim to be
33
+ # StorageError the store could not answer
34
+ # UnsupportedError this object cannot do that
35
+ # InvalidArgument a public method was called wrongly
36
+ # QueryError ...specifically, with an unusable query
37
+ # MissingKey a field or column that does not exist
38
+ #
39
+ # ### Why this is a module and not a class
40
+ #
41
+ # Because two of its members have to be something else as well. A caller who
42
+ # passes `threshold: 300` has made the mistake Ruby has had a class for since
43
+ # 1995, and `rescue ArgumentError` is what the code around this library
44
+ # already says; asking every host application to learn a private synonym for
45
+ # it would be this library exporting its own taxonomy into code that has no
46
+ # reason to care. So InvalidArgument is an `::ArgumentError` and MissingKey is
47
+ # a `::KeyError` -- and Ruby has one superclass to give. A module is what lets
48
+ # them be both, and `rescue ActiveSanction::Error` covers them anyway, because
49
+ # `rescue` matches with `===`, which a module answers.
50
+ #
51
+ # The trade is that `ActiveSanction::Error` cannot be raised or instantiated
52
+ # itself. That is not a loss: an error that says only "something in the
53
+ # sanctions library went wrong" is not one a caller could act on, and every
54
+ # member below names a response.
55
+ #
56
+ # ### Stability
57
+ #
58
+ # This hierarchy is public API. Within a major version an error will not move
59
+ # to a different parent, and an attribute will not be removed. New subclasses
60
+ # may be added under an existing parent -- that is what keeps `rescue
61
+ # ActiveSanction::FetchError` working when a new transport failure is given a
62
+ # name of its own -- so a `case` over error classes should carry an `else`.
63
+ module Error
64
+ extend T::Sig
65
+ extend T::Helpers
66
+
67
+ # Only ever mixed into an exception class -- this is a marker for the
68
+ # library's own failures, not a bag of attributes anything can wear.
69
+ requires_ancestor { Exception }
70
+
71
+ # Which list the failure belongs to, as the key its adapter declared, or
72
+ # nil for a failure that is not about one list -- a bad configuration, an
73
+ # unusable query, a store that will not open at all.
74
+ #
75
+ # Always set on a SourceError by the time it leaves the source, even when
76
+ # the layer that raised it could not know: an HTTP client knows a URL, not
77
+ # which sanctions list is at the other end of it. See #in_source.
78
+ sig { returns(T.nilable(Symbol)) }
79
+ attr_reader :source_id
80
+
81
+ # The HTTP status behind the failure, where there was one. Nil for
82
+ # everything that failed before a server answered -- a timeout, a refused
83
+ # connection -- and for everything that is not a fetch.
84
+ sig { returns(T.nilable(Integer)) }
85
+ attr_reader :status
86
+
87
+ # `retryable:` overrides whatever the subclass would have decided, for the
88
+ # cases only the raising code knows about. Everything else is a subclass's
89
+ # own answer; see #retryable?.
90
+ sig do
91
+ params(message: T.untyped, source_id: T.untyped, status: T.untyped, retryable: T.nilable(T::Boolean)).void
92
+ end
93
+ def initialize(message = nil, source_id: nil, status: nil, retryable: nil)
94
+ @source_id = T.let(source_id&.to_sym, T.nilable(Symbol))
95
+ @status = T.let(status&.to_i, T.nilable(Integer))
96
+ @retryable = T.let(retryable, T.nilable(T::Boolean))
97
+ super(message)
98
+ end
99
+
100
+ # Whether running the same call again could plausibly succeed.
101
+ #
102
+ # A first-class predicate rather than something a consumer reconstructs
103
+ # from the message, because backoff is the one decision a host application
104
+ # has to make in the request path and it should not be making it out of
105
+ # English. False is the default and the safe answer: a failure nobody has
106
+ # classified is one to look at rather than one to hammer.
107
+ sig { returns(T::Boolean) }
108
+ def retryable? = retryable_or(false)
109
+
110
+ # The failure as data, for a log line or a job record that has to survive
111
+ # the process.
112
+ sig { returns(T::Hash[Symbol, T.untyped]) }
113
+ def to_h
114
+ { error: self.class.name, message: message, source_id: source_id, status: status, retryable: retryable? }
115
+ .compact
116
+ end
117
+
118
+ # Stamps the list this failure belongs to onto an error raised by a layer
119
+ # that did not know it, and returns self so a rescue can re-raise in one
120
+ # line. Never overwrites a source already recorded -- the innermost layer
121
+ # that knew is the one that was right.
122
+ sig { params(key: T.untyped).returns(T.self_type) }
123
+ def in_source(key)
124
+ @source_id = T.let(key&.to_sym, T.nilable(Symbol)) if @source_id.nil?
125
+ self
126
+ end
127
+
128
+ private
129
+
130
+ # What a subclass answers with when the raising code did not override it.
131
+ # The override exists because the classification is occasionally something
132
+ # only the call site knows -- a 404 from a publisher that rotates its URLs
133
+ # weekly is worth another look, and one from a URL we hardcoded is not.
134
+ sig { params(default: T::Boolean).returns(T::Boolean) }
135
+ def retryable_or(default) = @retryable.nil? ? default : @retryable
136
+ end
137
+
138
+ # This installation is set up wrong: a blank User-Agent, a negative timeout,
139
+ # a source key nothing is registered under, an ActiveRecord store whose
140
+ # migration was never run.
141
+ #
142
+ # Never retryable, by definition -- nothing about waiting changes an
143
+ # initializer -- and raised as early as the bad value can be seen, which for
144
+ # a setting is where it is set rather than during a sync three hours later.
145
+ # Separate from InvalidArgument: this means "this installation is
146
+ # misconfigured", that means "this call is wrong", and only one of them is
147
+ # fixed by editing an initializer.
148
+ class ConfigurationError < StandardError
149
+ extend T::Sig
150
+ include Error
151
+
152
+ sig { returns(T::Boolean) }
153
+ def retryable? = false
154
+ end
155
+
156
+ # Something went wrong with one list. Carries #source_id, so a caller
157
+ # rescuing a whole sync knows which publisher to name.
158
+ #
159
+ # Raised directly only where the failure fits none of the three below --
160
+ # a source adapter that declares no URL, a payload the publisher confirmed
161
+ # but would not serve.
162
+ class SourceError < StandardError
163
+ include Error
164
+ end
165
+
166
+ # The bytes could not be obtained: a timeout, a refused connection, a
167
+ # redirect chain that does not terminate, a status the caller declared fatal.
168
+ #
169
+ # Carries #status where a server produced one, and answers #retryable? from
170
+ # it. This is the error a host application backs off on, and the reason the
171
+ # predicate exists: 503 and 429 are the publisher having a bad afternoon,
172
+ # 403 and 404 are a request that will be just as wrong in ten minutes.
173
+ class FetchError < SourceError
174
+ extend T::Sig
175
+
176
+ # 408 and 429 are the server asking to be asked again; 5xx is it failing to
177
+ # answer at all. A 4xx outside those two is never in here: a 403 for a
178
+ # missing User-Agent or a 404 for a retired URL says the request is wrong,
179
+ # and repeating it wastes the publisher's capacity to make the same point.
180
+ #
181
+ # @api private
182
+ RETRYABLE_STATUSES = T.let(([408, 425, 429] + (500..599).to_a).freeze, T::Array[Integer])
183
+
184
+ sig { returns(T::Boolean) }
185
+ def retryable? = retryable_or(RETRYABLE_STATUSES.include?(status))
186
+ end
187
+
188
+ # The bytes arrived and could not be read as the format they were declared
189
+ # to be: an HTML error page served under a `.xml` URL, a truncated download,
190
+ # an encoding that cannot be decoded, a ZIP member that will not inflate.
191
+ #
192
+ # Distinct from a Parsers::Warning, which is one *row* that could not be read
193
+ # while the rest of the file could. A list is not a file we control, and
194
+ # refusing 19,321 records because one of them is malformed fails exactly when
195
+ # the list is most needed -- so a bad row is a warning, and only a payload
196
+ # that cannot be read at all raises this.
197
+ #
198
+ # ### Where
199
+ #
200
+ # A 25 MB XML payload that "is not XML" is not a diagnosable complaint, so
201
+ # this carries a locator whenever the parser can produce one:
202
+ #
203
+ # rescue ActiveSanction::ParseError => e
204
+ # e.line # => 418_223
205
+ # e.record # => 12_004 (1-based, in the order the file yielded them)
206
+ # e.offset # => 8_388_608 (byte offset into the payload)
207
+ # e.locator # => "line 418223"
208
+ #
209
+ # All three are nil where the parser cannot say -- libxml2 reports no
210
+ # position for some failures, and a ZIP directory that ends inside an entry
211
+ # header has an offset but no line. An error that cannot point at a line
212
+ # still says what went wrong rather than pointing at the wrong one.
213
+ class ParseError < SourceError
214
+ extend T::Sig
215
+
216
+ # 1-based line within the payload, or nil.
217
+ sig { returns(T.nilable(Integer)) }
218
+ attr_reader :line
219
+
220
+ # 1-based index of the record being read, in the order the parser yielded
221
+ # them, or nil. The locator that means something for a format with no
222
+ # lines -- a spreadsheet, a stream of XML elements on one line.
223
+ sig { returns(T.nilable(Integer)) }
224
+ attr_reader :record
225
+
226
+ # Byte offset into the payload, or nil.
227
+ sig { returns(T.nilable(Integer)) }
228
+ attr_reader :offset
229
+
230
+ sig do
231
+ params(message: T.untyped, line: T.nilable(Integer), record: T.nilable(Integer), offset: T.nilable(Integer),
232
+ options: T.untyped).void
233
+ end
234
+ def initialize(message = nil, line: nil, record: nil, offset: nil, **options)
235
+ @line = T.let(line, T.nilable(Integer))
236
+ @record = T.let(record, T.nilable(Integer))
237
+ @offset = T.let(offset, T.nilable(Integer))
238
+ super(message, **options)
239
+ end
240
+
241
+ # Where in the payload, in the terms the parser could supply, or nil when
242
+ # it could supply none. Appended to #to_s -- and so to #message, which is
243
+ # defined in terms of it -- so a log line that records nothing but the
244
+ # message still says where.
245
+ sig { returns(T.nilable(String)) }
246
+ def locator
247
+ parts = []
248
+ parts << "line #{line}" if line
249
+ parts << "record #{record}" if record
250
+ parts << "byte #{offset}" if offset
251
+ parts.empty? ? nil : parts.join(", ")
252
+ end
253
+
254
+ sig { returns(String) }
255
+ def to_s
256
+ where = locator
257
+ where ? "#{super} (at #{where})" : super
258
+ end
259
+
260
+ sig { returns(T::Hash[Symbol, T.untyped]) }
261
+ def to_h = super.merge({ line: line, record: record, offset: offset }.compact)
262
+ end
263
+
264
+ # The bytes are not what they claim to be: content that no longer hashes to
265
+ # the checksum stored beside it, a snapshot filed under one source that says
266
+ # it is another.
267
+ #
268
+ # Never repaired, and never answered with whatever could still be read. A
269
+ # store that hands back the 8,000 records it managed to parse out of 19,015
270
+ # produces a report that looks exactly like a clean one, which is the most
271
+ # expensive thing this library can get wrong. An operator can always discard
272
+ # the copy and re-sync; nobody can recover a screening decision made against
273
+ # a list that was quietly half there.
274
+ class IntegrityError < SourceError; end
275
+
276
+ # The store could not answer: a list that has never been synced, a snapshot
277
+ # written under a schema this version does not read, a validator file that
278
+ # is not readable as validators.
279
+ #
280
+ # About the *store* rather than about the publisher, which is the difference
281
+ # that matters when deciding what to do: a FetchError is somebody else's
282
+ # outage, and this is local state to repair or re-sync.
283
+ class StorageError < StandardError
284
+ include Error
285
+ end
286
+
287
+ # This object cannot do that: a backend without the capability asked for, an
288
+ # abstract method a subclass never implemented.
289
+ #
290
+ # A StandardError rather than the `NotImplementedError` that reads more
291
+ # naturally for the second case, deliberately. `NotImplementedError` is a
292
+ # ScriptError, so it is not caught by `rescue StandardError` -- and one
293
+ # adapter forgetting `#parse` would take down a sync run that is supposed to
294
+ # isolate each source's failure from the others, which is the run's single
295
+ # most important property.
296
+ class UnsupportedError < StandardError
297
+ include Error
298
+ end
299
+
300
+ # A public method was called with something it cannot use: a threshold of
301
+ # 300, a Snapshot whose `record_count` disagrees with its entities, a name
302
+ # value object built with no value.
303
+ #
304
+ # This is the "bug in my call" branch, and it is an `::ArgumentError` as well
305
+ # as an ActiveSanction::Error so that it reads as one to code that has never
306
+ # heard of this library. Messages are written for whoever has to fix the
307
+ # call, or the record: `"limit must be at least 1, got 0"` rather than a type
308
+ # name.
309
+ class InvalidArgument < ::ArgumentError
310
+ include Error
311
+ end
312
+
313
+ # A screening query that cannot be run: an empty `sources:` list, a threshold
314
+ # outside 0..100, a limit of zero, a field spelled two ways at once.
315
+ #
316
+ # Its own class because a query is user input in a way the rest of this is
317
+ # not -- it is frequently built from a form or an API request -- and a
318
+ # service turning a bad query into a 422 and a bad configuration into a 500
319
+ # should not have to tell them apart by reading messages.
320
+ class QueryError < InvalidArgument; end
321
+
322
+ # A field or column that does not exist, asked for by name: `row[:nmae]`, or
323
+ # a path into an XML record that no element supplies.
324
+ #
325
+ # A `::KeyError`, because it is what `Hash#fetch` raises and these methods
326
+ # are `fetch` in everything but name. Almost always a typo in an adapter
327
+ # rather than a question about the data, so the message lists what *is*
328
+ # there.
329
+ class MissingKey < ::KeyError
330
+ include Error
331
+ end
332
+ end