active_sanction 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +538 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/CONTRIBUTING.md +312 -0
- data/LICENSE.txt +21 -0
- data/README.md +616 -0
- data/SECURITY.md +97 -0
- data/active_sanction.gemspec +98 -0
- data/docs/adding_a_source.md +1083 -0
- data/docs/api_stability.md +357 -0
- data/docs/bundle_format.md +407 -0
- data/lib/active_sanction/address.rb +170 -0
- data/lib/active_sanction/client.rb +341 -0
- data/lib/active_sanction/configuration.rb +721 -0
- data/lib/active_sanction/countries.txt +292 -0
- data/lib/active_sanction/country.rb +164 -0
- data/lib/active_sanction/deprecation.rb +168 -0
- data/lib/active_sanction/diff/change.rb +208 -0
- data/lib/active_sanction/diff.rb +348 -0
- data/lib/active_sanction/doctor/checkup.rb +356 -0
- data/lib/active_sanction/doctor/diagnosis.rb +245 -0
- data/lib/active_sanction/doctor/finding.rb +183 -0
- data/lib/active_sanction/doctor/profile.rb +384 -0
- data/lib/active_sanction/doctor/report.rb +227 -0
- data/lib/active_sanction/doctor.rb +347 -0
- data/lib/active_sanction/entity.rb +293 -0
- data/lib/active_sanction/error.rb +332 -0
- data/lib/active_sanction/fetcher/result.rb +116 -0
- data/lib/active_sanction/fetcher.rb +242 -0
- data/lib/active_sanction/http_client/errors.rb +94 -0
- data/lib/active_sanction/http_client/response.rb +129 -0
- data/lib/active_sanction/http_client.rb +324 -0
- data/lib/active_sanction/identifier.rb +212 -0
- data/lib/active_sanction/index/builder.rb +89 -0
- data/lib/active_sanction/index/candidate.rb +63 -0
- data/lib/active_sanction/index/entry.rb +66 -0
- data/lib/active_sanction/index/features.rb +112 -0
- data/lib/active_sanction/index.rb +381 -0
- data/lib/active_sanction/match_result.rb +345 -0
- data/lib/active_sanction/matcher.rb +388 -0
- data/lib/active_sanction/name.rb +183 -0
- data/lib/active_sanction/normalizer/cache.rb +82 -0
- data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
- data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
- data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
- data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
- data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
- data/lib/active_sanction/normalizer/dictionary.rb +243 -0
- data/lib/active_sanction/normalizer/form.rb +279 -0
- data/lib/active_sanction/normalizer.rb +137 -0
- data/lib/active_sanction/parsers/column_shape.rb +274 -0
- data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
- data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
- data/lib/active_sanction/parsers/delimited_table.rb +151 -0
- data/lib/active_sanction/parsers/format.rb +107 -0
- data/lib/active_sanction/parsers/join.rb +149 -0
- data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
- data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
- data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
- data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
- data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
- data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
- data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
- data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
- data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
- data/lib/active_sanction/parsers/xml_records.rb +140 -0
- data/lib/active_sanction/parsers.rb +94 -0
- data/lib/active_sanction/partial_date/parser.rb +124 -0
- data/lib/active_sanction/partial_date.rb +303 -0
- data/lib/active_sanction/payload_cache/checksum.rb +66 -0
- data/lib/active_sanction/payload_cache/entry.rb +304 -0
- data/lib/active_sanction/payload_cache.rb +366 -0
- data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
- data/lib/active_sanction/phonetics.rb +40 -0
- data/lib/active_sanction/query.rb +290 -0
- data/lib/active_sanction/rescreen/alert.rb +314 -0
- data/lib/active_sanction/rescreen.rb +332 -0
- data/lib/active_sanction/scorer/adjustments.rb +241 -0
- data/lib/active_sanction/scorer/name_score.rb +279 -0
- data/lib/active_sanction/scorer/reason.rb +136 -0
- data/lib/active_sanction/scorer/result.rb +129 -0
- data/lib/active_sanction/scorer/subject.rb +235 -0
- data/lib/active_sanction/scorer/weights.rb +306 -0
- data/lib/active_sanction/scorer.rb +303 -0
- data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
- data/lib/active_sanction/similarity/levenshtein.rb +179 -0
- data/lib/active_sanction/similarity/token_set.rb +159 -0
- data/lib/active_sanction/similarity/token_sort.rb +120 -0
- data/lib/active_sanction/similarity.rb +183 -0
- data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
- data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
- data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
- data/lib/active_sanction/snapshot/bundle.rb +381 -0
- data/lib/active_sanction/snapshot.rb +305 -0
- data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
- data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
- data/lib/active_sanction/sources/australia_dfat.rb +291 -0
- data/lib/active_sanction/sources/base.rb +295 -0
- data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
- data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
- data/lib/active_sanction/sources/canada_sema.rb +153 -0
- data/lib/active_sanction/sources/definition.rb +360 -0
- data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
- data/lib/active_sanction/sources/eu_fsf.rb +203 -0
- data/lib/active_sanction/sources/ofac/record.rb +251 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
- data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
- data/lib/active_sanction/sources/ofac.rb +248 -0
- data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
- data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
- data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
- data/lib/active_sanction/sources/remarks.rb +84 -0
- data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
- data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
- data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
- data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
- data/lib/active_sanction/sources/un_consolidated.rb +140 -0
- data/lib/active_sanction/sources.rb +196 -0
- data/lib/active_sanction/storage/active_record/reader.rb +124 -0
- data/lib/active_sanction/storage/active_record/row.rb +175 -0
- data/lib/active_sanction/storage/active_record/writer.rb +141 -0
- data/lib/active_sanction/storage/active_record.rb +277 -0
- data/lib/active_sanction/storage/base.rb +216 -0
- data/lib/active_sanction/storage/file_system.rb +373 -0
- data/lib/active_sanction/storage/memory.rb +67 -0
- data/lib/active_sanction/storage/meta.rb +188 -0
- data/lib/active_sanction/storage.rb +74 -0
- data/lib/active_sanction/subject.rb +271 -0
- data/lib/active_sanction/sync/report.rb +245 -0
- data/lib/active_sanction/sync/result.rb +296 -0
- data/lib/active_sanction/sync.rb +390 -0
- data/lib/active_sanction/validator_store/file_system.rb +103 -0
- data/lib/active_sanction/validator_store/memory.rb +41 -0
- data/lib/active_sanction/validator_store.rb +119 -0
- data/lib/active_sanction/validators.rb +195 -0
- data/lib/active_sanction/version.rb +23 -0
- data/lib/active_sanction.rb +331 -0
- data/lib/generators/active_sanction/install/install_generator.rb +60 -0
- data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
- metadata +236 -0
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
module ActiveSanction
|
|
7
|
+
# The single source-agnostic record every adapter produces. Nothing
|
|
8
|
+
# downstream -- storage, index, matcher -- should ever need to know which
|
|
9
|
+
# government published a record.
|
|
10
|
+
#
|
|
11
|
+
# ActiveSanction::Entity.new(
|
|
12
|
+
# id: "ofac_sdn:2674",
|
|
13
|
+
# source: :ofac_sdn,
|
|
14
|
+
# source_ref: "2674",
|
|
15
|
+
# type: :individual,
|
|
16
|
+
# names: [Name, ...],
|
|
17
|
+
# addresses: [Address, ...],
|
|
18
|
+
# identifiers: [Identifier, ...],
|
|
19
|
+
# dates_of_birth: [PartialDate, ...],
|
|
20
|
+
# nationalities: ["EG"],
|
|
21
|
+
# programs: ["SDGT"],
|
|
22
|
+
# listed_on: PartialDate,
|
|
23
|
+
# remarks: "..."
|
|
24
|
+
# )
|
|
25
|
+
#
|
|
26
|
+
# Instances are frozen on construction and compare by value.
|
|
27
|
+
class Entity
|
|
28
|
+
extend T::Sig
|
|
29
|
+
|
|
30
|
+
# `vessel` and `aircraft` are first-class because they are ~10% of the OFAC
|
|
31
|
+
# SDN list (1,540 vessels, 342 aircraft) and carry name-like strings. Without
|
|
32
|
+
# a distinct type a search for a person can rank a ship.
|
|
33
|
+
TYPES = T.let(%i[individual organization vessel aircraft].freeze, T::Array[Symbol])
|
|
34
|
+
|
|
35
|
+
# Canonical member order. Snapshot (#8) checksums the serialized form, so
|
|
36
|
+
# #to_h must lay its keys out the same way every time.
|
|
37
|
+
#
|
|
38
|
+
# @api private
|
|
39
|
+
MEMBERS = T.let(%i[
|
|
40
|
+
id source source_ref type names addresses identifiers dates_of_birth
|
|
41
|
+
nationalities programs listed_on remarks
|
|
42
|
+
].freeze, T::Array[Symbol])
|
|
43
|
+
|
|
44
|
+
# Plural, and it is not a hedge. The UN publishes more than one date of
|
|
45
|
+
# birth for 140 of its 736 individuals and as many as ten for one of them,
|
|
46
|
+
# because that is the honest state of the intelligence: several
|
|
47
|
+
# governments reported several dates and the Committee listed all of them.
|
|
48
|
+
# Collapsing that to one would mean choosing, on no evidence, which
|
|
49
|
+
# report to believe -- and a screening decision that clears someone whose
|
|
50
|
+
# DOB matched the discarded one is exactly the failure this library exists
|
|
51
|
+
# to prevent. The scorer (#32) reads them the way PartialDate#overlaps?
|
|
52
|
+
# already reads a single imprecise date: any of them matching is a match.
|
|
53
|
+
#
|
|
54
|
+
# @api private
|
|
55
|
+
DATE_MEMBERS = T.let(%i[dates_of_birth].freeze, T::Array[Symbol])
|
|
56
|
+
|
|
57
|
+
# Which class rebuilds each nested member from a hash. The names are
|
|
58
|
+
# strings, resolved lazily through `const_get`, so this file depends on
|
|
59
|
+
# none of those four classes at load time -- and Sorbet cannot see through
|
|
60
|
+
# that, deliberately. What the checker holds instead is the other end:
|
|
61
|
+
# #initialize declares all four member types, so a hash that rebuilds into
|
|
62
|
+
# the wrong thing is caught where the entity is built rather than here.
|
|
63
|
+
#
|
|
64
|
+
# @api private
|
|
65
|
+
COLLECTION_TYPES = T.let({
|
|
66
|
+
names: "ActiveSanction::Name",
|
|
67
|
+
addresses: "ActiveSanction::Address",
|
|
68
|
+
identifiers: "ActiveSanction::Identifier",
|
|
69
|
+
dates_of_birth: "ActiveSanction::PartialDate"
|
|
70
|
+
}.freeze, T::Hash[Symbol, String])
|
|
71
|
+
|
|
72
|
+
# @api private
|
|
73
|
+
SCALAR_TYPES = T.let({ listed_on: "ActiveSanction::PartialDate" }.freeze, T::Hash[Symbol, String])
|
|
74
|
+
|
|
75
|
+
# Namespaced, and never nil: #initialize derives one from the source and
|
|
76
|
+
# the publisher's own reference when the caller gives none.
|
|
77
|
+
sig { returns(String) }
|
|
78
|
+
attr_reader :id
|
|
79
|
+
|
|
80
|
+
sig { returns(Symbol) }
|
|
81
|
+
attr_reader :source
|
|
82
|
+
|
|
83
|
+
sig { returns(T.nilable(String)) }
|
|
84
|
+
attr_reader :source_ref
|
|
85
|
+
|
|
86
|
+
sig { returns(Symbol) }
|
|
87
|
+
attr_reader :type
|
|
88
|
+
|
|
89
|
+
sig { returns(T::Array[Name]) }
|
|
90
|
+
attr_reader :names
|
|
91
|
+
|
|
92
|
+
sig { returns(T::Array[Address]) }
|
|
93
|
+
attr_reader :addresses
|
|
94
|
+
|
|
95
|
+
sig { returns(T::Array[Identifier]) }
|
|
96
|
+
attr_reader :identifiers
|
|
97
|
+
|
|
98
|
+
sig { returns(T::Array[PartialDate]) }
|
|
99
|
+
attr_reader :dates_of_birth
|
|
100
|
+
|
|
101
|
+
sig { returns(T::Array[String]) }
|
|
102
|
+
attr_reader :nationalities
|
|
103
|
+
|
|
104
|
+
sig { returns(T::Array[String]) }
|
|
105
|
+
attr_reader :programs
|
|
106
|
+
|
|
107
|
+
sig { returns(T.nilable(PartialDate)) }
|
|
108
|
+
attr_reader :listed_on
|
|
109
|
+
|
|
110
|
+
sig { returns(T.nilable(String)) }
|
|
111
|
+
attr_reader :remarks
|
|
112
|
+
|
|
113
|
+
# Rebuilds an entity from #to_h output. Accepts string keys too, so a record
|
|
114
|
+
# that has been through JSON round-trips without a separate coercion step.
|
|
115
|
+
sig { params(hash: T.untyped).returns(T.attached_class) }
|
|
116
|
+
def self.from_h(hash)
|
|
117
|
+
attributes = hash.to_h.transform_keys(&:to_sym)
|
|
118
|
+
unknown = attributes.keys - MEMBERS
|
|
119
|
+
raise InvalidArgument, "unknown Entity attribute(s): #{unknown.join(", ")}" if unknown.any?
|
|
120
|
+
|
|
121
|
+
# `new(**hash)` past required keyword parameters is one of the few
|
|
122
|
+
# things Sorbet cannot check statically. The hash is validated on the two
|
|
123
|
+
# lines above and by #initialize below, so what is lost here is only the
|
|
124
|
+
# checker's ability to see it happen.
|
|
125
|
+
T.unsafe(self).new(**coerce_members(attributes))
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
sig { params(attributes: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
|
|
129
|
+
def self.coerce_members(attributes)
|
|
130
|
+
COLLECTION_TYPES.each do |member, class_name|
|
|
131
|
+
attributes[member] &&= attributes[member].map { |value| build(class_name, value) }
|
|
132
|
+
end
|
|
133
|
+
SCALAR_TYPES.each { |member, class_name| attributes[member] &&= build(class_name, attributes[member]) }
|
|
134
|
+
attributes
|
|
135
|
+
end
|
|
136
|
+
private_class_method :coerce_members
|
|
137
|
+
|
|
138
|
+
# Values that are already value objects pass through untouched, so from_h is
|
|
139
|
+
# safe to call on a half-deserialized hash.
|
|
140
|
+
sig { params(class_name: String, value: T.untyped).returns(T.untyped) }
|
|
141
|
+
def self.build(class_name, value)
|
|
142
|
+
return value unless value.is_a?(Hash)
|
|
143
|
+
|
|
144
|
+
Object.const_get(class_name).from_h(value)
|
|
145
|
+
end
|
|
146
|
+
private_class_method :build
|
|
147
|
+
|
|
148
|
+
# Each collection is nilable because nil is how "the publisher listed
|
|
149
|
+
# none" arrives from a store or a half-built hash; #list! turns it into the
|
|
150
|
+
# empty array the reader hands back.
|
|
151
|
+
#
|
|
152
|
+
# The four collection members and `listed_on` are declared, and the rest is
|
|
153
|
+
# `T.untyped` on purpose. The difference is who wrote the value: the nested
|
|
154
|
+
# members are canonical objects an adapter builds, and declaring them is
|
|
155
|
+
# what makes `srb tc` refuse an adapter that hands over the string a
|
|
156
|
+
# publisher wrote where a PartialDate belongs. The runtime check that comes
|
|
157
|
+
# with the signature is shallow -- it sees the Array and not what is in it
|
|
158
|
+
# -- so the adapter conformance group goes on asserting the element types
|
|
159
|
+
# per fixture, which is what covers an adapter written outside this repo.
|
|
160
|
+
#
|
|
161
|
+
# Everything else is the publisher's own text arriving as whatever the
|
|
162
|
+
# parser made of it, and the coercions below say what happens to it in
|
|
163
|
+
# messages written for whoever has to fix the record. A type error would
|
|
164
|
+
# say less.
|
|
165
|
+
sig do
|
|
166
|
+
params(
|
|
167
|
+
source: T.untyped,
|
|
168
|
+
type: T.untyped,
|
|
169
|
+
id: T.untyped,
|
|
170
|
+
source_ref: T.untyped,
|
|
171
|
+
names: T.nilable(T::Array[Name]),
|
|
172
|
+
addresses: T.nilable(T::Array[Address]),
|
|
173
|
+
identifiers: T.nilable(T::Array[Identifier]),
|
|
174
|
+
dates_of_birth: T.nilable(T::Array[PartialDate]),
|
|
175
|
+
nationalities: T.untyped,
|
|
176
|
+
programs: T.untyped,
|
|
177
|
+
listed_on: T.nilable(PartialDate),
|
|
178
|
+
remarks: T.untyped
|
|
179
|
+
).void
|
|
180
|
+
end
|
|
181
|
+
def initialize(source:, type:, id: nil, source_ref: nil, names: [], addresses: [], identifiers: [],
|
|
182
|
+
dates_of_birth: [], nationalities: [], programs: [], listed_on: nil, remarks: nil)
|
|
183
|
+
@source = T.let(symbol!(:source, source), Symbol)
|
|
184
|
+
@type = T.let(type!(type), Symbol)
|
|
185
|
+
@source_ref = T.let(string_or_nil(source_ref), T.nilable(String))
|
|
186
|
+
@id = T.let(string_or_nil(id) || derived_id, String)
|
|
187
|
+
@names = T.let(list!(:names, names), T::Array[Name])
|
|
188
|
+
@addresses = T.let(list!(:addresses, addresses), T::Array[Address])
|
|
189
|
+
@identifiers = T.let(list!(:identifiers, identifiers), T::Array[Identifier])
|
|
190
|
+
@dates_of_birth = T.let(list!(:dates_of_birth, dates_of_birth), T::Array[PartialDate])
|
|
191
|
+
@nationalities = T.let(strings!(:nationalities, nationalities), T::Array[String])
|
|
192
|
+
@programs = T.let(strings!(:programs, programs), T::Array[String])
|
|
193
|
+
@listed_on = T.let(listed_on, T.nilable(PartialDate))
|
|
194
|
+
# original free text, always retained verbatim
|
|
195
|
+
@remarks = T.let(string_or_nil(remarks), T.nilable(String))
|
|
196
|
+
freeze
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# True when the publisher gave no date at all, which is most of OFAC --
|
|
200
|
+
# its dates are prose in Remarks and stay there until #19 reads them.
|
|
201
|
+
sig { returns(T::Boolean) }
|
|
202
|
+
def dates_of_birth? = dates_of_birth.any?
|
|
203
|
+
|
|
204
|
+
# The name an adapter marked `:primary`, falling back to the first name for
|
|
205
|
+
# sources such as Canada that publish no alias kinds at all.
|
|
206
|
+
sig { returns(T.nilable(Name)) }
|
|
207
|
+
def primary_name
|
|
208
|
+
names.find(&:primary?) || names.first
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
212
|
+
def to_h
|
|
213
|
+
{
|
|
214
|
+
id: id,
|
|
215
|
+
source: source,
|
|
216
|
+
source_ref: source_ref,
|
|
217
|
+
type: type,
|
|
218
|
+
names: names.map(&:to_h),
|
|
219
|
+
addresses: addresses.map(&:to_h),
|
|
220
|
+
identifiers: identifiers.map(&:to_h),
|
|
221
|
+
dates_of_birth: dates_of_birth.map(&:to_h),
|
|
222
|
+
nationalities: nationalities,
|
|
223
|
+
programs: programs,
|
|
224
|
+
listed_on: listed_on&.to_h,
|
|
225
|
+
remarks: remarks
|
|
226
|
+
}
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Compared through #to_h so nested members only have to serialize, not
|
|
230
|
+
# implement value equality themselves. Class is part of the comparison to
|
|
231
|
+
# keep #== and #hash agreeing, which is what Hash and Set rely on.
|
|
232
|
+
sig { params(other: T.untyped).returns(T::Boolean) }
|
|
233
|
+
def ==(other)
|
|
234
|
+
return false unless other.instance_of?(self.class)
|
|
235
|
+
|
|
236
|
+
to_h == other.to_h
|
|
237
|
+
end
|
|
238
|
+
alias eql? ==
|
|
239
|
+
|
|
240
|
+
sig { returns(Integer) }
|
|
241
|
+
def hash
|
|
242
|
+
[self.class, to_h].hash
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
sig { returns(String) }
|
|
246
|
+
def inspect
|
|
247
|
+
"#<#{self.class} id=#{id.inspect} type=#{type.inspect} names=#{names.size}>"
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
private
|
|
251
|
+
|
|
252
|
+
sig { returns(String) }
|
|
253
|
+
def derived_id
|
|
254
|
+
raise InvalidArgument, "id is required when source_ref is nil" if source_ref.nil?
|
|
255
|
+
|
|
256
|
+
# Namespaced so ids stay unique and stable across sources.
|
|
257
|
+
-"#{source}:#{source_ref}"
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
sig { params(value: T.untyped).returns(Symbol) }
|
|
261
|
+
def type!(value)
|
|
262
|
+
type = symbol!(:type, value)
|
|
263
|
+
return type if TYPES.include?(type)
|
|
264
|
+
|
|
265
|
+
raise InvalidArgument, "unknown type #{type.inspect}, expected one of #{TYPES.join(", ")}"
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
sig { params(member: Symbol, value: T.untyped).returns(Symbol) }
|
|
269
|
+
def symbol!(member, value)
|
|
270
|
+
raise InvalidArgument, "#{member} is required" if value.nil? || value.to_s.empty?
|
|
271
|
+
|
|
272
|
+
value.to_sym
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
sig { params(value: T.untyped).returns(T.nilable(String)) }
|
|
276
|
+
def string_or_nil(value)
|
|
277
|
+
value.nil? ? nil : -value.to_s
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
sig { params(member: Symbol, value: T.untyped).returns(T.untyped) }
|
|
281
|
+
def list!(member, value)
|
|
282
|
+
return [].freeze if value.nil?
|
|
283
|
+
raise InvalidArgument, "#{member} must be an Array" unless value.is_a?(Array)
|
|
284
|
+
|
|
285
|
+
value.dup.freeze
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
sig { params(member: Symbol, value: T.untyped).returns(T::Array[String]) }
|
|
289
|
+
def strings!(member, value)
|
|
290
|
+
list!(member, value).map { |item| -item.to_s }.freeze
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
end
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
module ActiveSanction
|
|
7
|
+
# The one rescue that covers this library.
|
|
8
|
+
#
|
|
9
|
+
# begin
|
|
10
|
+
# ActiveSanction.sync!
|
|
11
|
+
# rescue ActiveSanction::Error => e
|
|
12
|
+
# raise unless e.retryable?
|
|
13
|
+
#
|
|
14
|
+
# RetryLater.enqueue(e.source_id)
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# Every error raised out of a public method answers to this, and carries
|
|
18
|
+
# what a caller needs to decide between the only three responses there are
|
|
19
|
+
# to a screening failure: **retry this** (`retryable?`), **alert somebody**
|
|
20
|
+
# (anything else that is not the caller's fault), and **this is a bug in my
|
|
21
|
+
# call** (ConfigurationError, InvalidArgument, QueryError). None of those
|
|
22
|
+
# decisions should be made by matching on a message string, so none of them
|
|
23
|
+
# has to be.
|
|
24
|
+
#
|
|
25
|
+
# ### The hierarchy
|
|
26
|
+
#
|
|
27
|
+
# Error the marker; rescue this
|
|
28
|
+
# ConfigurationError this installation is set up wrong; never retry
|
|
29
|
+
# SourceError something went wrong with one list
|
|
30
|
+
# FetchError the bytes could not be obtained
|
|
31
|
+
# ParseError the bytes could not be read
|
|
32
|
+
# IntegrityError the bytes are not what they claim to be
|
|
33
|
+
# StorageError the store could not answer
|
|
34
|
+
# UnsupportedError this object cannot do that
|
|
35
|
+
# InvalidArgument a public method was called wrongly
|
|
36
|
+
# QueryError ...specifically, with an unusable query
|
|
37
|
+
# MissingKey a field or column that does not exist
|
|
38
|
+
#
|
|
39
|
+
# ### Why this is a module and not a class
|
|
40
|
+
#
|
|
41
|
+
# Because two of its members have to be something else as well. A caller who
|
|
42
|
+
# passes `threshold: 300` has made the mistake Ruby has had a class for since
|
|
43
|
+
# 1995, and `rescue ArgumentError` is what the code around this library
|
|
44
|
+
# already says; asking every host application to learn a private synonym for
|
|
45
|
+
# it would be this library exporting its own taxonomy into code that has no
|
|
46
|
+
# reason to care. So InvalidArgument is an `::ArgumentError` and MissingKey is
|
|
47
|
+
# a `::KeyError` -- and Ruby has one superclass to give. A module is what lets
|
|
48
|
+
# them be both, and `rescue ActiveSanction::Error` covers them anyway, because
|
|
49
|
+
# `rescue` matches with `===`, which a module answers.
|
|
50
|
+
#
|
|
51
|
+
# The trade is that `ActiveSanction::Error` cannot be raised or instantiated
|
|
52
|
+
# itself. That is not a loss: an error that says only "something in the
|
|
53
|
+
# sanctions library went wrong" is not one a caller could act on, and every
|
|
54
|
+
# member below names a response.
|
|
55
|
+
#
|
|
56
|
+
# ### Stability
|
|
57
|
+
#
|
|
58
|
+
# This hierarchy is public API. Within a major version an error will not move
|
|
59
|
+
# to a different parent, and an attribute will not be removed. New subclasses
|
|
60
|
+
# may be added under an existing parent -- that is what keeps `rescue
|
|
61
|
+
# ActiveSanction::FetchError` working when a new transport failure is given a
|
|
62
|
+
# name of its own -- so a `case` over error classes should carry an `else`.
|
|
63
|
+
module Error
|
|
64
|
+
extend T::Sig
|
|
65
|
+
extend T::Helpers
|
|
66
|
+
|
|
67
|
+
# Only ever mixed into an exception class -- this is a marker for the
|
|
68
|
+
# library's own failures, not a bag of attributes anything can wear.
|
|
69
|
+
requires_ancestor { Exception }
|
|
70
|
+
|
|
71
|
+
# Which list the failure belongs to, as the key its adapter declared, or
|
|
72
|
+
# nil for a failure that is not about one list -- a bad configuration, an
|
|
73
|
+
# unusable query, a store that will not open at all.
|
|
74
|
+
#
|
|
75
|
+
# Always set on a SourceError by the time it leaves the source, even when
|
|
76
|
+
# the layer that raised it could not know: an HTTP client knows a URL, not
|
|
77
|
+
# which sanctions list is at the other end of it. See #in_source.
|
|
78
|
+
sig { returns(T.nilable(Symbol)) }
|
|
79
|
+
attr_reader :source_id
|
|
80
|
+
|
|
81
|
+
# The HTTP status behind the failure, where there was one. Nil for
|
|
82
|
+
# everything that failed before a server answered -- a timeout, a refused
|
|
83
|
+
# connection -- and for everything that is not a fetch.
|
|
84
|
+
sig { returns(T.nilable(Integer)) }
|
|
85
|
+
attr_reader :status
|
|
86
|
+
|
|
87
|
+
# `retryable:` overrides whatever the subclass would have decided, for the
|
|
88
|
+
# cases only the raising code knows about. Everything else is a subclass's
|
|
89
|
+
# own answer; see #retryable?.
|
|
90
|
+
sig do
|
|
91
|
+
params(message: T.untyped, source_id: T.untyped, status: T.untyped, retryable: T.nilable(T::Boolean)).void
|
|
92
|
+
end
|
|
93
|
+
def initialize(message = nil, source_id: nil, status: nil, retryable: nil)
|
|
94
|
+
@source_id = T.let(source_id&.to_sym, T.nilable(Symbol))
|
|
95
|
+
@status = T.let(status&.to_i, T.nilable(Integer))
|
|
96
|
+
@retryable = T.let(retryable, T.nilable(T::Boolean))
|
|
97
|
+
super(message)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Whether running the same call again could plausibly succeed.
|
|
101
|
+
#
|
|
102
|
+
# A first-class predicate rather than something a consumer reconstructs
|
|
103
|
+
# from the message, because backoff is the one decision a host application
|
|
104
|
+
# has to make in the request path and it should not be making it out of
|
|
105
|
+
# English. False is the default and the safe answer: a failure nobody has
|
|
106
|
+
# classified is one to look at rather than one to hammer.
|
|
107
|
+
sig { returns(T::Boolean) }
|
|
108
|
+
def retryable? = retryable_or(false)
|
|
109
|
+
|
|
110
|
+
# The failure as data, for a log line or a job record that has to survive
|
|
111
|
+
# the process.
|
|
112
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
113
|
+
def to_h
|
|
114
|
+
{ error: self.class.name, message: message, source_id: source_id, status: status, retryable: retryable? }
|
|
115
|
+
.compact
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Stamps the list this failure belongs to onto an error raised by a layer
|
|
119
|
+
# that did not know it, and returns self so a rescue can re-raise in one
|
|
120
|
+
# line. Never overwrites a source already recorded -- the innermost layer
|
|
121
|
+
# that knew is the one that was right.
|
|
122
|
+
sig { params(key: T.untyped).returns(T.self_type) }
|
|
123
|
+
def in_source(key)
|
|
124
|
+
@source_id = T.let(key&.to_sym, T.nilable(Symbol)) if @source_id.nil?
|
|
125
|
+
self
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
private
|
|
129
|
+
|
|
130
|
+
# What a subclass answers with when the raising code did not override it.
|
|
131
|
+
# The override exists because the classification is occasionally something
|
|
132
|
+
# only the call site knows -- a 404 from a publisher that rotates its URLs
|
|
133
|
+
# weekly is worth another look, and one from a URL we hardcoded is not.
|
|
134
|
+
sig { params(default: T::Boolean).returns(T::Boolean) }
|
|
135
|
+
def retryable_or(default) = @retryable.nil? ? default : @retryable
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# This installation is set up wrong: a blank User-Agent, a negative timeout,
|
|
139
|
+
# a source key nothing is registered under, an ActiveRecord store whose
|
|
140
|
+
# migration was never run.
|
|
141
|
+
#
|
|
142
|
+
# Never retryable, by definition -- nothing about waiting changes an
|
|
143
|
+
# initializer -- and raised as early as the bad value can be seen, which for
|
|
144
|
+
# a setting is where it is set rather than during a sync three hours later.
|
|
145
|
+
# Separate from InvalidArgument: this means "this installation is
|
|
146
|
+
# misconfigured", that means "this call is wrong", and only one of them is
|
|
147
|
+
# fixed by editing an initializer.
|
|
148
|
+
class ConfigurationError < StandardError
|
|
149
|
+
extend T::Sig
|
|
150
|
+
include Error
|
|
151
|
+
|
|
152
|
+
sig { returns(T::Boolean) }
|
|
153
|
+
def retryable? = false
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Something went wrong with one list. Carries #source_id, so a caller
|
|
157
|
+
# rescuing a whole sync knows which publisher to name.
|
|
158
|
+
#
|
|
159
|
+
# Raised directly only where the failure fits none of the three below --
|
|
160
|
+
# a source adapter that declares no URL, a payload the publisher confirmed
|
|
161
|
+
# but would not serve.
|
|
162
|
+
class SourceError < StandardError
|
|
163
|
+
include Error
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# The bytes could not be obtained: a timeout, a refused connection, a
|
|
167
|
+
# redirect chain that does not terminate, a status the caller declared fatal.
|
|
168
|
+
#
|
|
169
|
+
# Carries #status where a server produced one, and answers #retryable? from
|
|
170
|
+
# it. This is the error a host application backs off on, and the reason the
|
|
171
|
+
# predicate exists: 503 and 429 are the publisher having a bad afternoon,
|
|
172
|
+
# 403 and 404 are a request that will be just as wrong in ten minutes.
|
|
173
|
+
class FetchError < SourceError
|
|
174
|
+
extend T::Sig
|
|
175
|
+
|
|
176
|
+
# 408 and 429 are the server asking to be asked again; 5xx is it failing to
|
|
177
|
+
# answer at all. A 4xx outside those two is never in here: a 403 for a
|
|
178
|
+
# missing User-Agent or a 404 for a retired URL says the request is wrong,
|
|
179
|
+
# and repeating it wastes the publisher's capacity to make the same point.
|
|
180
|
+
#
|
|
181
|
+
# @api private
|
|
182
|
+
RETRYABLE_STATUSES = T.let(([408, 425, 429] + (500..599).to_a).freeze, T::Array[Integer])
|
|
183
|
+
|
|
184
|
+
sig { returns(T::Boolean) }
|
|
185
|
+
def retryable? = retryable_or(RETRYABLE_STATUSES.include?(status))
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# The bytes arrived and could not be read as the format they were declared
|
|
189
|
+
# to be: an HTML error page served under a `.xml` URL, a truncated download,
|
|
190
|
+
# an encoding that cannot be decoded, a ZIP member that will not inflate.
|
|
191
|
+
#
|
|
192
|
+
# Distinct from a Parsers::Warning, which is one *row* that could not be read
|
|
193
|
+
# while the rest of the file could. A list is not a file we control, and
|
|
194
|
+
# refusing 19,321 records because one of them is malformed fails exactly when
|
|
195
|
+
# the list is most needed -- so a bad row is a warning, and only a payload
|
|
196
|
+
# that cannot be read at all raises this.
|
|
197
|
+
#
|
|
198
|
+
# ### Where
|
|
199
|
+
#
|
|
200
|
+
# A 25 MB XML payload that "is not XML" is not a diagnosable complaint, so
|
|
201
|
+
# this carries a locator whenever the parser can produce one:
|
|
202
|
+
#
|
|
203
|
+
# rescue ActiveSanction::ParseError => e
|
|
204
|
+
# e.line # => 418_223
|
|
205
|
+
# e.record # => 12_004 (1-based, in the order the file yielded them)
|
|
206
|
+
# e.offset # => 8_388_608 (byte offset into the payload)
|
|
207
|
+
# e.locator # => "line 418223"
|
|
208
|
+
#
|
|
209
|
+
# All three are nil where the parser cannot say -- libxml2 reports no
|
|
210
|
+
# position for some failures, and a ZIP directory that ends inside an entry
|
|
211
|
+
# header has an offset but no line. An error that cannot point at a line
|
|
212
|
+
# still says what went wrong rather than pointing at the wrong one.
|
|
213
|
+
class ParseError < SourceError
|
|
214
|
+
extend T::Sig
|
|
215
|
+
|
|
216
|
+
# 1-based line within the payload, or nil.
|
|
217
|
+
sig { returns(T.nilable(Integer)) }
|
|
218
|
+
attr_reader :line
|
|
219
|
+
|
|
220
|
+
# 1-based index of the record being read, in the order the parser yielded
|
|
221
|
+
# them, or nil. The locator that means something for a format with no
|
|
222
|
+
# lines -- a spreadsheet, a stream of XML elements on one line.
|
|
223
|
+
sig { returns(T.nilable(Integer)) }
|
|
224
|
+
attr_reader :record
|
|
225
|
+
|
|
226
|
+
# Byte offset into the payload, or nil.
|
|
227
|
+
sig { returns(T.nilable(Integer)) }
|
|
228
|
+
attr_reader :offset
|
|
229
|
+
|
|
230
|
+
sig do
|
|
231
|
+
params(message: T.untyped, line: T.nilable(Integer), record: T.nilable(Integer), offset: T.nilable(Integer),
|
|
232
|
+
options: T.untyped).void
|
|
233
|
+
end
|
|
234
|
+
def initialize(message = nil, line: nil, record: nil, offset: nil, **options)
|
|
235
|
+
@line = T.let(line, T.nilable(Integer))
|
|
236
|
+
@record = T.let(record, T.nilable(Integer))
|
|
237
|
+
@offset = T.let(offset, T.nilable(Integer))
|
|
238
|
+
super(message, **options)
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# Where in the payload, in the terms the parser could supply, or nil when
|
|
242
|
+
# it could supply none. Appended to #to_s -- and so to #message, which is
|
|
243
|
+
# defined in terms of it -- so a log line that records nothing but the
|
|
244
|
+
# message still says where.
|
|
245
|
+
sig { returns(T.nilable(String)) }
|
|
246
|
+
def locator
|
|
247
|
+
parts = []
|
|
248
|
+
parts << "line #{line}" if line
|
|
249
|
+
parts << "record #{record}" if record
|
|
250
|
+
parts << "byte #{offset}" if offset
|
|
251
|
+
parts.empty? ? nil : parts.join(", ")
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
sig { returns(String) }
|
|
255
|
+
def to_s
|
|
256
|
+
where = locator
|
|
257
|
+
where ? "#{super} (at #{where})" : super
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
261
|
+
def to_h = super.merge({ line: line, record: record, offset: offset }.compact)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# The bytes are not what they claim to be: content that no longer hashes to
|
|
265
|
+
# the checksum stored beside it, a snapshot filed under one source that says
|
|
266
|
+
# it is another.
|
|
267
|
+
#
|
|
268
|
+
# Never repaired, and never answered with whatever could still be read. A
|
|
269
|
+
# store that hands back the 8,000 records it managed to parse out of 19,015
|
|
270
|
+
# produces a report that looks exactly like a clean one, which is the most
|
|
271
|
+
# expensive thing this library can get wrong. An operator can always discard
|
|
272
|
+
# the copy and re-sync; nobody can recover a screening decision made against
|
|
273
|
+
# a list that was quietly half there.
|
|
274
|
+
class IntegrityError < SourceError; end
|
|
275
|
+
|
|
276
|
+
# The store could not answer: a list that has never been synced, a snapshot
|
|
277
|
+
# written under a schema this version does not read, a validator file that
|
|
278
|
+
# is not readable as validators.
|
|
279
|
+
#
|
|
280
|
+
# About the *store* rather than about the publisher, which is the difference
|
|
281
|
+
# that matters when deciding what to do: a FetchError is somebody else's
|
|
282
|
+
# outage, and this is local state to repair or re-sync.
|
|
283
|
+
class StorageError < StandardError
|
|
284
|
+
include Error
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# This object cannot do that: a backend without the capability asked for, an
|
|
288
|
+
# abstract method a subclass never implemented.
|
|
289
|
+
#
|
|
290
|
+
# A StandardError rather than the `NotImplementedError` that reads more
|
|
291
|
+
# naturally for the second case, deliberately. `NotImplementedError` is a
|
|
292
|
+
# ScriptError, so it is not caught by `rescue StandardError` -- and one
|
|
293
|
+
# adapter forgetting `#parse` would take down a sync run that is supposed to
|
|
294
|
+
# isolate each source's failure from the others, which is the run's single
|
|
295
|
+
# most important property.
|
|
296
|
+
class UnsupportedError < StandardError
|
|
297
|
+
include Error
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# A public method was called with something it cannot use: a threshold of
|
|
301
|
+
# 300, a Snapshot whose `record_count` disagrees with its entities, a name
|
|
302
|
+
# value object built with no value.
|
|
303
|
+
#
|
|
304
|
+
# This is the "bug in my call" branch, and it is an `::ArgumentError` as well
|
|
305
|
+
# as an ActiveSanction::Error so that it reads as one to code that has never
|
|
306
|
+
# heard of this library. Messages are written for whoever has to fix the
|
|
307
|
+
# call, or the record: `"limit must be at least 1, got 0"` rather than a type
|
|
308
|
+
# name.
|
|
309
|
+
class InvalidArgument < ::ArgumentError
|
|
310
|
+
include Error
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# A screening query that cannot be run: an empty `sources:` list, a threshold
|
|
314
|
+
# outside 0..100, a limit of zero, a field spelled two ways at once.
|
|
315
|
+
#
|
|
316
|
+
# Its own class because a query is user input in a way the rest of this is
|
|
317
|
+
# not -- it is frequently built from a form or an API request -- and a
|
|
318
|
+
# service turning a bad query into a 422 and a bad configuration into a 500
|
|
319
|
+
# should not have to tell them apart by reading messages.
|
|
320
|
+
class QueryError < InvalidArgument; end
|
|
321
|
+
|
|
322
|
+
# A field or column that does not exist, asked for by name: `row[:nmae]`, or
|
|
323
|
+
# a path into an XML record that no element supplies.
|
|
324
|
+
#
|
|
325
|
+
# A `::KeyError`, because it is what `Hash#fetch` raises and these methods
|
|
326
|
+
# are `fetch` in everything but name. Almost always a typo in an adapter
|
|
327
|
+
# rather than a question about the data, so the message lists what *is*
|
|
328
|
+
# there.
|
|
329
|
+
class MissingKey < ::KeyError
|
|
330
|
+
include Error
|
|
331
|
+
end
|
|
332
|
+
end
|