active_sanction 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +538 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/CONTRIBUTING.md +312 -0
- data/LICENSE.txt +21 -0
- data/README.md +616 -0
- data/SECURITY.md +97 -0
- data/active_sanction.gemspec +98 -0
- data/docs/adding_a_source.md +1083 -0
- data/docs/api_stability.md +357 -0
- data/docs/bundle_format.md +407 -0
- data/lib/active_sanction/address.rb +170 -0
- data/lib/active_sanction/client.rb +341 -0
- data/lib/active_sanction/configuration.rb +721 -0
- data/lib/active_sanction/countries.txt +292 -0
- data/lib/active_sanction/country.rb +164 -0
- data/lib/active_sanction/deprecation.rb +168 -0
- data/lib/active_sanction/diff/change.rb +208 -0
- data/lib/active_sanction/diff.rb +348 -0
- data/lib/active_sanction/doctor/checkup.rb +356 -0
- data/lib/active_sanction/doctor/diagnosis.rb +245 -0
- data/lib/active_sanction/doctor/finding.rb +183 -0
- data/lib/active_sanction/doctor/profile.rb +384 -0
- data/lib/active_sanction/doctor/report.rb +227 -0
- data/lib/active_sanction/doctor.rb +347 -0
- data/lib/active_sanction/entity.rb +293 -0
- data/lib/active_sanction/error.rb +332 -0
- data/lib/active_sanction/fetcher/result.rb +116 -0
- data/lib/active_sanction/fetcher.rb +242 -0
- data/lib/active_sanction/http_client/errors.rb +94 -0
- data/lib/active_sanction/http_client/response.rb +129 -0
- data/lib/active_sanction/http_client.rb +324 -0
- data/lib/active_sanction/identifier.rb +212 -0
- data/lib/active_sanction/index/builder.rb +89 -0
- data/lib/active_sanction/index/candidate.rb +63 -0
- data/lib/active_sanction/index/entry.rb +66 -0
- data/lib/active_sanction/index/features.rb +112 -0
- data/lib/active_sanction/index.rb +381 -0
- data/lib/active_sanction/match_result.rb +345 -0
- data/lib/active_sanction/matcher.rb +388 -0
- data/lib/active_sanction/name.rb +183 -0
- data/lib/active_sanction/normalizer/cache.rb +82 -0
- data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
- data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
- data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
- data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
- data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
- data/lib/active_sanction/normalizer/dictionary.rb +243 -0
- data/lib/active_sanction/normalizer/form.rb +279 -0
- data/lib/active_sanction/normalizer.rb +137 -0
- data/lib/active_sanction/parsers/column_shape.rb +274 -0
- data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
- data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
- data/lib/active_sanction/parsers/delimited_table.rb +151 -0
- data/lib/active_sanction/parsers/format.rb +107 -0
- data/lib/active_sanction/parsers/join.rb +149 -0
- data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
- data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
- data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
- data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
- data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
- data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
- data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
- data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
- data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
- data/lib/active_sanction/parsers/xml_records.rb +140 -0
- data/lib/active_sanction/parsers.rb +94 -0
- data/lib/active_sanction/partial_date/parser.rb +124 -0
- data/lib/active_sanction/partial_date.rb +303 -0
- data/lib/active_sanction/payload_cache/checksum.rb +66 -0
- data/lib/active_sanction/payload_cache/entry.rb +304 -0
- data/lib/active_sanction/payload_cache.rb +366 -0
- data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
- data/lib/active_sanction/phonetics.rb +40 -0
- data/lib/active_sanction/query.rb +290 -0
- data/lib/active_sanction/rescreen/alert.rb +314 -0
- data/lib/active_sanction/rescreen.rb +332 -0
- data/lib/active_sanction/scorer/adjustments.rb +241 -0
- data/lib/active_sanction/scorer/name_score.rb +279 -0
- data/lib/active_sanction/scorer/reason.rb +136 -0
- data/lib/active_sanction/scorer/result.rb +129 -0
- data/lib/active_sanction/scorer/subject.rb +235 -0
- data/lib/active_sanction/scorer/weights.rb +306 -0
- data/lib/active_sanction/scorer.rb +303 -0
- data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
- data/lib/active_sanction/similarity/levenshtein.rb +179 -0
- data/lib/active_sanction/similarity/token_set.rb +159 -0
- data/lib/active_sanction/similarity/token_sort.rb +120 -0
- data/lib/active_sanction/similarity.rb +183 -0
- data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
- data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
- data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
- data/lib/active_sanction/snapshot/bundle.rb +381 -0
- data/lib/active_sanction/snapshot.rb +305 -0
- data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
- data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
- data/lib/active_sanction/sources/australia_dfat.rb +291 -0
- data/lib/active_sanction/sources/base.rb +295 -0
- data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
- data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
- data/lib/active_sanction/sources/canada_sema.rb +153 -0
- data/lib/active_sanction/sources/definition.rb +360 -0
- data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
- data/lib/active_sanction/sources/eu_fsf.rb +203 -0
- data/lib/active_sanction/sources/ofac/record.rb +251 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
- data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
- data/lib/active_sanction/sources/ofac.rb +248 -0
- data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
- data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
- data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
- data/lib/active_sanction/sources/remarks.rb +84 -0
- data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
- data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
- data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
- data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
- data/lib/active_sanction/sources/un_consolidated.rb +140 -0
- data/lib/active_sanction/sources.rb +196 -0
- data/lib/active_sanction/storage/active_record/reader.rb +124 -0
- data/lib/active_sanction/storage/active_record/row.rb +175 -0
- data/lib/active_sanction/storage/active_record/writer.rb +141 -0
- data/lib/active_sanction/storage/active_record.rb +277 -0
- data/lib/active_sanction/storage/base.rb +216 -0
- data/lib/active_sanction/storage/file_system.rb +373 -0
- data/lib/active_sanction/storage/memory.rb +67 -0
- data/lib/active_sanction/storage/meta.rb +188 -0
- data/lib/active_sanction/storage.rb +74 -0
- data/lib/active_sanction/subject.rb +271 -0
- data/lib/active_sanction/sync/report.rb +245 -0
- data/lib/active_sanction/sync/result.rb +296 -0
- data/lib/active_sanction/sync.rb +390 -0
- data/lib/active_sanction/validator_store/file_system.rb +103 -0
- data/lib/active_sanction/validator_store/memory.rb +41 -0
- data/lib/active_sanction/validator_store.rb +119 -0
- data/lib/active_sanction/validators.rb +195 -0
- data/lib/active_sanction/version.rb +23 -0
- data/lib/active_sanction.rb +331 -0
- data/lib/generators/active_sanction/install/install_generator.rb +60 -0
- data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
- metadata +236 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/entity"
|
|
7
|
+
require "active_sanction/name"
|
|
8
|
+
require "active_sanction/address"
|
|
9
|
+
require "active_sanction/identifier"
|
|
10
|
+
require "active_sanction/partial_date"
|
|
11
|
+
require "active_sanction/sources/remarks"
|
|
12
|
+
|
|
13
|
+
module ActiveSanction
|
|
14
|
+
module Sources
|
|
15
|
+
class UnConsolidated < Base
|
|
16
|
+
# One `<INDIVIDUAL>` or `<ENTITY>` turned into an Entity.
|
|
17
|
+
#
|
|
18
|
+
# Separate from the adapter because they are two jobs: the adapter says
|
|
19
|
+
# what the list is and where it lives, and this says what the UN's
|
|
20
|
+
# elements mean. The mapping is where all the judgment sits, so it is
|
|
21
|
+
# worth being able to read it on its own.
|
|
22
|
+
#
|
|
23
|
+
# The two record shapes are mapped by one class rather than two. Every
|
|
24
|
+
# element that differs between them differs only by prefix --
|
|
25
|
+
# `INDIVIDUAL_ALIAS` against `ENTITY_ALIAS`, `INDIVIDUAL_ADDRESS` against
|
|
26
|
+
# `ENTITY_ADDRESS` -- so the prefix is taken from the record's own name.
|
|
27
|
+
# An element only individuals have then resolves to a path entities do
|
|
28
|
+
# not carry, which reads as absent, which is what it is.
|
|
29
|
+
#
|
|
30
|
+
# @api private
|
|
31
|
+
class Record
|
|
32
|
+
extend T::Sig
|
|
33
|
+
|
|
34
|
+
TYPES = T.let({ UnConsolidated::INDIVIDUAL => :individual,
|
|
35
|
+
UnConsolidated::ENTITY => :organization }.freeze, T::Hash[String, Symbol])
|
|
36
|
+
|
|
37
|
+
# The UN splits one name across up to four numbered elements, and any
|
|
38
|
+
# subset may be present: 736 individuals have a FIRST_NAME, 170 have a
|
|
39
|
+
# FOURTH_NAME, and an entity's whole name is in FIRST_NAME alone.
|
|
40
|
+
NAME_PARTS = T.let(%w[FIRST_NAME SECOND_NAME THIRD_NAME FOURTH_NAME].freeze, T::Array[String])
|
|
41
|
+
|
|
42
|
+
# `QUALITY` under `<INDIVIDUAL_ALIAS>` grades the alias. Published as
|
|
43
|
+
# Good (1,534), Low (629), or blank (169).
|
|
44
|
+
ALIAS_QUALITIES = T.let({ "good" => :good, "low" => :low }.freeze, T::Hash[String, Symbol])
|
|
45
|
+
|
|
46
|
+
# `QUALITY` under `<ENTITY_ALIAS>` is not a grade at all -- it is the
|
|
47
|
+
# alias kind, published as a.k.a. (585), f.k.a. (19), or blank (125).
|
|
48
|
+
# One element name, two vocabularies, in one document. Both tables are
|
|
49
|
+
# consulted for every alias and each misses the other's values, so
|
|
50
|
+
# neither shape needs a branch and neither loses what it publishes.
|
|
51
|
+
ALIAS_KINDS = T.let(
|
|
52
|
+
{ "a.k.a." => :aka, "f.k.a." => :fka, "n.k.a." => :nka }.freeze, T::Hash[String, Symbol]
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
# `TYPE_OF_DOCUMENT` is free text, and the UN writes it in three
|
|
56
|
+
# languages. Anything unrecognised is :other, which is a real answer:
|
|
57
|
+
# a document we cannot classify still matches on its number.
|
|
58
|
+
DOCUMENT_KINDS = T.let({
|
|
59
|
+
"passport" => :passport,
|
|
60
|
+
"numéro de passeport" => :passport,
|
|
61
|
+
"número de pasaporte" => :passport,
|
|
62
|
+
"national identification number" => :national_id
|
|
63
|
+
}.freeze, T::Hash[String, Symbol])
|
|
64
|
+
|
|
65
|
+
# `TYPE_OF_DATE`, of which only these two change how the date is read.
|
|
66
|
+
# EXACT and a blank value are the same instruction: read what is there.
|
|
67
|
+
BETWEEN = T.let("BETWEEN", String)
|
|
68
|
+
APPROXIMATELY = T.let("APPROXIMATELY", String)
|
|
69
|
+
|
|
70
|
+
# Elements the canonical model has no home for, appended to remarks
|
|
71
|
+
# rather than dropped. A gender and a place of birth are real screening
|
|
72
|
+
# signal, and losing them to keep a schema tidy is the wrong trade.
|
|
73
|
+
EXTRA_FIELDS = T.let({
|
|
74
|
+
"VERSIONNUM" => "Version", "REFERENCE_NUMBER" => "Reference", "GENDER" => "Gender",
|
|
75
|
+
"TITLE/VALUE" => "Title", "DESIGNATION/VALUE" => "Designation"
|
|
76
|
+
}.freeze, T::Hash[String, String])
|
|
77
|
+
|
|
78
|
+
PLACE_OF_BIRTH_PARTS = T.let(%w[STREET CITY STATE_PROVINCE COUNTRY NOTE].freeze, T::Array[String])
|
|
79
|
+
|
|
80
|
+
WHITESPACE = T.let(/\s+/, Regexp)
|
|
81
|
+
|
|
82
|
+
sig { returns(Parsers::XmlRecords::Record) }
|
|
83
|
+
attr_reader :node
|
|
84
|
+
|
|
85
|
+
sig { params(node: Parsers::XmlRecords::Record).void }
|
|
86
|
+
def initialize(node)
|
|
87
|
+
@node = T.let(node, Parsers::XmlRecords::Record)
|
|
88
|
+
@names = T.let(nil, T.nilable(T::Array[Name]))
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The entity, or nil for a record with no name -- which cannot be
|
|
92
|
+
# screened against and is never what the Committee meant to publish.
|
|
93
|
+
sig { returns(T.nilable(Entity)) }
|
|
94
|
+
def entity
|
|
95
|
+
return nil if names.empty?
|
|
96
|
+
|
|
97
|
+
Entity.new(source: :un_consolidated, source_ref: source_ref, type: type, names: names,
|
|
98
|
+
addresses: addresses, identifiers: identifiers, dates_of_birth: dates_of_birth,
|
|
99
|
+
nationalities: nationalities, programs: programs, listed_on: listed_on,
|
|
100
|
+
remarks: remarks)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
sig { returns(Symbol) }
|
|
104
|
+
def type = TYPES.fetch(node.name, :organization)
|
|
105
|
+
|
|
106
|
+
sig { returns(T.nilable(String)) }
|
|
107
|
+
def source_ref = node["DATAID"]
|
|
108
|
+
|
|
109
|
+
sig { returns(T::Array[String]) }
|
|
110
|
+
def nationalities = node.values("NATIONALITY/VALUE")
|
|
111
|
+
|
|
112
|
+
sig { returns(T.nilable(PartialDate)) }
|
|
113
|
+
def listed_on = PartialDate.parse(node["LISTED_ON"])
|
|
114
|
+
|
|
115
|
+
# The joined published name first, then the original script, then every
|
|
116
|
+
# alias in the order the Committee filed it.
|
|
117
|
+
sig { returns(T::Array[Name]) }
|
|
118
|
+
def names
|
|
119
|
+
@names ||= [published_name, original_script].compact + alias_names
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
sig { returns(T::Array[PartialDate]) }
|
|
123
|
+
def dates_of_birth
|
|
124
|
+
each("DATE_OF_BIRTH").filter_map { |born| date_of_birth(born) }
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
sig { returns(T::Array[Identifier]) }
|
|
128
|
+
def identifiers
|
|
129
|
+
each("DOCUMENT").filter_map { |document| identifier(document) }
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
sig { returns(T::Array[Address]) }
|
|
133
|
+
def addresses
|
|
134
|
+
each("ADDRESS").filter_map { |place| address(place) }
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
sig { returns(T::Array[String]) }
|
|
138
|
+
def programs = node.values("UN_LIST_TYPE")
|
|
139
|
+
|
|
140
|
+
# The UN's own comment verbatim, then the elements that have nowhere
|
|
141
|
+
# else to go, behind the marker that makes them trivial to strip again.
|
|
142
|
+
sig { returns(T.nilable(String)) }
|
|
143
|
+
def remarks
|
|
144
|
+
Remarks.build(node["COMMENTS1"], extras)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
private
|
|
148
|
+
|
|
149
|
+
# The child elements of a record, which the UN names after the record
|
|
150
|
+
# they hang off. See the class comment for why that is worth exploiting.
|
|
151
|
+
sig { params(suffix: String).returns(T::Array[Parsers::XmlRecords::Record]) }
|
|
152
|
+
def each(suffix) = node.nodes("#{node.name}_#{suffix}")
|
|
153
|
+
|
|
154
|
+
sig { returns(T.nilable(Name)) }
|
|
155
|
+
def published_name
|
|
156
|
+
value = collapse(NAME_PARTS.filter_map { |part| node[part] }.join(" "))
|
|
157
|
+
value && Name.new(value: value, kind: :primary)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# The name as the publisher's own script writes it -- Arabic for 338 of
|
|
161
|
+
# the individuals. Filed as an alias with no `script` declared: which
|
|
162
|
+
# script a string is in is a question about its characters, and
|
|
163
|
+
# answering it from the element's name would be a guess. The normalizer
|
|
164
|
+
# (#26) reads the characters and is the right place for it.
|
|
165
|
+
sig { returns(T.nilable(Name)) }
|
|
166
|
+
def original_script
|
|
167
|
+
value = collapse(node["NAME_ORIGINAL_SCRIPT"])
|
|
168
|
+
value && Name.new(value: value, kind: :aka)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# 294 of the 3,061 alias elements are placeholders: `<QUALITY/>` and
|
|
172
|
+
# `<ALIAS_NAME/>` and nothing else. They must produce no name at all --
|
|
173
|
+
# a blank-valued Name would be a record that matches everything.
|
|
174
|
+
sig { returns(T::Array[Name]) }
|
|
175
|
+
def alias_names
|
|
176
|
+
each("ALIAS").filter_map do |alt|
|
|
177
|
+
value = collapse(alt["ALIAS_NAME"])
|
|
178
|
+
next nil if value.nil?
|
|
179
|
+
|
|
180
|
+
published = alt["QUALITY"].to_s.downcase
|
|
181
|
+
Name.new(value: value, kind: ALIAS_KINDS.fetch(published, :aka),
|
|
182
|
+
quality: ALIAS_QUALITIES[published])
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# 55 of the 949 date elements carry a TYPE_OF_DATE and no date, which
|
|
187
|
+
# reads as nil rather than as a date of unknown value.
|
|
188
|
+
sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(PartialDate)) }
|
|
189
|
+
def date_of_birth(born)
|
|
190
|
+
type_of_date = born["TYPE_OF_DATE"].to_s.upcase
|
|
191
|
+
return between(born) if type_of_date == BETWEEN
|
|
192
|
+
|
|
193
|
+
approximate(PartialDate.parse(born["DATE"] || born["YEAR"]), type_of_date)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(PartialDate)) }
|
|
197
|
+
def between(born)
|
|
198
|
+
from = born["FROM_YEAR"]
|
|
199
|
+
to = born["TO_YEAR"]
|
|
200
|
+
return nil if from.nil? || to.nil?
|
|
201
|
+
|
|
202
|
+
PartialDate.range(from, to)
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# "Circa 1962" and "1963" are the same claim about a person, and
|
|
206
|
+
# PartialDate already knows to compare an approximate date loosely --
|
|
207
|
+
# but only if the adapter tells it the publisher said approximately.
|
|
208
|
+
sig { params(date: T.nilable(PartialDate), type_of_date: String).returns(T.nilable(PartialDate)) }
|
|
209
|
+
def approximate(date, type_of_date)
|
|
210
|
+
return date if date.nil? || type_of_date != APPROXIMATELY
|
|
211
|
+
|
|
212
|
+
PartialDate.from_h(date.to_h.merge(approximate: true))
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# 447 of the UN's 954 document elements name a document type and give
|
|
216
|
+
# no number. There is nothing there to match on, so they are dropped
|
|
217
|
+
# rather than kept as identifiers with no identity.
|
|
218
|
+
sig { params(document: Parsers::XmlRecords::Record).returns(T.nilable(Identifier)) }
|
|
219
|
+
def identifier(document)
|
|
220
|
+
return nil if document.null?("NUMBER")
|
|
221
|
+
|
|
222
|
+
Identifier.new(kind: document_kind(document), value: document["NUMBER"],
|
|
223
|
+
country: document["ISSUING_COUNTRY"] || document["COUNTRY_OF_ISSUE"],
|
|
224
|
+
issued_on: PartialDate.parse(document["DATE_OF_ISSUE"]),
|
|
225
|
+
expires_on: PartialDate.parse(document["DATE_OF_EXPIRY"]),
|
|
226
|
+
note: document_note(document))
|
|
227
|
+
rescue ArgumentError
|
|
228
|
+
nil
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
sig { params(document: Parsers::XmlRecords::Record).returns(Symbol) }
|
|
232
|
+
def document_kind(document)
|
|
233
|
+
DOCUMENT_KINDS.fetch(collapse(document["TYPE_OF_DOCUMENT"]).to_s.downcase, :other)
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
sig { params(document: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
|
|
237
|
+
def document_note(document)
|
|
238
|
+
parts = [collapse(document["TYPE_OF_DOCUMENT2"]), document["CITY_OF_ISSUE"], document["NOTE"]].compact
|
|
239
|
+
parts.empty? ? nil : parts.join("; ")
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# An address element that located nothing is dropped rather than kept
|
|
243
|
+
# empty: it cannot be screened on and would only inflate the count.
|
|
244
|
+
sig { params(place: Parsers::XmlRecords::Record).returns(T.nilable(Address)) }
|
|
245
|
+
def address(place)
|
|
246
|
+
Address.new(street: place["STREET"], city: place["CITY"], state_province: place["STATE_PROVINCE"],
|
|
247
|
+
postal_code: place["ZIP_CODE"], country: place["COUNTRY"], note: place["NOTE"])
|
|
248
|
+
rescue ArgumentError
|
|
249
|
+
nil
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Label/value pairs for Remarks.build, which drops the ones the record
|
|
253
|
+
# left blank. A value may be several -- the UN files three designations
|
|
254
|
+
# under one element -- and arrives as the Array it published.
|
|
255
|
+
sig { returns(T::Array[T.untyped]) }
|
|
256
|
+
def extras
|
|
257
|
+
EXTRA_FIELDS.map { |path, label| [label, node.values(path)] } + places_of_birth
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
sig { returns(T::Array[T.untyped]) }
|
|
261
|
+
def places_of_birth
|
|
262
|
+
each("PLACE_OF_BIRTH").map do |place|
|
|
263
|
+
["Place of birth", PLACE_OF_BIRTH_PARTS.filter_map { |part| place[part] }.join(", ")]
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
# The UN's own line wrapping arrives inside values: two names in the
|
|
268
|
+
# published file carry a newline and fifty spaces mid-string, and 39
|
|
269
|
+
# name parts are padded on one side. Joined verbatim those produce
|
|
270
|
+
# "JOHN IMANI", which is a name nothing will ever match.
|
|
271
|
+
sig { params(value: T.untyped).returns(T.nilable(String)) }
|
|
272
|
+
def collapse(value)
|
|
273
|
+
string = value.to_s.split(WHITESPACE).join(" ")
|
|
274
|
+
string.empty? ? nil : -string
|
|
275
|
+
end
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
end
|
|
279
|
+
end
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/parsers"
|
|
7
|
+
require "active_sanction/sources"
|
|
8
|
+
require "active_sanction/sources/base"
|
|
9
|
+
|
|
10
|
+
module ActiveSanction
|
|
11
|
+
module Sources
|
|
12
|
+
# The United Nations Security Council consolidated list: every individual
|
|
13
|
+
# and entity subject to a UN sanctions regime, in one document.
|
|
14
|
+
#
|
|
15
|
+
# snapshot = ActiveSanction::Sources[:un_consolidated].new.sync
|
|
16
|
+
#
|
|
17
|
+
# ### One file, two record shapes
|
|
18
|
+
#
|
|
19
|
+
# https://scsanctions.un.org/resources/xml/en/consolidated.xml 2.2 MB
|
|
20
|
+
#
|
|
21
|
+
# <INDIVIDUALS><INDIVIDUAL> 736 people
|
|
22
|
+
# <ENTITIES><ENTITY> 275 organizations
|
|
23
|
+
#
|
|
24
|
+
# Both are read in a single streaming pass. The file is small enough today
|
|
25
|
+
# to have been loaded whole without anyone noticing, which is exactly why
|
|
26
|
+
# it was not: the toolkit this adapter is written against (#15) is the one
|
|
27
|
+
# that will read OFAC's 126 MB advanced XML, and an adapter that quietly
|
|
28
|
+
# depended on holding the document would have to be rewritten then.
|
|
29
|
+
#
|
|
30
|
+
# ### The trap in this list
|
|
31
|
+
#
|
|
32
|
+
# `QUALITY` appears under both alias elements and means something different
|
|
33
|
+
# under each. Under `<INDIVIDUAL_ALIAS>` it grades the alias -- Good or Low
|
|
34
|
+
# -- which is a matching signal the scorer (#32) penalizes on. Under
|
|
35
|
+
# `<ENTITY_ALIAS>` it is not a grade at all; it is `a.k.a.` or `f.k.a.`,
|
|
36
|
+
# which is an alias *kind*. Reading one as the other silently either throws
|
|
37
|
+
# away every entity's alias kind or grades 585 organization aliases on a
|
|
38
|
+
# scale that was never applied to them. Record maps them separately, and
|
|
39
|
+
# the counts above come from the published file rather than from a guess.
|
|
40
|
+
#
|
|
41
|
+
# ### What this adapter does not do
|
|
42
|
+
#
|
|
43
|
+
# `INDIVIDUAL_PLACE_OF_BIRTH`, `GENDER`, `TITLE` and `DESIGNATION` have no
|
|
44
|
+
# home in the canonical model. They are appended to remarks after a marker
|
|
45
|
+
# rather than dropped -- a place of birth is real screening signal, and
|
|
46
|
+
# losing it to keep a schema tidy is the wrong trade -- and are left for a
|
|
47
|
+
# later issue to structure if a matcher turns out to want them.
|
|
48
|
+
class UnConsolidated < Base
|
|
49
|
+
extend T::Sig
|
|
50
|
+
|
|
51
|
+
key :un_consolidated
|
|
52
|
+
jurisdiction :un
|
|
53
|
+
authority "United Nations Security Council"
|
|
54
|
+
format :xml
|
|
55
|
+
|
|
56
|
+
# The most restrictive of the seven on paper. The UN's copyright page
|
|
57
|
+
# reserves reproduction of material on its sites, and the Consolidated
|
|
58
|
+
# List carries no separate open licence -- it is published so that
|
|
59
|
+
# member states and their regulated entities can implement the
|
|
60
|
+
# measures, which is the use this library is for. Redistributing it,
|
|
61
|
+
# as a bundle or through an API, is the case to take to counsel.
|
|
62
|
+
licence_notice "Copyright reserved by the United Nations. Published " \
|
|
63
|
+
"for implementation of Security Council measures; no " \
|
|
64
|
+
"open reuse licence is stated. Verified 2026-09-11."
|
|
65
|
+
licence_url "https://www.un.org/en/about-us/copyright"
|
|
66
|
+
|
|
67
|
+
url :main, "https://scsanctions.un.org/resources/xml/en/consolidated.xml"
|
|
68
|
+
|
|
69
|
+
# @api private
|
|
70
|
+
INDIVIDUAL = T.let("INDIVIDUAL", String)
|
|
71
|
+
# @api private
|
|
72
|
+
ENTITY = T.let("ENTITY", String)
|
|
73
|
+
|
|
74
|
+
# @api private
|
|
75
|
+
LIST = T.let(Parsers::XmlRecords.new(records: [INDIVIDUAL, ENTITY]), Parsers::XmlRecords)
|
|
76
|
+
|
|
77
|
+
# The generation timestamp the UN stamps on the document element. More
|
|
78
|
+
# precise than the Last-Modified header Base falls back to, and it is the
|
|
79
|
+
# string that appears on the UN's own site, so it is the one an examiner
|
|
80
|
+
# asking "which version was this screened against" will recognise.
|
|
81
|
+
#
|
|
82
|
+
# @api private
|
|
83
|
+
GENERATED_AT = T.let("dateGenerated", String)
|
|
84
|
+
|
|
85
|
+
# Records that could not be used. Read after #parse; sync orchestration
|
|
86
|
+
# (#34) reports them.
|
|
87
|
+
sig { returns(T::Array[Parsers::Warning]) }
|
|
88
|
+
attr_reader :warnings
|
|
89
|
+
|
|
90
|
+
sig { params(args: T.untyped, options: T.untyped).void }
|
|
91
|
+
def initialize(*args, **options)
|
|
92
|
+
super
|
|
93
|
+
@warnings = T.let([], T::Array[Parsers::Warning])
|
|
94
|
+
@unmapped = T.let([], T::Array[Parsers::Warning])
|
|
95
|
+
@generated_at = T.let(nil, T.nilable(String))
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
sig { override.params(raw: T.untyped).returns(T::Array[Entity]) }
|
|
99
|
+
def parse(raw)
|
|
100
|
+
reader = LIST.read(raw)
|
|
101
|
+
entities = build(reader)
|
|
102
|
+
@generated_at = reader.root[GENERATED_AT]
|
|
103
|
+
@warnings = reader.warnings + @unmapped
|
|
104
|
+
entities
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
sig { override.returns(T.nilable(String)) }
|
|
108
|
+
def source_version = @generated_at || super
|
|
109
|
+
|
|
110
|
+
private
|
|
111
|
+
|
|
112
|
+
sig { params(reader: Parsers::XmlRecords::Reader).returns(T::Array[Entity]) }
|
|
113
|
+
def build(reader)
|
|
114
|
+
@unmapped = []
|
|
115
|
+
reader.filter_map do |node|
|
|
116
|
+
record = Record.new(node)
|
|
117
|
+
entity = record.entity
|
|
118
|
+
entity.nil? ? note_nameless(node) : entity
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# A record with no name cannot be screened against and is never what the
|
|
123
|
+
# Committee meant to publish. None of the 1,011 published today is
|
|
124
|
+
# nameless; the warning exists so that the day one is, it is visible
|
|
125
|
+
# rather than absent.
|
|
126
|
+
sig { params(node: Parsers::XmlRecords::Record).returns(NilClass) }
|
|
127
|
+
def note_nameless(node)
|
|
128
|
+
@unmapped << Parsers::Warning.new(
|
|
129
|
+
line: node.line,
|
|
130
|
+
message: "<#{node.name}> #{node["DATAID"].inspect} has no name and was skipped"
|
|
131
|
+
)
|
|
132
|
+
nil
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
require "active_sanction/sources/un_consolidated/record"
|
|
139
|
+
|
|
140
|
+
ActiveSanction::Sources.register(ActiveSanction::Sources::UnConsolidated)
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/configuration"
|
|
7
|
+
require "active_sanction/error"
|
|
8
|
+
|
|
9
|
+
module ActiveSanction
|
|
10
|
+
# Every sanctions list this library knows how to read, filed under the key
|
|
11
|
+
# its adapter declares.
|
|
12
|
+
#
|
|
13
|
+
# ActiveSanction::Sources[:un_consolidated] # => the adapter class
|
|
14
|
+
# ActiveSanction::Sources.keys # => [:ofac_sdn, :un_consolidated, ...]
|
|
15
|
+
#
|
|
16
|
+
# The registry is open, and that is the point of this milestone. A bank with
|
|
17
|
+
# an internal watchlist, or a gem adding a jurisdiction this one has not got
|
|
18
|
+
# to, registers an adapter of its own without forking:
|
|
19
|
+
#
|
|
20
|
+
# ActiveSanction::Sources.register(MyCompany::InternalWatchlist)
|
|
21
|
+
# ActiveSanction.configure { |c| c.sources = %i[ofac_sdn my_internal_watchlist] }
|
|
22
|
+
#
|
|
23
|
+
# Nothing here requires Sources::Base. Registration is duck-typed on `.key`
|
|
24
|
+
# and `.new`, so a source backed by a database table rather than a published
|
|
25
|
+
# file -- which has no URL to declare and no payload to fetch -- is a
|
|
26
|
+
# first-class citizen rather than something that has to pretend to be a file
|
|
27
|
+
# download. Base is the convenient way to write an adapter, not the price of
|
|
28
|
+
# admission.
|
|
29
|
+
#
|
|
30
|
+
# Built-in adapters register themselves at the bottom of their own file, one
|
|
31
|
+
# explicit `Sources.register(self)` line each, rather than being enrolled by
|
|
32
|
+
# `inherited`. Auto-registering every subclass would also enrol the abstract
|
|
33
|
+
# intermediates that adapters sharing a publisher will want (an OFAC base
|
|
34
|
+
# holding the authority and the format for SDN and Consolidated both) and
|
|
35
|
+
# every throwaway subclass a test defines.
|
|
36
|
+
module Sources
|
|
37
|
+
# Two lists cannot answer to one name. Raised at load time, which is where
|
|
38
|
+
# this collision is cheap to fix.
|
|
39
|
+
#
|
|
40
|
+
# A ConfigurationError, like the two below it: all three are an
|
|
41
|
+
# installation wired up wrong -- a key typed twice, a key typed wrong, an
|
|
42
|
+
# adapter that never declared what it is -- and none of them is fixed by
|
|
43
|
+
# waiting and trying again.
|
|
44
|
+
class DuplicateKey < ConfigurationError; end
|
|
45
|
+
|
|
46
|
+
# A key nothing is registered under: a typo in `config.sources`, a CLI
|
|
47
|
+
# argument, or an adapter whose file was never required.
|
|
48
|
+
class UnknownSource < ConfigurationError; end
|
|
49
|
+
|
|
50
|
+
# An adapter that does not declare what the contract requires, or is asked
|
|
51
|
+
# for a declaration it never made.
|
|
52
|
+
class DeclarationError < ConfigurationError; end
|
|
53
|
+
|
|
54
|
+
# The bytes of one of a source's files could not be obtained -- the
|
|
55
|
+
# publisher confirmed a copy we do not hold, and re-asking for it in full
|
|
56
|
+
# did not produce one either.
|
|
57
|
+
#
|
|
58
|
+
# Retryable: it is a publisher or an intermediary cache in a state it will
|
|
59
|
+
# not be in an hour from now, and the next run usually just works.
|
|
60
|
+
class MissingPayload < FetchError
|
|
61
|
+
extend T::Sig
|
|
62
|
+
|
|
63
|
+
sig { returns(T::Boolean) }
|
|
64
|
+
def retryable? = retryable_or(true)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# @api private
|
|
68
|
+
MUTEX = T.let(Mutex.new, Mutex)
|
|
69
|
+
private_constant :MUTEX
|
|
70
|
+
|
|
71
|
+
# Built at load rather than memoized on first use. A lazily created Hash
|
|
72
|
+
# is itself a write, and the first two adapters to register in two threads
|
|
73
|
+
# would each create one and each keep the half of the registry the other
|
|
74
|
+
# could not see. Everything that writes to it holds MUTEX; reads do not,
|
|
75
|
+
# because a registry is written at load and read for the life of the
|
|
76
|
+
# process.
|
|
77
|
+
#
|
|
78
|
+
# @api private
|
|
79
|
+
REGISTRY = T.let({}, T::Hash[Symbol, T.untyped])
|
|
80
|
+
private_constant :REGISTRY
|
|
81
|
+
|
|
82
|
+
class << self
|
|
83
|
+
extend T::Sig
|
|
84
|
+
|
|
85
|
+
# An adapter is anything answering `.key` and `.new`, which is why every
|
|
86
|
+
# signature here says `T.untyped` where a class goes and none of them
|
|
87
|
+
# says `T.class_of(Sources::Base)`. That is the milestone this registry
|
|
88
|
+
# exists for: a bank's internal watchlist, backed by a database table
|
|
89
|
+
# with no URL and no payload, is a first-class source rather than
|
|
90
|
+
# something pretending to be a file download. Narrowing these types would
|
|
91
|
+
# quietly close what the class comment above promises is open.
|
|
92
|
+
|
|
93
|
+
# Adds a source to the registry and returns it. Registering the same
|
|
94
|
+
# class twice is a no-op, so a file that manages to get loaded under two
|
|
95
|
+
# paths does not take the whole process down with it.
|
|
96
|
+
sig { params(source: T.untyped).returns(T.untyped) }
|
|
97
|
+
def register(source)
|
|
98
|
+
key = registrable!(source)
|
|
99
|
+
MUTEX.synchronize do
|
|
100
|
+
claimed = registry[key]
|
|
101
|
+
raise DuplicateKey, duplicate_message(key, source, claimed) if claimed && claimed != source
|
|
102
|
+
|
|
103
|
+
registry[key] = source
|
|
104
|
+
end
|
|
105
|
+
source
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The adapter registered under a key.
|
|
109
|
+
#
|
|
110
|
+
# Raises rather than returning nil, because every caller of this method
|
|
111
|
+
# is resolving a name a human typed -- into `config.sources`, into
|
|
112
|
+
# `activesanction sync ofac_sdb` -- and a nil surfaces three layers later
|
|
113
|
+
# as a NoMethodError that says nothing about the misspelling. The message
|
|
114
|
+
# lists what *is* registered, which is also the answer to "why is my
|
|
115
|
+
# adapter not being picked up" (its file was never required).
|
|
116
|
+
sig { params(key: T.untyped).returns(T.untyped) }
|
|
117
|
+
def [](key)
|
|
118
|
+
name = key.to_sym
|
|
119
|
+
registry.fetch(name) do
|
|
120
|
+
raise UnknownSource, "no source registered as #{name.inspect}. Registered: #{list}"
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
sig { params(key: T.untyped).returns(T::Boolean) }
|
|
125
|
+
def registered?(key) = registry.key?(key.to_sym)
|
|
126
|
+
|
|
127
|
+
# Every registered adapter, ordered by key so a CLI listing and a sync
|
|
128
|
+
# summary do not reshuffle themselves between runs.
|
|
129
|
+
sig { returns(T::Array[T.untyped]) }
|
|
130
|
+
def all = registry.keys.sort.map { |key| registry[key] }
|
|
131
|
+
|
|
132
|
+
sig { returns(T::Array[Symbol]) }
|
|
133
|
+
def keys = registry.keys.sort
|
|
134
|
+
|
|
135
|
+
sig { returns(Integer) }
|
|
136
|
+
def size = registry.size
|
|
137
|
+
|
|
138
|
+
sig { returns(T::Boolean) }
|
|
139
|
+
def empty? = registry.empty?
|
|
140
|
+
|
|
141
|
+
# The adapters a sync should run: what `config.sources` names, or every
|
|
142
|
+
# registered source when it names nothing. An unknown key raises here,
|
|
143
|
+
# at the start of the run, rather than after the other lists have been
|
|
144
|
+
# downloaded.
|
|
145
|
+
sig { params(configured: T.nilable(T::Array[Symbol])).returns(T::Array[T.untyped]) }
|
|
146
|
+
def enabled(configured = ActiveSanction.config.sources)
|
|
147
|
+
return all if configured.nil?
|
|
148
|
+
|
|
149
|
+
configured.map { |key| self[key] }
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Drops a key and returns what was registered there, or nil. The
|
|
153
|
+
# supported way to replace a built-in adapter with a patched one:
|
|
154
|
+
#
|
|
155
|
+
# ActiveSanction::Sources.unregister(:ofac_sdn)
|
|
156
|
+
# ActiveSanction::Sources.register(MyCompany::PatchedOfacSdn)
|
|
157
|
+
#
|
|
158
|
+
# There is deliberately no `clear!`. Built-in adapters register on
|
|
159
|
+
# require, and `require` runs once per process: a suite that emptied the
|
|
160
|
+
# registry between examples would leave every later example running
|
|
161
|
+
# against a library that has forgotten its own sources.
|
|
162
|
+
sig { params(key: T.untyped).returns(T.untyped) }
|
|
163
|
+
def unregister(key)
|
|
164
|
+
MUTEX.synchronize { registry.delete(key.to_sym) }
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
private
|
|
168
|
+
|
|
169
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
170
|
+
def registry = REGISTRY
|
|
171
|
+
|
|
172
|
+
sig { returns(String) }
|
|
173
|
+
def list = registry.empty? ? "(nothing)" : keys.join(", ")
|
|
174
|
+
|
|
175
|
+
sig { params(source: T.untyped).returns(Symbol) }
|
|
176
|
+
def registrable!(source)
|
|
177
|
+
unless source.respond_to?(:key) && source.respond_to?(:new)
|
|
178
|
+
raise InvalidArgument, "a source must answer .key and .new, got #{source.inspect}"
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
Definition.key!(source.key)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
sig { params(key: Symbol, source: T.untyped, claimed: T.untyped).returns(String) }
|
|
185
|
+
def duplicate_message(key, source, claimed)
|
|
186
|
+
"cannot register #{source} as #{key.inspect}: #{claimed} already claims that key. A key is the public " \
|
|
187
|
+
"name of a list -- in configuration, in stored snapshots, in every match result that cites it -- so two " \
|
|
188
|
+
"lists cannot share one. Give the new source a different key, or call " \
|
|
189
|
+
"ActiveSanction::Sources.unregister(#{key.inspect}) first if you mean to replace #{claimed}."
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
require "active_sanction/sources/definition"
|
|
196
|
+
require "active_sanction/sources/base"
|