active_sanction 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +538 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/CONTRIBUTING.md +312 -0
- data/LICENSE.txt +21 -0
- data/README.md +616 -0
- data/SECURITY.md +97 -0
- data/active_sanction.gemspec +98 -0
- data/docs/adding_a_source.md +1083 -0
- data/docs/api_stability.md +357 -0
- data/docs/bundle_format.md +407 -0
- data/lib/active_sanction/address.rb +170 -0
- data/lib/active_sanction/client.rb +341 -0
- data/lib/active_sanction/configuration.rb +721 -0
- data/lib/active_sanction/countries.txt +292 -0
- data/lib/active_sanction/country.rb +164 -0
- data/lib/active_sanction/deprecation.rb +168 -0
- data/lib/active_sanction/diff/change.rb +208 -0
- data/lib/active_sanction/diff.rb +348 -0
- data/lib/active_sanction/doctor/checkup.rb +356 -0
- data/lib/active_sanction/doctor/diagnosis.rb +245 -0
- data/lib/active_sanction/doctor/finding.rb +183 -0
- data/lib/active_sanction/doctor/profile.rb +384 -0
- data/lib/active_sanction/doctor/report.rb +227 -0
- data/lib/active_sanction/doctor.rb +347 -0
- data/lib/active_sanction/entity.rb +293 -0
- data/lib/active_sanction/error.rb +332 -0
- data/lib/active_sanction/fetcher/result.rb +116 -0
- data/lib/active_sanction/fetcher.rb +242 -0
- data/lib/active_sanction/http_client/errors.rb +94 -0
- data/lib/active_sanction/http_client/response.rb +129 -0
- data/lib/active_sanction/http_client.rb +324 -0
- data/lib/active_sanction/identifier.rb +212 -0
- data/lib/active_sanction/index/builder.rb +89 -0
- data/lib/active_sanction/index/candidate.rb +63 -0
- data/lib/active_sanction/index/entry.rb +66 -0
- data/lib/active_sanction/index/features.rb +112 -0
- data/lib/active_sanction/index.rb +381 -0
- data/lib/active_sanction/match_result.rb +345 -0
- data/lib/active_sanction/matcher.rb +388 -0
- data/lib/active_sanction/name.rb +183 -0
- data/lib/active_sanction/normalizer/cache.rb +82 -0
- data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
- data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
- data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
- data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
- data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
- data/lib/active_sanction/normalizer/dictionary.rb +243 -0
- data/lib/active_sanction/normalizer/form.rb +279 -0
- data/lib/active_sanction/normalizer.rb +137 -0
- data/lib/active_sanction/parsers/column_shape.rb +274 -0
- data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
- data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
- data/lib/active_sanction/parsers/delimited_table.rb +151 -0
- data/lib/active_sanction/parsers/format.rb +107 -0
- data/lib/active_sanction/parsers/join.rb +149 -0
- data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
- data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
- data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
- data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
- data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
- data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
- data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
- data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
- data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
- data/lib/active_sanction/parsers/xml_records.rb +140 -0
- data/lib/active_sanction/parsers.rb +94 -0
- data/lib/active_sanction/partial_date/parser.rb +124 -0
- data/lib/active_sanction/partial_date.rb +303 -0
- data/lib/active_sanction/payload_cache/checksum.rb +66 -0
- data/lib/active_sanction/payload_cache/entry.rb +304 -0
- data/lib/active_sanction/payload_cache.rb +366 -0
- data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
- data/lib/active_sanction/phonetics.rb +40 -0
- data/lib/active_sanction/query.rb +290 -0
- data/lib/active_sanction/rescreen/alert.rb +314 -0
- data/lib/active_sanction/rescreen.rb +332 -0
- data/lib/active_sanction/scorer/adjustments.rb +241 -0
- data/lib/active_sanction/scorer/name_score.rb +279 -0
- data/lib/active_sanction/scorer/reason.rb +136 -0
- data/lib/active_sanction/scorer/result.rb +129 -0
- data/lib/active_sanction/scorer/subject.rb +235 -0
- data/lib/active_sanction/scorer/weights.rb +306 -0
- data/lib/active_sanction/scorer.rb +303 -0
- data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
- data/lib/active_sanction/similarity/levenshtein.rb +179 -0
- data/lib/active_sanction/similarity/token_set.rb +159 -0
- data/lib/active_sanction/similarity/token_sort.rb +120 -0
- data/lib/active_sanction/similarity.rb +183 -0
- data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
- data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
- data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
- data/lib/active_sanction/snapshot/bundle.rb +381 -0
- data/lib/active_sanction/snapshot.rb +305 -0
- data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
- data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
- data/lib/active_sanction/sources/australia_dfat.rb +291 -0
- data/lib/active_sanction/sources/base.rb +295 -0
- data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
- data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
- data/lib/active_sanction/sources/canada_sema.rb +153 -0
- data/lib/active_sanction/sources/definition.rb +360 -0
- data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
- data/lib/active_sanction/sources/eu_fsf.rb +203 -0
- data/lib/active_sanction/sources/ofac/record.rb +251 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
- data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
- data/lib/active_sanction/sources/ofac.rb +248 -0
- data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
- data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
- data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
- data/lib/active_sanction/sources/remarks.rb +84 -0
- data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
- data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
- data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
- data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
- data/lib/active_sanction/sources/un_consolidated.rb +140 -0
- data/lib/active_sanction/sources.rb +196 -0
- data/lib/active_sanction/storage/active_record/reader.rb +124 -0
- data/lib/active_sanction/storage/active_record/row.rb +175 -0
- data/lib/active_sanction/storage/active_record/writer.rb +141 -0
- data/lib/active_sanction/storage/active_record.rb +277 -0
- data/lib/active_sanction/storage/base.rb +216 -0
- data/lib/active_sanction/storage/file_system.rb +373 -0
- data/lib/active_sanction/storage/memory.rb +67 -0
- data/lib/active_sanction/storage/meta.rb +188 -0
- data/lib/active_sanction/storage.rb +74 -0
- data/lib/active_sanction/subject.rb +271 -0
- data/lib/active_sanction/sync/report.rb +245 -0
- data/lib/active_sanction/sync/result.rb +296 -0
- data/lib/active_sanction/sync.rb +390 -0
- data/lib/active_sanction/validator_store/file_system.rb +103 -0
- data/lib/active_sanction/validator_store/memory.rb +41 -0
- data/lib/active_sanction/validator_store.rb +119 -0
- data/lib/active_sanction/validators.rb +195 -0
- data/lib/active_sanction/version.rb +23 -0
- data/lib/active_sanction.rb +331 -0
- data/lib/generators/active_sanction/install/install_generator.rb +60 -0
- data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
- metadata +236 -0
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/identifier"
|
|
7
|
+
require "active_sanction/name"
|
|
8
|
+
require "active_sanction/partial_date"
|
|
9
|
+
|
|
10
|
+
module ActiveSanction
|
|
11
|
+
module Sources
|
|
12
|
+
class Ofac < Base
|
|
13
|
+
# Reads the fields OFAC publishes no columns for out of the free text it
|
|
14
|
+
# packs them into.
|
|
15
|
+
#
|
|
16
|
+
# parsed = RemarksParser.new("DOB 10 Dec 1948; POB Egypt; Passport 123456 (Egypt)")
|
|
17
|
+
# parsed.dates_of_birth # => [#<PartialDate 1948-12-10>]
|
|
18
|
+
# parsed.places_of_birth # => ["Egypt"]
|
|
19
|
+
# parsed.identifiers # => [#<Identifier :passport "123456" country="Egypt">]
|
|
20
|
+
# parsed.unrecognized # => []
|
|
21
|
+
#
|
|
22
|
+
# SDN.CSV has no date of birth, place of birth, nationality or passport
|
|
23
|
+
# column. All of it -- 88,827 semicolon-delimited segments across 19,015
|
|
24
|
+
# remarks -- is prose in one field, written for a human reading a page.
|
|
25
|
+
# Until it is read, every OFAC entity carries names and nothing else, and
|
|
26
|
+
# names alone are what makes a screening tool cry wolf: the secondary
|
|
27
|
+
# identifiers that clear a false positive (#32) are all in here.
|
|
28
|
+
#
|
|
29
|
+
# ### Nothing is ever removed from the remark
|
|
30
|
+
#
|
|
31
|
+
# Extraction is additive. `Entity#remarks` keeps the publisher's whole
|
|
32
|
+
# string byte for byte whether this class understood it or not, so a
|
|
33
|
+
# pattern that drifts costs structure and never content. That is the one
|
|
34
|
+
# rule this file must not break: these are heuristics against text a
|
|
35
|
+
# government writes for people, they will go stale, and silent data loss
|
|
36
|
+
# in a compliance tool is the worst failure there is. A segment we cannot
|
|
37
|
+
# read is still in front of the user, in the publisher's own words.
|
|
38
|
+
#
|
|
39
|
+
# ### How a segment is read
|
|
40
|
+
#
|
|
41
|
+
# Segments are split on `;` and matched against a label vocabulary. A
|
|
42
|
+
# segment either extracts a value, or matches a shape known to be prose
|
|
43
|
+
# (`Secondary sanctions risk: ...`, which is 12% of the file and carries
|
|
44
|
+
# nothing structured), or is unrecognized -- and #unrecognized is what
|
|
45
|
+
# Coverage counts, so drift shows up as a number rather than as a bug
|
|
46
|
+
# report years later.
|
|
47
|
+
#
|
|
48
|
+
# Nothing here raises. A malformed segment -- `Passport issued in
|
|
49
|
+
# Sarajevo`, which names no passport at all -- reads as unrecognized and
|
|
50
|
+
# stays in the remark, because one unparseable clause must never cost the
|
|
51
|
+
# entity around it.
|
|
52
|
+
class RemarksParser
|
|
53
|
+
extend T::Sig
|
|
54
|
+
|
|
55
|
+
SEPARATOR = T.let(";", String)
|
|
56
|
+
|
|
57
|
+
# `alt.` prefixes a repeat: "DOB 1955; alt. DOB 1956" is one person two
|
|
58
|
+
# governments reported differently. It marks repetition and nothing
|
|
59
|
+
# else, so it is stripped and what follows is matched normally -- which
|
|
60
|
+
# is how multiple values of one kind fall out without a second rule.
|
|
61
|
+
ALTERNATE = T.let(/\Aalt\.\s+/i, Regexp)
|
|
62
|
+
|
|
63
|
+
# The full stop that ends a remark belongs to the sentence, not to the
|
|
64
|
+
# value: "Gender Male." is not a gender spelled with a period.
|
|
65
|
+
TRAILING_STOP = T.let(/\s*\.\z/, Regexp)
|
|
66
|
+
|
|
67
|
+
# OFAC publishes wallet addresses as `Digital Currency Address - XBT
|
|
68
|
+
# 1abc...`, one label per currency. They are matched by shape rather
|
|
69
|
+
# than enumerated: the list of currencies grows every time a new one is
|
|
70
|
+
# designated, and none of them changes how the address is read.
|
|
71
|
+
DIGITAL_CURRENCY = T.let(
|
|
72
|
+
/\ADigital Currency Address\s*-\s*(?<currency>[[:alnum:]]+)\s+(?<value>\S+)\z/i, Regexp
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
# The parenthesised qualifiers a document segment ends in: `Passport
|
|
76
|
+
# 123456 (Egypt)`, and sometimes two -- `Folio Mercantil No. 22839
|
|
77
|
+
# (Jalisco) (Mexico)` is a state and then a country. The space is what
|
|
78
|
+
# makes it a qualifier rather than part of the number: Hong Kong writes
|
|
79
|
+
# its ID numbers as `D489833(9)`, and splitting that off would leave a
|
|
80
|
+
# document number the issuing government would not recognize.
|
|
81
|
+
TRAILING_QUALIFIER = T.let(/\s+\(([^)]*)\)\s*\z/, Regexp)
|
|
82
|
+
|
|
83
|
+
# `Passport ZG4109521 (Pakistan) issued 07 Jun 2008 expires 06 Jun 2013`
|
|
84
|
+
# -- either clause may be absent and both may be present, so the value
|
|
85
|
+
# ends wherever the first one starts.
|
|
86
|
+
DATE_KEYWORDS = T.let(/\b(?:issued|expires|expired)\b/i, Regexp)
|
|
87
|
+
DATE_CLAUSE = T.let(/(issued|expires|expired)\s+(?:on\s+)?(.*?)(?=\s+#{DATE_KEYWORDS}|\z)/i, Regexp)
|
|
88
|
+
|
|
89
|
+
# A document number is a code, not a sentence. `Passport 265 216` and
|
|
90
|
+
# `SWIFT/BIC SBERRUMM` are numbers; `License to operate` is prose that
|
|
91
|
+
# happened to open with a label, and without this rule it would become
|
|
92
|
+
# an identifier that matches nothing and misleads everyone.
|
|
93
|
+
CODE = T.let(%r{\A[[:alnum:]][[:alnum:]\s._()/-]*\z}, Regexp)
|
|
94
|
+
DIGIT = T.let(/\d/, Regexp)
|
|
95
|
+
|
|
96
|
+
# No government issues a document number this long -- China's 18-digit
|
|
97
|
+
# social credit code is the longest in the file. What exceeds it is a
|
|
98
|
+
# clause that ran on: "Passport OR801168 and Kuwaiti National ID No.
|
|
99
|
+
# 281020505755 issued under the name ..." is one segment OFAC wrote as
|
|
100
|
+
# a sentence, and an identifier built from all of it would match
|
|
101
|
+
# nothing and mislead whoever read it.
|
|
102
|
+
MAX_CODE_LENGTH = T.let(40, Integer)
|
|
103
|
+
|
|
104
|
+
QUOTES = T.let(/\A['"“”‘’]|['"“”‘’]\z/, Regexp)
|
|
105
|
+
|
|
106
|
+
# The remark as published, and how this parser read it: every segment,
|
|
107
|
+
# those it extracted something from, those it recognized as prose, and
|
|
108
|
+
# those it could not read -- which is what Coverage counts.
|
|
109
|
+
sig { returns(String) }
|
|
110
|
+
attr_reader :text
|
|
111
|
+
|
|
112
|
+
sig { returns(T::Array[String]) }
|
|
113
|
+
attr_reader :segments
|
|
114
|
+
|
|
115
|
+
sig { returns(T::Array[String]) }
|
|
116
|
+
attr_reader :extracted
|
|
117
|
+
|
|
118
|
+
sig { returns(T::Array[String]) }
|
|
119
|
+
attr_reader :prose
|
|
120
|
+
|
|
121
|
+
sig { returns(T::Array[String]) }
|
|
122
|
+
attr_reader :unrecognized
|
|
123
|
+
|
|
124
|
+
sig { returns(T::Array[PartialDate]) }
|
|
125
|
+
attr_reader :dates_of_birth
|
|
126
|
+
|
|
127
|
+
sig { returns(T::Array[String]) }
|
|
128
|
+
attr_reader :places_of_birth
|
|
129
|
+
|
|
130
|
+
sig { returns(T::Array[String]) }
|
|
131
|
+
attr_reader :nationalities
|
|
132
|
+
|
|
133
|
+
sig { returns(T::Array[String]) }
|
|
134
|
+
attr_reader :genders
|
|
135
|
+
|
|
136
|
+
sig { returns(T::Array[Name]) }
|
|
137
|
+
attr_reader :aliases
|
|
138
|
+
|
|
139
|
+
sig { returns(T::Array[Identifier]) }
|
|
140
|
+
attr_reader :identifiers
|
|
141
|
+
|
|
142
|
+
sig { params(text: T.untyped).void }
|
|
143
|
+
def initialize(text)
|
|
144
|
+
@text = T.let(text.to_s, String)
|
|
145
|
+
@segments = T.let([], T::Array[String])
|
|
146
|
+
@extracted = T.let([], T::Array[String])
|
|
147
|
+
@prose = T.let([], T::Array[String])
|
|
148
|
+
@unrecognized = T.let([], T::Array[String])
|
|
149
|
+
@dates_of_birth = T.let([], T::Array[PartialDate])
|
|
150
|
+
@places_of_birth = T.let([], T::Array[String])
|
|
151
|
+
@nationalities = T.let([], T::Array[String])
|
|
152
|
+
@genders = T.let([], T::Array[String])
|
|
153
|
+
@aliases = T.let([], T::Array[Name])
|
|
154
|
+
@identifiers = T.let([], T::Array[Identifier])
|
|
155
|
+
read
|
|
156
|
+
freeze
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# True when the remark yielded anything structured at all.
|
|
160
|
+
sig { returns(T::Boolean) }
|
|
161
|
+
def any? = extracted.any?
|
|
162
|
+
|
|
163
|
+
private
|
|
164
|
+
|
|
165
|
+
sig { void }
|
|
166
|
+
def read
|
|
167
|
+
text.split(SEPARATOR).each do |raw|
|
|
168
|
+
segment = raw.strip
|
|
169
|
+
next if segment.empty?
|
|
170
|
+
|
|
171
|
+
@segments << segment
|
|
172
|
+
classify(segment)
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
sig { params(segment: String).void }
|
|
177
|
+
def classify(segment)
|
|
178
|
+
body = segment.sub(ALTERNATE, "")
|
|
179
|
+
if field(body) || document(body) || digital_currency(body)
|
|
180
|
+
@extracted << segment
|
|
181
|
+
elsif Vocabulary::PROSE_PATTERN.match?(body)
|
|
182
|
+
@prose << segment
|
|
183
|
+
else
|
|
184
|
+
@unrecognized << segment
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
sig { params(body: String).returns(T.untyped) }
|
|
189
|
+
def field(body)
|
|
190
|
+
match = Vocabulary::FIELD_PATTERN.match(body)
|
|
191
|
+
return nil unless match
|
|
192
|
+
|
|
193
|
+
value = trim(match[:value])
|
|
194
|
+
return nil if value.nil?
|
|
195
|
+
|
|
196
|
+
case Vocabulary::FIELD_KINDS.fetch(T.must(match[:label]).downcase)
|
|
197
|
+
when :date_of_birth then born(value)
|
|
198
|
+
when :place_of_birth then keep(@places_of_birth, value)
|
|
199
|
+
when :nationality then keep(@nationalities, value)
|
|
200
|
+
when :gender then keep(@genders, value)
|
|
201
|
+
else known_as(Vocabulary::FIELD_KINDS.fetch(T.must(match[:label]).downcase), value)
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# An unreadable date is not a date. It reads as unrecognized rather
|
|
206
|
+
# than as a nil the entity would carry around, which puts it in the
|
|
207
|
+
# coverage histogram where a new OFAC spelling can be seen.
|
|
208
|
+
sig { params(value: String).returns(T.untyped) }
|
|
209
|
+
def born(value)
|
|
210
|
+
date = PartialDate.parse(value)
|
|
211
|
+
return nil if date.nil?
|
|
212
|
+
|
|
213
|
+
keep(@dates_of_birth, date)
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# `a.k.a. 'EL SENOR'` -- quoted, and the quotes are OFAC's punctuation
|
|
217
|
+
# rather than part of the name. 4,325 of the 4,349 inline aliases
|
|
218
|
+
# appear nowhere in ALT.CSV, so these are names the list publishes here
|
|
219
|
+
# and only here.
|
|
220
|
+
sig { params(kind: Symbol, value: String).returns(T.untyped) }
|
|
221
|
+
def known_as(kind, value)
|
|
222
|
+
keep(@aliases, Name.new(value: value.sub(QUOTES, "").sub(QUOTES, ""), kind: kind))
|
|
223
|
+
rescue ArgumentError
|
|
224
|
+
nil
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
sig { params(body: String).returns(T.untyped) }
|
|
228
|
+
def document(body)
|
|
229
|
+
match = Vocabulary::DOCUMENT_PATTERN.match(body)
|
|
230
|
+
return nil unless match
|
|
231
|
+
|
|
232
|
+
rest = trim(match[:rest])
|
|
233
|
+
return nil if rest.nil?
|
|
234
|
+
|
|
235
|
+
label = T.must(match[:label])
|
|
236
|
+
identify(label, Vocabulary::DOCUMENT_KINDS.fetch(label.downcase), rest)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
sig { params(label: String, kind: Symbol, rest: String).returns(T.untyped) }
|
|
240
|
+
def identify(label, kind, rest)
|
|
241
|
+
head, issued_on, expires_on = split(rest)
|
|
242
|
+
value, qualifiers = unwrap(head)
|
|
243
|
+
return nil unless code?(value)
|
|
244
|
+
|
|
245
|
+
keep(@identifiers, Identifier.new(kind: kind, value: value, country: qualifiers.last,
|
|
246
|
+
issued_on: issued_on, expires_on: expires_on,
|
|
247
|
+
note: note(label, qualifiers)))
|
|
248
|
+
rescue ArgumentError
|
|
249
|
+
nil
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
sig { params(rest: String).returns([String, T.nilable(PartialDate), T.nilable(PartialDate)]) }
|
|
253
|
+
def split(rest)
|
|
254
|
+
dates = T.let({}, T::Hash[String, T.nilable(PartialDate)])
|
|
255
|
+
rest.scan(DATE_CLAUSE) { |keyword, date| dates[keyword.downcase] ||= PartialDate.parse(date) }
|
|
256
|
+
[rest.split(/\s+#{DATE_KEYWORDS}/, 2).first.to_s, dates["issued"], dates["expires"] || dates["expired"]]
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
# Outermost qualifier last, which is the one Identifier has a country
|
|
260
|
+
# field for. Anything inside it -- a state, a province -- has no home
|
|
261
|
+
# on the record and goes to the note rather than being dropped.
|
|
262
|
+
sig { params(head: String).returns([String, T::Array[String]]) }
|
|
263
|
+
def unwrap(head)
|
|
264
|
+
qualifiers = T.let([], T::Array[String])
|
|
265
|
+
text = head.dup
|
|
266
|
+
qualifiers.unshift(T.must(Regexp.last_match(1))) while text.sub!(TRAILING_QUALIFIER, "")
|
|
267
|
+
[text.strip, qualifiers]
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
sig { params(label: String, qualifiers: T::Array[String]).returns(String) }
|
|
271
|
+
def note(label, qualifiers)
|
|
272
|
+
inner = qualifiers[0..-2].to_a
|
|
273
|
+
inner.empty? ? label : "#{label} (#{inner.join(", ")})"
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
sig { params(body: String).returns(T.untyped) }
|
|
277
|
+
def digital_currency(body)
|
|
278
|
+
match = DIGITAL_CURRENCY.match(trim(body).to_s)
|
|
279
|
+
return nil unless match
|
|
280
|
+
|
|
281
|
+
currency = T.must(match[:currency]).upcase
|
|
282
|
+
keep(@identifiers, Identifier.new(kind: :other, value: match[:value], note: "#{currency} address"))
|
|
283
|
+
rescue ArgumentError
|
|
284
|
+
nil
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
sig { params(value: String).returns(T::Boolean) }
|
|
288
|
+
def code?(value)
|
|
289
|
+
return false if value.empty? || value.length > MAX_CODE_LENGTH || !CODE.match?(value)
|
|
290
|
+
|
|
291
|
+
DIGIT.match?(value) || value == value.upcase
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
# Returns the list, which is truthy: a caller reads "something was
|
|
295
|
+
# kept" from it, and a miss answers nil.
|
|
296
|
+
sig { params(list: T::Array[T.untyped], value: T.untyped).returns(T::Array[T.untyped]) }
|
|
297
|
+
def keep(list, value)
|
|
298
|
+
list << value
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
sig { params(value: T.untyped).returns(T.nilable(String)) }
|
|
302
|
+
def trim(value)
|
|
303
|
+
string = value.to_s.sub(TRAILING_STOP, "").strip
|
|
304
|
+
string.empty? ? nil : string
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
require "active_sanction/sources/ofac/remarks_parser/vocabulary"
|
|
312
|
+
require "active_sanction/sources/ofac/remarks_parser/coverage"
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/parsers"
|
|
7
|
+
require "active_sanction/sources"
|
|
8
|
+
require "active_sanction/sources/base"
|
|
9
|
+
|
|
10
|
+
module ActiveSanction
|
|
11
|
+
module Sources
|
|
12
|
+
# What the two OFAC lists have in common, which is nearly everything.
|
|
13
|
+
#
|
|
14
|
+
# OFAC publishes the SDN list and the Consolidated (non-SDN) list in
|
|
15
|
+
# exactly the same shape: three headerless CSVs joined on `ent_num`, the
|
|
16
|
+
# primary file carrying twelve columns, ALT carrying five and ADD six, all
|
|
17
|
+
# of them in Windows-1252 with `-0- ` for null. Only the file names, the
|
|
18
|
+
# sizes and the legal consequence of being on one differ.
|
|
19
|
+
#
|
|
20
|
+
# SDN.CSV 19,321 rows CONS_PRIM.CSV 481 rows
|
|
21
|
+
# ALT.CSV 20,147 rows CONS_ALT.CSV 1,109 rows
|
|
22
|
+
# ADD.CSV 25,078 rows CONS_ADD.CSV 614 rows
|
|
23
|
+
#
|
|
24
|
+
# So the reading is written once, here, and an adapter says only which
|
|
25
|
+
# list it is and where its three files live. Abstract: it declares no key
|
|
26
|
+
# and does not register, because there is no such list as "OFAC".
|
|
27
|
+
#
|
|
28
|
+
# ### The fields OFAC publishes no columns for
|
|
29
|
+
#
|
|
30
|
+
# Neither file has a date of birth, place of birth, nationality or
|
|
31
|
+
# passport column. All of it is prose in `Remarks`:
|
|
32
|
+
#
|
|
33
|
+
# "DOB 10 Dec 1948; POB Egypt; nationality Egypt; Passport 123456 (Egypt)"
|
|
34
|
+
#
|
|
35
|
+
# RemarksParser reads it, and #remarks_coverage reports how much of it it
|
|
36
|
+
# understood -- 97% of the SDN file's 88,827 segments, which is a figure to
|
|
37
|
+
# watch rather than a guarantee. The remark is kept verbatim either way, so
|
|
38
|
+
# a pattern that goes stale costs structure and never content.
|
|
39
|
+
#
|
|
40
|
+
# This class is written entirely against the public extension points:
|
|
41
|
+
# declarations from Definition, fetch and cache and checksum from Base,
|
|
42
|
+
# reading and joining from Parsers. It required no change to any of them,
|
|
43
|
+
# which is the property M4 exists to prove.
|
|
44
|
+
#
|
|
45
|
+
# @api private
|
|
46
|
+
class Ofac < Base
|
|
47
|
+
extend T::Sig
|
|
48
|
+
|
|
49
|
+
jurisdiction :us
|
|
50
|
+
authority "U.S. Department of the Treasury, Office of Foreign Assets Control"
|
|
51
|
+
format :csv
|
|
52
|
+
|
|
53
|
+
# A work of the United States government, which 17 U.S.C. Section 105
|
|
54
|
+
# puts outside copyright protection. Treasury restricts its seals and
|
|
55
|
+
# its name rather than the data, which is why the notice mentions them:
|
|
56
|
+
# republishing the list is unencumbered, implying Treasury endorsed you
|
|
57
|
+
# is not.
|
|
58
|
+
licence_notice "A work of the U.S. government and not subject to " \
|
|
59
|
+
"domestic copyright (17 U.S.C. 105). Treasury's seals " \
|
|
60
|
+
"and marks are restricted separately, and nothing here " \
|
|
61
|
+
"may imply endorsement. Verified 2026-09-11."
|
|
62
|
+
licence_url "https://home.treasury.gov/subfooter/site-policies-and-notices"
|
|
63
|
+
|
|
64
|
+
# The one floor either OFAC list commits to, and only for the run that
|
|
65
|
+
# has no previous snapshot to compare against. RemarksParser reads about
|
|
66
|
+
# 97% of the SDN file's segments; 90% is a long way below anything the
|
|
67
|
+
# file has done and still far above what a re-spelled document label
|
|
68
|
+
# would leave. There is deliberately no floor on the record count: the
|
|
69
|
+
# SDN list has roughly tripled since 2010, so any bound wide enough to
|
|
70
|
+
# have survived that is too wide to catch a truncated download, and the
|
|
71
|
+
# previous snapshot catches one for nothing. See Definition#floor.
|
|
72
|
+
floor :remarks_coverage, 0.90
|
|
73
|
+
|
|
74
|
+
# OFAC serves Windows-1252, not UTF-8, and says nothing about it in a
|
|
75
|
+
# header. Read as UTF-8 the accented names in the list -- and there are
|
|
76
|
+
# thousands -- arrive as replacement characters.
|
|
77
|
+
ENCODING = T.let(Encoding::WINDOWS_1252, Encoding)
|
|
78
|
+
|
|
79
|
+
# OFAC writes "-0- " for null, with a trailing space, in every one of its
|
|
80
|
+
# files and roughly a quarter of a million times overall.
|
|
81
|
+
NULL = T.let("-0-", String)
|
|
82
|
+
|
|
83
|
+
# Column names, positional: all six files ship without a header row.
|
|
84
|
+
# Declaring them here also pins each file's width, so a column inserted
|
|
85
|
+
# upstream surfaces as a warning on every row rather than as 19,321
|
|
86
|
+
# entities quietly built from shifted fields.
|
|
87
|
+
PRIMARY_COLUMNS = T.let(%i[
|
|
88
|
+
ent_num sdn_name sdn_type program title call_sign vessel_type
|
|
89
|
+
tonnage gross_registered_tonnage vessel_flag vessel_owner remarks
|
|
90
|
+
].freeze, T::Array[Symbol])
|
|
91
|
+
|
|
92
|
+
ALT_COLUMNS = T.let(%i[ent_num alt_num alt_type alt_name alt_remarks].freeze, T::Array[Symbol])
|
|
93
|
+
|
|
94
|
+
ADD_COLUMNS = T.let(
|
|
95
|
+
%i[ent_num add_num address city_state_province_postal_code country add_remarks].freeze,
|
|
96
|
+
T::Array[Symbol]
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
PRIMARY = T.let(
|
|
100
|
+
Parsers::DelimitedTable.new(columns: PRIMARY_COLUMNS, null: NULL, encoding: ENCODING),
|
|
101
|
+
Parsers::DelimitedTable
|
|
102
|
+
)
|
|
103
|
+
ALT = T.let(
|
|
104
|
+
Parsers::DelimitedTable.new(columns: ALT_COLUMNS, null: NULL, encoding: ENCODING),
|
|
105
|
+
Parsers::DelimitedTable
|
|
106
|
+
)
|
|
107
|
+
ADD = T.let(
|
|
108
|
+
Parsers::DelimitedTable.new(columns: ADD_COLUMNS, null: NULL, encoding: ENCODING),
|
|
109
|
+
Parsers::DelimitedTable
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
# OFAC's join key. Every row of all six files carries one, and it is the
|
|
113
|
+
# only column on the primary file whose *values* have a shape.
|
|
114
|
+
ENT_NUM = T.let(/\A\d+\z/, Regexp)
|
|
115
|
+
|
|
116
|
+
# Rows that could not be read, and child rows that matched no entity.
|
|
117
|
+
# Populated by #parse and read afterwards -- sync orchestration (#34)
|
|
118
|
+
# reports them, and a nonzero orphan count is the signal that the three
|
|
119
|
+
# files were downloaded at different moments and no longer agree.
|
|
120
|
+
sig { returns(T::Array[Parsers::Warning]) }
|
|
121
|
+
attr_reader :warnings
|
|
122
|
+
|
|
123
|
+
# Child rows that matched no entity, by file -- a nonzero count means the
|
|
124
|
+
# three files were downloaded at different moments.
|
|
125
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
126
|
+
attr_reader :orphans
|
|
127
|
+
|
|
128
|
+
# How much of OFAC's free text the last #parse understood. Not a warning,
|
|
129
|
+
# because an unread segment is not an error -- it is still in the remark,
|
|
130
|
+
# in the publisher's own words -- but a figure to watch: it is the only
|
|
131
|
+
# thing that moves when OFAC changes how it writes a passport line.
|
|
132
|
+
sig { returns(RemarksParser::Coverage) }
|
|
133
|
+
attr_reader :remarks_coverage
|
|
134
|
+
|
|
135
|
+
sig { params(args: T.untyped, options: T.untyped).void }
|
|
136
|
+
def initialize(*args, **options)
|
|
137
|
+
super
|
|
138
|
+
@warnings = T.let([], T::Array[Parsers::Warning])
|
|
139
|
+
@orphans = T.let({}, T::Hash[Symbol, T.untyped])
|
|
140
|
+
@remarks_coverage = T.let(RemarksParser::Coverage.new, RemarksParser::Coverage)
|
|
141
|
+
@unmapped = T.let([], T::Array[Parsers::Warning])
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
sig { override.params(raw: T.untyped).returns(T::Array[Entity]) }
|
|
145
|
+
def parse(raw)
|
|
146
|
+
join = Parsers::Join.new(on: :ent_num, aliases: ALT.read(raw[:alt]), addresses: ADD.read(raw[:add]))
|
|
147
|
+
entities = build(join, PRIMARY.read(raw[primary_file]))
|
|
148
|
+
@warnings = join.warnings + @unmapped
|
|
149
|
+
@orphans = join.orphans
|
|
150
|
+
entities
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# What the two assertable columns of the primary file hold.
|
|
154
|
+
#
|
|
155
|
+
# The width declared in PRIMARY_COLUMNS catches a column OFAC *inserts*:
|
|
156
|
+
# every row arrives the wrong width and every row says so. Nothing about
|
|
157
|
+
# the width catches one OFAC *reorders*, which parses cleanly and builds
|
|
158
|
+
# 19,321 entities out of shifted fields. These are what catches that.
|
|
159
|
+
#
|
|
160
|
+
# Two of twelve, because ten of the columns are free text a government
|
|
161
|
+
# writes for people and there is nothing to assert about them. See
|
|
162
|
+
# Parsers::ColumnShape.
|
|
163
|
+
#
|
|
164
|
+
# `sdn_type` is held to three quarters rather than the 99% `ent_num` is,
|
|
165
|
+
# and the difference is what each one would cost to get wrong. OFAC does
|
|
166
|
+
# publish types this adapter has never seen -- that is what
|
|
167
|
+
# Record#unknown_type? exists for, and a new one is an `info` finding and
|
|
168
|
+
# a mapping to write, not a broken file. Demanding 99% would turn the
|
|
169
|
+
# week OFAC coins a word into an `error` on a list that is entirely fine,
|
|
170
|
+
# and a diagnostic that cries wolf about a healthy list is one nobody
|
|
171
|
+
# reads the week it is right. What three quarters still catches is the
|
|
172
|
+
# only thing worth an error here: a column that has stopped being the
|
|
173
|
+
# type column at all, which reads as zero.
|
|
174
|
+
sig { returns(T::Array[Parsers::ColumnShape]) }
|
|
175
|
+
def column_assertions
|
|
176
|
+
[Parsers::ColumnShape.new(name: :ent_num, matches: ENT_NUM, description: "numeric"),
|
|
177
|
+
Parsers::ColumnShape.new(name: :sdn_type, allowing: record_class.published_types, at_least: 0.75,
|
|
178
|
+
description: "a published SDN_Type")]
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# One extra pass over the primary file, for the doctor and nothing else.
|
|
182
|
+
# It is the same reader the parse used, asked a different question.
|
|
183
|
+
sig { override.params(raw: T.untyped).returns(T::Array[Parsers::ColumnShape::Tally]) }
|
|
184
|
+
def column_shapes(raw)
|
|
185
|
+
rows = PRIMARY.read(raw[primary_file]).to_a
|
|
186
|
+
column_assertions.map { |shape| shape.tally(rows.map { |row| row[shape.name] }) }
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# The declaration name of the file carrying one row per entity -- `:sdn`
|
|
190
|
+
# for the SDN list, `:prim` for the consolidated one. The first file an
|
|
191
|
+
# adapter declares, by the convention that the primary comes before the
|
|
192
|
+
# children it is joined to.
|
|
193
|
+
sig { returns(Symbol) }
|
|
194
|
+
def primary_file = T.must(urls.keys.first)
|
|
195
|
+
|
|
196
|
+
private
|
|
197
|
+
|
|
198
|
+
# The class each joined row is handed to. Overridden by an adapter whose
|
|
199
|
+
# list publishes something the shared mapping does not know about.
|
|
200
|
+
sig { returns(T.untyped) }
|
|
201
|
+
def record_class = Record
|
|
202
|
+
|
|
203
|
+
# Coverage is folded over every row, the nameless ones that produce no
|
|
204
|
+
# entity included: the question it answers is how much of the file this
|
|
205
|
+
# parser can read, and a row we drop is still a row OFAC published.
|
|
206
|
+
sig { params(join: Parsers::Join, rows: T.untyped).returns(T::Array[Entity]) }
|
|
207
|
+
def build(join, rows)
|
|
208
|
+
@unmapped = []
|
|
209
|
+
@remarks_coverage = RemarksParser::Coverage.new
|
|
210
|
+
entities = []
|
|
211
|
+
join.each(rows) do |row, related|
|
|
212
|
+
record = record_class.new(row: row, source: key, aliases: related[:aliases],
|
|
213
|
+
addresses: related[:addresses])
|
|
214
|
+
note(record)
|
|
215
|
+
entity = record.entity
|
|
216
|
+
entity.nil? ? note_nameless(row) : entities << entity
|
|
217
|
+
end
|
|
218
|
+
entities
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Everything worth saying about one record before it becomes an entity.
|
|
222
|
+
# A subclass adds to it rather than replacing it, so a list that can
|
|
223
|
+
# complain about more still complains about the same things.
|
|
224
|
+
sig { params(record: T.untyped).void }
|
|
225
|
+
def note(record)
|
|
226
|
+
note_unknown_type(record) if record.unknown_type?
|
|
227
|
+
@remarks_coverage.record(record.parsed_remarks)
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
sig { params(record: T.untyped).void }
|
|
231
|
+
def note_unknown_type(record)
|
|
232
|
+
@unmapped << Parsers::Warning.new(
|
|
233
|
+
line: record.row.line,
|
|
234
|
+
message: "unknown SDN_Type #{record.row[:sdn_type].inspect}; treated as an organization"
|
|
235
|
+
)
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
sig { params(row: Parsers::DelimitedTable::Row).void }
|
|
239
|
+
def note_nameless(row)
|
|
240
|
+
@unmapped << Parsers::Warning.new(
|
|
241
|
+
line: row.line, message: "row #{row[:ent_num].inspect} has no SDN_Name and was skipped"
|
|
242
|
+
)
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
require "active_sanction/sources/ofac/record"
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/sources/ofac"
|
|
7
|
+
|
|
8
|
+
module ActiveSanction
|
|
9
|
+
module Sources
|
|
10
|
+
class OfacConsolidated < Ofac
|
|
11
|
+
# A consolidated row, read exactly as an SDN row is, plus the one thing
|
|
12
|
+
# the SDN list does not have to say: which of OFAC's six non-SDN lists
|
|
13
|
+
# this record is on.
|
|
14
|
+
#
|
|
15
|
+
# The attribution is derived from the programs rather than stored beside
|
|
16
|
+
# them, so that it stays derivable from a stored Entity long after this
|
|
17
|
+
# object is gone -- see OfacConsolidated.lists. What the record adds is
|
|
18
|
+
# the human-readable form, appended to remarks ahead of the other
|
|
19
|
+
# source fields because it is the first thing an examiner looking at a
|
|
20
|
+
# hit needs to know.
|
|
21
|
+
#
|
|
22
|
+
# @api private
|
|
23
|
+
class Record < Ofac::Record
|
|
24
|
+
extend T::Sig
|
|
25
|
+
|
|
26
|
+
sig { returns(T::Array[Symbol]) }
|
|
27
|
+
def lists = OfacConsolidated.lists(programs)
|
|
28
|
+
|
|
29
|
+
sig { returns(T::Array[String]) }
|
|
30
|
+
def list_names = lists.map { |list| OfacConsolidated::NAMES.fetch(list) }
|
|
31
|
+
|
|
32
|
+
sig { override.returns(T::Array[T.untyped]) }
|
|
33
|
+
def remark_fields = [["List", list_names]] + super
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|