active_sanction 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +538 -0
  3. data/CODE_OF_CONDUCT.md +74 -0
  4. data/CONTRIBUTING.md +312 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +616 -0
  7. data/SECURITY.md +97 -0
  8. data/active_sanction.gemspec +98 -0
  9. data/docs/adding_a_source.md +1083 -0
  10. data/docs/api_stability.md +357 -0
  11. data/docs/bundle_format.md +407 -0
  12. data/lib/active_sanction/address.rb +170 -0
  13. data/lib/active_sanction/client.rb +341 -0
  14. data/lib/active_sanction/configuration.rb +721 -0
  15. data/lib/active_sanction/countries.txt +292 -0
  16. data/lib/active_sanction/country.rb +164 -0
  17. data/lib/active_sanction/deprecation.rb +168 -0
  18. data/lib/active_sanction/diff/change.rb +208 -0
  19. data/lib/active_sanction/diff.rb +348 -0
  20. data/lib/active_sanction/doctor/checkup.rb +356 -0
  21. data/lib/active_sanction/doctor/diagnosis.rb +245 -0
  22. data/lib/active_sanction/doctor/finding.rb +183 -0
  23. data/lib/active_sanction/doctor/profile.rb +384 -0
  24. data/lib/active_sanction/doctor/report.rb +227 -0
  25. data/lib/active_sanction/doctor.rb +347 -0
  26. data/lib/active_sanction/entity.rb +293 -0
  27. data/lib/active_sanction/error.rb +332 -0
  28. data/lib/active_sanction/fetcher/result.rb +116 -0
  29. data/lib/active_sanction/fetcher.rb +242 -0
  30. data/lib/active_sanction/http_client/errors.rb +94 -0
  31. data/lib/active_sanction/http_client/response.rb +129 -0
  32. data/lib/active_sanction/http_client.rb +324 -0
  33. data/lib/active_sanction/identifier.rb +212 -0
  34. data/lib/active_sanction/index/builder.rb +89 -0
  35. data/lib/active_sanction/index/candidate.rb +63 -0
  36. data/lib/active_sanction/index/entry.rb +66 -0
  37. data/lib/active_sanction/index/features.rb +112 -0
  38. data/lib/active_sanction/index.rb +381 -0
  39. data/lib/active_sanction/match_result.rb +345 -0
  40. data/lib/active_sanction/matcher.rb +388 -0
  41. data/lib/active_sanction/name.rb +183 -0
  42. data/lib/active_sanction/normalizer/cache.rb +82 -0
  43. data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
  44. data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
  45. data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
  46. data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
  47. data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
  48. data/lib/active_sanction/normalizer/dictionary.rb +243 -0
  49. data/lib/active_sanction/normalizer/form.rb +279 -0
  50. data/lib/active_sanction/normalizer.rb +137 -0
  51. data/lib/active_sanction/parsers/column_shape.rb +274 -0
  52. data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
  53. data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
  54. data/lib/active_sanction/parsers/delimited_table.rb +151 -0
  55. data/lib/active_sanction/parsers/format.rb +107 -0
  56. data/lib/active_sanction/parsers/join.rb +149 -0
  57. data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
  58. data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
  59. data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
  60. data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
  61. data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
  62. data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
  63. data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
  64. data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
  65. data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
  66. data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
  67. data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
  68. data/lib/active_sanction/parsers/xml_records.rb +140 -0
  69. data/lib/active_sanction/parsers.rb +94 -0
  70. data/lib/active_sanction/partial_date/parser.rb +124 -0
  71. data/lib/active_sanction/partial_date.rb +303 -0
  72. data/lib/active_sanction/payload_cache/checksum.rb +66 -0
  73. data/lib/active_sanction/payload_cache/entry.rb +304 -0
  74. data/lib/active_sanction/payload_cache.rb +366 -0
  75. data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
  76. data/lib/active_sanction/phonetics.rb +40 -0
  77. data/lib/active_sanction/query.rb +290 -0
  78. data/lib/active_sanction/rescreen/alert.rb +314 -0
  79. data/lib/active_sanction/rescreen.rb +332 -0
  80. data/lib/active_sanction/scorer/adjustments.rb +241 -0
  81. data/lib/active_sanction/scorer/name_score.rb +279 -0
  82. data/lib/active_sanction/scorer/reason.rb +136 -0
  83. data/lib/active_sanction/scorer/result.rb +129 -0
  84. data/lib/active_sanction/scorer/subject.rb +235 -0
  85. data/lib/active_sanction/scorer/weights.rb +306 -0
  86. data/lib/active_sanction/scorer.rb +303 -0
  87. data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
  88. data/lib/active_sanction/similarity/levenshtein.rb +179 -0
  89. data/lib/active_sanction/similarity/token_set.rb +159 -0
  90. data/lib/active_sanction/similarity/token_sort.rb +120 -0
  91. data/lib/active_sanction/similarity.rb +183 -0
  92. data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
  93. data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
  94. data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
  95. data/lib/active_sanction/snapshot/bundle.rb +381 -0
  96. data/lib/active_sanction/snapshot.rb +305 -0
  97. data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
  98. data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
  99. data/lib/active_sanction/sources/australia_dfat.rb +291 -0
  100. data/lib/active_sanction/sources/base.rb +295 -0
  101. data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
  102. data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
  103. data/lib/active_sanction/sources/canada_sema.rb +153 -0
  104. data/lib/active_sanction/sources/definition.rb +360 -0
  105. data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
  106. data/lib/active_sanction/sources/eu_fsf.rb +203 -0
  107. data/lib/active_sanction/sources/ofac/record.rb +251 -0
  108. data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
  109. data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
  110. data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
  111. data/lib/active_sanction/sources/ofac.rb +248 -0
  112. data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
  113. data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
  114. data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
  115. data/lib/active_sanction/sources/remarks.rb +84 -0
  116. data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
  117. data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
  118. data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
  119. data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
  120. data/lib/active_sanction/sources/un_consolidated.rb +140 -0
  121. data/lib/active_sanction/sources.rb +196 -0
  122. data/lib/active_sanction/storage/active_record/reader.rb +124 -0
  123. data/lib/active_sanction/storage/active_record/row.rb +175 -0
  124. data/lib/active_sanction/storage/active_record/writer.rb +141 -0
  125. data/lib/active_sanction/storage/active_record.rb +277 -0
  126. data/lib/active_sanction/storage/base.rb +216 -0
  127. data/lib/active_sanction/storage/file_system.rb +373 -0
  128. data/lib/active_sanction/storage/memory.rb +67 -0
  129. data/lib/active_sanction/storage/meta.rb +188 -0
  130. data/lib/active_sanction/storage.rb +74 -0
  131. data/lib/active_sanction/subject.rb +271 -0
  132. data/lib/active_sanction/sync/report.rb +245 -0
  133. data/lib/active_sanction/sync/result.rb +296 -0
  134. data/lib/active_sanction/sync.rb +390 -0
  135. data/lib/active_sanction/validator_store/file_system.rb +103 -0
  136. data/lib/active_sanction/validator_store/memory.rb +41 -0
  137. data/lib/active_sanction/validator_store.rb +119 -0
  138. data/lib/active_sanction/validators.rb +195 -0
  139. data/lib/active_sanction/version.rb +23 -0
  140. data/lib/active_sanction.rb +331 -0
  141. data/lib/generators/active_sanction/install/install_generator.rb +60 -0
  142. data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
  143. metadata +236 -0
@@ -0,0 +1,312 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/identifier"
7
+ require "active_sanction/name"
8
+ require "active_sanction/partial_date"
9
+
10
+ module ActiveSanction
11
+ module Sources
12
+ class Ofac < Base
13
+ # Reads the fields OFAC publishes no columns for out of the free text it
14
+ # packs them into.
15
+ #
16
+ # parsed = RemarksParser.new("DOB 10 Dec 1948; POB Egypt; Passport 123456 (Egypt)")
17
+ # parsed.dates_of_birth # => [#<PartialDate 1948-12-10>]
18
+ # parsed.places_of_birth # => ["Egypt"]
19
+ # parsed.identifiers # => [#<Identifier :passport "123456" country="Egypt">]
20
+ # parsed.unrecognized # => []
21
+ #
22
+ # SDN.CSV has no date of birth, place of birth, nationality or passport
23
+ # column. All of it -- 88,827 semicolon-delimited segments across 19,015
24
+ # remarks -- is prose in one field, written for a human reading a page.
25
+ # Until it is read, every OFAC entity carries names and nothing else, and
26
+ # names alone are what makes a screening tool cry wolf: the secondary
27
+ # identifiers that clear a false positive (#32) are all in here.
28
+ #
29
+ # ### Nothing is ever removed from the remark
30
+ #
31
+ # Extraction is additive. `Entity#remarks` keeps the publisher's whole
32
+ # string byte for byte whether this class understood it or not, so a
33
+ # pattern that drifts costs structure and never content. That is the one
34
+ # rule this file must not break: these are heuristics against text a
35
+ # government writes for people, they will go stale, and silent data loss
36
+ # in a compliance tool is the worst failure there is. A segment we cannot
37
+ # read is still in front of the user, in the publisher's own words.
38
+ #
39
+ # ### How a segment is read
40
+ #
41
+ # Segments are split on `;` and matched against a label vocabulary. A
42
+ # segment either extracts a value, or matches a shape known to be prose
43
+ # (`Secondary sanctions risk: ...`, which is 12% of the file and carries
44
+ # nothing structured), or is unrecognized -- and #unrecognized is what
45
+ # Coverage counts, so drift shows up as a number rather than as a bug
46
+ # report years later.
47
+ #
48
+ # Nothing here raises. A malformed segment -- `Passport issued in
49
+ # Sarajevo`, which names no passport at all -- reads as unrecognized and
50
+ # stays in the remark, because one unparseable clause must never cost the
51
+ # entity around it.
52
+ class RemarksParser
53
+ extend T::Sig
54
+
55
+ SEPARATOR = T.let(";", String)
56
+
57
+ # `alt.` prefixes a repeat: "DOB 1955; alt. DOB 1956" is one person two
58
+ # governments reported differently. It marks repetition and nothing
59
+ # else, so it is stripped and what follows is matched normally -- which
60
+ # is how multiple values of one kind fall out without a second rule.
61
+ ALTERNATE = T.let(/\Aalt\.\s+/i, Regexp)
62
+
63
+ # The full stop that ends a remark belongs to the sentence, not to the
64
+ # value: "Gender Male." is not a gender spelled with a period.
65
+ TRAILING_STOP = T.let(/\s*\.\z/, Regexp)
66
+
67
+ # OFAC publishes wallet addresses as `Digital Currency Address - XBT
68
+ # 1abc...`, one label per currency. They are matched by shape rather
69
+ # than enumerated: the list of currencies grows every time a new one is
70
+ # designated, and none of them changes how the address is read.
71
+ DIGITAL_CURRENCY = T.let(
72
+ /\ADigital Currency Address\s*-\s*(?<currency>[[:alnum:]]+)\s+(?<value>\S+)\z/i, Regexp
73
+ )
74
+
75
+ # The parenthesised qualifiers a document segment ends in: `Passport
76
+ # 123456 (Egypt)`, and sometimes two -- `Folio Mercantil No. 22839
77
+ # (Jalisco) (Mexico)` is a state and then a country. The space is what
78
+ # makes it a qualifier rather than part of the number: Hong Kong writes
79
+ # its ID numbers as `D489833(9)`, and splitting that off would leave a
80
+ # document number the issuing government would not recognize.
81
+ TRAILING_QUALIFIER = T.let(/\s+\(([^)]*)\)\s*\z/, Regexp)
82
+
83
+ # `Passport ZG4109521 (Pakistan) issued 07 Jun 2008 expires 06 Jun 2013`
84
+ # -- either clause may be absent and both may be present, so the value
85
+ # ends wherever the first one starts.
86
+ DATE_KEYWORDS = T.let(/\b(?:issued|expires|expired)\b/i, Regexp)
87
+ DATE_CLAUSE = T.let(/(issued|expires|expired)\s+(?:on\s+)?(.*?)(?=\s+#{DATE_KEYWORDS}|\z)/i, Regexp)
88
+
89
+ # A document number is a code, not a sentence. `Passport 265 216` and
90
+ # `SWIFT/BIC SBERRUMM` are numbers; `License to operate` is prose that
91
+ # happened to open with a label, and without this rule it would become
92
+ # an identifier that matches nothing and misleads everyone.
93
+ CODE = T.let(%r{\A[[:alnum:]][[:alnum:]\s._()/-]*\z}, Regexp)
94
+ DIGIT = T.let(/\d/, Regexp)
95
+
96
+ # No government issues a document number this long -- China's 18-digit
97
+ # social credit code is the longest in the file. What exceeds it is a
98
+ # clause that ran on: "Passport OR801168 and Kuwaiti National ID No.
99
+ # 281020505755 issued under the name ..." is one segment OFAC wrote as
100
+ # a sentence, and an identifier built from all of it would match
101
+ # nothing and mislead whoever read it.
102
+ MAX_CODE_LENGTH = T.let(40, Integer)
103
+
104
+ QUOTES = T.let(/\A['"“”‘’]|['"“”‘’]\z/, Regexp)
105
+
106
+ # The remark as published, and how this parser read it: every segment,
107
+ # those it extracted something from, those it recognized as prose, and
108
+ # those it could not read -- which is what Coverage counts.
109
+ sig { returns(String) }
110
+ attr_reader :text
111
+
112
+ sig { returns(T::Array[String]) }
113
+ attr_reader :segments
114
+
115
+ sig { returns(T::Array[String]) }
116
+ attr_reader :extracted
117
+
118
+ sig { returns(T::Array[String]) }
119
+ attr_reader :prose
120
+
121
+ sig { returns(T::Array[String]) }
122
+ attr_reader :unrecognized
123
+
124
+ sig { returns(T::Array[PartialDate]) }
125
+ attr_reader :dates_of_birth
126
+
127
+ sig { returns(T::Array[String]) }
128
+ attr_reader :places_of_birth
129
+
130
+ sig { returns(T::Array[String]) }
131
+ attr_reader :nationalities
132
+
133
+ sig { returns(T::Array[String]) }
134
+ attr_reader :genders
135
+
136
+ sig { returns(T::Array[Name]) }
137
+ attr_reader :aliases
138
+
139
+ sig { returns(T::Array[Identifier]) }
140
+ attr_reader :identifiers
141
+
142
+ sig { params(text: T.untyped).void }
143
+ def initialize(text)
144
+ @text = T.let(text.to_s, String)
145
+ @segments = T.let([], T::Array[String])
146
+ @extracted = T.let([], T::Array[String])
147
+ @prose = T.let([], T::Array[String])
148
+ @unrecognized = T.let([], T::Array[String])
149
+ @dates_of_birth = T.let([], T::Array[PartialDate])
150
+ @places_of_birth = T.let([], T::Array[String])
151
+ @nationalities = T.let([], T::Array[String])
152
+ @genders = T.let([], T::Array[String])
153
+ @aliases = T.let([], T::Array[Name])
154
+ @identifiers = T.let([], T::Array[Identifier])
155
+ read
156
+ freeze
157
+ end
158
+
159
+ # True when the remark yielded anything structured at all.
160
+ sig { returns(T::Boolean) }
161
+ def any? = extracted.any?
162
+
163
+ private
164
+
165
+ sig { void }
166
+ def read
167
+ text.split(SEPARATOR).each do |raw|
168
+ segment = raw.strip
169
+ next if segment.empty?
170
+
171
+ @segments << segment
172
+ classify(segment)
173
+ end
174
+ end
175
+
176
+ sig { params(segment: String).void }
177
+ def classify(segment)
178
+ body = segment.sub(ALTERNATE, "")
179
+ if field(body) || document(body) || digital_currency(body)
180
+ @extracted << segment
181
+ elsif Vocabulary::PROSE_PATTERN.match?(body)
182
+ @prose << segment
183
+ else
184
+ @unrecognized << segment
185
+ end
186
+ end
187
+
188
+ sig { params(body: String).returns(T.untyped) }
189
+ def field(body)
190
+ match = Vocabulary::FIELD_PATTERN.match(body)
191
+ return nil unless match
192
+
193
+ value = trim(match[:value])
194
+ return nil if value.nil?
195
+
196
+ case Vocabulary::FIELD_KINDS.fetch(T.must(match[:label]).downcase)
197
+ when :date_of_birth then born(value)
198
+ when :place_of_birth then keep(@places_of_birth, value)
199
+ when :nationality then keep(@nationalities, value)
200
+ when :gender then keep(@genders, value)
201
+ else known_as(Vocabulary::FIELD_KINDS.fetch(T.must(match[:label]).downcase), value)
202
+ end
203
+ end
204
+
205
+ # An unreadable date is not a date. It reads as unrecognized rather
206
+ # than as a nil the entity would carry around, which puts it in the
207
+ # coverage histogram where a new OFAC spelling can be seen.
208
+ sig { params(value: String).returns(T.untyped) }
209
+ def born(value)
210
+ date = PartialDate.parse(value)
211
+ return nil if date.nil?
212
+
213
+ keep(@dates_of_birth, date)
214
+ end
215
+
216
+ # `a.k.a. 'EL SENOR'` -- quoted, and the quotes are OFAC's punctuation
217
+ # rather than part of the name. 4,325 of the 4,349 inline aliases
218
+ # appear nowhere in ALT.CSV, so these are names the list publishes here
219
+ # and only here.
220
+ sig { params(kind: Symbol, value: String).returns(T.untyped) }
221
+ def known_as(kind, value)
222
+ keep(@aliases, Name.new(value: value.sub(QUOTES, "").sub(QUOTES, ""), kind: kind))
223
+ rescue ArgumentError
224
+ nil
225
+ end
226
+
227
+ sig { params(body: String).returns(T.untyped) }
228
+ def document(body)
229
+ match = Vocabulary::DOCUMENT_PATTERN.match(body)
230
+ return nil unless match
231
+
232
+ rest = trim(match[:rest])
233
+ return nil if rest.nil?
234
+
235
+ label = T.must(match[:label])
236
+ identify(label, Vocabulary::DOCUMENT_KINDS.fetch(label.downcase), rest)
237
+ end
238
+
239
+ sig { params(label: String, kind: Symbol, rest: String).returns(T.untyped) }
240
+ def identify(label, kind, rest)
241
+ head, issued_on, expires_on = split(rest)
242
+ value, qualifiers = unwrap(head)
243
+ return nil unless code?(value)
244
+
245
+ keep(@identifiers, Identifier.new(kind: kind, value: value, country: qualifiers.last,
246
+ issued_on: issued_on, expires_on: expires_on,
247
+ note: note(label, qualifiers)))
248
+ rescue ArgumentError
249
+ nil
250
+ end
251
+
252
+ sig { params(rest: String).returns([String, T.nilable(PartialDate), T.nilable(PartialDate)]) }
253
+ def split(rest)
254
+ dates = T.let({}, T::Hash[String, T.nilable(PartialDate)])
255
+ rest.scan(DATE_CLAUSE) { |keyword, date| dates[keyword.downcase] ||= PartialDate.parse(date) }
256
+ [rest.split(/\s+#{DATE_KEYWORDS}/, 2).first.to_s, dates["issued"], dates["expires"] || dates["expired"]]
257
+ end
258
+
259
+ # Outermost qualifier last, which is the one Identifier has a country
260
+ # field for. Anything inside it -- a state, a province -- has no home
261
+ # on the record and goes to the note rather than being dropped.
262
+ sig { params(head: String).returns([String, T::Array[String]]) }
263
+ def unwrap(head)
264
+ qualifiers = T.let([], T::Array[String])
265
+ text = head.dup
266
+ qualifiers.unshift(T.must(Regexp.last_match(1))) while text.sub!(TRAILING_QUALIFIER, "")
267
+ [text.strip, qualifiers]
268
+ end
269
+
270
+ sig { params(label: String, qualifiers: T::Array[String]).returns(String) }
271
+ def note(label, qualifiers)
272
+ inner = qualifiers[0..-2].to_a
273
+ inner.empty? ? label : "#{label} (#{inner.join(", ")})"
274
+ end
275
+
276
+ sig { params(body: String).returns(T.untyped) }
277
+ def digital_currency(body)
278
+ match = DIGITAL_CURRENCY.match(trim(body).to_s)
279
+ return nil unless match
280
+
281
+ currency = T.must(match[:currency]).upcase
282
+ keep(@identifiers, Identifier.new(kind: :other, value: match[:value], note: "#{currency} address"))
283
+ rescue ArgumentError
284
+ nil
285
+ end
286
+
287
+ sig { params(value: String).returns(T::Boolean) }
288
+ def code?(value)
289
+ return false if value.empty? || value.length > MAX_CODE_LENGTH || !CODE.match?(value)
290
+
291
+ DIGIT.match?(value) || value == value.upcase
292
+ end
293
+
294
+ # Returns the list, which is truthy: a caller reads "something was
295
+ # kept" from it, and a miss answers nil.
296
+ sig { params(list: T::Array[T.untyped], value: T.untyped).returns(T::Array[T.untyped]) }
297
+ def keep(list, value)
298
+ list << value
299
+ end
300
+
301
+ sig { params(value: T.untyped).returns(T.nilable(String)) }
302
+ def trim(value)
303
+ string = value.to_s.sub(TRAILING_STOP, "").strip
304
+ string.empty? ? nil : string
305
+ end
306
+ end
307
+ end
308
+ end
309
+ end
310
+
311
+ require "active_sanction/sources/ofac/remarks_parser/vocabulary"
312
+ require "active_sanction/sources/ofac/remarks_parser/coverage"
@@ -0,0 +1,248 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/parsers"
7
+ require "active_sanction/sources"
8
+ require "active_sanction/sources/base"
9
+
10
+ module ActiveSanction
11
+ module Sources
12
+ # What the two OFAC lists have in common, which is nearly everything.
13
+ #
14
+ # OFAC publishes the SDN list and the Consolidated (non-SDN) list in
15
+ # exactly the same shape: three headerless CSVs joined on `ent_num`, the
16
+ # primary file carrying twelve columns, ALT carrying five and ADD six, all
17
+ # of them in Windows-1252 with `-0- ` for null. Only the file names, the
18
+ # sizes and the legal consequence of being on one differ.
19
+ #
20
+ # SDN.CSV 19,321 rows CONS_PRIM.CSV 481 rows
21
+ # ALT.CSV 20,147 rows CONS_ALT.CSV 1,109 rows
22
+ # ADD.CSV 25,078 rows CONS_ADD.CSV 614 rows
23
+ #
24
+ # So the reading is written once, here, and an adapter says only which
25
+ # list it is and where its three files live. Abstract: it declares no key
26
+ # and does not register, because there is no such list as "OFAC".
27
+ #
28
+ # ### The fields OFAC publishes no columns for
29
+ #
30
+ # Neither file has a date of birth, place of birth, nationality or
31
+ # passport column. All of it is prose in `Remarks`:
32
+ #
33
+ # "DOB 10 Dec 1948; POB Egypt; nationality Egypt; Passport 123456 (Egypt)"
34
+ #
35
+ # RemarksParser reads it, and #remarks_coverage reports how much of it it
36
+ # understood -- 97% of the SDN file's 88,827 segments, which is a figure to
37
+ # watch rather than a guarantee. The remark is kept verbatim either way, so
38
+ # a pattern that goes stale costs structure and never content.
39
+ #
40
+ # This class is written entirely against the public extension points:
41
+ # declarations from Definition, fetch and cache and checksum from Base,
42
+ # reading and joining from Parsers. It required no change to any of them,
43
+ # which is the property M4 exists to prove.
44
+ #
45
+ # @api private
46
+ class Ofac < Base
47
+ extend T::Sig
48
+
49
+ jurisdiction :us
50
+ authority "U.S. Department of the Treasury, Office of Foreign Assets Control"
51
+ format :csv
52
+
53
+ # A work of the United States government, which 17 U.S.C. Section 105
54
+ # puts outside copyright protection. Treasury restricts its seals and
55
+ # its name rather than the data, which is why the notice mentions them:
56
+ # republishing the list is unencumbered, implying Treasury endorsed you
57
+ # is not.
58
+ licence_notice "A work of the U.S. government and not subject to " \
59
+ "domestic copyright (17 U.S.C. 105). Treasury's seals " \
60
+ "and marks are restricted separately, and nothing here " \
61
+ "may imply endorsement. Verified 2026-09-11."
62
+ licence_url "https://home.treasury.gov/subfooter/site-policies-and-notices"
63
+
64
+ # The one floor either OFAC list commits to, and only for the run that
65
+ # has no previous snapshot to compare against. RemarksParser reads about
66
+ # 97% of the SDN file's segments; 90% is a long way below anything the
67
+ # file has done and still far above what a re-spelled document label
68
+ # would leave. There is deliberately no floor on the record count: the
69
+ # SDN list has roughly tripled since 2010, so any bound wide enough to
70
+ # have survived that is too wide to catch a truncated download, and the
71
+ # previous snapshot catches one for nothing. See Definition#floor.
72
+ floor :remarks_coverage, 0.90
73
+
74
+ # OFAC serves Windows-1252, not UTF-8, and says nothing about it in a
75
+ # header. Read as UTF-8 the accented names in the list -- and there are
76
+ # thousands -- arrive as replacement characters.
77
+ ENCODING = T.let(Encoding::WINDOWS_1252, Encoding)
78
+
79
+ # OFAC writes "-0- " for null, with a trailing space, in every one of its
80
+ # files and roughly a quarter of a million times overall.
81
+ NULL = T.let("-0-", String)
82
+
83
+ # Column names, positional: all six files ship without a header row.
84
+ # Declaring them here also pins each file's width, so a column inserted
85
+ # upstream surfaces as a warning on every row rather than as 19,321
86
+ # entities quietly built from shifted fields.
87
+ PRIMARY_COLUMNS = T.let(%i[
88
+ ent_num sdn_name sdn_type program title call_sign vessel_type
89
+ tonnage gross_registered_tonnage vessel_flag vessel_owner remarks
90
+ ].freeze, T::Array[Symbol])
91
+
92
+ ALT_COLUMNS = T.let(%i[ent_num alt_num alt_type alt_name alt_remarks].freeze, T::Array[Symbol])
93
+
94
+ ADD_COLUMNS = T.let(
95
+ %i[ent_num add_num address city_state_province_postal_code country add_remarks].freeze,
96
+ T::Array[Symbol]
97
+ )
98
+
99
+ PRIMARY = T.let(
100
+ Parsers::DelimitedTable.new(columns: PRIMARY_COLUMNS, null: NULL, encoding: ENCODING),
101
+ Parsers::DelimitedTable
102
+ )
103
+ ALT = T.let(
104
+ Parsers::DelimitedTable.new(columns: ALT_COLUMNS, null: NULL, encoding: ENCODING),
105
+ Parsers::DelimitedTable
106
+ )
107
+ ADD = T.let(
108
+ Parsers::DelimitedTable.new(columns: ADD_COLUMNS, null: NULL, encoding: ENCODING),
109
+ Parsers::DelimitedTable
110
+ )
111
+
112
+ # OFAC's join key. Every row of all six files carries one, and it is the
113
+ # only column on the primary file whose *values* have a shape.
114
+ ENT_NUM = T.let(/\A\d+\z/, Regexp)
115
+
116
+ # Rows that could not be read, and child rows that matched no entity.
117
+ # Populated by #parse and read afterwards -- sync orchestration (#34)
118
+ # reports them, and a nonzero orphan count is the signal that the three
119
+ # files were downloaded at different moments and no longer agree.
120
+ sig { returns(T::Array[Parsers::Warning]) }
121
+ attr_reader :warnings
122
+
123
+ # Child rows that matched no entity, by file -- a nonzero count means the
124
+ # three files were downloaded at different moments.
125
+ sig { returns(T::Hash[Symbol, T.untyped]) }
126
+ attr_reader :orphans
127
+
128
+ # How much of OFAC's free text the last #parse understood. Not a warning,
129
+ # because an unread segment is not an error -- it is still in the remark,
130
+ # in the publisher's own words -- but a figure to watch: it is the only
131
+ # thing that moves when OFAC changes how it writes a passport line.
132
+ sig { returns(RemarksParser::Coverage) }
133
+ attr_reader :remarks_coverage
134
+
135
+ sig { params(args: T.untyped, options: T.untyped).void }
136
+ def initialize(*args, **options)
137
+ super
138
+ @warnings = T.let([], T::Array[Parsers::Warning])
139
+ @orphans = T.let({}, T::Hash[Symbol, T.untyped])
140
+ @remarks_coverage = T.let(RemarksParser::Coverage.new, RemarksParser::Coverage)
141
+ @unmapped = T.let([], T::Array[Parsers::Warning])
142
+ end
143
+
144
+ sig { override.params(raw: T.untyped).returns(T::Array[Entity]) }
145
+ def parse(raw)
146
+ join = Parsers::Join.new(on: :ent_num, aliases: ALT.read(raw[:alt]), addresses: ADD.read(raw[:add]))
147
+ entities = build(join, PRIMARY.read(raw[primary_file]))
148
+ @warnings = join.warnings + @unmapped
149
+ @orphans = join.orphans
150
+ entities
151
+ end
152
+
153
+ # What the two assertable columns of the primary file hold.
154
+ #
155
+ # The width declared in PRIMARY_COLUMNS catches a column OFAC *inserts*:
156
+ # every row arrives the wrong width and every row says so. Nothing about
157
+ # the width catches one OFAC *reorders*, which parses cleanly and builds
158
+ # 19,321 entities out of shifted fields. These are what catches that.
159
+ #
160
+ # Two of twelve, because ten of the columns are free text a government
161
+ # writes for people and there is nothing to assert about them. See
162
+ # Parsers::ColumnShape.
163
+ #
164
+ # `sdn_type` is held to three quarters rather than the 99% `ent_num` is,
165
+ # and the difference is what each one would cost to get wrong. OFAC does
166
+ # publish types this adapter has never seen -- that is what
167
+ # Record#unknown_type? exists for, and a new one is an `info` finding and
168
+ # a mapping to write, not a broken file. Demanding 99% would turn the
169
+ # week OFAC coins a word into an `error` on a list that is entirely fine,
170
+ # and a diagnostic that cries wolf about a healthy list is one nobody
171
+ # reads the week it is right. What three quarters still catches is the
172
+ # only thing worth an error here: a column that has stopped being the
173
+ # type column at all, which reads as zero.
174
+ sig { returns(T::Array[Parsers::ColumnShape]) }
175
+ def column_assertions
176
+ [Parsers::ColumnShape.new(name: :ent_num, matches: ENT_NUM, description: "numeric"),
177
+ Parsers::ColumnShape.new(name: :sdn_type, allowing: record_class.published_types, at_least: 0.75,
178
+ description: "a published SDN_Type")]
179
+ end
180
+
181
+ # One extra pass over the primary file, for the doctor and nothing else.
182
+ # It is the same reader the parse used, asked a different question.
183
+ sig { override.params(raw: T.untyped).returns(T::Array[Parsers::ColumnShape::Tally]) }
184
+ def column_shapes(raw)
185
+ rows = PRIMARY.read(raw[primary_file]).to_a
186
+ column_assertions.map { |shape| shape.tally(rows.map { |row| row[shape.name] }) }
187
+ end
188
+
189
+ # The declaration name of the file carrying one row per entity -- `:sdn`
190
+ # for the SDN list, `:prim` for the consolidated one. The first file an
191
+ # adapter declares, by the convention that the primary comes before the
192
+ # children it is joined to.
193
+ sig { returns(Symbol) }
194
+ def primary_file = T.must(urls.keys.first)
195
+
196
+ private
197
+
198
+ # The class each joined row is handed to. Overridden by an adapter whose
199
+ # list publishes something the shared mapping does not know about.
200
+ sig { returns(T.untyped) }
201
+ def record_class = Record
202
+
203
+ # Coverage is folded over every row, the nameless ones that produce no
204
+ # entity included: the question it answers is how much of the file this
205
+ # parser can read, and a row we drop is still a row OFAC published.
206
+ sig { params(join: Parsers::Join, rows: T.untyped).returns(T::Array[Entity]) }
207
+ def build(join, rows)
208
+ @unmapped = []
209
+ @remarks_coverage = RemarksParser::Coverage.new
210
+ entities = []
211
+ join.each(rows) do |row, related|
212
+ record = record_class.new(row: row, source: key, aliases: related[:aliases],
213
+ addresses: related[:addresses])
214
+ note(record)
215
+ entity = record.entity
216
+ entity.nil? ? note_nameless(row) : entities << entity
217
+ end
218
+ entities
219
+ end
220
+
221
+ # Everything worth saying about one record before it becomes an entity.
222
+ # A subclass adds to it rather than replacing it, so a list that can
223
+ # complain about more still complains about the same things.
224
+ sig { params(record: T.untyped).void }
225
+ def note(record)
226
+ note_unknown_type(record) if record.unknown_type?
227
+ @remarks_coverage.record(record.parsed_remarks)
228
+ end
229
+
230
+ sig { params(record: T.untyped).void }
231
+ def note_unknown_type(record)
232
+ @unmapped << Parsers::Warning.new(
233
+ line: record.row.line,
234
+ message: "unknown SDN_Type #{record.row[:sdn_type].inspect}; treated as an organization"
235
+ )
236
+ end
237
+
238
+ sig { params(row: Parsers::DelimitedTable::Row).void }
239
+ def note_nameless(row)
240
+ @unmapped << Parsers::Warning.new(
241
+ line: row.line, message: "row #{row[:ent_num].inspect} has no SDN_Name and was skipped"
242
+ )
243
+ end
244
+ end
245
+ end
246
+ end
247
+
248
+ require "active_sanction/sources/ofac/record"
@@ -0,0 +1,37 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/sources/ofac"
7
+
8
+ module ActiveSanction
9
+ module Sources
10
+ class OfacConsolidated < Ofac
11
+ # A consolidated row, read exactly as an SDN row is, plus the one thing
12
+ # the SDN list does not have to say: which of OFAC's six non-SDN lists
13
+ # this record is on.
14
+ #
15
+ # The attribution is derived from the programs rather than stored beside
16
+ # them, so that it stays derivable from a stored Entity long after this
17
+ # object is gone -- see OfacConsolidated.lists. What the record adds is
18
+ # the human-readable form, appended to remarks ahead of the other
19
+ # source fields because it is the first thing an examiner looking at a
20
+ # hit needs to know.
21
+ #
22
+ # @api private
23
+ class Record < Ofac::Record
24
+ extend T::Sig
25
+
26
+ sig { returns(T::Array[Symbol]) }
27
+ def lists = OfacConsolidated.lists(programs)
28
+
29
+ sig { returns(T::Array[String]) }
30
+ def list_names = lists.map { |list| OfacConsolidated::NAMES.fetch(list) }
31
+
32
+ sig { override.returns(T::Array[T.untyped]) }
33
+ def remark_fields = [["List", list_names]] + super
34
+ end
35
+ end
36
+ end
37
+ end