active_sanction 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +538 -0
  3. data/CODE_OF_CONDUCT.md +74 -0
  4. data/CONTRIBUTING.md +312 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +616 -0
  7. data/SECURITY.md +97 -0
  8. data/active_sanction.gemspec +98 -0
  9. data/docs/adding_a_source.md +1083 -0
  10. data/docs/api_stability.md +357 -0
  11. data/docs/bundle_format.md +407 -0
  12. data/lib/active_sanction/address.rb +170 -0
  13. data/lib/active_sanction/client.rb +341 -0
  14. data/lib/active_sanction/configuration.rb +721 -0
  15. data/lib/active_sanction/countries.txt +292 -0
  16. data/lib/active_sanction/country.rb +164 -0
  17. data/lib/active_sanction/deprecation.rb +168 -0
  18. data/lib/active_sanction/diff/change.rb +208 -0
  19. data/lib/active_sanction/diff.rb +348 -0
  20. data/lib/active_sanction/doctor/checkup.rb +356 -0
  21. data/lib/active_sanction/doctor/diagnosis.rb +245 -0
  22. data/lib/active_sanction/doctor/finding.rb +183 -0
  23. data/lib/active_sanction/doctor/profile.rb +384 -0
  24. data/lib/active_sanction/doctor/report.rb +227 -0
  25. data/lib/active_sanction/doctor.rb +347 -0
  26. data/lib/active_sanction/entity.rb +293 -0
  27. data/lib/active_sanction/error.rb +332 -0
  28. data/lib/active_sanction/fetcher/result.rb +116 -0
  29. data/lib/active_sanction/fetcher.rb +242 -0
  30. data/lib/active_sanction/http_client/errors.rb +94 -0
  31. data/lib/active_sanction/http_client/response.rb +129 -0
  32. data/lib/active_sanction/http_client.rb +324 -0
  33. data/lib/active_sanction/identifier.rb +212 -0
  34. data/lib/active_sanction/index/builder.rb +89 -0
  35. data/lib/active_sanction/index/candidate.rb +63 -0
  36. data/lib/active_sanction/index/entry.rb +66 -0
  37. data/lib/active_sanction/index/features.rb +112 -0
  38. data/lib/active_sanction/index.rb +381 -0
  39. data/lib/active_sanction/match_result.rb +345 -0
  40. data/lib/active_sanction/matcher.rb +388 -0
  41. data/lib/active_sanction/name.rb +183 -0
  42. data/lib/active_sanction/normalizer/cache.rb +82 -0
  43. data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
  44. data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
  45. data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
  46. data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
  47. data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
  48. data/lib/active_sanction/normalizer/dictionary.rb +243 -0
  49. data/lib/active_sanction/normalizer/form.rb +279 -0
  50. data/lib/active_sanction/normalizer.rb +137 -0
  51. data/lib/active_sanction/parsers/column_shape.rb +274 -0
  52. data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
  53. data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
  54. data/lib/active_sanction/parsers/delimited_table.rb +151 -0
  55. data/lib/active_sanction/parsers/format.rb +107 -0
  56. data/lib/active_sanction/parsers/join.rb +149 -0
  57. data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
  58. data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
  59. data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
  60. data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
  61. data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
  62. data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
  63. data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
  64. data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
  65. data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
  66. data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
  67. data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
  68. data/lib/active_sanction/parsers/xml_records.rb +140 -0
  69. data/lib/active_sanction/parsers.rb +94 -0
  70. data/lib/active_sanction/partial_date/parser.rb +124 -0
  71. data/lib/active_sanction/partial_date.rb +303 -0
  72. data/lib/active_sanction/payload_cache/checksum.rb +66 -0
  73. data/lib/active_sanction/payload_cache/entry.rb +304 -0
  74. data/lib/active_sanction/payload_cache.rb +366 -0
  75. data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
  76. data/lib/active_sanction/phonetics.rb +40 -0
  77. data/lib/active_sanction/query.rb +290 -0
  78. data/lib/active_sanction/rescreen/alert.rb +314 -0
  79. data/lib/active_sanction/rescreen.rb +332 -0
  80. data/lib/active_sanction/scorer/adjustments.rb +241 -0
  81. data/lib/active_sanction/scorer/name_score.rb +279 -0
  82. data/lib/active_sanction/scorer/reason.rb +136 -0
  83. data/lib/active_sanction/scorer/result.rb +129 -0
  84. data/lib/active_sanction/scorer/subject.rb +235 -0
  85. data/lib/active_sanction/scorer/weights.rb +306 -0
  86. data/lib/active_sanction/scorer.rb +303 -0
  87. data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
  88. data/lib/active_sanction/similarity/levenshtein.rb +179 -0
  89. data/lib/active_sanction/similarity/token_set.rb +159 -0
  90. data/lib/active_sanction/similarity/token_sort.rb +120 -0
  91. data/lib/active_sanction/similarity.rb +183 -0
  92. data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
  93. data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
  94. data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
  95. data/lib/active_sanction/snapshot/bundle.rb +381 -0
  96. data/lib/active_sanction/snapshot.rb +305 -0
  97. data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
  98. data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
  99. data/lib/active_sanction/sources/australia_dfat.rb +291 -0
  100. data/lib/active_sanction/sources/base.rb +295 -0
  101. data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
  102. data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
  103. data/lib/active_sanction/sources/canada_sema.rb +153 -0
  104. data/lib/active_sanction/sources/definition.rb +360 -0
  105. data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
  106. data/lib/active_sanction/sources/eu_fsf.rb +203 -0
  107. data/lib/active_sanction/sources/ofac/record.rb +251 -0
  108. data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
  109. data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
  110. data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
  111. data/lib/active_sanction/sources/ofac.rb +248 -0
  112. data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
  113. data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
  114. data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
  115. data/lib/active_sanction/sources/remarks.rb +84 -0
  116. data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
  117. data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
  118. data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
  119. data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
  120. data/lib/active_sanction/sources/un_consolidated.rb +140 -0
  121. data/lib/active_sanction/sources.rb +196 -0
  122. data/lib/active_sanction/storage/active_record/reader.rb +124 -0
  123. data/lib/active_sanction/storage/active_record/row.rb +175 -0
  124. data/lib/active_sanction/storage/active_record/writer.rb +141 -0
  125. data/lib/active_sanction/storage/active_record.rb +277 -0
  126. data/lib/active_sanction/storage/base.rb +216 -0
  127. data/lib/active_sanction/storage/file_system.rb +373 -0
  128. data/lib/active_sanction/storage/memory.rb +67 -0
  129. data/lib/active_sanction/storage/meta.rb +188 -0
  130. data/lib/active_sanction/storage.rb +74 -0
  131. data/lib/active_sanction/subject.rb +271 -0
  132. data/lib/active_sanction/sync/report.rb +245 -0
  133. data/lib/active_sanction/sync/result.rb +296 -0
  134. data/lib/active_sanction/sync.rb +390 -0
  135. data/lib/active_sanction/validator_store/file_system.rb +103 -0
  136. data/lib/active_sanction/validator_store/memory.rb +41 -0
  137. data/lib/active_sanction/validator_store.rb +119 -0
  138. data/lib/active_sanction/validators.rb +195 -0
  139. data/lib/active_sanction/version.rb +23 -0
  140. data/lib/active_sanction.rb +331 -0
  141. data/lib/generators/active_sanction/install/install_generator.rb +60 -0
  142. data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
  143. metadata +236 -0
@@ -0,0 +1,467 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/entity"
7
+ require "active_sanction/name"
8
+ require "active_sanction/address"
9
+ require "active_sanction/identifier"
10
+ require "active_sanction/partial_date"
11
+ require "active_sanction/sources/remarks"
12
+
13
+ module ActiveSanction
14
+ module Sources
15
+ class EuFsf < Base
16
+ # One `<sanctionEntity>` turned into an Entity.
17
+ #
18
+ # Separate from the adapter because they are two jobs: the adapter says
19
+ # what the list is and where it lives, and this says what the
20
+ # Commission's elements mean. The mapping is where all the judgment sits,
21
+ # so it is worth being able to read it on its own.
22
+ #
23
+ # ### Nearly everything is an attribute
24
+ #
25
+ # The FSF export puts its data in attributes and its prose in elements:
26
+ # a name is `<nameAlias wholeName="...">` and the only text nodes in the
27
+ # whole 25.7 MB file are `<remark>` and `<publicationUrl>`. So most reads
28
+ # here are the toolkit's `@attribute` paths, and the `<remark>` child of
29
+ # a name, a date, an address or a document is the one place the
30
+ # Commission writes free prose about that one field.
31
+ #
32
+ # ### The regulation trail this deliberately flattens
33
+ #
34
+ # Every repeated element carries a `<regulationSummary>` naming the act
35
+ # that put it there, so the file records not just that a person has an
36
+ # address but which regulation added it. That is a provenance graph, and
37
+ # the canonical Entity is not one: what survives here is the entity's own
38
+ # current `<regulation>` -- its programme, its number and its Official
39
+ # Journal URL -- and the per-field trail is dropped rather than
40
+ # flattened into 31,053 remark fragments nothing reads.
41
+ #
42
+ # @api private
43
+ class Record
44
+ extend T::Sig
45
+
46
+ TYPES = T.let({ "person" => :individual, "enterprise" => :organization }.freeze,
47
+ T::Hash[String, Symbol])
48
+
49
+ # `identificationTypeCode` is a closed vocabulary of 18 values, which
50
+ # is why this maps codes rather than the free-text description beside
51
+ # them. Anything unrecognised is :other, which is a real answer: a
52
+ # document we cannot classify still matches on its number, and the
53
+ # Commission's own description is kept in the note either way.
54
+ DOCUMENT_KINDS = T.let({
55
+ "passport" => :passport,
56
+ "id" => :national_id, "ssn" => :national_id, "unssn" => :national_id,
57
+ "fiscalcode" => :tax_id, "taxid" => :tax_id, "euvat" => :tax_id,
58
+ "regnumber" => :registration_number, "tradelic" => :registration_number
59
+ }.freeze, T::Hash[String, Symbol])
60
+
61
+ # The EU publishes no alias-quality column and no alias-kind column.
62
+ # What it publishes instead is prose in a name's `<remark>`: "low
63
+ # quality alias", "Good quality a.k.a.", "formerly known as", "Maiden
64
+ # name: Al Akhras". Read, because a low-quality alias is a weaker
65
+ # signal and the scorer (#32) penalizes it, and because a former name
66
+ # is a real hit that should not rank as a current one.
67
+ #
68
+ # ### Why these are anchored to the start of a clause
69
+ #
70
+ # Searching the remark for the phrase anywhere is what a first version
71
+ # did, and it is wrong on real records. The remark on Zadna
72
+ # International's own name is five lines about the company, one of
73
+ # which reads "99 % owned by the Special Fund ..., formerly known as
74
+ # the Charity Organisation for the Support of the Armed Forces" -- a
75
+ # sentence about the *owner*. Matched loosely it files the company's
76
+ # published English name as a former name and promotes the French
77
+ # translation in its place, on a record where the Commission said no
78
+ # such thing.
79
+ #
80
+ # A grading is a whole annotation, so it starts one: the remark is cut
81
+ # into clauses at newlines and semicolons -- which is how the
82
+ # Commission writes them, "born 11.8.1960 in Libya\ngood quality
83
+ # alias" -- and a clause grades the name only if it begins with one of
84
+ # these. Every embedded occurrence in the published file is prose about
85
+ # some other entity, and every standalone one is a grading.
86
+ LOW_QUALITY = T.let(/\Alow[[:space:]]+quality[[:space:]]+(?:alias|a\.k\.a)/i, Regexp)
87
+ GOOD_QUALITY = T.let(/\A(?:good|high)[[:space:]]+quality[[:space:]]+(?:alias|a\.k\.a)/i, Regexp)
88
+ FORMER_NAME = T.let(
89
+ /\A(?:formerly[[:space:]]+known[[:space:]]+as|former[[:space:]]+name|f\.k\.a|maiden[[:space:]]+name)/i,
90
+ Regexp
91
+ )
92
+
93
+ # What the Commission separates annotations with, and the punctuation
94
+ # it opens one with -- `(formerly known as State enterprise ...)`.
95
+ CLAUSE = T.let(/[\n;]/, Regexp)
96
+ CLAUSE_OPENING = T.let(/\A[[:space:]("'\u2018]+/, Regexp)
97
+
98
+ # `calendarType`, and the one value that means the date components can
99
+ # be read as they are written. See the adapter's class comment for what
100
+ # reading a Hijri year as a Gregorian one does to a score.
101
+ GREGORIAN = T.let("GREGORIAN", String)
102
+
103
+ # The Commission's own sentinel for "not stated", published in the
104
+ # ISO 3166 field with `countryDescription="UNKNOWN"` beside it. 1,743
105
+ # of the 4,373 birth dates carry it, and passing it through would file
106
+ # 1,743 people as citizens of a country called `00`.
107
+ UNKNOWN_COUNTRY = T.let("00", String)
108
+
109
+ UNKNOWN_COUNTRY_NAME = T.let("UNKNOWN", String)
110
+
111
+ # Every boolean in the export is the lowercase string, on the
112
+ # attribute rather than as a present-or-absent flag: `circa="false"`
113
+ # appears 4,176 times. So a flag is read by comparing, and the absent
114
+ # attribute -- which is a different thing from `"false"` -- is nil and
115
+ # compares false without a branch.
116
+ BOOLEAN_TRUE = T.let("true", String)
117
+
118
+ # Attributes that say something about a document rather than
119
+ # identifying it, kept in the identifier's note. `knownFalse` is on one
120
+ # document in the whole file and is the most important of them: it is
121
+ # the Commission saying the number is a forgery.
122
+ DOCUMENT_FLAGS = T.let({
123
+ "knownFalse" => "known false", "knownExpired" => "known expired",
124
+ "revokedByIssuer" => "revoked by issuer", "reportedLost" => "reported lost",
125
+ "diplomatic" => "diplomatic"
126
+ }.freeze, T::Hash[String, String])
127
+
128
+ # The parts of a place of birth, which the canonical model has no home
129
+ # for and which are on 2,462 of the 4,373 birthdate elements. Ordered
130
+ # from the most specific to the least, the way an address reads.
131
+ PLACE_PARTS = T.let(%w[@place @city @region].freeze, T::Array[String])
132
+
133
+ WHITESPACE = T.let(/[[:space:]]+/, Regexp)
134
+
135
+ sig { returns(Parsers::XmlRecords::Record) }
136
+ attr_reader :node
137
+
138
+ sig { params(node: Parsers::XmlRecords::Record).void }
139
+ def initialize(node)
140
+ @node = T.let(node, Parsers::XmlRecords::Record)
141
+ @names = T.let(nil, T.nilable(T::Array[Name]))
142
+ @dates_of_birth = T.let(nil, T.nilable(T::Array[PartialDate]))
143
+ @unread_dates = T.let([], T::Array[String])
144
+ end
145
+
146
+ # The entity, or nil for a record with no name -- which cannot be
147
+ # screened against and is never what the Commission meant to publish.
148
+ sig { returns(T.nilable(Entity)) }
149
+ def entity
150
+ return nil if names.empty?
151
+
152
+ Entity.new(source: :eu_fsf, source_ref: source_ref, type: type, names: names,
153
+ addresses: addresses, identifiers: identifiers, dates_of_birth: dates_of_birth,
154
+ nationalities: nationalities, programs: programs, listed_on: listed_on,
155
+ remarks: remarks)
156
+ end
157
+
158
+ sig { returns(Symbol) }
159
+ def type = TYPES.fetch(node["subjectType/@code"].to_s, :organization)
160
+
161
+ # `euReferenceNumber` rather than `logicalId`: both are unique across
162
+ # all 6,234 records, but the reference number is the one the
163
+ # Commission prints in its own consolidated list and the one a person
164
+ # checking a hit against the official file can search for. The internal
165
+ # id is kept in remarks so the two can still be reconciled.
166
+ sig { returns(T.nilable(String)) }
167
+ def source_ref = node["@euReferenceNumber"]
168
+
169
+ # ISO 3166 alpha-2, which is what the scorer compares on, rather than
170
+ # the shouty description beside it. Three withdrawn codes are in the
171
+ # file -- AN, CS and YU, for states that no longer exist -- and
172
+ # Country resolves none of them, which Scorer::Adjustments reads as
173
+ # absent rather than as a conflict. That is the right answer: nobody
174
+ # holds a Yugoslav passport to contradict.
175
+ sig { returns(T::Array[String]) }
176
+ def nationalities
177
+ node.values("citizenship/@countryIso2Code").reject { |code| code == UNKNOWN_COUNTRY }.uniq
178
+ end
179
+
180
+ # The date the Council designated the entity, which 5,654 of the 6,234
181
+ # records carry. Not the regulation's publication date: a record is
182
+ # amended by later acts, so `regulation/@publicationDate` moves while
183
+ # this does not.
184
+ sig { returns(T.nilable(PartialDate)) }
185
+ def listed_on = PartialDate.parse(node["@designationDate"])
186
+
187
+ sig { returns(T::Array[String]) }
188
+ def programs = node.values("regulation/@programme").uniq
189
+
190
+ sig { returns(T::Array[Name]) }
191
+ def names
192
+ @names ||= promote(published_names)
193
+ end
194
+
195
+ # Memoized, because reading them is also what fills the list of dates
196
+ # this could not use -- a Hijri year with no Gregorian equivalent, a
197
+ # half-open span -- which #remarks then keeps rather than losing.
198
+ sig { returns(T::Array[PartialDate]) }
199
+ def dates_of_birth
200
+ @dates_of_birth ||= begin
201
+ @unread_dates = []
202
+ # Deduped: one Saudi record publishes the same day three times,
203
+ # twice as itself and once as the Gregorian rendering of a Hijri
204
+ # date. Three copies of one date are three ways to say the same
205
+ # thing to a comparison that already reads it once.
206
+ node.nodes("birthdate").filter_map { |born| date_of_birth(born) }.uniq
207
+ end
208
+ end
209
+
210
+ sig { returns(T::Array[Identifier]) }
211
+ def identifiers
212
+ node.nodes("identification").filter_map { |document| identifier(document) }
213
+ end
214
+
215
+ sig { returns(T::Array[Address]) }
216
+ def addresses
217
+ node.nodes("address").filter_map { |place| address(place) }
218
+ end
219
+
220
+ # The Commission's own comment verbatim, then the elements that have
221
+ # nowhere else to go, behind the marker that makes them trivial to
222
+ # strip again.
223
+ sig { returns(T.nilable(String)) }
224
+ def remarks
225
+ dates_of_birth # for its side effect: it is what fills @unread_dates, which #extras keeps
226
+ Remarks.build(node["remark"], extras)
227
+ end
228
+
229
+ private
230
+
231
+ # Every `<nameAlias>` the record carries, in document order, deduped on
232
+ # the published spelling: 222 records file the same string twice under
233
+ # two `nameLanguage` values, and two identical Names are two index
234
+ # entries that can only ever fire together.
235
+ sig { returns(T::Array[Name]) }
236
+ def published_names
237
+ seen = T.let({}, T::Hash[String, Name])
238
+ node.nodes("nameAlias").each do |alt|
239
+ value = collapse(alt["@wholeName"])
240
+ next if value.nil? || seen.key?(value)
241
+
242
+ annotation = clauses(alt["remark"])
243
+ seen[value] = Name.new(value: value, kind: alias_kind(annotation),
244
+ quality: alias_quality(annotation))
245
+ end
246
+ seen.values
247
+ end
248
+
249
+ # The EU marks no name as the official one, so one is chosen: the first
250
+ # the Commission published that it did not itself annotate as an alias.
251
+ # 24 records lead with a name their own remark calls a low-quality
252
+ # alias or a former name, and promoting one of those would report a hit
253
+ # under a spelling the Commission had flagged as weak.
254
+ #
255
+ # The script is left unstated throughout, the same choice the UN
256
+ # adapter makes: which script a string is in is a question about its
257
+ # characters, and `nameLanguage` does not answer it -- record
258
+ # EU.2797.3 files a Cyrillic spelling under `nameLanguage="EN"`.
259
+ sig { params(published: T::Array[Name]).returns(T::Array[Name]) }
260
+ def promote(published)
261
+ chosen = published.find { |name| name.kind == :aka && name.quality.nil? } || published.first
262
+ published.map do |name|
263
+ name.equal?(chosen) ? Name.new(value: name.value, kind: :primary, quality: name.quality) : name
264
+ end
265
+ end
266
+
267
+ sig { params(annotation: T.untyped).returns(T::Array[String]) }
268
+ def clauses(annotation)
269
+ annotation.to_s.split(CLAUSE).map { |clause| clause.sub(CLAUSE_OPENING, "") }
270
+ end
271
+
272
+ sig { params(annotation: T::Array[String]).returns(Symbol) }
273
+ def alias_kind(annotation)
274
+ annotation.any? { |clause| clause.match?(FORMER_NAME) } ? :fka : :aka
275
+ end
276
+
277
+ sig { params(annotation: T::Array[String]).returns(T.nilable(Symbol)) }
278
+ def alias_quality(annotation)
279
+ return :low if annotation.any? { |clause| clause.match?(LOW_QUALITY) }
280
+
281
+ :good if annotation.any? { |clause| clause.match?(GOOD_QUALITY) }
282
+ end
283
+
284
+ # A Hijri date is not a Gregorian one, and the components hold the
285
+ # Hijri reading: `year="1343"` for a man born in 1964. Where the
286
+ # Commission also supplied `birthdate` -- which it does on one of the
287
+ # four -- that attribute is already converted and is read; where it did
288
+ # not, there is no date here at all, and the published one goes to
289
+ # remarks rather than into a comparison it would poison.
290
+ sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(PartialDate)) }
291
+ def date_of_birth(born)
292
+ return converted(born) unless born["@calendarType"] == GREGORIAN
293
+
294
+ approximate = born["@circa"] == BOOLEAN_TRUE
295
+ point(born, approximate) || span(born, approximate) || note_unread(born)
296
+ end
297
+
298
+ sig { params(born: Parsers::XmlRecords::Record, approximate: T::Boolean).returns(T.nilable(PartialDate)) }
299
+ def point(born, approximate)
300
+ year = born["@year"]
301
+ return nil if year.nil?
302
+
303
+ PartialDate.new(year: year, month: born["@monthOfYear"], day: born["@dayOfMonth"],
304
+ approximate: approximate)
305
+ rescue ArgumentError
306
+ nil
307
+ end
308
+
309
+ # `yearRangeFrom` and `yearRangeTo`, on 55 records. One record in the
310
+ # file carries only the `to` half, which is a claim about an interval
311
+ # with no beginning: PartialDate has no shape for it, and inventing one
312
+ # end of a span is exactly the false precision it exists to prevent.
313
+ sig { params(born: Parsers::XmlRecords::Record, approximate: T::Boolean).returns(T.nilable(PartialDate)) }
314
+ def span(born, approximate)
315
+ from = born["@yearRangeFrom"]
316
+ to = born["@yearRangeTo"]
317
+ return nil if from.nil? || to.nil?
318
+
319
+ PartialDate.range(from, to, approximate: approximate)
320
+ rescue ArgumentError
321
+ nil
322
+ end
323
+
324
+ sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(PartialDate)) }
325
+ def converted(born)
326
+ PartialDate.parse(born["@birthdate"]) || note_unread(born)
327
+ end
328
+
329
+ # A birthdate element that gave no usable date. 110 of them are a place
330
+ # of birth and nothing else, which #extras already keeps; what is
331
+ # recorded here is the rest -- the Hijri dates and the half-open span
332
+ # -- so that nothing the Commission published about when a person was
333
+ # born disappears silently.
334
+ sig { params(born: Parsers::XmlRecords::Record).returns(NilClass) }
335
+ def note_unread(born)
336
+ published = [born["@birthdate"], born["@year"], span_text(born)].compact
337
+ return nil if published.empty?
338
+
339
+ calendar = born["@calendarType"].to_s
340
+ suffix = " (#{calendar.downcase} calendar)" unless calendar == GREGORIAN
341
+ @unread_dates << "#{published.join(" ")}#{suffix}"
342
+ nil
343
+ end
344
+
345
+ # A span with one end missing, written so that the missing end is
346
+ # visible rather than implied.
347
+ sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
348
+ def span_text(born)
349
+ from = born["@yearRangeFrom"]
350
+ to = born["@yearRangeTo"]
351
+ return nil if from.nil? && to.nil?
352
+
353
+ "#{from || "?"} to #{to || "?"}"
354
+ end
355
+
356
+ # `number` is populated on all 3,013 identification elements, but six
357
+ # of them hold `-`, which is the Commission writing "no number" in the
358
+ # number field. Identifier refuses a value with no alphanumerics and
359
+ # those six are dropped: a document with a type and no identity is
360
+ # nothing to match on, which is the same call the UN adapter makes
361
+ # about the 447 numberless documents on its list.
362
+ #
363
+ # The kind that could not be classified still keeps the Commission's
364
+ # own description, which is what a person reading the hit needs.
365
+ sig { params(document: Parsers::XmlRecords::Record).returns(T.nilable(Identifier)) }
366
+ def identifier(document)
367
+ Identifier.new(kind: document_kind(document), value: document["@number"],
368
+ country: country(document),
369
+ issued_on: PartialDate.parse(document["@issueDate"] || document["@validFrom"]),
370
+ expires_on: PartialDate.parse(document["@validTo"]),
371
+ note: document_note(document))
372
+ rescue ArgumentError
373
+ nil
374
+ end
375
+
376
+ sig { params(document: Parsers::XmlRecords::Record).returns(Symbol) }
377
+ def document_kind(document)
378
+ DOCUMENT_KINDS.fetch(document["@identificationTypeCode"].to_s.downcase, :other)
379
+ end
380
+
381
+ sig { params(document: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
382
+ def document_note(document)
383
+ flags = DOCUMENT_FLAGS.filter_map { |attribute, label| label if document["@#{attribute}"] == BOOLEAN_TRUE }
384
+ parts = [document["@identificationTypeDescription"], document["@issuedBy"], document["@region"],
385
+ document["remark"], *flags]
386
+ joined = parts.compact.join("; ")
387
+ joined.empty? ? nil : joined
388
+ end
389
+
390
+ # An address element that located nothing is dropped rather than kept
391
+ # empty: it cannot be screened on and would only inflate the count.
392
+ # 87 of the 2,648 carry the `00` country sentinel and nothing else.
393
+ sig { params(place: Parsers::XmlRecords::Record).returns(T.nilable(Address)) }
394
+ def address(place)
395
+ Address.new(street: place["@street"], city: place["@city"], state_province: place["@region"],
396
+ postal_code: place["@zipCode"], country: country(place), note: address_note(place))
397
+ rescue ArgumentError
398
+ nil
399
+ end
400
+
401
+ # `poBox` and `place` have no member of their own on Address, and
402
+ # `contactInfo` -- a web site, a phone number, an email address, on
403
+ # 1,681 of the addresses -- has nowhere at all. All of it is real
404
+ # locating detail, so it goes to the note rather than over the side.
405
+ sig { params(place: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
406
+ def address_note(place)
407
+ contacts = place.nodes("contactInfo").map { |info| "#{info["@key"]}: #{info["@value"]}" }
408
+ box = place["@poBox"]
409
+ parts = [box && "P.O. Box #{box}", place["@place"], place["remark"], *contacts,
410
+ ("as at listing time" if place["@asAtListingTime"] == BOOLEAN_TRUE)]
411
+ joined = parts.compact.join("; ")
412
+ joined.empty? ? nil : joined
413
+ end
414
+
415
+ # The Commission's prose name for a country, or nil for its sentinel.
416
+ # Prose rather than the code beside it because Address#country and
417
+ # Identifier#country are what a reviewer reads, and `IQ` is not.
418
+ sig { params(element: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
419
+ def country(element)
420
+ return nil if element["@countryIso2Code"] == UNKNOWN_COUNTRY
421
+
422
+ name = element["@countryDescription"]
423
+ name == UNKNOWN_COUNTRY_NAME ? nil : name
424
+ end
425
+
426
+ # Label/value pairs for Remarks.build, which drops the ones the record
427
+ # left blank. A value may be several -- one person's function is filed
428
+ # once per name -- and arrives as the Array it published.
429
+ sig { returns(T::Array[T.untyped]) }
430
+ def extras
431
+ [["EU logical id", node["@logicalId"]],
432
+ ["UN reference", node["@unitedNationId"]],
433
+ ["Designation details", node["@designationDetails"]],
434
+ ["Regulation", node["regulation/@numberTitle"]],
435
+ ["Official Journal", node["regulation/publicationUrl"]],
436
+ ["Function", node.values("nameAlias/@function").uniq],
437
+ ["Title", node.values("nameAlias/@title").uniq],
438
+ ["Gender", node.values("nameAlias/@gender").uniq],
439
+ ["Date of birth as published", @unread_dates]] + places_of_birth
440
+ end
441
+
442
+ # A place of birth is real screening signal and the canonical model has
443
+ # no home for it, so it is kept rather than dropped to keep a schema
444
+ # tidy. One line per birthdate element, because a person with two
445
+ # reported birth dates usually has two reported birth places.
446
+ sig { returns(T::Array[T.untyped]) }
447
+ def places_of_birth
448
+ places = node.nodes("birthdate").filter_map do |born|
449
+ parts = PLACE_PARTS.filter_map { |part| born[part] } + [country(born)].compact
450
+ parts.join(", ") if parts.any?
451
+ end
452
+ places.uniq.map { |place| ["Place of birth", place] }
453
+ end
454
+
455
+ # The Commission's own line wrapping arrives inside attribute values:
456
+ # 2,149 of the 31,053 names carry a newline, a run of spaces, or
457
+ # padding on one side. Left verbatim those produce "Ivan Ivanov",
458
+ # which is a name nothing will ever match.
459
+ sig { params(value: T.untyped).returns(T.nilable(String)) }
460
+ def collapse(value)
461
+ string = value.to_s.split(WHITESPACE).join(" ")
462
+ string.empty? ? nil : -string
463
+ end
464
+ end
465
+ end
466
+ end
467
+ end
@@ -0,0 +1,203 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/parsers"
7
+ require "active_sanction/sources"
8
+ require "active_sanction/sources/base"
9
+
10
+ module ActiveSanction
11
+ module Sources
12
+ # The EU Consolidated Financial Sanctions List: every person and entity
13
+ # subject to an EU financial sanction, in one document, as the Financial
14
+ # Sanctions Files (FSF) export.
15
+ #
16
+ # snapshot = ActiveSanction::Sources[:eu_fsf].new.sync
17
+ #
18
+ # ### One file, one record shape, ten times the size
19
+ #
20
+ # https://webgate.ec.europa.eu/fsd/fsf/public/files/xmlFullSanctionsList_1_1/content
21
+ #
22
+ # <export><sanctionEntity> 6,234 records, 25.7 MB
23
+ #
24
+ # An order of magnitude larger than the UN's 2.2 MB or Canada's 2.9 MB, and
25
+ # the first list to actually exercise the streaming interface the XML
26
+ # toolkit was built around. It reads in one pass at roughly 90 MB of
27
+ # resident memory, most of which is the payload itself; nothing here ever
28
+ # holds two records at once.
29
+ #
30
+ # ### The token in the URL
31
+ #
32
+ # The public endpoint takes a `token` query parameter. Omitting it answers
33
+ # 403 and a wrong one answers 500, so it is not optional -- but it is also
34
+ # not a credential: `dG9rZW4tMjAxNw` is base64 for `token-2017`, it is the
35
+ # value the Commission's own public download page has carried since that
36
+ # year, and it is the same string for every caller. It is declared inline
37
+ # rather than configured, the way any other part of a published URL is.
38
+ #
39
+ # If the Commission ever rotates it, no release of this gem is needed:
40
+ #
41
+ # ActiveSanction::Sources::EuFsf.token = "..." # or
42
+ # ActiveSanction::Sources::EuFsf.url :main, "https://..."
43
+ #
44
+ # ### Conditional GET does not work here, and that is the publisher's doing
45
+ #
46
+ # The endpoint serves `Last-Modified` but answers `If-Modified-Since` with
47
+ # 200 and the whole file, and sends no `ETag` and `Cache-Control:
48
+ # no-store`. So every sync of this list downloads 25.7 MB where the other
49
+ # four usually download nothing. The conditional request is still sent --
50
+ # it costs nothing and the day the Commission honours it, it works -- and
51
+ # the snapshot checksum is unchanged when the content is, so a re-download
52
+ # of identical bytes still diffs (#35) to nothing.
53
+ #
54
+ # ### The two traps in this list
55
+ #
56
+ # **No name is the name.** The EU marks no `<nameAlias>` as the official
57
+ # one: all 31,053 carry `strong="true"`, and every other candidate signal
58
+ # is wrong somewhere in the file. Document order files Qusay Hussein's
59
+ # French transliteration ahead of his English name; `nameLanguage` files a
60
+ # Cyrillic spelling of Anatoliy Sidorov's name under `EN`; ordering by
61
+ # `logicalId` picks a non-Latin name for 3,203 of the 5,502 multi-name
62
+ # records. So Record picks one by a stated rule -- see #primary_name -- and
63
+ # what "primary" means for this source is *the first name the EU published
64
+ # that the EU did not itself annotate as an alias*, which is weaker than
65
+ # what it means for OFAC. It costs nothing in score: the scorer takes the
66
+ # best of an entity's names and the kind only reaches the reason line.
67
+ #
68
+ # **Four birth dates are not in the Gregorian calendar.** `calendarType`
69
+ # is `ISLAMIC` on four records, where `year`, `monthOfYear` and
70
+ # `dayOfMonth` hold a Hijri date -- `year="1343"` for a man born in 1964.
71
+ # Read as published those become dates in the fourteenth century, and a
72
+ # fourteenth-century date does not merely fail to match a real one, it
73
+ # *conflicts* with it, and the scorer penalizes the record. Three of the
74
+ # four carry no Gregorian equivalent at all, so they produce no date of
75
+ # birth and the published Hijri date goes to remarks instead.
76
+ #
77
+ # ### What a clean EU result is worth
78
+ #
79
+ # More than a Canadian one and less than an OFAC one. 3,013 identification
80
+ # documents across 6,234 records is real corroborating signal, and 2,764
81
+ # citizenships arrive as ISO codes rather than as prose. What is thin is
82
+ # the alias grading: the EU publishes no quality column, and the 499
83
+ # gradings this adapter does read are prose the Commission happened to put
84
+ # in a `<remark>` -- so most EU aliases arrive ungraded, which the scorer
85
+ # correctly treats as unstated rather than as good.
86
+ #
87
+ # ### What this adapter does not do
88
+ #
89
+ # It does not infer a vessel. `subjectType` publishes only `person` and
90
+ # `enterprise`, and the 41 records carrying an `imo` document are shipping
91
+ # *companies* holding IMO company numbers -- Chongchongang Shipping, Korea
92
+ # Ansan Shipping -- not the ships themselves. Typing those as :vessel on
93
+ # the strength of the document kind would be a guess that hides them from
94
+ # every organization search.
95
+ class EuFsf < Base
96
+ extend T::Sig
97
+
98
+ key :eu_fsf
99
+ jurisdiction :eu
100
+ authority "European Commission"
101
+ format :xml
102
+
103
+ # Commission Decision 2011/833/EU, which permits reuse of Commission
104
+ # documents provided the source is acknowledged and the reuse does not
105
+ # suggest the Commission endorses it.
106
+ licence_notice "Reusable under Commission Decision 2011/833/EU " \
107
+ "provided the source is acknowledged and no Commission " \
108
+ "endorsement is implied. Verified 2026-09-11."
109
+ licence_url "https://eur-lex.europa.eu/eli/dec/2011/833/oj/eng"
110
+
111
+ # Split out so #token= can rebuild the URL around a rotated value. See
112
+ # the class comment for why the token is not a credential.
113
+ #
114
+ # @api private
115
+ ENDPOINT = T.let(
116
+ "https://webgate.ec.europa.eu/fsd/fsf/public/files/xmlFullSanctionsList_1_1/content", String
117
+ )
118
+
119
+ # @api private
120
+ PUBLIC_TOKEN = T.let("dG9rZW4tMjAxNw", String)
121
+
122
+ url :main, "#{ENDPOINT}?token=#{PUBLIC_TOKEN}"
123
+
124
+ # @api private
125
+ SANCTION_ENTITY = T.let("sanctionEntity", String)
126
+
127
+ # @api private
128
+ LIST = T.let(Parsers::XmlRecords.new(records: SANCTION_ENTITY), Parsers::XmlRecords)
129
+
130
+ # The generation timestamp the Commission stamps on the document element,
131
+ # to the millisecond. More precise than the Last-Modified header Base
132
+ # falls back to, and the string the FSF download page itself shows, so it
133
+ # is the one an examiner asking "which version was this screened
134
+ # against" will recognise.
135
+ #
136
+ # @api private
137
+ GENERATED_AT = T.let("generationDate", String)
138
+
139
+ # Re-points the list at the same endpoint with a different token, for the
140
+ # day the Commission rotates the one baked in above.
141
+ #
142
+ # ActiveSanction::Sources::EuFsf.token = "..."
143
+ sig { params(value: T.untyped).returns(String) }
144
+ def self.token=(value)
145
+ url :main, "#{ENDPOINT}?token=#{value}"
146
+ end
147
+
148
+ # Records that could not be used, and fields that could not be read.
149
+ # Read after #parse; sync orchestration (#34) reports them.
150
+ sig { returns(T::Array[Parsers::Warning]) }
151
+ attr_reader :warnings
152
+
153
+ sig { params(args: T.untyped, options: T.untyped).void }
154
+ def initialize(*args, **options)
155
+ super
156
+ @warnings = T.let([], T::Array[Parsers::Warning])
157
+ @unmapped = T.let([], T::Array[Parsers::Warning])
158
+ @generated_at = T.let(nil, T.nilable(String))
159
+ end
160
+
161
+ sig { override.params(raw: T.untyped).returns(T::Array[Entity]) }
162
+ def parse(raw)
163
+ reader = LIST.read(raw)
164
+ entities = build(reader)
165
+ @generated_at = reader.root[GENERATED_AT]
166
+ @warnings = reader.warnings + @unmapped
167
+ entities
168
+ end
169
+
170
+ sig { override.returns(T.nilable(String)) }
171
+ def source_version = @generated_at || super
172
+
173
+ private
174
+
175
+ sig { params(reader: Parsers::XmlRecords::Reader).returns(T::Array[Entity]) }
176
+ def build(reader)
177
+ @unmapped = []
178
+ reader.filter_map do |node|
179
+ record = Record.new(node)
180
+ entity = record.entity
181
+ entity.nil? ? note_nameless(node) : entity
182
+ end
183
+ end
184
+
185
+ # A record with no name cannot be screened against and is never what the
186
+ # Commission meant to publish. Every one of the 6,234 published today
187
+ # carries at least one `<nameAlias>`; the warning exists so that the day
188
+ # one does not, it is visible rather than absent.
189
+ sig { params(node: Parsers::XmlRecords::Record).returns(NilClass) }
190
+ def note_nameless(node)
191
+ @unmapped << Parsers::Warning.new(
192
+ line: node.line,
193
+ message: "<#{node.name}> #{node["@euReferenceNumber"].inspect} has no name and was skipped"
194
+ )
195
+ nil
196
+ end
197
+ end
198
+ end
199
+ end
200
+
201
+ require "active_sanction/sources/eu_fsf/record"
202
+
203
+ ActiveSanction::Sources.register(ActiveSanction::Sources::EuFsf)