active_sanction 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +538 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/CONTRIBUTING.md +312 -0
- data/LICENSE.txt +21 -0
- data/README.md +616 -0
- data/SECURITY.md +97 -0
- data/active_sanction.gemspec +98 -0
- data/docs/adding_a_source.md +1083 -0
- data/docs/api_stability.md +357 -0
- data/docs/bundle_format.md +407 -0
- data/lib/active_sanction/address.rb +170 -0
- data/lib/active_sanction/client.rb +341 -0
- data/lib/active_sanction/configuration.rb +721 -0
- data/lib/active_sanction/countries.txt +292 -0
- data/lib/active_sanction/country.rb +164 -0
- data/lib/active_sanction/deprecation.rb +168 -0
- data/lib/active_sanction/diff/change.rb +208 -0
- data/lib/active_sanction/diff.rb +348 -0
- data/lib/active_sanction/doctor/checkup.rb +356 -0
- data/lib/active_sanction/doctor/diagnosis.rb +245 -0
- data/lib/active_sanction/doctor/finding.rb +183 -0
- data/lib/active_sanction/doctor/profile.rb +384 -0
- data/lib/active_sanction/doctor/report.rb +227 -0
- data/lib/active_sanction/doctor.rb +347 -0
- data/lib/active_sanction/entity.rb +293 -0
- data/lib/active_sanction/error.rb +332 -0
- data/lib/active_sanction/fetcher/result.rb +116 -0
- data/lib/active_sanction/fetcher.rb +242 -0
- data/lib/active_sanction/http_client/errors.rb +94 -0
- data/lib/active_sanction/http_client/response.rb +129 -0
- data/lib/active_sanction/http_client.rb +324 -0
- data/lib/active_sanction/identifier.rb +212 -0
- data/lib/active_sanction/index/builder.rb +89 -0
- data/lib/active_sanction/index/candidate.rb +63 -0
- data/lib/active_sanction/index/entry.rb +66 -0
- data/lib/active_sanction/index/features.rb +112 -0
- data/lib/active_sanction/index.rb +381 -0
- data/lib/active_sanction/match_result.rb +345 -0
- data/lib/active_sanction/matcher.rb +388 -0
- data/lib/active_sanction/name.rb +183 -0
- data/lib/active_sanction/normalizer/cache.rb +82 -0
- data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
- data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
- data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
- data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
- data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
- data/lib/active_sanction/normalizer/dictionary.rb +243 -0
- data/lib/active_sanction/normalizer/form.rb +279 -0
- data/lib/active_sanction/normalizer.rb +137 -0
- data/lib/active_sanction/parsers/column_shape.rb +274 -0
- data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
- data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
- data/lib/active_sanction/parsers/delimited_table.rb +151 -0
- data/lib/active_sanction/parsers/format.rb +107 -0
- data/lib/active_sanction/parsers/join.rb +149 -0
- data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
- data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
- data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
- data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
- data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
- data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
- data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
- data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
- data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
- data/lib/active_sanction/parsers/xml_records.rb +140 -0
- data/lib/active_sanction/parsers.rb +94 -0
- data/lib/active_sanction/partial_date/parser.rb +124 -0
- data/lib/active_sanction/partial_date.rb +303 -0
- data/lib/active_sanction/payload_cache/checksum.rb +66 -0
- data/lib/active_sanction/payload_cache/entry.rb +304 -0
- data/lib/active_sanction/payload_cache.rb +366 -0
- data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
- data/lib/active_sanction/phonetics.rb +40 -0
- data/lib/active_sanction/query.rb +290 -0
- data/lib/active_sanction/rescreen/alert.rb +314 -0
- data/lib/active_sanction/rescreen.rb +332 -0
- data/lib/active_sanction/scorer/adjustments.rb +241 -0
- data/lib/active_sanction/scorer/name_score.rb +279 -0
- data/lib/active_sanction/scorer/reason.rb +136 -0
- data/lib/active_sanction/scorer/result.rb +129 -0
- data/lib/active_sanction/scorer/subject.rb +235 -0
- data/lib/active_sanction/scorer/weights.rb +306 -0
- data/lib/active_sanction/scorer.rb +303 -0
- data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
- data/lib/active_sanction/similarity/levenshtein.rb +179 -0
- data/lib/active_sanction/similarity/token_set.rb +159 -0
- data/lib/active_sanction/similarity/token_sort.rb +120 -0
- data/lib/active_sanction/similarity.rb +183 -0
- data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
- data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
- data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
- data/lib/active_sanction/snapshot/bundle.rb +381 -0
- data/lib/active_sanction/snapshot.rb +305 -0
- data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
- data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
- data/lib/active_sanction/sources/australia_dfat.rb +291 -0
- data/lib/active_sanction/sources/base.rb +295 -0
- data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
- data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
- data/lib/active_sanction/sources/canada_sema.rb +153 -0
- data/lib/active_sanction/sources/definition.rb +360 -0
- data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
- data/lib/active_sanction/sources/eu_fsf.rb +203 -0
- data/lib/active_sanction/sources/ofac/record.rb +251 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
- data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
- data/lib/active_sanction/sources/ofac.rb +248 -0
- data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
- data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
- data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
- data/lib/active_sanction/sources/remarks.rb +84 -0
- data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
- data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
- data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
- data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
- data/lib/active_sanction/sources/un_consolidated.rb +140 -0
- data/lib/active_sanction/sources.rb +196 -0
- data/lib/active_sanction/storage/active_record/reader.rb +124 -0
- data/lib/active_sanction/storage/active_record/row.rb +175 -0
- data/lib/active_sanction/storage/active_record/writer.rb +141 -0
- data/lib/active_sanction/storage/active_record.rb +277 -0
- data/lib/active_sanction/storage/base.rb +216 -0
- data/lib/active_sanction/storage/file_system.rb +373 -0
- data/lib/active_sanction/storage/memory.rb +67 -0
- data/lib/active_sanction/storage/meta.rb +188 -0
- data/lib/active_sanction/storage.rb +74 -0
- data/lib/active_sanction/subject.rb +271 -0
- data/lib/active_sanction/sync/report.rb +245 -0
- data/lib/active_sanction/sync/result.rb +296 -0
- data/lib/active_sanction/sync.rb +390 -0
- data/lib/active_sanction/validator_store/file_system.rb +103 -0
- data/lib/active_sanction/validator_store/memory.rb +41 -0
- data/lib/active_sanction/validator_store.rb +119 -0
- data/lib/active_sanction/validators.rb +195 -0
- data/lib/active_sanction/version.rb +23 -0
- data/lib/active_sanction.rb +331 -0
- data/lib/generators/active_sanction/install/install_generator.rb +60 -0
- data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
- metadata +236 -0
|
@@ -0,0 +1,467 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/entity"
|
|
7
|
+
require "active_sanction/name"
|
|
8
|
+
require "active_sanction/address"
|
|
9
|
+
require "active_sanction/identifier"
|
|
10
|
+
require "active_sanction/partial_date"
|
|
11
|
+
require "active_sanction/sources/remarks"
|
|
12
|
+
|
|
13
|
+
module ActiveSanction
|
|
14
|
+
module Sources
|
|
15
|
+
class EuFsf < Base
|
|
16
|
+
# One `<sanctionEntity>` turned into an Entity.
|
|
17
|
+
#
|
|
18
|
+
# Separate from the adapter because they are two jobs: the adapter says
|
|
19
|
+
# what the list is and where it lives, and this says what the
|
|
20
|
+
# Commission's elements mean. The mapping is where all the judgment sits,
|
|
21
|
+
# so it is worth being able to read it on its own.
|
|
22
|
+
#
|
|
23
|
+
# ### Nearly everything is an attribute
|
|
24
|
+
#
|
|
25
|
+
# The FSF export puts its data in attributes and its prose in elements:
|
|
26
|
+
# a name is `<nameAlias wholeName="...">` and the only text nodes in the
|
|
27
|
+
# whole 25.7 MB file are `<remark>` and `<publicationUrl>`. So most reads
|
|
28
|
+
# here are the toolkit's `@attribute` paths, and the `<remark>` child of
|
|
29
|
+
# a name, a date, an address or a document is the one place the
|
|
30
|
+
# Commission writes free prose about that one field.
|
|
31
|
+
#
|
|
32
|
+
# ### The regulation trail this deliberately flattens
|
|
33
|
+
#
|
|
34
|
+
# Every repeated element carries a `<regulationSummary>` naming the act
|
|
35
|
+
# that put it there, so the file records not just that a person has an
|
|
36
|
+
# address but which regulation added it. That is a provenance graph, and
|
|
37
|
+
# the canonical Entity is not one: what survives here is the entity's own
|
|
38
|
+
# current `<regulation>` -- its programme, its number and its Official
|
|
39
|
+
# Journal URL -- and the per-field trail is dropped rather than
|
|
40
|
+
# flattened into 31,053 remark fragments nothing reads.
|
|
41
|
+
#
|
|
42
|
+
# @api private
|
|
43
|
+
class Record
|
|
44
|
+
extend T::Sig
|
|
45
|
+
|
|
46
|
+
TYPES = T.let({ "person" => :individual, "enterprise" => :organization }.freeze,
|
|
47
|
+
T::Hash[String, Symbol])
|
|
48
|
+
|
|
49
|
+
# `identificationTypeCode` is a closed vocabulary of 18 values, which
|
|
50
|
+
# is why this maps codes rather than the free-text description beside
|
|
51
|
+
# them. Anything unrecognised is :other, which is a real answer: a
|
|
52
|
+
# document we cannot classify still matches on its number, and the
|
|
53
|
+
# Commission's own description is kept in the note either way.
|
|
54
|
+
DOCUMENT_KINDS = T.let({
|
|
55
|
+
"passport" => :passport,
|
|
56
|
+
"id" => :national_id, "ssn" => :national_id, "unssn" => :national_id,
|
|
57
|
+
"fiscalcode" => :tax_id, "taxid" => :tax_id, "euvat" => :tax_id,
|
|
58
|
+
"regnumber" => :registration_number, "tradelic" => :registration_number
|
|
59
|
+
}.freeze, T::Hash[String, Symbol])
|
|
60
|
+
|
|
61
|
+
# The EU publishes no alias-quality column and no alias-kind column.
|
|
62
|
+
# What it publishes instead is prose in a name's `<remark>`: "low
|
|
63
|
+
# quality alias", "Good quality a.k.a.", "formerly known as", "Maiden
|
|
64
|
+
# name: Al Akhras". Read, because a low-quality alias is a weaker
|
|
65
|
+
# signal and the scorer (#32) penalizes it, and because a former name
|
|
66
|
+
# is a real hit that should not rank as a current one.
|
|
67
|
+
#
|
|
68
|
+
# ### Why these are anchored to the start of a clause
|
|
69
|
+
#
|
|
70
|
+
# Searching the remark for the phrase anywhere is what a first version
|
|
71
|
+
# did, and it is wrong on real records. The remark on Zadna
|
|
72
|
+
# International's own name is five lines about the company, one of
|
|
73
|
+
# which reads "99 % owned by the Special Fund ..., formerly known as
|
|
74
|
+
# the Charity Organisation for the Support of the Armed Forces" -- a
|
|
75
|
+
# sentence about the *owner*. Matched loosely it files the company's
|
|
76
|
+
# published English name as a former name and promotes the French
|
|
77
|
+
# translation in its place, on a record where the Commission said no
|
|
78
|
+
# such thing.
|
|
79
|
+
#
|
|
80
|
+
# A grading is a whole annotation, so it starts one: the remark is cut
|
|
81
|
+
# into clauses at newlines and semicolons -- which is how the
|
|
82
|
+
# Commission writes them, "born 11.8.1960 in Libya\ngood quality
|
|
83
|
+
# alias" -- and a clause grades the name only if it begins with one of
|
|
84
|
+
# these. Every embedded occurrence in the published file is prose about
|
|
85
|
+
# some other entity, and every standalone one is a grading.
|
|
86
|
+
LOW_QUALITY = T.let(/\Alow[[:space:]]+quality[[:space:]]+(?:alias|a\.k\.a)/i, Regexp)
|
|
87
|
+
GOOD_QUALITY = T.let(/\A(?:good|high)[[:space:]]+quality[[:space:]]+(?:alias|a\.k\.a)/i, Regexp)
|
|
88
|
+
FORMER_NAME = T.let(
|
|
89
|
+
/\A(?:formerly[[:space:]]+known[[:space:]]+as|former[[:space:]]+name|f\.k\.a|maiden[[:space:]]+name)/i,
|
|
90
|
+
Regexp
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
# What the Commission separates annotations with, and the punctuation
|
|
94
|
+
# it opens one with -- `(formerly known as State enterprise ...)`.
|
|
95
|
+
CLAUSE = T.let(/[\n;]/, Regexp)
|
|
96
|
+
CLAUSE_OPENING = T.let(/\A[[:space:]("'\u2018]+/, Regexp)
|
|
97
|
+
|
|
98
|
+
# `calendarType`, and the one value that means the date components can
|
|
99
|
+
# be read as they are written. See the adapter's class comment for what
|
|
100
|
+
# reading a Hijri year as a Gregorian one does to a score.
|
|
101
|
+
GREGORIAN = T.let("GREGORIAN", String)
|
|
102
|
+
|
|
103
|
+
# The Commission's own sentinel for "not stated", published in the
|
|
104
|
+
# ISO 3166 field with `countryDescription="UNKNOWN"` beside it. 1,743
|
|
105
|
+
# of the 4,373 birth dates carry it, and passing it through would file
|
|
106
|
+
# 1,743 people as citizens of a country called `00`.
|
|
107
|
+
UNKNOWN_COUNTRY = T.let("00", String)
|
|
108
|
+
|
|
109
|
+
UNKNOWN_COUNTRY_NAME = T.let("UNKNOWN", String)
|
|
110
|
+
|
|
111
|
+
# Every boolean in the export is the lowercase string, on the
|
|
112
|
+
# attribute rather than as a present-or-absent flag: `circa="false"`
|
|
113
|
+
# appears 4,176 times. So a flag is read by comparing, and the absent
|
|
114
|
+
# attribute -- which is a different thing from `"false"` -- is nil and
|
|
115
|
+
# compares false without a branch.
|
|
116
|
+
BOOLEAN_TRUE = T.let("true", String)
|
|
117
|
+
|
|
118
|
+
# Attributes that say something about a document rather than
|
|
119
|
+
# identifying it, kept in the identifier's note. `knownFalse` is on one
|
|
120
|
+
# document in the whole file and is the most important of them: it is
|
|
121
|
+
# the Commission saying the number is a forgery.
|
|
122
|
+
DOCUMENT_FLAGS = T.let({
|
|
123
|
+
"knownFalse" => "known false", "knownExpired" => "known expired",
|
|
124
|
+
"revokedByIssuer" => "revoked by issuer", "reportedLost" => "reported lost",
|
|
125
|
+
"diplomatic" => "diplomatic"
|
|
126
|
+
}.freeze, T::Hash[String, String])
|
|
127
|
+
|
|
128
|
+
# The parts of a place of birth, which the canonical model has no home
|
|
129
|
+
# for and which are on 2,462 of the 4,373 birthdate elements. Ordered
|
|
130
|
+
# from the most specific to the least, the way an address reads.
|
|
131
|
+
PLACE_PARTS = T.let(%w[@place @city @region].freeze, T::Array[String])
|
|
132
|
+
|
|
133
|
+
WHITESPACE = T.let(/[[:space:]]+/, Regexp)
|
|
134
|
+
|
|
135
|
+
sig { returns(Parsers::XmlRecords::Record) }
|
|
136
|
+
attr_reader :node
|
|
137
|
+
|
|
138
|
+
sig { params(node: Parsers::XmlRecords::Record).void }
|
|
139
|
+
def initialize(node)
|
|
140
|
+
@node = T.let(node, Parsers::XmlRecords::Record)
|
|
141
|
+
@names = T.let(nil, T.nilable(T::Array[Name]))
|
|
142
|
+
@dates_of_birth = T.let(nil, T.nilable(T::Array[PartialDate]))
|
|
143
|
+
@unread_dates = T.let([], T::Array[String])
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# The entity, or nil for a record with no name -- which cannot be
|
|
147
|
+
# screened against and is never what the Commission meant to publish.
|
|
148
|
+
sig { returns(T.nilable(Entity)) }
|
|
149
|
+
def entity
|
|
150
|
+
return nil if names.empty?
|
|
151
|
+
|
|
152
|
+
Entity.new(source: :eu_fsf, source_ref: source_ref, type: type, names: names,
|
|
153
|
+
addresses: addresses, identifiers: identifiers, dates_of_birth: dates_of_birth,
|
|
154
|
+
nationalities: nationalities, programs: programs, listed_on: listed_on,
|
|
155
|
+
remarks: remarks)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
sig { returns(Symbol) }
|
|
159
|
+
def type = TYPES.fetch(node["subjectType/@code"].to_s, :organization)
|
|
160
|
+
|
|
161
|
+
# `euReferenceNumber` rather than `logicalId`: both are unique across
|
|
162
|
+
# all 6,234 records, but the reference number is the one the
|
|
163
|
+
# Commission prints in its own consolidated list and the one a person
|
|
164
|
+
# checking a hit against the official file can search for. The internal
|
|
165
|
+
# id is kept in remarks so the two can still be reconciled.
|
|
166
|
+
sig { returns(T.nilable(String)) }
|
|
167
|
+
def source_ref = node["@euReferenceNumber"]
|
|
168
|
+
|
|
169
|
+
# ISO 3166 alpha-2, which is what the scorer compares on, rather than
|
|
170
|
+
# the shouty description beside it. Three withdrawn codes are in the
|
|
171
|
+
# file -- AN, CS and YU, for states that no longer exist -- and
|
|
172
|
+
# Country resolves none of them, which Scorer::Adjustments reads as
|
|
173
|
+
# absent rather than as a conflict. That is the right answer: nobody
|
|
174
|
+
# holds a Yugoslav passport to contradict.
|
|
175
|
+
sig { returns(T::Array[String]) }
|
|
176
|
+
def nationalities
|
|
177
|
+
node.values("citizenship/@countryIso2Code").reject { |code| code == UNKNOWN_COUNTRY }.uniq
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# The date the Council designated the entity, which 5,654 of the 6,234
|
|
181
|
+
# records carry. Not the regulation's publication date: a record is
|
|
182
|
+
# amended by later acts, so `regulation/@publicationDate` moves while
|
|
183
|
+
# this does not.
|
|
184
|
+
sig { returns(T.nilable(PartialDate)) }
|
|
185
|
+
def listed_on = PartialDate.parse(node["@designationDate"])
|
|
186
|
+
|
|
187
|
+
sig { returns(T::Array[String]) }
|
|
188
|
+
def programs = node.values("regulation/@programme").uniq
|
|
189
|
+
|
|
190
|
+
sig { returns(T::Array[Name]) }
|
|
191
|
+
def names
|
|
192
|
+
@names ||= promote(published_names)
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Memoized, because reading them is also what fills the list of dates
|
|
196
|
+
# this could not use -- a Hijri year with no Gregorian equivalent, a
|
|
197
|
+
# half-open span -- which #remarks then keeps rather than losing.
|
|
198
|
+
sig { returns(T::Array[PartialDate]) }
|
|
199
|
+
def dates_of_birth
|
|
200
|
+
@dates_of_birth ||= begin
|
|
201
|
+
@unread_dates = []
|
|
202
|
+
# Deduped: one Saudi record publishes the same day three times,
|
|
203
|
+
# twice as itself and once as the Gregorian rendering of a Hijri
|
|
204
|
+
# date. Three copies of one date are three ways to say the same
|
|
205
|
+
# thing to a comparison that already reads it once.
|
|
206
|
+
node.nodes("birthdate").filter_map { |born| date_of_birth(born) }.uniq
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
sig { returns(T::Array[Identifier]) }
|
|
211
|
+
def identifiers
|
|
212
|
+
node.nodes("identification").filter_map { |document| identifier(document) }
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
sig { returns(T::Array[Address]) }
|
|
216
|
+
def addresses
|
|
217
|
+
node.nodes("address").filter_map { |place| address(place) }
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# The Commission's own comment verbatim, then the elements that have
|
|
221
|
+
# nowhere else to go, behind the marker that makes them trivial to
|
|
222
|
+
# strip again.
|
|
223
|
+
sig { returns(T.nilable(String)) }
|
|
224
|
+
def remarks
|
|
225
|
+
dates_of_birth # for its side effect: it is what fills @unread_dates, which #extras keeps
|
|
226
|
+
Remarks.build(node["remark"], extras)
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
private
|
|
230
|
+
|
|
231
|
+
# Every `<nameAlias>` the record carries, in document order, deduped on
|
|
232
|
+
# the published spelling: 222 records file the same string twice under
|
|
233
|
+
# two `nameLanguage` values, and two identical Names are two index
|
|
234
|
+
# entries that can only ever fire together.
|
|
235
|
+
sig { returns(T::Array[Name]) }
|
|
236
|
+
def published_names
|
|
237
|
+
seen = T.let({}, T::Hash[String, Name])
|
|
238
|
+
node.nodes("nameAlias").each do |alt|
|
|
239
|
+
value = collapse(alt["@wholeName"])
|
|
240
|
+
next if value.nil? || seen.key?(value)
|
|
241
|
+
|
|
242
|
+
annotation = clauses(alt["remark"])
|
|
243
|
+
seen[value] = Name.new(value: value, kind: alias_kind(annotation),
|
|
244
|
+
quality: alias_quality(annotation))
|
|
245
|
+
end
|
|
246
|
+
seen.values
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# The EU marks no name as the official one, so one is chosen: the first
|
|
250
|
+
# the Commission published that it did not itself annotate as an alias.
|
|
251
|
+
# 24 records lead with a name their own remark calls a low-quality
|
|
252
|
+
# alias or a former name, and promoting one of those would report a hit
|
|
253
|
+
# under a spelling the Commission had flagged as weak.
|
|
254
|
+
#
|
|
255
|
+
# The script is left unstated throughout, the same choice the UN
|
|
256
|
+
# adapter makes: which script a string is in is a question about its
|
|
257
|
+
# characters, and `nameLanguage` does not answer it -- record
|
|
258
|
+
# EU.2797.3 files a Cyrillic spelling under `nameLanguage="EN"`.
|
|
259
|
+
sig { params(published: T::Array[Name]).returns(T::Array[Name]) }
|
|
260
|
+
def promote(published)
|
|
261
|
+
chosen = published.find { |name| name.kind == :aka && name.quality.nil? } || published.first
|
|
262
|
+
published.map do |name|
|
|
263
|
+
name.equal?(chosen) ? Name.new(value: name.value, kind: :primary, quality: name.quality) : name
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
sig { params(annotation: T.untyped).returns(T::Array[String]) }
|
|
268
|
+
def clauses(annotation)
|
|
269
|
+
annotation.to_s.split(CLAUSE).map { |clause| clause.sub(CLAUSE_OPENING, "") }
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
sig { params(annotation: T::Array[String]).returns(Symbol) }
|
|
273
|
+
def alias_kind(annotation)
|
|
274
|
+
annotation.any? { |clause| clause.match?(FORMER_NAME) } ? :fka : :aka
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
sig { params(annotation: T::Array[String]).returns(T.nilable(Symbol)) }
|
|
278
|
+
def alias_quality(annotation)
|
|
279
|
+
return :low if annotation.any? { |clause| clause.match?(LOW_QUALITY) }
|
|
280
|
+
|
|
281
|
+
:good if annotation.any? { |clause| clause.match?(GOOD_QUALITY) }
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# A Hijri date is not a Gregorian one, and the components hold the
|
|
285
|
+
# Hijri reading: `year="1343"` for a man born in 1964. Where the
|
|
286
|
+
# Commission also supplied `birthdate` -- which it does on one of the
|
|
287
|
+
# four -- that attribute is already converted and is read; where it did
|
|
288
|
+
# not, there is no date here at all, and the published one goes to
|
|
289
|
+
# remarks rather than into a comparison it would poison.
|
|
290
|
+
sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(PartialDate)) }
|
|
291
|
+
def date_of_birth(born)
|
|
292
|
+
return converted(born) unless born["@calendarType"] == GREGORIAN
|
|
293
|
+
|
|
294
|
+
approximate = born["@circa"] == BOOLEAN_TRUE
|
|
295
|
+
point(born, approximate) || span(born, approximate) || note_unread(born)
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
sig { params(born: Parsers::XmlRecords::Record, approximate: T::Boolean).returns(T.nilable(PartialDate)) }
|
|
299
|
+
def point(born, approximate)
|
|
300
|
+
year = born["@year"]
|
|
301
|
+
return nil if year.nil?
|
|
302
|
+
|
|
303
|
+
PartialDate.new(year: year, month: born["@monthOfYear"], day: born["@dayOfMonth"],
|
|
304
|
+
approximate: approximate)
|
|
305
|
+
rescue ArgumentError
|
|
306
|
+
nil
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# `yearRangeFrom` and `yearRangeTo`, on 55 records. One record in the
|
|
310
|
+
# file carries only the `to` half, which is a claim about an interval
|
|
311
|
+
# with no beginning: PartialDate has no shape for it, and inventing one
|
|
312
|
+
# end of a span is exactly the false precision it exists to prevent.
|
|
313
|
+
sig { params(born: Parsers::XmlRecords::Record, approximate: T::Boolean).returns(T.nilable(PartialDate)) }
|
|
314
|
+
def span(born, approximate)
|
|
315
|
+
from = born["@yearRangeFrom"]
|
|
316
|
+
to = born["@yearRangeTo"]
|
|
317
|
+
return nil if from.nil? || to.nil?
|
|
318
|
+
|
|
319
|
+
PartialDate.range(from, to, approximate: approximate)
|
|
320
|
+
rescue ArgumentError
|
|
321
|
+
nil
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(PartialDate)) }
|
|
325
|
+
def converted(born)
|
|
326
|
+
PartialDate.parse(born["@birthdate"]) || note_unread(born)
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# A birthdate element that gave no usable date. 110 of them are a place
|
|
330
|
+
# of birth and nothing else, which #extras already keeps; what is
|
|
331
|
+
# recorded here is the rest -- the Hijri dates and the half-open span
|
|
332
|
+
# -- so that nothing the Commission published about when a person was
|
|
333
|
+
# born disappears silently.
|
|
334
|
+
sig { params(born: Parsers::XmlRecords::Record).returns(NilClass) }
|
|
335
|
+
def note_unread(born)
|
|
336
|
+
published = [born["@birthdate"], born["@year"], span_text(born)].compact
|
|
337
|
+
return nil if published.empty?
|
|
338
|
+
|
|
339
|
+
calendar = born["@calendarType"].to_s
|
|
340
|
+
suffix = " (#{calendar.downcase} calendar)" unless calendar == GREGORIAN
|
|
341
|
+
@unread_dates << "#{published.join(" ")}#{suffix}"
|
|
342
|
+
nil
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
# A span with one end missing, written so that the missing end is
|
|
346
|
+
# visible rather than implied.
|
|
347
|
+
sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
|
|
348
|
+
def span_text(born)
|
|
349
|
+
from = born["@yearRangeFrom"]
|
|
350
|
+
to = born["@yearRangeTo"]
|
|
351
|
+
return nil if from.nil? && to.nil?
|
|
352
|
+
|
|
353
|
+
"#{from || "?"} to #{to || "?"}"
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
# `number` is populated on all 3,013 identification elements, but six
|
|
357
|
+
# of them hold `-`, which is the Commission writing "no number" in the
|
|
358
|
+
# number field. Identifier refuses a value with no alphanumerics and
|
|
359
|
+
# those six are dropped: a document with a type and no identity is
|
|
360
|
+
# nothing to match on, which is the same call the UN adapter makes
|
|
361
|
+
# about the 447 numberless documents on its list.
|
|
362
|
+
#
|
|
363
|
+
# The kind that could not be classified still keeps the Commission's
|
|
364
|
+
# own description, which is what a person reading the hit needs.
|
|
365
|
+
sig { params(document: Parsers::XmlRecords::Record).returns(T.nilable(Identifier)) }
|
|
366
|
+
def identifier(document)
|
|
367
|
+
Identifier.new(kind: document_kind(document), value: document["@number"],
|
|
368
|
+
country: country(document),
|
|
369
|
+
issued_on: PartialDate.parse(document["@issueDate"] || document["@validFrom"]),
|
|
370
|
+
expires_on: PartialDate.parse(document["@validTo"]),
|
|
371
|
+
note: document_note(document))
|
|
372
|
+
rescue ArgumentError
|
|
373
|
+
nil
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
sig { params(document: Parsers::XmlRecords::Record).returns(Symbol) }
|
|
377
|
+
def document_kind(document)
|
|
378
|
+
DOCUMENT_KINDS.fetch(document["@identificationTypeCode"].to_s.downcase, :other)
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
sig { params(document: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
|
|
382
|
+
def document_note(document)
|
|
383
|
+
flags = DOCUMENT_FLAGS.filter_map { |attribute, label| label if document["@#{attribute}"] == BOOLEAN_TRUE }
|
|
384
|
+
parts = [document["@identificationTypeDescription"], document["@issuedBy"], document["@region"],
|
|
385
|
+
document["remark"], *flags]
|
|
386
|
+
joined = parts.compact.join("; ")
|
|
387
|
+
joined.empty? ? nil : joined
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
# An address element that located nothing is dropped rather than kept
|
|
391
|
+
# empty: it cannot be screened on and would only inflate the count.
|
|
392
|
+
# 87 of the 2,648 carry the `00` country sentinel and nothing else.
|
|
393
|
+
sig { params(place: Parsers::XmlRecords::Record).returns(T.nilable(Address)) }
|
|
394
|
+
def address(place)
|
|
395
|
+
Address.new(street: place["@street"], city: place["@city"], state_province: place["@region"],
|
|
396
|
+
postal_code: place["@zipCode"], country: country(place), note: address_note(place))
|
|
397
|
+
rescue ArgumentError
|
|
398
|
+
nil
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
# `poBox` and `place` have no member of their own on Address, and
|
|
402
|
+
# `contactInfo` -- a web site, a phone number, an email address, on
|
|
403
|
+
# 1,681 of the addresses -- has nowhere at all. All of it is real
|
|
404
|
+
# locating detail, so it goes to the note rather than over the side.
|
|
405
|
+
sig { params(place: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
|
|
406
|
+
def address_note(place)
|
|
407
|
+
contacts = place.nodes("contactInfo").map { |info| "#{info["@key"]}: #{info["@value"]}" }
|
|
408
|
+
box = place["@poBox"]
|
|
409
|
+
parts = [box && "P.O. Box #{box}", place["@place"], place["remark"], *contacts,
|
|
410
|
+
("as at listing time" if place["@asAtListingTime"] == BOOLEAN_TRUE)]
|
|
411
|
+
joined = parts.compact.join("; ")
|
|
412
|
+
joined.empty? ? nil : joined
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
# The Commission's prose name for a country, or nil for its sentinel.
|
|
416
|
+
# Prose rather than the code beside it because Address#country and
|
|
417
|
+
# Identifier#country are what a reviewer reads, and `IQ` is not.
|
|
418
|
+
sig { params(element: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
|
|
419
|
+
def country(element)
|
|
420
|
+
return nil if element["@countryIso2Code"] == UNKNOWN_COUNTRY
|
|
421
|
+
|
|
422
|
+
name = element["@countryDescription"]
|
|
423
|
+
name == UNKNOWN_COUNTRY_NAME ? nil : name
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
# Label/value pairs for Remarks.build, which drops the ones the record
|
|
427
|
+
# left blank. A value may be several -- one person's function is filed
|
|
428
|
+
# once per name -- and arrives as the Array it published.
|
|
429
|
+
sig { returns(T::Array[T.untyped]) }
|
|
430
|
+
def extras
|
|
431
|
+
[["EU logical id", node["@logicalId"]],
|
|
432
|
+
["UN reference", node["@unitedNationId"]],
|
|
433
|
+
["Designation details", node["@designationDetails"]],
|
|
434
|
+
["Regulation", node["regulation/@numberTitle"]],
|
|
435
|
+
["Official Journal", node["regulation/publicationUrl"]],
|
|
436
|
+
["Function", node.values("nameAlias/@function").uniq],
|
|
437
|
+
["Title", node.values("nameAlias/@title").uniq],
|
|
438
|
+
["Gender", node.values("nameAlias/@gender").uniq],
|
|
439
|
+
["Date of birth as published", @unread_dates]] + places_of_birth
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
# A place of birth is real screening signal and the canonical model has
|
|
443
|
+
# no home for it, so it is kept rather than dropped to keep a schema
|
|
444
|
+
# tidy. One line per birthdate element, because a person with two
|
|
445
|
+
# reported birth dates usually has two reported birth places.
|
|
446
|
+
sig { returns(T::Array[T.untyped]) }
|
|
447
|
+
def places_of_birth
|
|
448
|
+
places = node.nodes("birthdate").filter_map do |born|
|
|
449
|
+
parts = PLACE_PARTS.filter_map { |part| born[part] } + [country(born)].compact
|
|
450
|
+
parts.join(", ") if parts.any?
|
|
451
|
+
end
|
|
452
|
+
places.uniq.map { |place| ["Place of birth", place] }
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
# The Commission's own line wrapping arrives inside attribute values:
|
|
456
|
+
# 2,149 of the 31,053 names carry a newline, a run of spaces, or
|
|
457
|
+
# padding on one side. Left verbatim those produce "Ivan Ivanov",
|
|
458
|
+
# which is a name nothing will ever match.
|
|
459
|
+
sig { params(value: T.untyped).returns(T.nilable(String)) }
|
|
460
|
+
def collapse(value)
|
|
461
|
+
string = value.to_s.split(WHITESPACE).join(" ")
|
|
462
|
+
string.empty? ? nil : -string
|
|
463
|
+
end
|
|
464
|
+
end
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
end
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/parsers"
|
|
7
|
+
require "active_sanction/sources"
|
|
8
|
+
require "active_sanction/sources/base"
|
|
9
|
+
|
|
10
|
+
module ActiveSanction
|
|
11
|
+
module Sources
|
|
12
|
+
# The EU Consolidated Financial Sanctions List: every person and entity
|
|
13
|
+
# subject to an EU financial sanction, in one document, as the Financial
|
|
14
|
+
# Sanctions Files (FSF) export.
|
|
15
|
+
#
|
|
16
|
+
# snapshot = ActiveSanction::Sources[:eu_fsf].new.sync
|
|
17
|
+
#
|
|
18
|
+
# ### One file, one record shape, ten times the size
|
|
19
|
+
#
|
|
20
|
+
# https://webgate.ec.europa.eu/fsd/fsf/public/files/xmlFullSanctionsList_1_1/content
|
|
21
|
+
#
|
|
22
|
+
# <export><sanctionEntity> 6,234 records, 25.7 MB
|
|
23
|
+
#
|
|
24
|
+
# An order of magnitude larger than the UN's 2.2 MB or Canada's 2.9 MB, and
|
|
25
|
+
# the first list to actually exercise the streaming interface the XML
|
|
26
|
+
# toolkit was built around. It reads in one pass at roughly 90 MB of
|
|
27
|
+
# resident memory, most of which is the payload itself; nothing here ever
|
|
28
|
+
# holds two records at once.
|
|
29
|
+
#
|
|
30
|
+
# ### The token in the URL
|
|
31
|
+
#
|
|
32
|
+
# The public endpoint takes a `token` query parameter. Omitting it answers
|
|
33
|
+
# 403 and a wrong one answers 500, so it is not optional -- but it is also
|
|
34
|
+
# not a credential: `dG9rZW4tMjAxNw` is base64 for `token-2017`, it is the
|
|
35
|
+
# value the Commission's own public download page has carried since that
|
|
36
|
+
# year, and it is the same string for every caller. It is declared inline
|
|
37
|
+
# rather than configured, the way any other part of a published URL is.
|
|
38
|
+
#
|
|
39
|
+
# If the Commission ever rotates it, no release of this gem is needed:
|
|
40
|
+
#
|
|
41
|
+
# ActiveSanction::Sources::EuFsf.token = "..." # or
|
|
42
|
+
# ActiveSanction::Sources::EuFsf.url :main, "https://..."
|
|
43
|
+
#
|
|
44
|
+
# ### Conditional GET does not work here, and that is the publisher's doing
|
|
45
|
+
#
|
|
46
|
+
# The endpoint serves `Last-Modified` but answers `If-Modified-Since` with
|
|
47
|
+
# 200 and the whole file, and sends no `ETag` and `Cache-Control:
|
|
48
|
+
# no-store`. So every sync of this list downloads 25.7 MB where the other
|
|
49
|
+
# four usually download nothing. The conditional request is still sent --
|
|
50
|
+
# it costs nothing and the day the Commission honours it, it works -- and
|
|
51
|
+
# the snapshot checksum is unchanged when the content is, so a re-download
|
|
52
|
+
# of identical bytes still diffs (#35) to nothing.
|
|
53
|
+
#
|
|
54
|
+
# ### The two traps in this list
|
|
55
|
+
#
|
|
56
|
+
# **No name is the name.** The EU marks no `<nameAlias>` as the official
|
|
57
|
+
# one: all 31,053 carry `strong="true"`, and every other candidate signal
|
|
58
|
+
# is wrong somewhere in the file. Document order files Qusay Hussein's
|
|
59
|
+
# French transliteration ahead of his English name; `nameLanguage` files a
|
|
60
|
+
# Cyrillic spelling of Anatoliy Sidorov's name under `EN`; ordering by
|
|
61
|
+
# `logicalId` picks a non-Latin name for 3,203 of the 5,502 multi-name
|
|
62
|
+
# records. So Record picks one by a stated rule -- see #primary_name -- and
|
|
63
|
+
# what "primary" means for this source is *the first name the EU published
|
|
64
|
+
# that the EU did not itself annotate as an alias*, which is weaker than
|
|
65
|
+
# what it means for OFAC. It costs nothing in score: the scorer takes the
|
|
66
|
+
# best of an entity's names and the kind only reaches the reason line.
|
|
67
|
+
#
|
|
68
|
+
# **Four birth dates are not in the Gregorian calendar.** `calendarType`
|
|
69
|
+
# is `ISLAMIC` on four records, where `year`, `monthOfYear` and
|
|
70
|
+
# `dayOfMonth` hold a Hijri date -- `year="1343"` for a man born in 1964.
|
|
71
|
+
# Read as published those become dates in the fourteenth century, and a
|
|
72
|
+
# fourteenth-century date does not merely fail to match a real one, it
|
|
73
|
+
# *conflicts* with it, and the scorer penalizes the record. Three of the
|
|
74
|
+
# four carry no Gregorian equivalent at all, so they produce no date of
|
|
75
|
+
# birth and the published Hijri date goes to remarks instead.
|
|
76
|
+
#
|
|
77
|
+
# ### What a clean EU result is worth
|
|
78
|
+
#
|
|
79
|
+
# More than a Canadian one and less than an OFAC one. 3,013 identification
|
|
80
|
+
# documents across 6,234 records is real corroborating signal, and 2,764
|
|
81
|
+
# citizenships arrive as ISO codes rather than as prose. What is thin is
|
|
82
|
+
# the alias grading: the EU publishes no quality column, and the 499
|
|
83
|
+
# gradings this adapter does read are prose the Commission happened to put
|
|
84
|
+
# in a `<remark>` -- so most EU aliases arrive ungraded, which the scorer
|
|
85
|
+
# correctly treats as unstated rather than as good.
|
|
86
|
+
#
|
|
87
|
+
# ### What this adapter does not do
|
|
88
|
+
#
|
|
89
|
+
# It does not infer a vessel. `subjectType` publishes only `person` and
|
|
90
|
+
# `enterprise`, and the 41 records carrying an `imo` document are shipping
|
|
91
|
+
# *companies* holding IMO company numbers -- Chongchongang Shipping, Korea
|
|
92
|
+
# Ansan Shipping -- not the ships themselves. Typing those as :vessel on
|
|
93
|
+
# the strength of the document kind would be a guess that hides them from
|
|
94
|
+
# every organization search.
|
|
95
|
+
class EuFsf < Base
|
|
96
|
+
extend T::Sig
|
|
97
|
+
|
|
98
|
+
key :eu_fsf
|
|
99
|
+
jurisdiction :eu
|
|
100
|
+
authority "European Commission"
|
|
101
|
+
format :xml
|
|
102
|
+
|
|
103
|
+
# Commission Decision 2011/833/EU, which permits reuse of Commission
|
|
104
|
+
# documents provided the source is acknowledged and the reuse does not
|
|
105
|
+
# suggest the Commission endorses it.
|
|
106
|
+
licence_notice "Reusable under Commission Decision 2011/833/EU " \
|
|
107
|
+
"provided the source is acknowledged and no Commission " \
|
|
108
|
+
"endorsement is implied. Verified 2026-09-11."
|
|
109
|
+
licence_url "https://eur-lex.europa.eu/eli/dec/2011/833/oj/eng"
|
|
110
|
+
|
|
111
|
+
# Split out so #token= can rebuild the URL around a rotated value. See
|
|
112
|
+
# the class comment for why the token is not a credential.
|
|
113
|
+
#
|
|
114
|
+
# @api private
|
|
115
|
+
ENDPOINT = T.let(
|
|
116
|
+
"https://webgate.ec.europa.eu/fsd/fsf/public/files/xmlFullSanctionsList_1_1/content", String
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# @api private
|
|
120
|
+
PUBLIC_TOKEN = T.let("dG9rZW4tMjAxNw", String)
|
|
121
|
+
|
|
122
|
+
url :main, "#{ENDPOINT}?token=#{PUBLIC_TOKEN}"
|
|
123
|
+
|
|
124
|
+
# @api private
|
|
125
|
+
SANCTION_ENTITY = T.let("sanctionEntity", String)
|
|
126
|
+
|
|
127
|
+
# @api private
|
|
128
|
+
LIST = T.let(Parsers::XmlRecords.new(records: SANCTION_ENTITY), Parsers::XmlRecords)
|
|
129
|
+
|
|
130
|
+
# The generation timestamp the Commission stamps on the document element,
|
|
131
|
+
# to the millisecond. More precise than the Last-Modified header Base
|
|
132
|
+
# falls back to, and the string the FSF download page itself shows, so it
|
|
133
|
+
# is the one an examiner asking "which version was this screened
|
|
134
|
+
# against" will recognise.
|
|
135
|
+
#
|
|
136
|
+
# @api private
|
|
137
|
+
GENERATED_AT = T.let("generationDate", String)
|
|
138
|
+
|
|
139
|
+
# Re-points the list at the same endpoint with a different token, for the
|
|
140
|
+
# day the Commission rotates the one baked in above.
|
|
141
|
+
#
|
|
142
|
+
# ActiveSanction::Sources::EuFsf.token = "..."
|
|
143
|
+
sig { params(value: T.untyped).returns(String) }
|
|
144
|
+
def self.token=(value)
|
|
145
|
+
url :main, "#{ENDPOINT}?token=#{value}"
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Records that could not be used, and fields that could not be read.
|
|
149
|
+
# Read after #parse; sync orchestration (#34) reports them.
|
|
150
|
+
sig { returns(T::Array[Parsers::Warning]) }
|
|
151
|
+
attr_reader :warnings
|
|
152
|
+
|
|
153
|
+
sig { params(args: T.untyped, options: T.untyped).void }
|
|
154
|
+
def initialize(*args, **options)
|
|
155
|
+
super
|
|
156
|
+
@warnings = T.let([], T::Array[Parsers::Warning])
|
|
157
|
+
@unmapped = T.let([], T::Array[Parsers::Warning])
|
|
158
|
+
@generated_at = T.let(nil, T.nilable(String))
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
sig { override.params(raw: T.untyped).returns(T::Array[Entity]) }
|
|
162
|
+
def parse(raw)
|
|
163
|
+
reader = LIST.read(raw)
|
|
164
|
+
entities = build(reader)
|
|
165
|
+
@generated_at = reader.root[GENERATED_AT]
|
|
166
|
+
@warnings = reader.warnings + @unmapped
|
|
167
|
+
entities
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
sig { override.returns(T.nilable(String)) }
|
|
171
|
+
def source_version = @generated_at || super
|
|
172
|
+
|
|
173
|
+
private
|
|
174
|
+
|
|
175
|
+
sig { params(reader: Parsers::XmlRecords::Reader).returns(T::Array[Entity]) }
|
|
176
|
+
def build(reader)
|
|
177
|
+
@unmapped = []
|
|
178
|
+
reader.filter_map do |node|
|
|
179
|
+
record = Record.new(node)
|
|
180
|
+
entity = record.entity
|
|
181
|
+
entity.nil? ? note_nameless(node) : entity
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# A record with no name cannot be screened against and is never what the
|
|
186
|
+
# Commission meant to publish. Every one of the 6,234 published today
|
|
187
|
+
# carries at least one `<nameAlias>`; the warning exists so that the day
|
|
188
|
+
# one does not, it is visible rather than absent.
|
|
189
|
+
sig { params(node: Parsers::XmlRecords::Record).returns(NilClass) }
|
|
190
|
+
def note_nameless(node)
|
|
191
|
+
@unmapped << Parsers::Warning.new(
|
|
192
|
+
line: node.line,
|
|
193
|
+
message: "<#{node.name}> #{node["@euReferenceNumber"].inspect} has no name and was skipped"
|
|
194
|
+
)
|
|
195
|
+
nil
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
require "active_sanction/sources/eu_fsf/record"
|
|
202
|
+
|
|
203
|
+
ActiveSanction::Sources.register(ActiveSanction::Sources::EuFsf)
|