active_sanction 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +538 -0
  3. data/CODE_OF_CONDUCT.md +74 -0
  4. data/CONTRIBUTING.md +312 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +616 -0
  7. data/SECURITY.md +97 -0
  8. data/active_sanction.gemspec +98 -0
  9. data/docs/adding_a_source.md +1083 -0
  10. data/docs/api_stability.md +357 -0
  11. data/docs/bundle_format.md +407 -0
  12. data/lib/active_sanction/address.rb +170 -0
  13. data/lib/active_sanction/client.rb +341 -0
  14. data/lib/active_sanction/configuration.rb +721 -0
  15. data/lib/active_sanction/countries.txt +292 -0
  16. data/lib/active_sanction/country.rb +164 -0
  17. data/lib/active_sanction/deprecation.rb +168 -0
  18. data/lib/active_sanction/diff/change.rb +208 -0
  19. data/lib/active_sanction/diff.rb +348 -0
  20. data/lib/active_sanction/doctor/checkup.rb +356 -0
  21. data/lib/active_sanction/doctor/diagnosis.rb +245 -0
  22. data/lib/active_sanction/doctor/finding.rb +183 -0
  23. data/lib/active_sanction/doctor/profile.rb +384 -0
  24. data/lib/active_sanction/doctor/report.rb +227 -0
  25. data/lib/active_sanction/doctor.rb +347 -0
  26. data/lib/active_sanction/entity.rb +293 -0
  27. data/lib/active_sanction/error.rb +332 -0
  28. data/lib/active_sanction/fetcher/result.rb +116 -0
  29. data/lib/active_sanction/fetcher.rb +242 -0
  30. data/lib/active_sanction/http_client/errors.rb +94 -0
  31. data/lib/active_sanction/http_client/response.rb +129 -0
  32. data/lib/active_sanction/http_client.rb +324 -0
  33. data/lib/active_sanction/identifier.rb +212 -0
  34. data/lib/active_sanction/index/builder.rb +89 -0
  35. data/lib/active_sanction/index/candidate.rb +63 -0
  36. data/lib/active_sanction/index/entry.rb +66 -0
  37. data/lib/active_sanction/index/features.rb +112 -0
  38. data/lib/active_sanction/index.rb +381 -0
  39. data/lib/active_sanction/match_result.rb +345 -0
  40. data/lib/active_sanction/matcher.rb +388 -0
  41. data/lib/active_sanction/name.rb +183 -0
  42. data/lib/active_sanction/normalizer/cache.rb +82 -0
  43. data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
  44. data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
  45. data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
  46. data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
  47. data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
  48. data/lib/active_sanction/normalizer/dictionary.rb +243 -0
  49. data/lib/active_sanction/normalizer/form.rb +279 -0
  50. data/lib/active_sanction/normalizer.rb +137 -0
  51. data/lib/active_sanction/parsers/column_shape.rb +274 -0
  52. data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
  53. data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
  54. data/lib/active_sanction/parsers/delimited_table.rb +151 -0
  55. data/lib/active_sanction/parsers/format.rb +107 -0
  56. data/lib/active_sanction/parsers/join.rb +149 -0
  57. data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
  58. data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
  59. data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
  60. data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
  61. data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
  62. data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
  63. data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
  64. data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
  65. data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
  66. data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
  67. data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
  68. data/lib/active_sanction/parsers/xml_records.rb +140 -0
  69. data/lib/active_sanction/parsers.rb +94 -0
  70. data/lib/active_sanction/partial_date/parser.rb +124 -0
  71. data/lib/active_sanction/partial_date.rb +303 -0
  72. data/lib/active_sanction/payload_cache/checksum.rb +66 -0
  73. data/lib/active_sanction/payload_cache/entry.rb +304 -0
  74. data/lib/active_sanction/payload_cache.rb +366 -0
  75. data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
  76. data/lib/active_sanction/phonetics.rb +40 -0
  77. data/lib/active_sanction/query.rb +290 -0
  78. data/lib/active_sanction/rescreen/alert.rb +314 -0
  79. data/lib/active_sanction/rescreen.rb +332 -0
  80. data/lib/active_sanction/scorer/adjustments.rb +241 -0
  81. data/lib/active_sanction/scorer/name_score.rb +279 -0
  82. data/lib/active_sanction/scorer/reason.rb +136 -0
  83. data/lib/active_sanction/scorer/result.rb +129 -0
  84. data/lib/active_sanction/scorer/subject.rb +235 -0
  85. data/lib/active_sanction/scorer/weights.rb +306 -0
  86. data/lib/active_sanction/scorer.rb +303 -0
  87. data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
  88. data/lib/active_sanction/similarity/levenshtein.rb +179 -0
  89. data/lib/active_sanction/similarity/token_set.rb +159 -0
  90. data/lib/active_sanction/similarity/token_sort.rb +120 -0
  91. data/lib/active_sanction/similarity.rb +183 -0
  92. data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
  93. data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
  94. data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
  95. data/lib/active_sanction/snapshot/bundle.rb +381 -0
  96. data/lib/active_sanction/snapshot.rb +305 -0
  97. data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
  98. data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
  99. data/lib/active_sanction/sources/australia_dfat.rb +291 -0
  100. data/lib/active_sanction/sources/base.rb +295 -0
  101. data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
  102. data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
  103. data/lib/active_sanction/sources/canada_sema.rb +153 -0
  104. data/lib/active_sanction/sources/definition.rb +360 -0
  105. data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
  106. data/lib/active_sanction/sources/eu_fsf.rb +203 -0
  107. data/lib/active_sanction/sources/ofac/record.rb +251 -0
  108. data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
  109. data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
  110. data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
  111. data/lib/active_sanction/sources/ofac.rb +248 -0
  112. data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
  113. data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
  114. data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
  115. data/lib/active_sanction/sources/remarks.rb +84 -0
  116. data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
  117. data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
  118. data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
  119. data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
  120. data/lib/active_sanction/sources/un_consolidated.rb +140 -0
  121. data/lib/active_sanction/sources.rb +196 -0
  122. data/lib/active_sanction/storage/active_record/reader.rb +124 -0
  123. data/lib/active_sanction/storage/active_record/row.rb +175 -0
  124. data/lib/active_sanction/storage/active_record/writer.rb +141 -0
  125. data/lib/active_sanction/storage/active_record.rb +277 -0
  126. data/lib/active_sanction/storage/base.rb +216 -0
  127. data/lib/active_sanction/storage/file_system.rb +373 -0
  128. data/lib/active_sanction/storage/memory.rb +67 -0
  129. data/lib/active_sanction/storage/meta.rb +188 -0
  130. data/lib/active_sanction/storage.rb +74 -0
  131. data/lib/active_sanction/subject.rb +271 -0
  132. data/lib/active_sanction/sync/report.rb +245 -0
  133. data/lib/active_sanction/sync/result.rb +296 -0
  134. data/lib/active_sanction/sync.rb +390 -0
  135. data/lib/active_sanction/validator_store/file_system.rb +103 -0
  136. data/lib/active_sanction/validator_store/memory.rb +41 -0
  137. data/lib/active_sanction/validator_store.rb +119 -0
  138. data/lib/active_sanction/validators.rb +195 -0
  139. data/lib/active_sanction/version.rb +23 -0
  140. data/lib/active_sanction.rb +331 -0
  141. data/lib/generators/active_sanction/install/install_generator.rb +60 -0
  142. data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
  143. metadata +236 -0
@@ -0,0 +1,279 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/entity"
7
+ require "active_sanction/name"
8
+ require "active_sanction/address"
9
+ require "active_sanction/identifier"
10
+ require "active_sanction/partial_date"
11
+ require "active_sanction/sources/remarks"
12
+
13
+ module ActiveSanction
14
+ module Sources
15
+ class UnConsolidated < Base
16
+ # One `<INDIVIDUAL>` or `<ENTITY>` turned into an Entity.
17
+ #
18
+ # Separate from the adapter because they are two jobs: the adapter says
19
+ # what the list is and where it lives, and this says what the UN's
20
+ # elements mean. The mapping is where all the judgment sits, so it is
21
+ # worth being able to read it on its own.
22
+ #
23
+ # The two record shapes are mapped by one class rather than two. Every
24
+ # element that differs between them differs only by prefix --
25
+ # `INDIVIDUAL_ALIAS` against `ENTITY_ALIAS`, `INDIVIDUAL_ADDRESS` against
26
+ # `ENTITY_ADDRESS` -- so the prefix is taken from the record's own name.
27
+ # An element only individuals have then resolves to a path entities do
28
+ # not carry, which reads as absent, which is what it is.
29
+ #
30
+ # @api private
31
+ class Record
32
+ extend T::Sig
33
+
34
+ TYPES = T.let({ UnConsolidated::INDIVIDUAL => :individual,
35
+ UnConsolidated::ENTITY => :organization }.freeze, T::Hash[String, Symbol])
36
+
37
+ # The UN splits one name across up to four numbered elements, and any
38
+ # subset may be present: 736 individuals have a FIRST_NAME, 170 have a
39
+ # FOURTH_NAME, and an entity's whole name is in FIRST_NAME alone.
40
+ NAME_PARTS = T.let(%w[FIRST_NAME SECOND_NAME THIRD_NAME FOURTH_NAME].freeze, T::Array[String])
41
+
42
+ # `QUALITY` under `<INDIVIDUAL_ALIAS>` grades the alias. Published as
43
+ # Good (1,534), Low (629), or blank (169).
44
+ ALIAS_QUALITIES = T.let({ "good" => :good, "low" => :low }.freeze, T::Hash[String, Symbol])
45
+
46
+ # `QUALITY` under `<ENTITY_ALIAS>` is not a grade at all -- it is the
47
+ # alias kind, published as a.k.a. (585), f.k.a. (19), or blank (125).
48
+ # One element name, two vocabularies, in one document. Both tables are
49
+ # consulted for every alias and each misses the other's values, so
50
+ # neither shape needs a branch and neither loses what it publishes.
51
+ ALIAS_KINDS = T.let(
52
+ { "a.k.a." => :aka, "f.k.a." => :fka, "n.k.a." => :nka }.freeze, T::Hash[String, Symbol]
53
+ )
54
+
55
+ # `TYPE_OF_DOCUMENT` is free text, and the UN writes it in three
56
+ # languages. Anything unrecognised is :other, which is a real answer:
57
+ # a document we cannot classify still matches on its number.
58
+ DOCUMENT_KINDS = T.let({
59
+ "passport" => :passport,
60
+ "numéro de passeport" => :passport,
61
+ "número de pasaporte" => :passport,
62
+ "national identification number" => :national_id
63
+ }.freeze, T::Hash[String, Symbol])
64
+
65
+ # `TYPE_OF_DATE`, of which only these two change how the date is read.
66
+ # EXACT and a blank value are the same instruction: read what is there.
67
+ BETWEEN = T.let("BETWEEN", String)
68
+ APPROXIMATELY = T.let("APPROXIMATELY", String)
69
+
70
+ # Elements the canonical model has no home for, appended to remarks
71
+ # rather than dropped. A gender and a place of birth are real screening
72
+ # signal, and losing them to keep a schema tidy is the wrong trade.
73
+ EXTRA_FIELDS = T.let({
74
+ "VERSIONNUM" => "Version", "REFERENCE_NUMBER" => "Reference", "GENDER" => "Gender",
75
+ "TITLE/VALUE" => "Title", "DESIGNATION/VALUE" => "Designation"
76
+ }.freeze, T::Hash[String, String])
77
+
78
+ PLACE_OF_BIRTH_PARTS = T.let(%w[STREET CITY STATE_PROVINCE COUNTRY NOTE].freeze, T::Array[String])
79
+
80
+ WHITESPACE = T.let(/\s+/, Regexp)
81
+
82
+ sig { returns(Parsers::XmlRecords::Record) }
83
+ attr_reader :node
84
+
85
+ sig { params(node: Parsers::XmlRecords::Record).void }
86
+ def initialize(node)
87
+ @node = T.let(node, Parsers::XmlRecords::Record)
88
+ @names = T.let(nil, T.nilable(T::Array[Name]))
89
+ end
90
+
91
+ # The entity, or nil for a record with no name -- which cannot be
92
+ # screened against and is never what the Committee meant to publish.
93
+ sig { returns(T.nilable(Entity)) }
94
+ def entity
95
+ return nil if names.empty?
96
+
97
+ Entity.new(source: :un_consolidated, source_ref: source_ref, type: type, names: names,
98
+ addresses: addresses, identifiers: identifiers, dates_of_birth: dates_of_birth,
99
+ nationalities: nationalities, programs: programs, listed_on: listed_on,
100
+ remarks: remarks)
101
+ end
102
+
103
+ sig { returns(Symbol) }
104
+ def type = TYPES.fetch(node.name, :organization)
105
+
106
+ sig { returns(T.nilable(String)) }
107
+ def source_ref = node["DATAID"]
108
+
109
+ sig { returns(T::Array[String]) }
110
+ def nationalities = node.values("NATIONALITY/VALUE")
111
+
112
+ sig { returns(T.nilable(PartialDate)) }
113
+ def listed_on = PartialDate.parse(node["LISTED_ON"])
114
+
115
+ # The joined published name first, then the original script, then every
116
+ # alias in the order the Committee filed it.
117
+ sig { returns(T::Array[Name]) }
118
+ def names
119
+ @names ||= [published_name, original_script].compact + alias_names
120
+ end
121
+
122
+ sig { returns(T::Array[PartialDate]) }
123
+ def dates_of_birth
124
+ each("DATE_OF_BIRTH").filter_map { |born| date_of_birth(born) }
125
+ end
126
+
127
+ sig { returns(T::Array[Identifier]) }
128
+ def identifiers
129
+ each("DOCUMENT").filter_map { |document| identifier(document) }
130
+ end
131
+
132
+ sig { returns(T::Array[Address]) }
133
+ def addresses
134
+ each("ADDRESS").filter_map { |place| address(place) }
135
+ end
136
+
137
+ sig { returns(T::Array[String]) }
138
+ def programs = node.values("UN_LIST_TYPE")
139
+
140
+ # The UN's own comment verbatim, then the elements that have nowhere
141
+ # else to go, behind the marker that makes them trivial to strip again.
142
+ sig { returns(T.nilable(String)) }
143
+ def remarks
144
+ Remarks.build(node["COMMENTS1"], extras)
145
+ end
146
+
147
+ private
148
+
149
+ # The child elements of a record, which the UN names after the record
150
+ # they hang off. See the class comment for why that is worth exploiting.
151
+ sig { params(suffix: String).returns(T::Array[Parsers::XmlRecords::Record]) }
152
+ def each(suffix) = node.nodes("#{node.name}_#{suffix}")
153
+
154
+ sig { returns(T.nilable(Name)) }
155
+ def published_name
156
+ value = collapse(NAME_PARTS.filter_map { |part| node[part] }.join(" "))
157
+ value && Name.new(value: value, kind: :primary)
158
+ end
159
+
160
+ # The name as the publisher's own script writes it -- Arabic for 338 of
161
+ # the individuals. Filed as an alias with no `script` declared: which
162
+ # script a string is in is a question about its characters, and
163
+ # answering it from the element's name would be a guess. The normalizer
164
+ # (#26) reads the characters and is the right place for it.
165
+ sig { returns(T.nilable(Name)) }
166
+ def original_script
167
+ value = collapse(node["NAME_ORIGINAL_SCRIPT"])
168
+ value && Name.new(value: value, kind: :aka)
169
+ end
170
+
171
+ # 294 of the 3,061 alias elements are placeholders: `<QUALITY/>` and
172
+ # `<ALIAS_NAME/>` and nothing else. They must produce no name at all --
173
+ # a blank-valued Name would be a record that matches everything.
174
+ sig { returns(T::Array[Name]) }
175
+ def alias_names
176
+ each("ALIAS").filter_map do |alt|
177
+ value = collapse(alt["ALIAS_NAME"])
178
+ next nil if value.nil?
179
+
180
+ published = alt["QUALITY"].to_s.downcase
181
+ Name.new(value: value, kind: ALIAS_KINDS.fetch(published, :aka),
182
+ quality: ALIAS_QUALITIES[published])
183
+ end
184
+ end
185
+
186
+ # 55 of the 949 date elements carry a TYPE_OF_DATE and no date, which
187
+ # reads as nil rather than as a date of unknown value.
188
+ sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(PartialDate)) }
189
+ def date_of_birth(born)
190
+ type_of_date = born["TYPE_OF_DATE"].to_s.upcase
191
+ return between(born) if type_of_date == BETWEEN
192
+
193
+ approximate(PartialDate.parse(born["DATE"] || born["YEAR"]), type_of_date)
194
+ end
195
+
196
+ sig { params(born: Parsers::XmlRecords::Record).returns(T.nilable(PartialDate)) }
197
+ def between(born)
198
+ from = born["FROM_YEAR"]
199
+ to = born["TO_YEAR"]
200
+ return nil if from.nil? || to.nil?
201
+
202
+ PartialDate.range(from, to)
203
+ end
204
+
205
+ # "Circa 1962" and "1963" are the same claim about a person, and
206
+ # PartialDate already knows to compare an approximate date loosely --
207
+ # but only if the adapter tells it the publisher said approximately.
208
+ sig { params(date: T.nilable(PartialDate), type_of_date: String).returns(T.nilable(PartialDate)) }
209
+ def approximate(date, type_of_date)
210
+ return date if date.nil? || type_of_date != APPROXIMATELY
211
+
212
+ PartialDate.from_h(date.to_h.merge(approximate: true))
213
+ end
214
+
215
+ # 447 of the UN's 954 document elements name a document type and give
216
+ # no number. There is nothing there to match on, so they are dropped
217
+ # rather than kept as identifiers with no identity.
218
+ sig { params(document: Parsers::XmlRecords::Record).returns(T.nilable(Identifier)) }
219
+ def identifier(document)
220
+ return nil if document.null?("NUMBER")
221
+
222
+ Identifier.new(kind: document_kind(document), value: document["NUMBER"],
223
+ country: document["ISSUING_COUNTRY"] || document["COUNTRY_OF_ISSUE"],
224
+ issued_on: PartialDate.parse(document["DATE_OF_ISSUE"]),
225
+ expires_on: PartialDate.parse(document["DATE_OF_EXPIRY"]),
226
+ note: document_note(document))
227
+ rescue ArgumentError
228
+ nil
229
+ end
230
+
231
+ sig { params(document: Parsers::XmlRecords::Record).returns(Symbol) }
232
+ def document_kind(document)
233
+ DOCUMENT_KINDS.fetch(collapse(document["TYPE_OF_DOCUMENT"]).to_s.downcase, :other)
234
+ end
235
+
236
+ sig { params(document: Parsers::XmlRecords::Record).returns(T.nilable(String)) }
237
+ def document_note(document)
238
+ parts = [collapse(document["TYPE_OF_DOCUMENT2"]), document["CITY_OF_ISSUE"], document["NOTE"]].compact
239
+ parts.empty? ? nil : parts.join("; ")
240
+ end
241
+
242
+ # An address element that located nothing is dropped rather than kept
243
+ # empty: it cannot be screened on and would only inflate the count.
244
+ sig { params(place: Parsers::XmlRecords::Record).returns(T.nilable(Address)) }
245
+ def address(place)
246
+ Address.new(street: place["STREET"], city: place["CITY"], state_province: place["STATE_PROVINCE"],
247
+ postal_code: place["ZIP_CODE"], country: place["COUNTRY"], note: place["NOTE"])
248
+ rescue ArgumentError
249
+ nil
250
+ end
251
+
252
+ # Label/value pairs for Remarks.build, which drops the ones the record
253
+ # left blank. A value may be several -- the UN files three designations
254
+ # under one element -- and arrives as the Array it published.
255
+ sig { returns(T::Array[T.untyped]) }
256
+ def extras
257
+ EXTRA_FIELDS.map { |path, label| [label, node.values(path)] } + places_of_birth
258
+ end
259
+
260
+ sig { returns(T::Array[T.untyped]) }
261
+ def places_of_birth
262
+ each("PLACE_OF_BIRTH").map do |place|
263
+ ["Place of birth", PLACE_OF_BIRTH_PARTS.filter_map { |part| place[part] }.join(", ")]
264
+ end
265
+ end
266
+
267
+ # The UN's own line wrapping arrives inside values: two names in the
268
+ # published file carry a newline and fifty spaces mid-string, and 39
269
+ # name parts are padded on one side. Joined verbatim those produce
270
+ # "JOHN IMANI", which is a name nothing will ever match.
271
+ sig { params(value: T.untyped).returns(T.nilable(String)) }
272
+ def collapse(value)
273
+ string = value.to_s.split(WHITESPACE).join(" ")
274
+ string.empty? ? nil : -string
275
+ end
276
+ end
277
+ end
278
+ end
279
+ end
@@ -0,0 +1,140 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/parsers"
7
+ require "active_sanction/sources"
8
+ require "active_sanction/sources/base"
9
+
10
+ module ActiveSanction
11
+ module Sources
12
+ # The United Nations Security Council consolidated list: every individual
13
+ # and entity subject to a UN sanctions regime, in one document.
14
+ #
15
+ # snapshot = ActiveSanction::Sources[:un_consolidated].new.sync
16
+ #
17
+ # ### One file, two record shapes
18
+ #
19
+ # https://scsanctions.un.org/resources/xml/en/consolidated.xml 2.2 MB
20
+ #
21
+ # <INDIVIDUALS><INDIVIDUAL> 736 people
22
+ # <ENTITIES><ENTITY> 275 organizations
23
+ #
24
+ # Both are read in a single streaming pass. The file is small enough today
25
+ # to have been loaded whole without anyone noticing, which is exactly why
26
+ # it was not: the toolkit this adapter is written against (#15) is the one
27
+ # that will read OFAC's 126 MB advanced XML, and an adapter that quietly
28
+ # depended on holding the document would have to be rewritten then.
29
+ #
30
+ # ### The trap in this list
31
+ #
32
+ # `QUALITY` appears under both alias elements and means something different
33
+ # under each. Under `<INDIVIDUAL_ALIAS>` it grades the alias -- Good or Low
34
+ # -- which is a matching signal the scorer (#32) penalizes on. Under
35
+ # `<ENTITY_ALIAS>` it is not a grade at all; it is `a.k.a.` or `f.k.a.`,
36
+ # which is an alias *kind*. Reading one as the other silently either throws
37
+ # away every entity's alias kind or grades 585 organization aliases on a
38
+ # scale that was never applied to them. Record maps them separately, and
39
+ # the counts above come from the published file rather than from a guess.
40
+ #
41
+ # ### What this adapter does not do
42
+ #
43
+ # `INDIVIDUAL_PLACE_OF_BIRTH`, `GENDER`, `TITLE` and `DESIGNATION` have no
44
+ # home in the canonical model. They are appended to remarks after a marker
45
+ # rather than dropped -- a place of birth is real screening signal, and
46
+ # losing it to keep a schema tidy is the wrong trade -- and are left for a
47
+ # later issue to structure if a matcher turns out to want them.
48
+ class UnConsolidated < Base
49
+ extend T::Sig
50
+
51
+ key :un_consolidated
52
+ jurisdiction :un
53
+ authority "United Nations Security Council"
54
+ format :xml
55
+
56
+ # The most restrictive of the seven on paper. The UN's copyright page
57
+ # reserves reproduction of material on its sites, and the Consolidated
58
+ # List carries no separate open licence -- it is published so that
59
+ # member states and their regulated entities can implement the
60
+ # measures, which is the use this library is for. Redistributing it,
61
+ # as a bundle or through an API, is the case to take to counsel.
62
+ licence_notice "Copyright reserved by the United Nations. Published " \
63
+ "for implementation of Security Council measures; no " \
64
+ "open reuse licence is stated. Verified 2026-09-11."
65
+ licence_url "https://www.un.org/en/about-us/copyright"
66
+
67
+ url :main, "https://scsanctions.un.org/resources/xml/en/consolidated.xml"
68
+
69
+ # @api private
70
+ INDIVIDUAL = T.let("INDIVIDUAL", String)
71
+ # @api private
72
+ ENTITY = T.let("ENTITY", String)
73
+
74
+ # @api private
75
+ LIST = T.let(Parsers::XmlRecords.new(records: [INDIVIDUAL, ENTITY]), Parsers::XmlRecords)
76
+
77
+ # The generation timestamp the UN stamps on the document element. More
78
+ # precise than the Last-Modified header Base falls back to, and it is the
79
+ # string that appears on the UN's own site, so it is the one an examiner
80
+ # asking "which version was this screened against" will recognise.
81
+ #
82
+ # @api private
83
+ GENERATED_AT = T.let("dateGenerated", String)
84
+
85
+ # Records that could not be used. Read after #parse; sync orchestration
86
+ # (#34) reports them.
87
+ sig { returns(T::Array[Parsers::Warning]) }
88
+ attr_reader :warnings
89
+
90
+ sig { params(args: T.untyped, options: T.untyped).void }
91
+ def initialize(*args, **options)
92
+ super
93
+ @warnings = T.let([], T::Array[Parsers::Warning])
94
+ @unmapped = T.let([], T::Array[Parsers::Warning])
95
+ @generated_at = T.let(nil, T.nilable(String))
96
+ end
97
+
98
+ sig { override.params(raw: T.untyped).returns(T::Array[Entity]) }
99
+ def parse(raw)
100
+ reader = LIST.read(raw)
101
+ entities = build(reader)
102
+ @generated_at = reader.root[GENERATED_AT]
103
+ @warnings = reader.warnings + @unmapped
104
+ entities
105
+ end
106
+
107
+ sig { override.returns(T.nilable(String)) }
108
+ def source_version = @generated_at || super
109
+
110
+ private
111
+
112
+ sig { params(reader: Parsers::XmlRecords::Reader).returns(T::Array[Entity]) }
113
+ def build(reader)
114
+ @unmapped = []
115
+ reader.filter_map do |node|
116
+ record = Record.new(node)
117
+ entity = record.entity
118
+ entity.nil? ? note_nameless(node) : entity
119
+ end
120
+ end
121
+
122
+ # A record with no name cannot be screened against and is never what the
123
+ # Committee meant to publish. None of the 1,011 published today is
124
+ # nameless; the warning exists so that the day one is, it is visible
125
+ # rather than absent.
126
+ sig { params(node: Parsers::XmlRecords::Record).returns(NilClass) }
127
+ def note_nameless(node)
128
+ @unmapped << Parsers::Warning.new(
129
+ line: node.line,
130
+ message: "<#{node.name}> #{node["DATAID"].inspect} has no name and was skipped"
131
+ )
132
+ nil
133
+ end
134
+ end
135
+ end
136
+ end
137
+
138
+ require "active_sanction/sources/un_consolidated/record"
139
+
140
+ ActiveSanction::Sources.register(ActiveSanction::Sources::UnConsolidated)
@@ -0,0 +1,196 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/configuration"
7
+ require "active_sanction/error"
8
+
9
+ module ActiveSanction
10
+ # Every sanctions list this library knows how to read, filed under the key
11
+ # its adapter declares.
12
+ #
13
+ # ActiveSanction::Sources[:un_consolidated] # => the adapter class
14
+ # ActiveSanction::Sources.keys # => [:ofac_sdn, :un_consolidated, ...]
15
+ #
16
+ # The registry is open, and that is the point of this milestone. A bank with
17
+ # an internal watchlist, or a gem adding a jurisdiction this one has not got
18
+ # to, registers an adapter of its own without forking:
19
+ #
20
+ # ActiveSanction::Sources.register(MyCompany::InternalWatchlist)
21
+ # ActiveSanction.configure { |c| c.sources = %i[ofac_sdn my_internal_watchlist] }
22
+ #
23
+ # Nothing here requires Sources::Base. Registration is duck-typed on `.key`
24
+ # and `.new`, so a source backed by a database table rather than a published
25
+ # file -- which has no URL to declare and no payload to fetch -- is a
26
+ # first-class citizen rather than something that has to pretend to be a file
27
+ # download. Base is the convenient way to write an adapter, not the price of
28
+ # admission.
29
+ #
30
+ # Built-in adapters register themselves at the bottom of their own file, one
31
+ # explicit `Sources.register(self)` line each, rather than being enrolled by
32
+ # `inherited`. Auto-registering every subclass would also enrol the abstract
33
+ # intermediates that adapters sharing a publisher will want (an OFAC base
34
+ # holding the authority and the format for SDN and Consolidated both) and
35
+ # every throwaway subclass a test defines.
36
+ module Sources
37
+ # Two lists cannot answer to one name. Raised at load time, which is where
38
+ # this collision is cheap to fix.
39
+ #
40
+ # A ConfigurationError, like the two below it: all three are an
41
+ # installation wired up wrong -- a key typed twice, a key typed wrong, an
42
+ # adapter that never declared what it is -- and none of them is fixed by
43
+ # waiting and trying again.
44
+ class DuplicateKey < ConfigurationError; end
45
+
46
+ # A key nothing is registered under: a typo in `config.sources`, a CLI
47
+ # argument, or an adapter whose file was never required.
48
+ class UnknownSource < ConfigurationError; end
49
+
50
+ # An adapter that does not declare what the contract requires, or is asked
51
+ # for a declaration it never made.
52
+ class DeclarationError < ConfigurationError; end
53
+
54
+ # The bytes of one of a source's files could not be obtained -- the
55
+ # publisher confirmed a copy we do not hold, and re-asking for it in full
56
+ # did not produce one either.
57
+ #
58
+ # Retryable: it is a publisher or an intermediary cache in a state it will
59
+ # not be in an hour from now, and the next run usually just works.
60
+ class MissingPayload < FetchError
61
+ extend T::Sig
62
+
63
+ sig { returns(T::Boolean) }
64
+ def retryable? = retryable_or(true)
65
+ end
66
+
67
+ # @api private
68
+ MUTEX = T.let(Mutex.new, Mutex)
69
+ private_constant :MUTEX
70
+
71
+ # Built at load rather than memoized on first use. A lazily created Hash
72
+ # is itself a write, and the first two adapters to register in two threads
73
+ # would each create one and each keep the half of the registry the other
74
+ # could not see. Everything that writes to it holds MUTEX; reads do not,
75
+ # because a registry is written at load and read for the life of the
76
+ # process.
77
+ #
78
+ # @api private
79
+ REGISTRY = T.let({}, T::Hash[Symbol, T.untyped])
80
+ private_constant :REGISTRY
81
+
82
+ class << self
83
+ extend T::Sig
84
+
85
+ # An adapter is anything answering `.key` and `.new`, which is why every
86
+ # signature here says `T.untyped` where a class goes and none of them
87
+ # says `T.class_of(Sources::Base)`. That is the milestone this registry
88
+ # exists for: a bank's internal watchlist, backed by a database table
89
+ # with no URL and no payload, is a first-class source rather than
90
+ # something pretending to be a file download. Narrowing these types would
91
+ # quietly close what the class comment above promises is open.
92
+
93
+ # Adds a source to the registry and returns it. Registering the same
94
+ # class twice is a no-op, so a file that manages to get loaded under two
95
+ # paths does not take the whole process down with it.
96
+ sig { params(source: T.untyped).returns(T.untyped) }
97
+ def register(source)
98
+ key = registrable!(source)
99
+ MUTEX.synchronize do
100
+ claimed = registry[key]
101
+ raise DuplicateKey, duplicate_message(key, source, claimed) if claimed && claimed != source
102
+
103
+ registry[key] = source
104
+ end
105
+ source
106
+ end
107
+
108
+ # The adapter registered under a key.
109
+ #
110
+ # Raises rather than returning nil, because every caller of this method
111
+ # is resolving a name a human typed -- into `config.sources`, into
112
+ # `activesanction sync ofac_sdb` -- and a nil surfaces three layers later
113
+ # as a NoMethodError that says nothing about the misspelling. The message
114
+ # lists what *is* registered, which is also the answer to "why is my
115
+ # adapter not being picked up" (its file was never required).
116
+ sig { params(key: T.untyped).returns(T.untyped) }
117
+ def [](key)
118
+ name = key.to_sym
119
+ registry.fetch(name) do
120
+ raise UnknownSource, "no source registered as #{name.inspect}. Registered: #{list}"
121
+ end
122
+ end
123
+
124
+ sig { params(key: T.untyped).returns(T::Boolean) }
125
+ def registered?(key) = registry.key?(key.to_sym)
126
+
127
+ # Every registered adapter, ordered by key so a CLI listing and a sync
128
+ # summary do not reshuffle themselves between runs.
129
+ sig { returns(T::Array[T.untyped]) }
130
+ def all = registry.keys.sort.map { |key| registry[key] }
131
+
132
+ sig { returns(T::Array[Symbol]) }
133
+ def keys = registry.keys.sort
134
+
135
+ sig { returns(Integer) }
136
+ def size = registry.size
137
+
138
+ sig { returns(T::Boolean) }
139
+ def empty? = registry.empty?
140
+
141
+ # The adapters a sync should run: what `config.sources` names, or every
142
+ # registered source when it names nothing. An unknown key raises here,
143
+ # at the start of the run, rather than after the other lists have been
144
+ # downloaded.
145
+ sig { params(configured: T.nilable(T::Array[Symbol])).returns(T::Array[T.untyped]) }
146
+ def enabled(configured = ActiveSanction.config.sources)
147
+ return all if configured.nil?
148
+
149
+ configured.map { |key| self[key] }
150
+ end
151
+
152
+ # Drops a key and returns what was registered there, or nil. The
153
+ # supported way to replace a built-in adapter with a patched one:
154
+ #
155
+ # ActiveSanction::Sources.unregister(:ofac_sdn)
156
+ # ActiveSanction::Sources.register(MyCompany::PatchedOfacSdn)
157
+ #
158
+ # There is deliberately no `clear!`. Built-in adapters register on
159
+ # require, and `require` runs once per process: a suite that emptied the
160
+ # registry between examples would leave every later example running
161
+ # against a library that has forgotten its own sources.
162
+ sig { params(key: T.untyped).returns(T.untyped) }
163
+ def unregister(key)
164
+ MUTEX.synchronize { registry.delete(key.to_sym) }
165
+ end
166
+
167
+ private
168
+
169
+ sig { returns(T::Hash[Symbol, T.untyped]) }
170
+ def registry = REGISTRY
171
+
172
+ sig { returns(String) }
173
+ def list = registry.empty? ? "(nothing)" : keys.join(", ")
174
+
175
+ sig { params(source: T.untyped).returns(Symbol) }
176
+ def registrable!(source)
177
+ unless source.respond_to?(:key) && source.respond_to?(:new)
178
+ raise InvalidArgument, "a source must answer .key and .new, got #{source.inspect}"
179
+ end
180
+
181
+ Definition.key!(source.key)
182
+ end
183
+
184
+ sig { params(key: Symbol, source: T.untyped, claimed: T.untyped).returns(String) }
185
+ def duplicate_message(key, source, claimed)
186
+ "cannot register #{source} as #{key.inspect}: #{claimed} already claims that key. A key is the public " \
187
+ "name of a list -- in configuration, in stored snapshots, in every match result that cites it -- so two " \
188
+ "lists cannot share one. Give the new source a different key, or call " \
189
+ "ActiveSanction::Sources.unregister(#{key.inspect}) first if you mean to replace #{claimed}."
190
+ end
191
+ end
192
+ end
193
+ end
194
+
195
+ require "active_sanction/sources/definition"
196
+ require "active_sanction/sources/base"