active_sanction 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +538 -0
  3. data/CODE_OF_CONDUCT.md +74 -0
  4. data/CONTRIBUTING.md +312 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +616 -0
  7. data/SECURITY.md +97 -0
  8. data/active_sanction.gemspec +98 -0
  9. data/docs/adding_a_source.md +1083 -0
  10. data/docs/api_stability.md +357 -0
  11. data/docs/bundle_format.md +407 -0
  12. data/lib/active_sanction/address.rb +170 -0
  13. data/lib/active_sanction/client.rb +341 -0
  14. data/lib/active_sanction/configuration.rb +721 -0
  15. data/lib/active_sanction/countries.txt +292 -0
  16. data/lib/active_sanction/country.rb +164 -0
  17. data/lib/active_sanction/deprecation.rb +168 -0
  18. data/lib/active_sanction/diff/change.rb +208 -0
  19. data/lib/active_sanction/diff.rb +348 -0
  20. data/lib/active_sanction/doctor/checkup.rb +356 -0
  21. data/lib/active_sanction/doctor/diagnosis.rb +245 -0
  22. data/lib/active_sanction/doctor/finding.rb +183 -0
  23. data/lib/active_sanction/doctor/profile.rb +384 -0
  24. data/lib/active_sanction/doctor/report.rb +227 -0
  25. data/lib/active_sanction/doctor.rb +347 -0
  26. data/lib/active_sanction/entity.rb +293 -0
  27. data/lib/active_sanction/error.rb +332 -0
  28. data/lib/active_sanction/fetcher/result.rb +116 -0
  29. data/lib/active_sanction/fetcher.rb +242 -0
  30. data/lib/active_sanction/http_client/errors.rb +94 -0
  31. data/lib/active_sanction/http_client/response.rb +129 -0
  32. data/lib/active_sanction/http_client.rb +324 -0
  33. data/lib/active_sanction/identifier.rb +212 -0
  34. data/lib/active_sanction/index/builder.rb +89 -0
  35. data/lib/active_sanction/index/candidate.rb +63 -0
  36. data/lib/active_sanction/index/entry.rb +66 -0
  37. data/lib/active_sanction/index/features.rb +112 -0
  38. data/lib/active_sanction/index.rb +381 -0
  39. data/lib/active_sanction/match_result.rb +345 -0
  40. data/lib/active_sanction/matcher.rb +388 -0
  41. data/lib/active_sanction/name.rb +183 -0
  42. data/lib/active_sanction/normalizer/cache.rb +82 -0
  43. data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
  44. data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
  45. data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
  46. data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
  47. data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
  48. data/lib/active_sanction/normalizer/dictionary.rb +243 -0
  49. data/lib/active_sanction/normalizer/form.rb +279 -0
  50. data/lib/active_sanction/normalizer.rb +137 -0
  51. data/lib/active_sanction/parsers/column_shape.rb +274 -0
  52. data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
  53. data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
  54. data/lib/active_sanction/parsers/delimited_table.rb +151 -0
  55. data/lib/active_sanction/parsers/format.rb +107 -0
  56. data/lib/active_sanction/parsers/join.rb +149 -0
  57. data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
  58. data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
  59. data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
  60. data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
  61. data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
  62. data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
  63. data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
  64. data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
  65. data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
  66. data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
  67. data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
  68. data/lib/active_sanction/parsers/xml_records.rb +140 -0
  69. data/lib/active_sanction/parsers.rb +94 -0
  70. data/lib/active_sanction/partial_date/parser.rb +124 -0
  71. data/lib/active_sanction/partial_date.rb +303 -0
  72. data/lib/active_sanction/payload_cache/checksum.rb +66 -0
  73. data/lib/active_sanction/payload_cache/entry.rb +304 -0
  74. data/lib/active_sanction/payload_cache.rb +366 -0
  75. data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
  76. data/lib/active_sanction/phonetics.rb +40 -0
  77. data/lib/active_sanction/query.rb +290 -0
  78. data/lib/active_sanction/rescreen/alert.rb +314 -0
  79. data/lib/active_sanction/rescreen.rb +332 -0
  80. data/lib/active_sanction/scorer/adjustments.rb +241 -0
  81. data/lib/active_sanction/scorer/name_score.rb +279 -0
  82. data/lib/active_sanction/scorer/reason.rb +136 -0
  83. data/lib/active_sanction/scorer/result.rb +129 -0
  84. data/lib/active_sanction/scorer/subject.rb +235 -0
  85. data/lib/active_sanction/scorer/weights.rb +306 -0
  86. data/lib/active_sanction/scorer.rb +303 -0
  87. data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
  88. data/lib/active_sanction/similarity/levenshtein.rb +179 -0
  89. data/lib/active_sanction/similarity/token_set.rb +159 -0
  90. data/lib/active_sanction/similarity/token_sort.rb +120 -0
  91. data/lib/active_sanction/similarity.rb +183 -0
  92. data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
  93. data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
  94. data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
  95. data/lib/active_sanction/snapshot/bundle.rb +381 -0
  96. data/lib/active_sanction/snapshot.rb +305 -0
  97. data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
  98. data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
  99. data/lib/active_sanction/sources/australia_dfat.rb +291 -0
  100. data/lib/active_sanction/sources/base.rb +295 -0
  101. data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
  102. data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
  103. data/lib/active_sanction/sources/canada_sema.rb +153 -0
  104. data/lib/active_sanction/sources/definition.rb +360 -0
  105. data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
  106. data/lib/active_sanction/sources/eu_fsf.rb +203 -0
  107. data/lib/active_sanction/sources/ofac/record.rb +251 -0
  108. data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
  109. data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
  110. data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
  111. data/lib/active_sanction/sources/ofac.rb +248 -0
  112. data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
  113. data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
  114. data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
  115. data/lib/active_sanction/sources/remarks.rb +84 -0
  116. data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
  117. data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
  118. data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
  119. data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
  120. data/lib/active_sanction/sources/un_consolidated.rb +140 -0
  121. data/lib/active_sanction/sources.rb +196 -0
  122. data/lib/active_sanction/storage/active_record/reader.rb +124 -0
  123. data/lib/active_sanction/storage/active_record/row.rb +175 -0
  124. data/lib/active_sanction/storage/active_record/writer.rb +141 -0
  125. data/lib/active_sanction/storage/active_record.rb +277 -0
  126. data/lib/active_sanction/storage/base.rb +216 -0
  127. data/lib/active_sanction/storage/file_system.rb +373 -0
  128. data/lib/active_sanction/storage/memory.rb +67 -0
  129. data/lib/active_sanction/storage/meta.rb +188 -0
  130. data/lib/active_sanction/storage.rb +74 -0
  131. data/lib/active_sanction/subject.rb +271 -0
  132. data/lib/active_sanction/sync/report.rb +245 -0
  133. data/lib/active_sanction/sync/result.rb +296 -0
  134. data/lib/active_sanction/sync.rb +390 -0
  135. data/lib/active_sanction/validator_store/file_system.rb +103 -0
  136. data/lib/active_sanction/validator_store/memory.rb +41 -0
  137. data/lib/active_sanction/validator_store.rb +119 -0
  138. data/lib/active_sanction/validators.rb +195 -0
  139. data/lib/active_sanction/version.rb +23 -0
  140. data/lib/active_sanction.rb +331 -0
  141. data/lib/generators/active_sanction/install/install_generator.rb +60 -0
  142. data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
  143. metadata +236 -0
@@ -0,0 +1,332 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "time"
7
+ require "active_sanction/diff"
8
+ require "active_sanction/error"
9
+ require "active_sanction/index"
10
+ require "active_sanction/match_result"
11
+ require "active_sanction/scorer"
12
+ require "active_sanction/subject"
13
+ require "active_sanction/rescreen/alert"
14
+
15
+ module ActiveSanction
16
+ # Who a list change affects. Screening answers about a name; this answers
17
+ # about a book of business.
18
+ #
19
+ # book = [
20
+ # ActiveSanction::Subject.new(id: "cust_1", name: "Bosco Ntaganda", date_of_birth: "1973"),
21
+ # ActiveSanction::Subject.new(id: "cust_2", name: "Jane Miller")
22
+ # ]
23
+ #
24
+ # diff = ActiveSanction.diff(:ofac_sdn, from: yesterdays_snapshot)
25
+ # alerts = ActiveSanction.rescreen(book, diff: diff, threshold: 75)
26
+ #
27
+ # alerts.first.subject_id # => "cust_1"
28
+ # alerts.first.change # => :newly_listed
29
+ # alerts.first.result # => a full MatchResult, with its explanation
30
+ # alerts.first.previous_score # => nil, or what it scored before
31
+ #
32
+ # ### Why this is the primitive rather than a nightly full screen
33
+ #
34
+ # Screening a customer once is a checkbox. The obligation is ongoing:
35
+ # somebody cleared last month may be listed today, and a delisting matters
36
+ # just as much, because it is the entry that lets a customer back through
37
+ # the door. That is recurring work, and the naive way to do it -- every
38
+ # subject against every record, every night -- costs the whole book times
39
+ # the whole corpus and stops being nightly somewhere around a few thousand
40
+ # customers.
41
+ #
42
+ # A rescreen costs the whole book times *the handful of records that moved*.
43
+ # Diff (#35) computes what changed in the list; this computes who that
44
+ # change affects, and it is the step that turns a diff into an alert. On a
45
+ # typical day an OFAC diff is a few dozen records, so the index built here
46
+ # is a few dozen names rather than 46,000 -- see the numbers in the README.
47
+ #
48
+ # ### It is built once and then only read
49
+ #
50
+ # A Rescreen holds an index over the records the diff names, both versions
51
+ # of each amended one, the weights it scores with and the threshold it
52
+ # defaults to. All of it is fixed at construction and the object is frozen,
53
+ # so a large book is streamed past one of these, in batches, from as many
54
+ # threads as a host has:
55
+ #
56
+ # rescreening = ActiveSanction::Rescreen.new(diff: diff, threshold: 75)
57
+ # customers.find_each(batch_size: 1_000) do |batch|
58
+ # rescreening.call(batch) { |alert| AlertRecord.create!(alert.to_h) }
59
+ # end
60
+ #
61
+ # Nothing about a book is held: subjects are read one at a time and only
62
+ # alerts are kept, so memory is a function of how much moved rather than of
63
+ # how many customers there are. The block is what makes even that bounded --
64
+ # it is called with each alert as it is raised, and a host that writes them
65
+ # out as they arrive never accumulates the array at all.
66
+ #
67
+ # ### It does not touch the matcher
68
+ #
69
+ # A rescreen builds its own index, over the diff, and never reads the store
70
+ # or the client's matcher. That is the point: applying a diff to a book must
71
+ # not cost an index build over the whole corpus, and a host that has never
72
+ # screened anything in this process can rescreen without paying for one.
73
+ #
74
+ # ### An empty diff does no work at all
75
+ #
76
+ # A sync that changed nothing, or a first sync -- which is a baseline rather
77
+ # than a list of 19,015 additions, see Diff -- yields no alerts and scores
78
+ # nothing. Not one subject is folded. A host that syncs hourly and
79
+ # rescreens after each sync is paying for the hours that moved, which is
80
+ # what makes rescreening after every sync affordable.
81
+ #
82
+ # ### What it does not do
83
+ #
84
+ # It reports the changes; it does not remember them. There is no alert
85
+ # store, no deduplication against what was raised yesterday, and no
86
+ # disposition -- see the note on case management in the README. Two runs
87
+ # over the same diff produce the same alerts, which is the property that
88
+ # makes them re-derivable and the reason a host, not this library, owns the
89
+ # queue they go into.
90
+ #
91
+ # It also cannot find what was already there. A subject who matched a record
92
+ # that did not change is not in a diff at all, and no rescreen will report
93
+ # them: the first screening run against a book is a deliberate full screen
94
+ # (`Client#screen_all`), and this is what keeps it current afterwards.
95
+ class Rescreen
96
+ extend T::Sig
97
+
98
+ # The diff being applied, which is what the alerts are about.
99
+ sig { returns(Diff) }
100
+ attr_reader :diff
101
+
102
+ # The lowest score worth an alert, for a subject that does not name its
103
+ # own. Read once, at construction, so a configuration changed mid-run
104
+ # cannot produce a book screened half one way.
105
+ sig { returns(Float) }
106
+ attr_reader :threshold
107
+
108
+ # What each signal was worth for this run, and what every result it
109
+ # produces records.
110
+ sig { returns(Scorer::Weights) }
111
+ attr_reader :weights
112
+
113
+ # How many names the index hands the scorer per subject. It does not bind
114
+ # on a typical diff -- a few dozen records cannot exceed it -- and is here
115
+ # for the day a publisher reissues a whole list under new ids.
116
+ sig { returns(Integer) }
117
+ attr_reader :candidate_limit
118
+
119
+ sig { returns(Symbol) }
120
+ attr_reader :backend
121
+
122
+ # Sugar, and what ActiveSanction.rescreen calls:
123
+ #
124
+ # ActiveSanction::Rescreen.call(book, diff: diff, threshold: 75)
125
+ #
126
+ # Builds a Rescreen and applies it once. A host streaming a book in
127
+ # batches builds one with .new and calls it per batch instead, so the
128
+ # index is built once rather than per batch.
129
+ sig do
130
+ params(subjects: T.untyped, diff: T.untyped, options: T.untyped,
131
+ block: T.nilable(T.proc.params(alert: Alert).void)).returns(T::Array[Alert])
132
+ end
133
+ def self.call(subjects, diff:, **options, &block)
134
+ T.unsafe(self).new(diff: diff, **options).call(subjects, &block)
135
+ end
136
+
137
+ sig do
138
+ params(diff: T.untyped, threshold: T.untyped, weights: T.untyped, candidate_limit: T.untyped,
139
+ backend: T.untyped).void
140
+ end
141
+ def initialize(diff:, threshold: nil, weights: nil, candidate_limit: nil,
142
+ backend: MatchResult::DEFAULT_BACKEND)
143
+ @diff = T.let(diff!(diff), Diff)
144
+ @threshold = T.let(threshold!(threshold), Float)
145
+ @weights = T.let(Scorer::Weights.build(weights), Scorer::Weights)
146
+ @candidate_limit = T.let(candidate_limit!(candidate_limit), Integer)
147
+ @backend = T.let(backend.to_s.to_sym, Symbol)
148
+ @versions = T.let(versions, T::Hash[String, T::Array[T.untyped]])
149
+ @fields = T.let(amended, T::Hash[String, T::Array[Symbol]])
150
+ # Both versions of an amended record are indexed, because a subject may
151
+ # have matched only the alias that was taken away.
152
+ @index = T.let(Index.build(@versions.values.flatten.compact), Index)
153
+ freeze
154
+ end
155
+
156
+ # The alerts this diff raises against this book, highest score first
157
+ # within each subject and in the order the subjects arrived:
158
+ #
159
+ # rescreening.call(book)
160
+ # rescreening.call(book) { |alert| queue.push(alert) }
161
+ #
162
+ # `subjects` is anything that responds to `each`, so an Enumerator over a
163
+ # database cursor is streamed rather than materialized. Each entry is a
164
+ # Subject or the Hash one is built from.
165
+ #
166
+ # Every alert in a run carries one `screened_at`, because a rescreening
167
+ # of a book against a new list version is a single event in an audit
168
+ # trail rather than ten thousand of them a microsecond apart. A host that
169
+ # calls this once per batch is running one event per batch, which is the
170
+ # honest description of what it did.
171
+ sig do
172
+ params(subjects: T.untyped, block: T.nilable(T.proc.params(alert: Alert).void)).returns(T::Array[Alert])
173
+ end
174
+ def call(subjects, &block)
175
+ return [] if diff.empty?
176
+
177
+ screened_at = Time.now.utc
178
+ alerts = T.let([], T::Array[Alert])
179
+ each(subjects) do |value|
180
+ found(Subject.build(value), screened_at).each do |alert|
181
+ block&.call(alert)
182
+ alerts << alert
183
+ end
184
+ end
185
+ alerts
186
+ end
187
+
188
+ # The list this run is about, taken from the diff.
189
+ sig { returns(Symbol) }
190
+ def source = diff.source
191
+
192
+ # How many records moved, which is what a run costs per subject.
193
+ sig { returns(Integer) }
194
+ def size = diff.size
195
+
196
+ # Nothing moved, so nothing can be affected. See the class comment.
197
+ sig { returns(T::Boolean) }
198
+ def empty? = diff.empty?
199
+
200
+ sig { returns(String) }
201
+ def inspect = "#<#{self.class} #{source} #{size} changed records at #{threshold}>"
202
+
203
+ private
204
+
205
+ # The alerts one subject raises, highest score first and then by record
206
+ # id, so that two runs over the same diff and the same book produce the
207
+ # same output in the same order.
208
+ sig { params(subject: Subject, screened_at: Time).returns(T::Array[Alert]) }
209
+ def found(subject, screened_at)
210
+ cutoff = subject.threshold || threshold
211
+ readings = candidates(subject).filter_map { |id| compare(subject, id, cutoff) }
212
+ return [] if readings.empty?
213
+
214
+ query = subject.query(threshold: cutoff, sources: [source])
215
+ readings.map { |reading| alert(subject, reading, query, screened_at) }
216
+ .sort_by { |raised| [-(raised.score || T.must(raised.previous_score)), raised.entity_id] }
217
+ end
218
+
219
+ # The records worth scoring this subject against: the same retrieval stage
220
+ # a screening call uses, over a corpus the size of the diff. An entity
221
+ # reached through two of its names, or through both versions of itself, is
222
+ # one record to compare and not two.
223
+ sig { params(subject: Subject).returns(T::Array[String]) }
224
+ def candidates(subject)
225
+ @index.candidates(subject.form, limit: candidate_limit).map { |candidate| candidate.entity.id }.uniq
226
+ end
227
+
228
+ # What this subject scores against both versions of one record, or nil
229
+ # when it reaches the threshold against neither -- which is the answer for
230
+ # nearly every pair and is why the cutoff is passed to the scorer rather
231
+ # than applied afterwards. See Scorer on what a threshold buys.
232
+ sig { params(subject: Subject, id: String, cutoff: Float).returns(T.nilable(T::Array[T.untyped])) }
233
+ def compare(subject, id, cutoff)
234
+ previous, current = @versions.fetch(id)
235
+ before = score(subject, previous, cutoff)
236
+ after = score(subject, current, cutoff)
237
+ return nil if before.nil? && after.nil?
238
+
239
+ change = if before.nil? then :newly_listed
240
+ elsif after.nil? then :delisted
241
+ else :details_changed
242
+ end
243
+ # The side that did not clear is scored again without a cutoff, so an
244
+ # alert can say a subject moved from 71 to 94 rather than only that it
245
+ # now matches. Reached once per alert, which is why it is affordable.
246
+ [id, change, before || score(subject, previous, 0.0), after || score(subject, current, 0.0)]
247
+ end
248
+
249
+ sig { params(subject: Subject, entity: T.untyped, cutoff: Float).returns(T.nilable(Scorer::Result)) }
250
+ def score(subject, entity, cutoff)
251
+ return nil if entity.nil?
252
+
253
+ Scorer.call(subject.evidence, entity, weights: weights, threshold: cutoff)
254
+ end
255
+
256
+ sig do
257
+ params(subject: Subject, reading: T::Array[T.untyped], query: Query, screened_at: Time).returns(Alert)
258
+ end
259
+ def alert(subject, reading, query, screened_at)
260
+ id, change, before, after = reading
261
+ Alert.new(
262
+ subject: subject, change: change, fields: @fields.fetch(id, []),
263
+ result: stamp(after, diff.to.checksum, query, screened_at),
264
+ previous_result: stamp(before, T.must(diff.from).checksum, query, screened_at),
265
+ snapshot_id: diff.to.checksum, previous_snapshot_id: T.must(diff.from).checksum
266
+ )
267
+ end
268
+
269
+ # One scored record as the audit object it has to be, stamped with the
270
+ # checksum of the list version it was scored against -- which is the whole
271
+ # of why an alert holds two of these rather than one score and a delta.
272
+ sig do
273
+ params(result: T.nilable(Scorer::Result), snapshot_id: String, query: Query,
274
+ screened_at: Time).returns(T.nilable(MatchResult))
275
+ end
276
+ def stamp(result, snapshot_id, query, screened_at)
277
+ return nil if result.nil?
278
+
279
+ MatchResult.from_scorer(result, query: query, snapshot_id: snapshot_id, weights: weights,
280
+ screened_at: screened_at, backend: backend)
281
+ end
282
+
283
+ sig { params(subjects: T.untyped, block: T.proc.params(value: T.untyped).void).void }
284
+ def each(subjects, &block)
285
+ unless subjects.respond_to?(:each)
286
+ raise InvalidArgument,
287
+ "a book of subjects has to be enumerable, got #{subjects.class}. An Array, or anything that " \
288
+ "responds to #each -- an Enumerator over a database cursor is streamed rather than materialized"
289
+ end
290
+
291
+ subjects.each(&block)
292
+ end
293
+
294
+ # Every record the diff names, by id, as a pair: how the old list had it
295
+ # and how the new list has it. A record that was added has no first half
296
+ # and one that was withdrawn has no second, and an amended one has both --
297
+ # which is what lets a single comparison produce a score on either side of
298
+ # the change. The three sets are disjoint, because a diff joins its two
299
+ # snapshots by id.
300
+ sig { returns(T::Hash[String, T::Array[T.untyped]]) }
301
+ def versions
302
+ found = T.let({}, T::Hash[String, T::Array[T.untyped]])
303
+ diff.removed.each { |entity| found[entity.id] = [entity, nil].freeze }
304
+ diff.added.each { |entity| found[entity.id] = [nil, entity].freeze }
305
+ diff.modified.each { |change| found[change.id] = [change.previous, change.entity].freeze }
306
+ found.freeze
307
+ end
308
+
309
+ sig { params(value: T.untyped).returns(Float) }
310
+ def threshold!(value) = Scorer.threshold!(value || ActiveSanction.config.screening_threshold)
311
+
312
+ sig { params(value: T.untyped).returns(Integer) }
313
+ def candidate_limit!(value) = Integer(value || ActiveSanction.config.candidate_limit)
314
+
315
+ # Which fields moved, per amended record, so an alert can say whether a
316
+ # subject's status changed because the list did or because the record did.
317
+ sig { returns(T::Hash[String, T::Array[Symbol]]) }
318
+ def amended = diff.modified.to_h { |change| [change.id, change.fields] }.freeze
319
+
320
+ # A baseline diff is empty, so `from` is present on every diff that can
321
+ # raise an alert -- which is what lets an alert cite both checksums. The
322
+ # check is here rather than at the alert, where the missing one would read
323
+ # as a bug in this class.
324
+ sig { params(value: T.untyped).returns(Diff) }
325
+ def diff!(value)
326
+ raise InvalidArgument, "diff must be an ActiveSanction::Diff, got #{value.class}" unless value.is_a?(Diff)
327
+ return value if value.empty? || value.from
328
+
329
+ raise InvalidArgument, "a rescreen needs both list versions to cite, and this diff has no `from`"
330
+ end
331
+ end
332
+ end
@@ -0,0 +1,241 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/country"
7
+ require "active_sanction/scorer/reason"
8
+
9
+ module ActiveSanction
10
+ module Scorer
11
+ # What the record says about a subject other than their name, and what
12
+ # each agreement and each contradiction is worth.
13
+ #
14
+ # ActiveSanction::Scorer::Adjustments.call(subject, entity)
15
+ # # => [#<Reason +40.0 identifier: passport AB123456 matches>,
16
+ # # #<Reason -8.0 nationality: query RU vs listed EG>]
17
+ #
18
+ # Stage 4b, and the part that separates a usable screening tool from a
19
+ # name-similarity toy. A name score alone puts thousands of people on a
20
+ # list of a few hundred, because a quarter of the individuals on these
21
+ # lists share a handful of given names and every one of them scores in the
22
+ # seventies against every other. The passport number, the date of birth
23
+ # and the nationality are what tell those apart, and they are the fields a
24
+ # compliance officer already has in a customer record.
25
+ #
26
+ # ### Absent is not conflict, and it is the rule everything here obeys
27
+ #
28
+ # Most records lack most identifiers. Canada publishes no aliases and
29
+ # frequently no date of birth; OFAC's dates are prose in a remarks field
30
+ # and are missing wherever the sentence did not parse; the UN grades what
31
+ # it has and says nothing about what it does not. So every adjustment
32
+ # below fires only when *both* sides carry the field, and a field missing
33
+ # on either side produces no Reason at all -- not a small penalty, not a
34
+ # zero-valued reason.
35
+ #
36
+ # Treating absence as disagreement would penalize exactly the sparser
37
+ # lists, which means systematically under-scoring the jurisdictions that
38
+ # publish least and hiding real hits behind a threshold. It is the
39
+ # quietest way to build a screening tool that does not screen, and it is
40
+ # why this is stated as a rule rather than left to each branch.
41
+ #
42
+ # The same rule governs a country nobody can resolve. A nationality of
43
+ # `Ruritania` is not a contradiction of `Egypt`; it is a string this
44
+ # library does not recognize, and the difference between those two is a
45
+ # penalty applied to a match that was correct. See Country.
46
+ #
47
+ # ### The three signals, and why they are worth what they are
48
+ #
49
+ # **A document number is the only near-decisive field on a sanctions
50
+ # record.** Two people share a name; they do not share a passport number.
51
+ # It is the one place where agreement outranks everything the name said,
52
+ # which is what a boost of 40 points on a 100-point scale means: a name in
53
+ # the fifties plus the right passport clears any threshold this library
54
+ # would ship.
55
+ #
56
+ # **A date of birth is strong evidence in both directions.** An exact full
57
+ # date agreeing is worth a real boost; a genuine conflict -- two dates
58
+ # that cannot describe the same person -- is worth more against, because
59
+ # sharing a birthday with a listed person is a coincidence a few thousand
60
+ # people have and not having theirs is not. `PartialDate#overlaps?` is
61
+ # what decides which of the three cases a pair is in, and it is exact
62
+ # whatever precision either side carries: a year-only record does not
63
+ # conflict with a full date inside it, it agrees with it, weakly.
64
+ #
65
+ # **Nationality moves the score least.** People hold two passports, lists
66
+ # record the country a person was born in as readily as the one they are a
67
+ # citizen of, and the field is published as prose. It is real evidence and
68
+ # it is the softest of the three.
69
+ #
70
+ # ### Order is fixed, because an explanation is read
71
+ #
72
+ # Identifier, then date of birth, then nationality: strongest evidence
73
+ # first, so a reviewer reading an explanation downwards meets the reason
74
+ # the score is what it is before meeting the ones that adjusted it. The
75
+ # order is also what makes a score reproducible to the last decimal --
76
+ # see Scorer on why the sum has to be taken the same way every time.
77
+ #
78
+ # @api private
79
+ module Adjustments
80
+ extend T::Sig
81
+
82
+ # Below this many alphanumerics, a "document number" is not a document
83
+ # number. Nothing a government issues is three characters long, and an
84
+ # exact match on a short string is the one way a 40-point boost could
85
+ # land on a coincidence -- a remarks parser that read `Passport 12` out
86
+ # of a half-formed sentence would otherwise make two unrelated records
87
+ # decisive for each other.
88
+ MINIMUM_IDENTIFIER_LENGTH = T.let(4, Integer)
89
+
90
+ module_function
91
+
92
+ # Every adjustment both sides carry the evidence for, strongest first.
93
+ # An empty array is the ordinary answer for a record that publishes a
94
+ # name and nothing else.
95
+ sig do
96
+ params(subject: Subject, entity: Entity, weights: Weights)
97
+ .returns(T::Array[Reason]).checked(:tests)
98
+ end
99
+ def call(subject, entity, weights = Weights.default)
100
+ [identifier(subject, entity, weights), dob(subject, entity, weights),
101
+ nationality(subject, entity, weights)].compact
102
+ end
103
+
104
+ # The first document number both sides carry, in the order the caller
105
+ # and the publisher listed them. First rather than best: there is no
106
+ # "better" exact match on a document number, and iterating in a fixed
107
+ # order is what makes the explanation identical on every run.
108
+ sig { params(subject: Subject, entity: Entity, weights: Weights).returns(T.nilable(Reason)).checked(:tests) }
109
+ def identifier(subject, entity, weights)
110
+ subject.identifiers.each do |mine|
111
+ next if mine.normalized_value.length < MINIMUM_IDENTIFIER_LENGTH
112
+
113
+ theirs = entity.identifiers.find { |listed| same_document?(mine, listed) }
114
+ next if theirs.nil?
115
+
116
+ return Reason.new(factor: :identifier, contribution: weights.identifier_match,
117
+ detail: document_detail(mine, theirs))
118
+ end
119
+ nil
120
+ end
121
+
122
+ # Exact, overlapping, or contradictory -- in that order, over every
123
+ # pairing of the dates the two sides carry. Both plural: the UN
124
+ # publishes more than one date of birth for 140 of its individuals
125
+ # because several governments reported several dates, and any of them
126
+ # matching is a match. A conflict therefore means *no* pairing overlaps,
127
+ # which is the only reading that does not turn an honestly uncertain
128
+ # record into a penalty.
129
+ sig { params(subject: Subject, entity: Entity, weights: Weights).returns(T.nilable(Reason)).checked(:tests) }
130
+ def dob(subject, entity, weights)
131
+ mine = subject.dates_of_birth
132
+ theirs = entity.dates_of_birth
133
+ return nil if mine.empty? || theirs.empty?
134
+
135
+ agreement(mine.product(theirs), weights) ||
136
+ dob_reason(:dob_conflict, "#{mine.join(", ")} conflicts with listed #{theirs.join(", ")}", weights)
137
+ end
138
+
139
+ # The best of the pairings, or nil when none of them agree at all --
140
+ # which is what makes the caller above a conflict.
141
+ sig do
142
+ params(pairs: T::Array[T::Array[PartialDate]], weights: Weights)
143
+ .returns(T.nilable(Reason)).checked(:tests)
144
+ end
145
+ def agreement(pairs, weights)
146
+ exact = pairs.find { |left, right| same_day?(T.must(left), T.must(right)) }
147
+ return dob_reason(:dob_exact, "#{exact.first} matches", weights) if exact
148
+
149
+ overlap = pairs.find { |left, right| T.must(left).overlaps?(right) }
150
+ return nil if overlap.nil?
151
+
152
+ dob_reason(:dob_overlap, "#{overlap.first} overlaps listed #{overlap.last}", weights)
153
+ end
154
+
155
+ # Agreement, or a contradiction both sides are precise enough to make.
156
+ #
157
+ # A country is compared as the alpha-2 code it resolves to, falling back
158
+ # to its folded string so that two publishers writing the same
159
+ # unrecognized value still agree. A *conflict* needs more: every value
160
+ # on both sides has to have resolved, because "these two strings are not
161
+ # equal" is not evidence that two countries are different.
162
+ sig { params(subject: Subject, entity: Entity, weights: Weights).returns(T.nilable(Reason)).checked(:tests) }
163
+ def nationality(subject, entity, weights)
164
+ listed = entity.nationalities
165
+ return nil if subject.nationalities.empty? || listed.empty?
166
+
167
+ mine = keys(subject.nationalities)
168
+ theirs = keys(listed)
169
+ shared = mine & theirs
170
+ if shared.any?
171
+ return Reason.new(factor: :nationality, contribution: weights.nationality_match,
172
+ detail: "#{shared.join(", ")} matches")
173
+ end
174
+ return nil unless subject.countries? && resolved?(listed)
175
+
176
+ Reason.new(factor: :nationality, contribution: weights.nationality_conflict,
177
+ detail: "query #{mine.join(", ")} vs listed #{theirs.join(", ")}")
178
+ end
179
+
180
+ # Kind, number and country all have to be compatible, and two of the
181
+ # three treat "unstated" as compatible rather than as different.
182
+ #
183
+ # `:other` is the kind OFAC's remarks produce for a number its sentence
184
+ # did not classify, so requiring the kinds to be equal would throw away
185
+ # most of the document numbers this library extracts. Country is the
186
+ # opposite case and is why Identifier's own equality carries it: two
187
+ # passports with the same number from different countries are different
188
+ # documents, so a country stated on both sides and disagreeing is a
189
+ # mismatch -- while a country stated on neither, or on one, is not.
190
+ sig { params(mine: Identifier, theirs: Identifier).returns(T::Boolean).checked(:tests) }
191
+ def same_document?(mine, theirs)
192
+ return false unless mine.normalized_value == theirs.normalized_value
193
+ return false unless mine.kind == theirs.kind || mine.kind == :other || theirs.kind == :other
194
+
195
+ same_country?(mine.country, theirs.country)
196
+ end
197
+
198
+ sig { params(mine: T.nilable(String), theirs: T.nilable(String)).returns(T::Boolean).checked(:tests) }
199
+ def same_country?(mine, theirs)
200
+ return true if mine.nil? || theirs.nil?
201
+
202
+ key(mine) == key(theirs)
203
+ end
204
+
205
+ # A full date on both sides, neither hedged with a "circa", naming the
206
+ # same day. Anything less precise is an overlap: `1948` and `1948-12-10`
207
+ # agree, and saying they match exactly would claim a precision the
208
+ # publisher did not.
209
+ sig { params(mine: PartialDate, theirs: PartialDate).returns(T::Boolean).checked(:tests) }
210
+ def same_day?(mine, theirs)
211
+ mine.precision == :day && theirs.precision == :day &&
212
+ !mine.approximate? && !theirs.approximate? && mine.first_date == theirs.first_date
213
+ end
214
+
215
+ sig { params(weight: Symbol, detail: String, weights: Weights).returns(Reason).checked(:tests) }
216
+ def dob_reason(weight, detail, weights)
217
+ Reason.new(factor: :dob, detail: "date of birth #{detail}", contribution: weights.fetch(weight))
218
+ end
219
+
220
+ sig { params(mine: Identifier, theirs: Identifier).returns(String).checked(:tests) }
221
+ def document_detail(mine, theirs)
222
+ kind = mine.kind == :other ? theirs.kind : mine.kind
223
+ return "#{kind} #{mine.value} matches" if mine.value == theirs.value
224
+
225
+ "#{kind} #{mine.value} matches listed #{theirs.value}"
226
+ end
227
+
228
+ # The alpha-2 code, or the folded string when the table does not know
229
+ # the value. Never nil, so agreement can be found on a spelling neither
230
+ # side could resolve.
231
+ sig { params(value: String).returns(String).checked(:tests) }
232
+ def key(value) = Country.code(value) || Country.fold(value)
233
+
234
+ sig { params(values: T::Array[String]).returns(T::Array[String]).checked(:tests) }
235
+ def keys(values) = values.map { |value| key(value) }.uniq
236
+
237
+ sig { params(values: T::Array[String]).returns(T::Boolean).checked(:tests) }
238
+ def resolved?(values) = values.all? { |value| Country.code(value) }
239
+ end
240
+ end
241
+ end