active_sanction 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +538 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/CONTRIBUTING.md +312 -0
- data/LICENSE.txt +21 -0
- data/README.md +616 -0
- data/SECURITY.md +97 -0
- data/active_sanction.gemspec +98 -0
- data/docs/adding_a_source.md +1083 -0
- data/docs/api_stability.md +357 -0
- data/docs/bundle_format.md +407 -0
- data/lib/active_sanction/address.rb +170 -0
- data/lib/active_sanction/client.rb +341 -0
- data/lib/active_sanction/configuration.rb +721 -0
- data/lib/active_sanction/countries.txt +292 -0
- data/lib/active_sanction/country.rb +164 -0
- data/lib/active_sanction/deprecation.rb +168 -0
- data/lib/active_sanction/diff/change.rb +208 -0
- data/lib/active_sanction/diff.rb +348 -0
- data/lib/active_sanction/doctor/checkup.rb +356 -0
- data/lib/active_sanction/doctor/diagnosis.rb +245 -0
- data/lib/active_sanction/doctor/finding.rb +183 -0
- data/lib/active_sanction/doctor/profile.rb +384 -0
- data/lib/active_sanction/doctor/report.rb +227 -0
- data/lib/active_sanction/doctor.rb +347 -0
- data/lib/active_sanction/entity.rb +293 -0
- data/lib/active_sanction/error.rb +332 -0
- data/lib/active_sanction/fetcher/result.rb +116 -0
- data/lib/active_sanction/fetcher.rb +242 -0
- data/lib/active_sanction/http_client/errors.rb +94 -0
- data/lib/active_sanction/http_client/response.rb +129 -0
- data/lib/active_sanction/http_client.rb +324 -0
- data/lib/active_sanction/identifier.rb +212 -0
- data/lib/active_sanction/index/builder.rb +89 -0
- data/lib/active_sanction/index/candidate.rb +63 -0
- data/lib/active_sanction/index/entry.rb +66 -0
- data/lib/active_sanction/index/features.rb +112 -0
- data/lib/active_sanction/index.rb +381 -0
- data/lib/active_sanction/match_result.rb +345 -0
- data/lib/active_sanction/matcher.rb +388 -0
- data/lib/active_sanction/name.rb +183 -0
- data/lib/active_sanction/normalizer/cache.rb +82 -0
- data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
- data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
- data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
- data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
- data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
- data/lib/active_sanction/normalizer/dictionary.rb +243 -0
- data/lib/active_sanction/normalizer/form.rb +279 -0
- data/lib/active_sanction/normalizer.rb +137 -0
- data/lib/active_sanction/parsers/column_shape.rb +274 -0
- data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
- data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
- data/lib/active_sanction/parsers/delimited_table.rb +151 -0
- data/lib/active_sanction/parsers/format.rb +107 -0
- data/lib/active_sanction/parsers/join.rb +149 -0
- data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
- data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
- data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
- data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
- data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
- data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
- data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
- data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
- data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
- data/lib/active_sanction/parsers/xml_records.rb +140 -0
- data/lib/active_sanction/parsers.rb +94 -0
- data/lib/active_sanction/partial_date/parser.rb +124 -0
- data/lib/active_sanction/partial_date.rb +303 -0
- data/lib/active_sanction/payload_cache/checksum.rb +66 -0
- data/lib/active_sanction/payload_cache/entry.rb +304 -0
- data/lib/active_sanction/payload_cache.rb +366 -0
- data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
- data/lib/active_sanction/phonetics.rb +40 -0
- data/lib/active_sanction/query.rb +290 -0
- data/lib/active_sanction/rescreen/alert.rb +314 -0
- data/lib/active_sanction/rescreen.rb +332 -0
- data/lib/active_sanction/scorer/adjustments.rb +241 -0
- data/lib/active_sanction/scorer/name_score.rb +279 -0
- data/lib/active_sanction/scorer/reason.rb +136 -0
- data/lib/active_sanction/scorer/result.rb +129 -0
- data/lib/active_sanction/scorer/subject.rb +235 -0
- data/lib/active_sanction/scorer/weights.rb +306 -0
- data/lib/active_sanction/scorer.rb +303 -0
- data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
- data/lib/active_sanction/similarity/levenshtein.rb +179 -0
- data/lib/active_sanction/similarity/token_set.rb +159 -0
- data/lib/active_sanction/similarity/token_sort.rb +120 -0
- data/lib/active_sanction/similarity.rb +183 -0
- data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
- data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
- data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
- data/lib/active_sanction/snapshot/bundle.rb +381 -0
- data/lib/active_sanction/snapshot.rb +305 -0
- data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
- data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
- data/lib/active_sanction/sources/australia_dfat.rb +291 -0
- data/lib/active_sanction/sources/base.rb +295 -0
- data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
- data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
- data/lib/active_sanction/sources/canada_sema.rb +153 -0
- data/lib/active_sanction/sources/definition.rb +360 -0
- data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
- data/lib/active_sanction/sources/eu_fsf.rb +203 -0
- data/lib/active_sanction/sources/ofac/record.rb +251 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
- data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
- data/lib/active_sanction/sources/ofac.rb +248 -0
- data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
- data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
- data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
- data/lib/active_sanction/sources/remarks.rb +84 -0
- data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
- data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
- data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
- data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
- data/lib/active_sanction/sources/un_consolidated.rb +140 -0
- data/lib/active_sanction/sources.rb +196 -0
- data/lib/active_sanction/storage/active_record/reader.rb +124 -0
- data/lib/active_sanction/storage/active_record/row.rb +175 -0
- data/lib/active_sanction/storage/active_record/writer.rb +141 -0
- data/lib/active_sanction/storage/active_record.rb +277 -0
- data/lib/active_sanction/storage/base.rb +216 -0
- data/lib/active_sanction/storage/file_system.rb +373 -0
- data/lib/active_sanction/storage/memory.rb +67 -0
- data/lib/active_sanction/storage/meta.rb +188 -0
- data/lib/active_sanction/storage.rb +74 -0
- data/lib/active_sanction/subject.rb +271 -0
- data/lib/active_sanction/sync/report.rb +245 -0
- data/lib/active_sanction/sync/result.rb +296 -0
- data/lib/active_sanction/sync.rb +390 -0
- data/lib/active_sanction/validator_store/file_system.rb +103 -0
- data/lib/active_sanction/validator_store/memory.rb +41 -0
- data/lib/active_sanction/validator_store.rb +119 -0
- data/lib/active_sanction/validators.rb +195 -0
- data/lib/active_sanction/version.rb +23 -0
- data/lib/active_sanction.rb +331 -0
- data/lib/generators/active_sanction/install/install_generator.rb +60 -0
- data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
- metadata +236 -0
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "time"
|
|
7
|
+
require "active_sanction/diff"
|
|
8
|
+
require "active_sanction/error"
|
|
9
|
+
require "active_sanction/index"
|
|
10
|
+
require "active_sanction/match_result"
|
|
11
|
+
require "active_sanction/scorer"
|
|
12
|
+
require "active_sanction/subject"
|
|
13
|
+
require "active_sanction/rescreen/alert"
|
|
14
|
+
|
|
15
|
+
module ActiveSanction
|
|
16
|
+
# Who a list change affects. Screening answers about a name; this answers
|
|
17
|
+
# about a book of business.
|
|
18
|
+
#
|
|
19
|
+
# book = [
|
|
20
|
+
# ActiveSanction::Subject.new(id: "cust_1", name: "Bosco Ntaganda", date_of_birth: "1973"),
|
|
21
|
+
# ActiveSanction::Subject.new(id: "cust_2", name: "Jane Miller")
|
|
22
|
+
# ]
|
|
23
|
+
#
|
|
24
|
+
# diff = ActiveSanction.diff(:ofac_sdn, from: yesterdays_snapshot)
|
|
25
|
+
# alerts = ActiveSanction.rescreen(book, diff: diff, threshold: 75)
|
|
26
|
+
#
|
|
27
|
+
# alerts.first.subject_id # => "cust_1"
|
|
28
|
+
# alerts.first.change # => :newly_listed
|
|
29
|
+
# alerts.first.result # => a full MatchResult, with its explanation
|
|
30
|
+
# alerts.first.previous_score # => nil, or what it scored before
|
|
31
|
+
#
|
|
32
|
+
# ### Why this is the primitive rather than a nightly full screen
|
|
33
|
+
#
|
|
34
|
+
# Screening a customer once is a checkbox. The obligation is ongoing:
|
|
35
|
+
# somebody cleared last month may be listed today, and a delisting matters
|
|
36
|
+
# just as much, because it is the entry that lets a customer back through
|
|
37
|
+
# the door. That is recurring work, and the naive way to do it -- every
|
|
38
|
+
# subject against every record, every night -- costs the whole book times
|
|
39
|
+
# the whole corpus and stops being nightly somewhere around a few thousand
|
|
40
|
+
# customers.
|
|
41
|
+
#
|
|
42
|
+
# A rescreen costs the whole book times *the handful of records that moved*.
|
|
43
|
+
# Diff (#35) computes what changed in the list; this computes who that
|
|
44
|
+
# change affects, and it is the step that turns a diff into an alert. On a
|
|
45
|
+
# typical day an OFAC diff is a few dozen records, so the index built here
|
|
46
|
+
# is a few dozen names rather than 46,000 -- see the numbers in the README.
|
|
47
|
+
#
|
|
48
|
+
# ### It is built once and then only read
|
|
49
|
+
#
|
|
50
|
+
# A Rescreen holds an index over the records the diff names, both versions
|
|
51
|
+
# of each amended one, the weights it scores with and the threshold it
|
|
52
|
+
# defaults to. All of it is fixed at construction and the object is frozen,
|
|
53
|
+
# so a large book is streamed past one of these, in batches, from as many
|
|
54
|
+
# threads as a host has:
|
|
55
|
+
#
|
|
56
|
+
# rescreening = ActiveSanction::Rescreen.new(diff: diff, threshold: 75)
|
|
57
|
+
# customers.find_each(batch_size: 1_000) do |batch|
|
|
58
|
+
# rescreening.call(batch) { |alert| AlertRecord.create!(alert.to_h) }
|
|
59
|
+
# end
|
|
60
|
+
#
|
|
61
|
+
# Nothing about a book is held: subjects are read one at a time and only
|
|
62
|
+
# alerts are kept, so memory is a function of how much moved rather than of
|
|
63
|
+
# how many customers there are. The block is what makes even that bounded --
|
|
64
|
+
# it is called with each alert as it is raised, and a host that writes them
|
|
65
|
+
# out as they arrive never accumulates the array at all.
|
|
66
|
+
#
|
|
67
|
+
# ### It does not touch the matcher
|
|
68
|
+
#
|
|
69
|
+
# A rescreen builds its own index, over the diff, and never reads the store
|
|
70
|
+
# or the client's matcher. That is the point: applying a diff to a book must
|
|
71
|
+
# not cost an index build over the whole corpus, and a host that has never
|
|
72
|
+
# screened anything in this process can rescreen without paying for one.
|
|
73
|
+
#
|
|
74
|
+
# ### An empty diff does no work at all
|
|
75
|
+
#
|
|
76
|
+
# A sync that changed nothing, or a first sync -- which is a baseline rather
|
|
77
|
+
# than a list of 19,015 additions, see Diff -- yields no alerts and scores
|
|
78
|
+
# nothing. Not one subject is folded. A host that syncs hourly and
|
|
79
|
+
# rescreens after each sync is paying for the hours that moved, which is
|
|
80
|
+
# what makes rescreening after every sync affordable.
|
|
81
|
+
#
|
|
82
|
+
# ### What it does not do
|
|
83
|
+
#
|
|
84
|
+
# It reports the changes; it does not remember them. There is no alert
|
|
85
|
+
# store, no deduplication against what was raised yesterday, and no
|
|
86
|
+
# disposition -- see the note on case management in the README. Two runs
|
|
87
|
+
# over the same diff produce the same alerts, which is the property that
|
|
88
|
+
# makes them re-derivable and the reason a host, not this library, owns the
|
|
89
|
+
# queue they go into.
|
|
90
|
+
#
|
|
91
|
+
# It also cannot find what was already there. A subject who matched a record
|
|
92
|
+
# that did not change is not in a diff at all, and no rescreen will report
|
|
93
|
+
# them: the first screening run against a book is a deliberate full screen
|
|
94
|
+
# (`Client#screen_all`), and this is what keeps it current afterwards.
|
|
95
|
+
class Rescreen
|
|
96
|
+
extend T::Sig
|
|
97
|
+
|
|
98
|
+
# The diff being applied, which is what the alerts are about.
|
|
99
|
+
sig { returns(Diff) }
|
|
100
|
+
attr_reader :diff
|
|
101
|
+
|
|
102
|
+
# The lowest score worth an alert, for a subject that does not name its
|
|
103
|
+
# own. Read once, at construction, so a configuration changed mid-run
|
|
104
|
+
# cannot produce a book screened half one way.
|
|
105
|
+
sig { returns(Float) }
|
|
106
|
+
attr_reader :threshold
|
|
107
|
+
|
|
108
|
+
# What each signal was worth for this run, and what every result it
|
|
109
|
+
# produces records.
|
|
110
|
+
sig { returns(Scorer::Weights) }
|
|
111
|
+
attr_reader :weights
|
|
112
|
+
|
|
113
|
+
# How many names the index hands the scorer per subject. It does not bind
|
|
114
|
+
# on a typical diff -- a few dozen records cannot exceed it -- and is here
|
|
115
|
+
# for the day a publisher reissues a whole list under new ids.
|
|
116
|
+
sig { returns(Integer) }
|
|
117
|
+
attr_reader :candidate_limit
|
|
118
|
+
|
|
119
|
+
sig { returns(Symbol) }
|
|
120
|
+
attr_reader :backend
|
|
121
|
+
|
|
122
|
+
# Sugar, and what ActiveSanction.rescreen calls:
|
|
123
|
+
#
|
|
124
|
+
# ActiveSanction::Rescreen.call(book, diff: diff, threshold: 75)
|
|
125
|
+
#
|
|
126
|
+
# Builds a Rescreen and applies it once. A host streaming a book in
|
|
127
|
+
# batches builds one with .new and calls it per batch instead, so the
|
|
128
|
+
# index is built once rather than per batch.
|
|
129
|
+
sig do
|
|
130
|
+
params(subjects: T.untyped, diff: T.untyped, options: T.untyped,
|
|
131
|
+
block: T.nilable(T.proc.params(alert: Alert).void)).returns(T::Array[Alert])
|
|
132
|
+
end
|
|
133
|
+
def self.call(subjects, diff:, **options, &block)
|
|
134
|
+
T.unsafe(self).new(diff: diff, **options).call(subjects, &block)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
sig do
|
|
138
|
+
params(diff: T.untyped, threshold: T.untyped, weights: T.untyped, candidate_limit: T.untyped,
|
|
139
|
+
backend: T.untyped).void
|
|
140
|
+
end
|
|
141
|
+
def initialize(diff:, threshold: nil, weights: nil, candidate_limit: nil,
|
|
142
|
+
backend: MatchResult::DEFAULT_BACKEND)
|
|
143
|
+
@diff = T.let(diff!(diff), Diff)
|
|
144
|
+
@threshold = T.let(threshold!(threshold), Float)
|
|
145
|
+
@weights = T.let(Scorer::Weights.build(weights), Scorer::Weights)
|
|
146
|
+
@candidate_limit = T.let(candidate_limit!(candidate_limit), Integer)
|
|
147
|
+
@backend = T.let(backend.to_s.to_sym, Symbol)
|
|
148
|
+
@versions = T.let(versions, T::Hash[String, T::Array[T.untyped]])
|
|
149
|
+
@fields = T.let(amended, T::Hash[String, T::Array[Symbol]])
|
|
150
|
+
# Both versions of an amended record are indexed, because a subject may
|
|
151
|
+
# have matched only the alias that was taken away.
|
|
152
|
+
@index = T.let(Index.build(@versions.values.flatten.compact), Index)
|
|
153
|
+
freeze
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# The alerts this diff raises against this book, highest score first
|
|
157
|
+
# within each subject and in the order the subjects arrived:
|
|
158
|
+
#
|
|
159
|
+
# rescreening.call(book)
|
|
160
|
+
# rescreening.call(book) { |alert| queue.push(alert) }
|
|
161
|
+
#
|
|
162
|
+
# `subjects` is anything that responds to `each`, so an Enumerator over a
|
|
163
|
+
# database cursor is streamed rather than materialized. Each entry is a
|
|
164
|
+
# Subject or the Hash one is built from.
|
|
165
|
+
#
|
|
166
|
+
# Every alert in a run carries one `screened_at`, because a rescreening
|
|
167
|
+
# of a book against a new list version is a single event in an audit
|
|
168
|
+
# trail rather than ten thousand of them a microsecond apart. A host that
|
|
169
|
+
# calls this once per batch is running one event per batch, which is the
|
|
170
|
+
# honest description of what it did.
|
|
171
|
+
sig do
|
|
172
|
+
params(subjects: T.untyped, block: T.nilable(T.proc.params(alert: Alert).void)).returns(T::Array[Alert])
|
|
173
|
+
end
|
|
174
|
+
def call(subjects, &block)
|
|
175
|
+
return [] if diff.empty?
|
|
176
|
+
|
|
177
|
+
screened_at = Time.now.utc
|
|
178
|
+
alerts = T.let([], T::Array[Alert])
|
|
179
|
+
each(subjects) do |value|
|
|
180
|
+
found(Subject.build(value), screened_at).each do |alert|
|
|
181
|
+
block&.call(alert)
|
|
182
|
+
alerts << alert
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
alerts
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# The list this run is about, taken from the diff.
|
|
189
|
+
sig { returns(Symbol) }
|
|
190
|
+
def source = diff.source
|
|
191
|
+
|
|
192
|
+
# How many records moved, which is what a run costs per subject.
|
|
193
|
+
sig { returns(Integer) }
|
|
194
|
+
def size = diff.size
|
|
195
|
+
|
|
196
|
+
# Nothing moved, so nothing can be affected. See the class comment.
|
|
197
|
+
sig { returns(T::Boolean) }
|
|
198
|
+
def empty? = diff.empty?
|
|
199
|
+
|
|
200
|
+
sig { returns(String) }
|
|
201
|
+
def inspect = "#<#{self.class} #{source} #{size} changed records at #{threshold}>"
|
|
202
|
+
|
|
203
|
+
private
|
|
204
|
+
|
|
205
|
+
# The alerts one subject raises, highest score first and then by record
|
|
206
|
+
# id, so that two runs over the same diff and the same book produce the
|
|
207
|
+
# same output in the same order.
|
|
208
|
+
sig { params(subject: Subject, screened_at: Time).returns(T::Array[Alert]) }
|
|
209
|
+
def found(subject, screened_at)
|
|
210
|
+
cutoff = subject.threshold || threshold
|
|
211
|
+
readings = candidates(subject).filter_map { |id| compare(subject, id, cutoff) }
|
|
212
|
+
return [] if readings.empty?
|
|
213
|
+
|
|
214
|
+
query = subject.query(threshold: cutoff, sources: [source])
|
|
215
|
+
readings.map { |reading| alert(subject, reading, query, screened_at) }
|
|
216
|
+
.sort_by { |raised| [-(raised.score || T.must(raised.previous_score)), raised.entity_id] }
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# The records worth scoring this subject against: the same retrieval stage
|
|
220
|
+
# a screening call uses, over a corpus the size of the diff. An entity
|
|
221
|
+
# reached through two of its names, or through both versions of itself, is
|
|
222
|
+
# one record to compare and not two.
|
|
223
|
+
sig { params(subject: Subject).returns(T::Array[String]) }
|
|
224
|
+
def candidates(subject)
|
|
225
|
+
@index.candidates(subject.form, limit: candidate_limit).map { |candidate| candidate.entity.id }.uniq
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# What this subject scores against both versions of one record, or nil
|
|
229
|
+
# when it reaches the threshold against neither -- which is the answer for
|
|
230
|
+
# nearly every pair and is why the cutoff is passed to the scorer rather
|
|
231
|
+
# than applied afterwards. See Scorer on what a threshold buys.
|
|
232
|
+
sig { params(subject: Subject, id: String, cutoff: Float).returns(T.nilable(T::Array[T.untyped])) }
|
|
233
|
+
def compare(subject, id, cutoff)
|
|
234
|
+
previous, current = @versions.fetch(id)
|
|
235
|
+
before = score(subject, previous, cutoff)
|
|
236
|
+
after = score(subject, current, cutoff)
|
|
237
|
+
return nil if before.nil? && after.nil?
|
|
238
|
+
|
|
239
|
+
change = if before.nil? then :newly_listed
|
|
240
|
+
elsif after.nil? then :delisted
|
|
241
|
+
else :details_changed
|
|
242
|
+
end
|
|
243
|
+
# The side that did not clear is scored again without a cutoff, so an
|
|
244
|
+
# alert can say a subject moved from 71 to 94 rather than only that it
|
|
245
|
+
# now matches. Reached once per alert, which is why it is affordable.
|
|
246
|
+
[id, change, before || score(subject, previous, 0.0), after || score(subject, current, 0.0)]
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
sig { params(subject: Subject, entity: T.untyped, cutoff: Float).returns(T.nilable(Scorer::Result)) }
|
|
250
|
+
def score(subject, entity, cutoff)
|
|
251
|
+
return nil if entity.nil?
|
|
252
|
+
|
|
253
|
+
Scorer.call(subject.evidence, entity, weights: weights, threshold: cutoff)
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
sig do
|
|
257
|
+
params(subject: Subject, reading: T::Array[T.untyped], query: Query, screened_at: Time).returns(Alert)
|
|
258
|
+
end
|
|
259
|
+
def alert(subject, reading, query, screened_at)
|
|
260
|
+
id, change, before, after = reading
|
|
261
|
+
Alert.new(
|
|
262
|
+
subject: subject, change: change, fields: @fields.fetch(id, []),
|
|
263
|
+
result: stamp(after, diff.to.checksum, query, screened_at),
|
|
264
|
+
previous_result: stamp(before, T.must(diff.from).checksum, query, screened_at),
|
|
265
|
+
snapshot_id: diff.to.checksum, previous_snapshot_id: T.must(diff.from).checksum
|
|
266
|
+
)
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# One scored record as the audit object it has to be, stamped with the
|
|
270
|
+
# checksum of the list version it was scored against -- which is the whole
|
|
271
|
+
# of why an alert holds two of these rather than one score and a delta.
|
|
272
|
+
sig do
|
|
273
|
+
params(result: T.nilable(Scorer::Result), snapshot_id: String, query: Query,
|
|
274
|
+
screened_at: Time).returns(T.nilable(MatchResult))
|
|
275
|
+
end
|
|
276
|
+
def stamp(result, snapshot_id, query, screened_at)
|
|
277
|
+
return nil if result.nil?
|
|
278
|
+
|
|
279
|
+
MatchResult.from_scorer(result, query: query, snapshot_id: snapshot_id, weights: weights,
|
|
280
|
+
screened_at: screened_at, backend: backend)
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
sig { params(subjects: T.untyped, block: T.proc.params(value: T.untyped).void).void }
|
|
284
|
+
def each(subjects, &block)
|
|
285
|
+
unless subjects.respond_to?(:each)
|
|
286
|
+
raise InvalidArgument,
|
|
287
|
+
"a book of subjects has to be enumerable, got #{subjects.class}. An Array, or anything that " \
|
|
288
|
+
"responds to #each -- an Enumerator over a database cursor is streamed rather than materialized"
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
subjects.each(&block)
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
# Every record the diff names, by id, as a pair: how the old list had it
|
|
295
|
+
# and how the new list has it. A record that was added has no first half
|
|
296
|
+
# and one that was withdrawn has no second, and an amended one has both --
|
|
297
|
+
# which is what lets a single comparison produce a score on either side of
|
|
298
|
+
# the change. The three sets are disjoint, because a diff joins its two
|
|
299
|
+
# snapshots by id.
|
|
300
|
+
sig { returns(T::Hash[String, T::Array[T.untyped]]) }
|
|
301
|
+
def versions
|
|
302
|
+
found = T.let({}, T::Hash[String, T::Array[T.untyped]])
|
|
303
|
+
diff.removed.each { |entity| found[entity.id] = [entity, nil].freeze }
|
|
304
|
+
diff.added.each { |entity| found[entity.id] = [nil, entity].freeze }
|
|
305
|
+
diff.modified.each { |change| found[change.id] = [change.previous, change.entity].freeze }
|
|
306
|
+
found.freeze
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
sig { params(value: T.untyped).returns(Float) }
|
|
310
|
+
def threshold!(value) = Scorer.threshold!(value || ActiveSanction.config.screening_threshold)
|
|
311
|
+
|
|
312
|
+
sig { params(value: T.untyped).returns(Integer) }
|
|
313
|
+
def candidate_limit!(value) = Integer(value || ActiveSanction.config.candidate_limit)
|
|
314
|
+
|
|
315
|
+
# Which fields moved, per amended record, so an alert can say whether a
|
|
316
|
+
# subject's status changed because the list did or because the record did.
|
|
317
|
+
sig { returns(T::Hash[String, T::Array[Symbol]]) }
|
|
318
|
+
def amended = diff.modified.to_h { |change| [change.id, change.fields] }.freeze
|
|
319
|
+
|
|
320
|
+
# A baseline diff is empty, so `from` is present on every diff that can
|
|
321
|
+
# raise an alert -- which is what lets an alert cite both checksums. The
|
|
322
|
+
# check is here rather than at the alert, where the missing one would read
|
|
323
|
+
# as a bug in this class.
|
|
324
|
+
sig { params(value: T.untyped).returns(Diff) }
|
|
325
|
+
def diff!(value)
|
|
326
|
+
raise InvalidArgument, "diff must be an ActiveSanction::Diff, got #{value.class}" unless value.is_a?(Diff)
|
|
327
|
+
return value if value.empty? || value.from
|
|
328
|
+
|
|
329
|
+
raise InvalidArgument, "a rescreen needs both list versions to cite, and this diff has no `from`"
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
end
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/country"
|
|
7
|
+
require "active_sanction/scorer/reason"
|
|
8
|
+
|
|
9
|
+
module ActiveSanction
|
|
10
|
+
module Scorer
|
|
11
|
+
# What the record says about a subject other than their name, and what
|
|
12
|
+
# each agreement and each contradiction is worth.
|
|
13
|
+
#
|
|
14
|
+
# ActiveSanction::Scorer::Adjustments.call(subject, entity)
|
|
15
|
+
# # => [#<Reason +40.0 identifier: passport AB123456 matches>,
|
|
16
|
+
# # #<Reason -8.0 nationality: query RU vs listed EG>]
|
|
17
|
+
#
|
|
18
|
+
# Stage 4b, and the part that separates a usable screening tool from a
|
|
19
|
+
# name-similarity toy. A name score alone puts thousands of people on a
|
|
20
|
+
# list of a few hundred, because a quarter of the individuals on these
|
|
21
|
+
# lists share a handful of given names and every one of them scores in the
|
|
22
|
+
# seventies against every other. The passport number, the date of birth
|
|
23
|
+
# and the nationality are what tell those apart, and they are the fields a
|
|
24
|
+
# compliance officer already has in a customer record.
|
|
25
|
+
#
|
|
26
|
+
# ### Absent is not conflict, and it is the rule everything here obeys
|
|
27
|
+
#
|
|
28
|
+
# Most records lack most identifiers. Canada publishes no aliases and
|
|
29
|
+
# frequently no date of birth; OFAC's dates are prose in a remarks field
|
|
30
|
+
# and are missing wherever the sentence did not parse; the UN grades what
|
|
31
|
+
# it has and says nothing about what it does not. So every adjustment
|
|
32
|
+
# below fires only when *both* sides carry the field, and a field missing
|
|
33
|
+
# on either side produces no Reason at all -- not a small penalty, not a
|
|
34
|
+
# zero-valued reason.
|
|
35
|
+
#
|
|
36
|
+
# Treating absence as disagreement would penalize exactly the sparser
|
|
37
|
+
# lists, which means systematically under-scoring the jurisdictions that
|
|
38
|
+
# publish least and hiding real hits behind a threshold. It is the
|
|
39
|
+
# quietest way to build a screening tool that does not screen, and it is
|
|
40
|
+
# why this is stated as a rule rather than left to each branch.
|
|
41
|
+
#
|
|
42
|
+
# The same rule governs a country nobody can resolve. A nationality of
|
|
43
|
+
# `Ruritania` is not a contradiction of `Egypt`; it is a string this
|
|
44
|
+
# library does not recognize, and the difference between those two is a
|
|
45
|
+
# penalty applied to a match that was correct. See Country.
|
|
46
|
+
#
|
|
47
|
+
# ### The three signals, and why they are worth what they are
|
|
48
|
+
#
|
|
49
|
+
# **A document number is the only near-decisive field on a sanctions
|
|
50
|
+
# record.** Two people share a name; they do not share a passport number.
|
|
51
|
+
# It is the one place where agreement outranks everything the name said,
|
|
52
|
+
# which is what a boost of 40 points on a 100-point scale means: a name in
|
|
53
|
+
# the fifties plus the right passport clears any threshold this library
|
|
54
|
+
# would ship.
|
|
55
|
+
#
|
|
56
|
+
# **A date of birth is strong evidence in both directions.** An exact full
|
|
57
|
+
# date agreeing is worth a real boost; a genuine conflict -- two dates
|
|
58
|
+
# that cannot describe the same person -- is worth more against, because
|
|
59
|
+
# sharing a birthday with a listed person is a coincidence a few thousand
|
|
60
|
+
# people have and not having theirs is not. `PartialDate#overlaps?` is
|
|
61
|
+
# what decides which of the three cases a pair is in, and it is exact
|
|
62
|
+
# whatever precision either side carries: a year-only record does not
|
|
63
|
+
# conflict with a full date inside it, it agrees with it, weakly.
|
|
64
|
+
#
|
|
65
|
+
# **Nationality moves the score least.** People hold two passports, lists
|
|
66
|
+
# record the country a person was born in as readily as the one they are a
|
|
67
|
+
# citizen of, and the field is published as prose. It is real evidence and
|
|
68
|
+
# it is the softest of the three.
|
|
69
|
+
#
|
|
70
|
+
# ### Order is fixed, because an explanation is read
|
|
71
|
+
#
|
|
72
|
+
# Identifier, then date of birth, then nationality: strongest evidence
|
|
73
|
+
# first, so a reviewer reading an explanation downwards meets the reason
|
|
74
|
+
# the score is what it is before meeting the ones that adjusted it. The
|
|
75
|
+
# order is also what makes a score reproducible to the last decimal --
|
|
76
|
+
# see Scorer on why the sum has to be taken the same way every time.
|
|
77
|
+
#
|
|
78
|
+
# @api private
|
|
79
|
+
module Adjustments
|
|
80
|
+
extend T::Sig
|
|
81
|
+
|
|
82
|
+
# Below this many alphanumerics, a "document number" is not a document
|
|
83
|
+
# number. Nothing a government issues is three characters long, and an
|
|
84
|
+
# exact match on a short string is the one way a 40-point boost could
|
|
85
|
+
# land on a coincidence -- a remarks parser that read `Passport 12` out
|
|
86
|
+
# of a half-formed sentence would otherwise make two unrelated records
|
|
87
|
+
# decisive for each other.
|
|
88
|
+
MINIMUM_IDENTIFIER_LENGTH = T.let(4, Integer)
|
|
89
|
+
|
|
90
|
+
module_function
|
|
91
|
+
|
|
92
|
+
# Every adjustment both sides carry the evidence for, strongest first.
|
|
93
|
+
# An empty array is the ordinary answer for a record that publishes a
|
|
94
|
+
# name and nothing else.
|
|
95
|
+
sig do
|
|
96
|
+
params(subject: Subject, entity: Entity, weights: Weights)
|
|
97
|
+
.returns(T::Array[Reason]).checked(:tests)
|
|
98
|
+
end
|
|
99
|
+
def call(subject, entity, weights = Weights.default)
|
|
100
|
+
[identifier(subject, entity, weights), dob(subject, entity, weights),
|
|
101
|
+
nationality(subject, entity, weights)].compact
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The first document number both sides carry, in the order the caller
|
|
105
|
+
# and the publisher listed them. First rather than best: there is no
|
|
106
|
+
# "better" exact match on a document number, and iterating in a fixed
|
|
107
|
+
# order is what makes the explanation identical on every run.
|
|
108
|
+
sig { params(subject: Subject, entity: Entity, weights: Weights).returns(T.nilable(Reason)).checked(:tests) }
|
|
109
|
+
def identifier(subject, entity, weights)
|
|
110
|
+
subject.identifiers.each do |mine|
|
|
111
|
+
next if mine.normalized_value.length < MINIMUM_IDENTIFIER_LENGTH
|
|
112
|
+
|
|
113
|
+
theirs = entity.identifiers.find { |listed| same_document?(mine, listed) }
|
|
114
|
+
next if theirs.nil?
|
|
115
|
+
|
|
116
|
+
return Reason.new(factor: :identifier, contribution: weights.identifier_match,
|
|
117
|
+
detail: document_detail(mine, theirs))
|
|
118
|
+
end
|
|
119
|
+
nil
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Exact, overlapping, or contradictory -- in that order, over every
|
|
123
|
+
# pairing of the dates the two sides carry. Both plural: the UN
|
|
124
|
+
# publishes more than one date of birth for 140 of its individuals
|
|
125
|
+
# because several governments reported several dates, and any of them
|
|
126
|
+
# matching is a match. A conflict therefore means *no* pairing overlaps,
|
|
127
|
+
# which is the only reading that does not turn an honestly uncertain
|
|
128
|
+
# record into a penalty.
|
|
129
|
+
sig { params(subject: Subject, entity: Entity, weights: Weights).returns(T.nilable(Reason)).checked(:tests) }
|
|
130
|
+
def dob(subject, entity, weights)
|
|
131
|
+
mine = subject.dates_of_birth
|
|
132
|
+
theirs = entity.dates_of_birth
|
|
133
|
+
return nil if mine.empty? || theirs.empty?
|
|
134
|
+
|
|
135
|
+
agreement(mine.product(theirs), weights) ||
|
|
136
|
+
dob_reason(:dob_conflict, "#{mine.join(", ")} conflicts with listed #{theirs.join(", ")}", weights)
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# The best of the pairings, or nil when none of them agree at all --
|
|
140
|
+
# which is what makes the caller above a conflict.
|
|
141
|
+
sig do
|
|
142
|
+
params(pairs: T::Array[T::Array[PartialDate]], weights: Weights)
|
|
143
|
+
.returns(T.nilable(Reason)).checked(:tests)
|
|
144
|
+
end
|
|
145
|
+
def agreement(pairs, weights)
|
|
146
|
+
exact = pairs.find { |left, right| same_day?(T.must(left), T.must(right)) }
|
|
147
|
+
return dob_reason(:dob_exact, "#{exact.first} matches", weights) if exact
|
|
148
|
+
|
|
149
|
+
overlap = pairs.find { |left, right| T.must(left).overlaps?(right) }
|
|
150
|
+
return nil if overlap.nil?
|
|
151
|
+
|
|
152
|
+
dob_reason(:dob_overlap, "#{overlap.first} overlaps listed #{overlap.last}", weights)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Agreement, or a contradiction both sides are precise enough to make.
|
|
156
|
+
#
|
|
157
|
+
# A country is compared as the alpha-2 code it resolves to, falling back
|
|
158
|
+
# to its folded string so that two publishers writing the same
|
|
159
|
+
# unrecognized value still agree. A *conflict* needs more: every value
|
|
160
|
+
# on both sides has to have resolved, because "these two strings are not
|
|
161
|
+
# equal" is not evidence that two countries are different.
|
|
162
|
+
sig { params(subject: Subject, entity: Entity, weights: Weights).returns(T.nilable(Reason)).checked(:tests) }
|
|
163
|
+
def nationality(subject, entity, weights)
|
|
164
|
+
listed = entity.nationalities
|
|
165
|
+
return nil if subject.nationalities.empty? || listed.empty?
|
|
166
|
+
|
|
167
|
+
mine = keys(subject.nationalities)
|
|
168
|
+
theirs = keys(listed)
|
|
169
|
+
shared = mine & theirs
|
|
170
|
+
if shared.any?
|
|
171
|
+
return Reason.new(factor: :nationality, contribution: weights.nationality_match,
|
|
172
|
+
detail: "#{shared.join(", ")} matches")
|
|
173
|
+
end
|
|
174
|
+
return nil unless subject.countries? && resolved?(listed)
|
|
175
|
+
|
|
176
|
+
Reason.new(factor: :nationality, contribution: weights.nationality_conflict,
|
|
177
|
+
detail: "query #{mine.join(", ")} vs listed #{theirs.join(", ")}")
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Kind, number and country all have to be compatible, and two of the
|
|
181
|
+
# three treat "unstated" as compatible rather than as different.
|
|
182
|
+
#
|
|
183
|
+
# `:other` is the kind OFAC's remarks produce for a number its sentence
|
|
184
|
+
# did not classify, so requiring the kinds to be equal would throw away
|
|
185
|
+
# most of the document numbers this library extracts. Country is the
|
|
186
|
+
# opposite case and is why Identifier's own equality carries it: two
|
|
187
|
+
# passports with the same number from different countries are different
|
|
188
|
+
# documents, so a country stated on both sides and disagreeing is a
|
|
189
|
+
# mismatch -- while a country stated on neither, or on one, is not.
|
|
190
|
+
sig { params(mine: Identifier, theirs: Identifier).returns(T::Boolean).checked(:tests) }
|
|
191
|
+
def same_document?(mine, theirs)
|
|
192
|
+
return false unless mine.normalized_value == theirs.normalized_value
|
|
193
|
+
return false unless mine.kind == theirs.kind || mine.kind == :other || theirs.kind == :other
|
|
194
|
+
|
|
195
|
+
same_country?(mine.country, theirs.country)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
sig { params(mine: T.nilable(String), theirs: T.nilable(String)).returns(T::Boolean).checked(:tests) }
|
|
199
|
+
def same_country?(mine, theirs)
|
|
200
|
+
return true if mine.nil? || theirs.nil?
|
|
201
|
+
|
|
202
|
+
key(mine) == key(theirs)
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# A full date on both sides, neither hedged with a "circa", naming the
|
|
206
|
+
# same day. Anything less precise is an overlap: `1948` and `1948-12-10`
|
|
207
|
+
# agree, and saying they match exactly would claim a precision the
|
|
208
|
+
# publisher did not.
|
|
209
|
+
sig { params(mine: PartialDate, theirs: PartialDate).returns(T::Boolean).checked(:tests) }
|
|
210
|
+
def same_day?(mine, theirs)
|
|
211
|
+
mine.precision == :day && theirs.precision == :day &&
|
|
212
|
+
!mine.approximate? && !theirs.approximate? && mine.first_date == theirs.first_date
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
sig { params(weight: Symbol, detail: String, weights: Weights).returns(Reason).checked(:tests) }
|
|
216
|
+
def dob_reason(weight, detail, weights)
|
|
217
|
+
Reason.new(factor: :dob, detail: "date of birth #{detail}", contribution: weights.fetch(weight))
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
sig { params(mine: Identifier, theirs: Identifier).returns(String).checked(:tests) }
|
|
221
|
+
def document_detail(mine, theirs)
|
|
222
|
+
kind = mine.kind == :other ? theirs.kind : mine.kind
|
|
223
|
+
return "#{kind} #{mine.value} matches" if mine.value == theirs.value
|
|
224
|
+
|
|
225
|
+
"#{kind} #{mine.value} matches listed #{theirs.value}"
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# The alpha-2 code, or the folded string when the table does not know
|
|
229
|
+
# the value. Never nil, so agreement can be found on a spelling neither
|
|
230
|
+
# side could resolve.
|
|
231
|
+
sig { params(value: String).returns(String).checked(:tests) }
|
|
232
|
+
def key(value) = Country.code(value) || Country.fold(value)
|
|
233
|
+
|
|
234
|
+
sig { params(values: T::Array[String]).returns(T::Array[String]).checked(:tests) }
|
|
235
|
+
def keys(values) = values.map { |value| key(value) }.uniq
|
|
236
|
+
|
|
237
|
+
sig { params(values: T::Array[String]).returns(T::Boolean).checked(:tests) }
|
|
238
|
+
def resolved?(values) = values.all? { |value| Country.code(value) }
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|