active_sanction 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +538 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/CONTRIBUTING.md +312 -0
- data/LICENSE.txt +21 -0
- data/README.md +616 -0
- data/SECURITY.md +97 -0
- data/active_sanction.gemspec +98 -0
- data/docs/adding_a_source.md +1083 -0
- data/docs/api_stability.md +357 -0
- data/docs/bundle_format.md +407 -0
- data/lib/active_sanction/address.rb +170 -0
- data/lib/active_sanction/client.rb +341 -0
- data/lib/active_sanction/configuration.rb +721 -0
- data/lib/active_sanction/countries.txt +292 -0
- data/lib/active_sanction/country.rb +164 -0
- data/lib/active_sanction/deprecation.rb +168 -0
- data/lib/active_sanction/diff/change.rb +208 -0
- data/lib/active_sanction/diff.rb +348 -0
- data/lib/active_sanction/doctor/checkup.rb +356 -0
- data/lib/active_sanction/doctor/diagnosis.rb +245 -0
- data/lib/active_sanction/doctor/finding.rb +183 -0
- data/lib/active_sanction/doctor/profile.rb +384 -0
- data/lib/active_sanction/doctor/report.rb +227 -0
- data/lib/active_sanction/doctor.rb +347 -0
- data/lib/active_sanction/entity.rb +293 -0
- data/lib/active_sanction/error.rb +332 -0
- data/lib/active_sanction/fetcher/result.rb +116 -0
- data/lib/active_sanction/fetcher.rb +242 -0
- data/lib/active_sanction/http_client/errors.rb +94 -0
- data/lib/active_sanction/http_client/response.rb +129 -0
- data/lib/active_sanction/http_client.rb +324 -0
- data/lib/active_sanction/identifier.rb +212 -0
- data/lib/active_sanction/index/builder.rb +89 -0
- data/lib/active_sanction/index/candidate.rb +63 -0
- data/lib/active_sanction/index/entry.rb +66 -0
- data/lib/active_sanction/index/features.rb +112 -0
- data/lib/active_sanction/index.rb +381 -0
- data/lib/active_sanction/match_result.rb +345 -0
- data/lib/active_sanction/matcher.rb +388 -0
- data/lib/active_sanction/name.rb +183 -0
- data/lib/active_sanction/normalizer/cache.rb +82 -0
- data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
- data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
- data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
- data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
- data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
- data/lib/active_sanction/normalizer/dictionary.rb +243 -0
- data/lib/active_sanction/normalizer/form.rb +279 -0
- data/lib/active_sanction/normalizer.rb +137 -0
- data/lib/active_sanction/parsers/column_shape.rb +274 -0
- data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
- data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
- data/lib/active_sanction/parsers/delimited_table.rb +151 -0
- data/lib/active_sanction/parsers/format.rb +107 -0
- data/lib/active_sanction/parsers/join.rb +149 -0
- data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
- data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
- data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
- data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
- data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
- data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
- data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
- data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
- data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
- data/lib/active_sanction/parsers/xml_records.rb +140 -0
- data/lib/active_sanction/parsers.rb +94 -0
- data/lib/active_sanction/partial_date/parser.rb +124 -0
- data/lib/active_sanction/partial_date.rb +303 -0
- data/lib/active_sanction/payload_cache/checksum.rb +66 -0
- data/lib/active_sanction/payload_cache/entry.rb +304 -0
- data/lib/active_sanction/payload_cache.rb +366 -0
- data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
- data/lib/active_sanction/phonetics.rb +40 -0
- data/lib/active_sanction/query.rb +290 -0
- data/lib/active_sanction/rescreen/alert.rb +314 -0
- data/lib/active_sanction/rescreen.rb +332 -0
- data/lib/active_sanction/scorer/adjustments.rb +241 -0
- data/lib/active_sanction/scorer/name_score.rb +279 -0
- data/lib/active_sanction/scorer/reason.rb +136 -0
- data/lib/active_sanction/scorer/result.rb +129 -0
- data/lib/active_sanction/scorer/subject.rb +235 -0
- data/lib/active_sanction/scorer/weights.rb +306 -0
- data/lib/active_sanction/scorer.rb +303 -0
- data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
- data/lib/active_sanction/similarity/levenshtein.rb +179 -0
- data/lib/active_sanction/similarity/token_set.rb +159 -0
- data/lib/active_sanction/similarity/token_sort.rb +120 -0
- data/lib/active_sanction/similarity.rb +183 -0
- data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
- data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
- data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
- data/lib/active_sanction/snapshot/bundle.rb +381 -0
- data/lib/active_sanction/snapshot.rb +305 -0
- data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
- data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
- data/lib/active_sanction/sources/australia_dfat.rb +291 -0
- data/lib/active_sanction/sources/base.rb +295 -0
- data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
- data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
- data/lib/active_sanction/sources/canada_sema.rb +153 -0
- data/lib/active_sanction/sources/definition.rb +360 -0
- data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
- data/lib/active_sanction/sources/eu_fsf.rb +203 -0
- data/lib/active_sanction/sources/ofac/record.rb +251 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
- data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
- data/lib/active_sanction/sources/ofac.rb +248 -0
- data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
- data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
- data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
- data/lib/active_sanction/sources/remarks.rb +84 -0
- data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
- data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
- data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
- data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
- data/lib/active_sanction/sources/un_consolidated.rb +140 -0
- data/lib/active_sanction/sources.rb +196 -0
- data/lib/active_sanction/storage/active_record/reader.rb +124 -0
- data/lib/active_sanction/storage/active_record/row.rb +175 -0
- data/lib/active_sanction/storage/active_record/writer.rb +141 -0
- data/lib/active_sanction/storage/active_record.rb +277 -0
- data/lib/active_sanction/storage/base.rb +216 -0
- data/lib/active_sanction/storage/file_system.rb +373 -0
- data/lib/active_sanction/storage/memory.rb +67 -0
- data/lib/active_sanction/storage/meta.rb +188 -0
- data/lib/active_sanction/storage.rb +74 -0
- data/lib/active_sanction/subject.rb +271 -0
- data/lib/active_sanction/sync/report.rb +245 -0
- data/lib/active_sanction/sync/result.rb +296 -0
- data/lib/active_sanction/sync.rb +390 -0
- data/lib/active_sanction/validator_store/file_system.rb +103 -0
- data/lib/active_sanction/validator_store/memory.rb +41 -0
- data/lib/active_sanction/validator_store.rb +119 -0
- data/lib/active_sanction/validators.rb +195 -0
- data/lib/active_sanction/version.rb +23 -0
- data/lib/active_sanction.rb +331 -0
- data/lib/generators/active_sanction/install/install_generator.rb +60 -0
- data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
- metadata +236 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
module ActiveSanction
|
|
7
|
+
# What a name sounds like, as a key two spellings of it can share. Stage 3c
|
|
8
|
+
# of the matching pipeline.
|
|
9
|
+
#
|
|
10
|
+
# ActiveSanction::Phonetics::DoubleMetaphone.call("qaddafi") # => ["KTF"]
|
|
11
|
+
# ActiveSanction::Phonetics::DoubleMetaphone.call("gaddafi") # => ["KTF"]
|
|
12
|
+
#
|
|
13
|
+
# ### Why this is not in Similarity
|
|
14
|
+
#
|
|
15
|
+
# Because a key is not a score. Everything in Similarity answers "how close
|
|
16
|
+
# are these two names" with a number between 0 and 1, and the whole of that
|
|
17
|
+
# module's contract -- symmetric, never rounded, `threshold:` as an
|
|
18
|
+
# optimization, a `ceiling` that may never come in under a real score --
|
|
19
|
+
# is about keeping four such numbers comparable enough to blend.
|
|
20
|
+
#
|
|
21
|
+
# This stage answers a different question and gives a different kind of
|
|
22
|
+
# answer: it turns one name into the handful of strings that stand for how
|
|
23
|
+
# it sounds. Two names either share one of those strings or they do not.
|
|
24
|
+
# That is what makes it useful in the two places it is used --
|
|
25
|
+
#
|
|
26
|
+
# - the inverted index (#31) keys on it, so that a query for `GADDAFI`
|
|
27
|
+
# reaches a record spelled `QADHAFI` at all, which no amount of comparing
|
|
28
|
+
# would help with if the record is never fetched; and
|
|
29
|
+
# - the scorer (#32) reads it as one share of the blend it scores a pair on.
|
|
30
|
+
#
|
|
31
|
+
# -- and it is also why it cannot be blended with the other four. A shared
|
|
32
|
+
# key is evidence, at the strength #32 decides. It is never a match on its
|
|
33
|
+
# own: `HSN` is the key for `HUSSEIN`, and equally for `HASSAN`.
|
|
34
|
+
#
|
|
35
|
+
# @api private
|
|
36
|
+
module Phonetics
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
require "active_sanction/phonetics/double_metaphone"
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/scorer"
|
|
7
|
+
require "active_sanction/sources/definition"
|
|
8
|
+
|
|
9
|
+
module ActiveSanction
|
|
10
|
+
# One screening request: what is known about the subject, and how the
|
|
11
|
+
# search is to be run.
|
|
12
|
+
#
|
|
13
|
+
# query = ActiveSanction::Query.build(
|
|
14
|
+
# name: "Bosco Ntaganda",
|
|
15
|
+
# type: :individual,
|
|
16
|
+
# date_of_birth: "1973",
|
|
17
|
+
# countries: %w[CD],
|
|
18
|
+
# sources: %i[ofac_sdn un_consolidated],
|
|
19
|
+
# threshold: 75,
|
|
20
|
+
# limit: 10
|
|
21
|
+
# )
|
|
22
|
+
#
|
|
23
|
+
# query.subject # => Scorer::Subject, folded once and held
|
|
24
|
+
# query.threshold # => 75.0
|
|
25
|
+
#
|
|
26
|
+
# ### Two kinds of field, and the line between them matters
|
|
27
|
+
#
|
|
28
|
+
# **Evidence** -- the name, the type, the dates of birth, the nationalities,
|
|
29
|
+
# the identifiers -- is what the scorer compares against a record, and it
|
|
30
|
+
# belongs to Scorer::Subject, which this builds and holds. **Search
|
|
31
|
+
# options** -- `sources`, `threshold`, `limit` -- decide which records are
|
|
32
|
+
# looked at and how many come back, and they never touch a comparison.
|
|
33
|
+
#
|
|
34
|
+
# Keeping them apart is what lets a stored decision be re-derived: the
|
|
35
|
+
# evidence says what was screened and the options say what the run was
|
|
36
|
+
# willing to return, and an audit needs both separately. It is also why
|
|
37
|
+
# Subject carries no threshold -- see the note at the end of that class.
|
|
38
|
+
#
|
|
39
|
+
# ### Singular and plural spellings are both accepted
|
|
40
|
+
#
|
|
41
|
+
# `date_of_birth:` and `dates_of_birth:`, `country:`, `countries:` and
|
|
42
|
+
# `nationalities:`, `identifier:` and `identifiers:` all mean the same
|
|
43
|
+
# thing. A caller with one date writes the singular and a caller with three
|
|
44
|
+
# writes the plural, and neither should have to remember which this library
|
|
45
|
+
# prefers. `#to_h` emits the plural, canonical spelling.
|
|
46
|
+
#
|
|
47
|
+
# That resolution happens in `.build` and `.from_h`, which is what every
|
|
48
|
+
# screening call goes through -- `Matcher#screen` builds one of these out of
|
|
49
|
+
# whatever it was handed. `.new` takes the canonical names and nothing else,
|
|
50
|
+
# in the manner of Scorer::Weights: one constructor states the shape and one
|
|
51
|
+
# accepts what a caller wrote.
|
|
52
|
+
#
|
|
53
|
+
# ### Defaults come from configuration, once, here
|
|
54
|
+
#
|
|
55
|
+
# A query with no `threshold:` takes `config.screening_threshold` and one
|
|
56
|
+
# with no `limit:` takes `config.screening_limit`, both read at construction
|
|
57
|
+
# and then fixed. Nothing downstream reads a global: a Matcher screens the
|
|
58
|
+
# numbers on the query it was given, so a configuration changed mid-batch
|
|
59
|
+
# cannot produce a run that is half one threshold and half another.
|
|
60
|
+
#
|
|
61
|
+
# Instances are frozen on construction and compare by value.
|
|
62
|
+
class Query
|
|
63
|
+
extend T::Sig
|
|
64
|
+
|
|
65
|
+
# The canonical spelling of every field, and the shape `#to_h` emits.
|
|
66
|
+
#
|
|
67
|
+
# @api private
|
|
68
|
+
MEMBERS = T.let(
|
|
69
|
+
%i[name type dates_of_birth nationalities identifiers sources threshold limit].freeze,
|
|
70
|
+
T::Array[Symbol]
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
# The spellings a caller may write instead, and what each one means. See
|
|
74
|
+
# the note above.
|
|
75
|
+
#
|
|
76
|
+
# @api private
|
|
77
|
+
ALIASES = T.let(
|
|
78
|
+
{
|
|
79
|
+
date_of_birth: :dates_of_birth, dob: :dates_of_birth, dobs: :dates_of_birth,
|
|
80
|
+
country: :nationalities, countries: :nationalities, nationality: :nationalities,
|
|
81
|
+
identifier: :identifiers, source: :sources
|
|
82
|
+
}.freeze,
|
|
83
|
+
T::Hash[Symbol, Symbol]
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
# The members that are lists. A value that is not one becomes a list of
|
|
87
|
+
# one, which is what makes `identifier: { kind: :passport, value: "AB-1" }`
|
|
88
|
+
# a single identifier: `Array(hash)` reads a Hash as a list of its pairs,
|
|
89
|
+
# and a caller naming one document would get two nonsense identifiers back.
|
|
90
|
+
#
|
|
91
|
+
# @api private
|
|
92
|
+
COLLECTIONS = T.let(%i[dates_of_birth nationalities identifiers sources].freeze, T::Array[Symbol])
|
|
93
|
+
|
|
94
|
+
# The evidence, folded once. Every comparison in a screening run happens
|
|
95
|
+
# against this one object rather than against a name re-folded per
|
|
96
|
+
# candidate.
|
|
97
|
+
sig { returns(Scorer::Subject) }
|
|
98
|
+
attr_reader :subject
|
|
99
|
+
|
|
100
|
+
# Which lists to screen against, or nil for every list the matcher holds.
|
|
101
|
+
# Naming one it does not hold is an error rather than a shorter answer --
|
|
102
|
+
# see Matcher.
|
|
103
|
+
sig { returns(T.nilable(T::Array[Symbol])) }
|
|
104
|
+
attr_reader :sources
|
|
105
|
+
|
|
106
|
+
# 0..100. The lowest score worth reporting, and the number that decides
|
|
107
|
+
# what a screening call costs -- see Scorer.
|
|
108
|
+
sig { returns(Float) }
|
|
109
|
+
attr_reader :threshold
|
|
110
|
+
|
|
111
|
+
# How many results to return, highest score first.
|
|
112
|
+
sig { returns(Integer) }
|
|
113
|
+
attr_reader :limit
|
|
114
|
+
|
|
115
|
+
class << self
|
|
116
|
+
extend T::Sig
|
|
117
|
+
|
|
118
|
+
# Whatever a caller had, as a Query:
|
|
119
|
+
#
|
|
120
|
+
# Query.build("Bosco Ntaganda")
|
|
121
|
+
# Query.build(name: "Bosco Ntaganda", threshold: 80)
|
|
122
|
+
# Query.build(query, limit: 5) # the same query, with one option changed
|
|
123
|
+
#
|
|
124
|
+
# A bare String or Name is a query about that name and nothing else,
|
|
125
|
+
# which is what a batch of names is a list of.
|
|
126
|
+
sig { params(value: T.untyped, overrides: T.untyped).returns(Query) }
|
|
127
|
+
def build(value = nil, **overrides)
|
|
128
|
+
case value
|
|
129
|
+
when Query then overrides.empty? ? value : T.unsafe(value).merge(**overrides)
|
|
130
|
+
when Hash then from_h(normalize(value).merge(normalize(overrides)))
|
|
131
|
+
when nil then from_h(overrides)
|
|
132
|
+
else from_h(normalize(overrides).merge(name: value))
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Rebuilds a query from #to_h output, accepting string keys so one
|
|
137
|
+
# stored in an audit record survives the round-trip through JSON.
|
|
138
|
+
sig { params(hash: T.untyped).returns(Query) }
|
|
139
|
+
def from_h(hash)
|
|
140
|
+
attributes = normalize(hash)
|
|
141
|
+
unknown = attributes.keys - MEMBERS
|
|
142
|
+
raise QueryError, "unknown Query attribute(s): #{unknown.join(", ")}" if unknown.any?
|
|
143
|
+
|
|
144
|
+
# `new(**hash)` past a required keyword parameter is one of the few
|
|
145
|
+
# things Sorbet cannot check statically. #initialize validates what
|
|
146
|
+
# arrives, which is where a bad round-trip is caught.
|
|
147
|
+
T.unsafe(self).new(**attributes)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
private
|
|
151
|
+
|
|
152
|
+
# Symbol keys, with every accepted spelling resolved to its canonical
|
|
153
|
+
# one. A caller that wrote both spellings of the same field is a typo
|
|
154
|
+
# rather than a merge, so it is refused.
|
|
155
|
+
sig { params(hash: T.untyped).returns(T::Hash[Symbol, T.untyped]) }
|
|
156
|
+
def normalize(hash)
|
|
157
|
+
spellings = T.let({}, T::Hash[Symbol, Symbol])
|
|
158
|
+
hash.to_h.each_with_object({}) do |(key, value), attributes|
|
|
159
|
+
written = key.to_s.to_sym
|
|
160
|
+
member = ALIASES.fetch(written, written)
|
|
161
|
+
clash = spellings[member]
|
|
162
|
+
raise QueryError, "#{clash} and #{written} are the same field -- pass one" if conflict?(clash, written)
|
|
163
|
+
|
|
164
|
+
spellings[member] = written
|
|
165
|
+
attributes[member] = wrap(member, value)
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
sig { params(clash: T.nilable(Symbol), written: Symbol).returns(T::Boolean) }
|
|
170
|
+
def conflict?(clash, written) = !clash.nil? && clash != written
|
|
171
|
+
|
|
172
|
+
sig { params(member: Symbol, value: T.untyped).returns(T.untyped) }
|
|
173
|
+
def wrap(member, value)
|
|
174
|
+
return value unless COLLECTIONS.include?(member)
|
|
175
|
+
return value if value.nil? || value.is_a?(Array)
|
|
176
|
+
|
|
177
|
+
[value]
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Only `name` is required; see Scorer::Subject on why that is the shape of
|
|
182
|
+
# the problem rather than a convenience.
|
|
183
|
+
sig do
|
|
184
|
+
params(name: T.untyped, type: T.untyped, dates_of_birth: T.untyped, nationalities: T.untyped,
|
|
185
|
+
identifiers: T.untyped, sources: T.untyped, threshold: T.untyped, limit: T.untyped).void
|
|
186
|
+
end
|
|
187
|
+
def initialize(name:, type: nil, dates_of_birth: [], nationalities: [], identifiers: [],
|
|
188
|
+
sources: nil, threshold: nil, limit: nil)
|
|
189
|
+
@subject = T.let(
|
|
190
|
+
Scorer::Subject.new(name: name, type: type, dates_of_birth: dates_of_birth,
|
|
191
|
+
nationalities: nationalities, identifiers: identifiers),
|
|
192
|
+
Scorer::Subject
|
|
193
|
+
)
|
|
194
|
+
@sources = T.let(sources!(sources), T.nilable(T::Array[Symbol]))
|
|
195
|
+
@threshold = T.let(threshold!(threshold), Float)
|
|
196
|
+
@limit = T.let(limit!(limit), Integer)
|
|
197
|
+
freeze
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# This query with some fields replaced, which is how a batch applies one
|
|
201
|
+
# threshold to a list of names.
|
|
202
|
+
sig { params(overrides: T.untyped).returns(Query) }
|
|
203
|
+
def merge(**overrides) = self.class.build(to_h, **overrides)
|
|
204
|
+
|
|
205
|
+
# The name as the caller wrote it, which is what a report quotes back.
|
|
206
|
+
sig { returns(String) }
|
|
207
|
+
def name = subject.name
|
|
208
|
+
|
|
209
|
+
sig { returns(T.nilable(Symbol)) }
|
|
210
|
+
def type = subject.type
|
|
211
|
+
|
|
212
|
+
sig { returns(T::Array[PartialDate]) }
|
|
213
|
+
def dates_of_birth = subject.dates_of_birth
|
|
214
|
+
|
|
215
|
+
sig { returns(T::Array[String]) }
|
|
216
|
+
def nationalities = subject.nationalities
|
|
217
|
+
|
|
218
|
+
sig { returns(T::Array[Identifier]) }
|
|
219
|
+
def identifiers = subject.identifiers
|
|
220
|
+
|
|
221
|
+
# The folded name every comparison runs against, and what a Matcher hands
|
|
222
|
+
# the index rather than the string it came from.
|
|
223
|
+
sig { returns(Normalizer::Form) }
|
|
224
|
+
def form = subject.form
|
|
225
|
+
|
|
226
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
227
|
+
def to_h
|
|
228
|
+
subject.to_h.merge(sources: sources, threshold: threshold, limit: limit)
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
sig { params(other: T.untyped).returns(T::Boolean) }
|
|
232
|
+
def ==(other)
|
|
233
|
+
return false unless other.instance_of?(self.class)
|
|
234
|
+
|
|
235
|
+
to_h == other.to_h
|
|
236
|
+
end
|
|
237
|
+
alias eql? ==
|
|
238
|
+
|
|
239
|
+
sig { returns(Integer) }
|
|
240
|
+
def hash = [self.class, to_h].hash
|
|
241
|
+
|
|
242
|
+
sig { returns(String) }
|
|
243
|
+
def inspect = "#<#{self.class} #{name.inspect} threshold=#{threshold} limit=#{limit}>"
|
|
244
|
+
|
|
245
|
+
private
|
|
246
|
+
|
|
247
|
+
# nil means every list the matcher holds. An empty array does not: a
|
|
248
|
+
# caller that computed its source list and got nothing back is asking to
|
|
249
|
+
# screen against no lists at all, which returns a clean report for
|
|
250
|
+
# everybody, so it is refused rather than quietly read as "all".
|
|
251
|
+
sig { params(value: T.untyped).returns(T.nilable(T::Array[Symbol])) }
|
|
252
|
+
def sources!(value)
|
|
253
|
+
return nil if value.nil?
|
|
254
|
+
|
|
255
|
+
keys = Array(value).map { |key| Sources::Definition.key!(key) }.uniq
|
|
256
|
+
raise QueryError, "sources cannot be empty -- omit it to screen against every list" if keys.empty?
|
|
257
|
+
|
|
258
|
+
keys.freeze
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# Validated by the scorer's own check, so a threshold on the 0..1 scale is
|
|
262
|
+
# refused here with the message that explains it rather than three layers
|
|
263
|
+
# down.
|
|
264
|
+
sig { params(value: T.untyped).returns(Float) }
|
|
265
|
+
def threshold!(value)
|
|
266
|
+
return Scorer.threshold!(ActiveSanction.config.screening_threshold) if value.nil?
|
|
267
|
+
|
|
268
|
+
number = begin
|
|
269
|
+
Float(value)
|
|
270
|
+
rescue TypeError, ArgumentError
|
|
271
|
+
raise QueryError, "threshold must be a number between 0 and 100, got #{value.inspect}"
|
|
272
|
+
end
|
|
273
|
+
Scorer.threshold!(number)
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
sig { params(value: T.untyped).returns(Integer) }
|
|
277
|
+
def limit!(value)
|
|
278
|
+
return ActiveSanction.config.screening_limit if value.nil?
|
|
279
|
+
|
|
280
|
+
integer = begin
|
|
281
|
+
Integer(value)
|
|
282
|
+
rescue TypeError, ArgumentError
|
|
283
|
+
raise QueryError, "limit must be a whole number of results, got #{value.inspect}"
|
|
284
|
+
end
|
|
285
|
+
raise QueryError, "limit must be at least 1, got #{integer}" unless integer.positive?
|
|
286
|
+
|
|
287
|
+
integer
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
end
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/error"
|
|
7
|
+
require "active_sanction/match_result"
|
|
8
|
+
require "active_sanction/subject"
|
|
9
|
+
|
|
10
|
+
module ActiveSanction
|
|
11
|
+
class Rescreen
|
|
12
|
+
# One subject, one record, and what moved between two list versions.
|
|
13
|
+
#
|
|
14
|
+
# alert.subject_id # => "cust_1"
|
|
15
|
+
# alert.change # => :newly_listed
|
|
16
|
+
# alert.result # => MatchResult, scored against the record as it is now
|
|
17
|
+
# alert.previous_result # => nil -- there was no such record before
|
|
18
|
+
# alert.score # => 94.1
|
|
19
|
+
# alert.previous_score # => nil
|
|
20
|
+
#
|
|
21
|
+
# puts alert
|
|
22
|
+
# # => cust_1 newly listed un_consolidated:6908021 BOSCO TAGANDA 94.1
|
|
23
|
+
#
|
|
24
|
+
# ### The two sides, and why either may be missing
|
|
25
|
+
#
|
|
26
|
+
# An alert is a *change*, so it is two screening results rather than one:
|
|
27
|
+
# `previous_result` is what this subject scored against the record as the
|
|
28
|
+
# old list had it, and `result` is what it scores against the record as
|
|
29
|
+
# the new list has it. A newly listed record has no previous side. A
|
|
30
|
+
# delisted one has no current side. Everything else has both, and the pair
|
|
31
|
+
# is what lets an alert say a subject moved from 71 to 94 rather than
|
|
32
|
+
# merely that it now matches -- which is the difference between an analyst
|
|
33
|
+
# reading a record and an analyst reading a change to one.
|
|
34
|
+
#
|
|
35
|
+
# Both are full MatchResults, each stamped with the checksum of the list
|
|
36
|
+
# version it was scored against, so an alert is defensible the same way a
|
|
37
|
+
# screening decision is: the explanation is on it, and it adds up.
|
|
38
|
+
# `#evidence` is the side the alert was raised on, for a caller that wants
|
|
39
|
+
# the record and does not care which list version described it.
|
|
40
|
+
#
|
|
41
|
+
# The side that did *not* clear is scored again without a cutoff, so its
|
|
42
|
+
# result may sit below the threshold its own query names -- which is
|
|
43
|
+
# exactly what a subject moving into or out of range looks like, and is
|
|
44
|
+
# the whole reason the score is carried rather than only the fact of a
|
|
45
|
+
# match.
|
|
46
|
+
#
|
|
47
|
+
# ### What `change` says, and what it does not
|
|
48
|
+
#
|
|
49
|
+
# It is about **this subject's match**, not about the record's paperwork:
|
|
50
|
+
#
|
|
51
|
+
# - `:newly_listed` -- the subject did not reach the threshold against
|
|
52
|
+
# this record before and does now. Usually because the record is new;
|
|
53
|
+
# also because an existing record gained the alias, the identifier or
|
|
54
|
+
# the date of birth that brought the subject over the line, which is the
|
|
55
|
+
# same event for a compliance team and is why it is not filed
|
|
56
|
+
# separately.
|
|
57
|
+
# - `:delisted` -- it did reach the threshold before and does not now.
|
|
58
|
+
# Usually because the record was withdrawn; also because an amendment
|
|
59
|
+
# moved it out of range. This is the half of a rescreen that a run
|
|
60
|
+
# against new records only would miss, and it is the half that lets a
|
|
61
|
+
# customer back through the door.
|
|
62
|
+
# - `:details_changed` -- it matched before, it matches now, and the
|
|
63
|
+
# record moved underneath it. The score may be identical: a program
|
|
64
|
+
# added or an address corrected changes what a hit *means* without
|
|
65
|
+
# changing what it scores, and deciding that such a change is too small
|
|
66
|
+
# to report would be deciding which sanctions hits a host is willing to
|
|
67
|
+
# miss.
|
|
68
|
+
#
|
|
69
|
+
# Which of those two routes into `:newly_listed` and `:delisted` a given
|
|
70
|
+
# alert took is not guesswork -- `#fields` is empty when the record itself
|
|
71
|
+
# arrived or left, and names the fields that moved when it was amended.
|
|
72
|
+
#
|
|
73
|
+
# ### Both snapshot ids, on every alert
|
|
74
|
+
#
|
|
75
|
+
# A MatchResult cites the one list version it was scored against, and an
|
|
76
|
+
# alert is about two. So it carries both: `previous_snapshot_id` and
|
|
77
|
+
# `snapshot_id` are the checksums the diff was computed over, which is
|
|
78
|
+
# what makes an alert reproducible under audit -- keep the pair and the
|
|
79
|
+
# whole run can be derived again, from lists that can be identified rather
|
|
80
|
+
# than from a copy of an answer nobody can check.
|
|
81
|
+
#
|
|
82
|
+
# Instances are frozen on construction and compare by value.
|
|
83
|
+
class Alert
|
|
84
|
+
extend T::Sig
|
|
85
|
+
|
|
86
|
+
# What happened to this subject's match. See the class comment.
|
|
87
|
+
CHANGES = T.let(%i[newly_listed delisted details_changed].freeze, T::Array[Symbol])
|
|
88
|
+
|
|
89
|
+
# @api private
|
|
90
|
+
MEMBERS = T.let(
|
|
91
|
+
%i[subject change fields result previous_result snapshot_id previous_snapshot_id].freeze,
|
|
92
|
+
T::Array[Symbol]
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
# The book entry this alert is about, as the caller supplied it.
|
|
96
|
+
sig { returns(Subject) }
|
|
97
|
+
attr_reader :subject
|
|
98
|
+
|
|
99
|
+
sig { returns(Symbol) }
|
|
100
|
+
attr_reader :change
|
|
101
|
+
|
|
102
|
+
# The fields of the record that moved, in Entity's member order, or an
|
|
103
|
+
# empty array when the record was added or withdrawn whole. See
|
|
104
|
+
# Diff::Change.
|
|
105
|
+
sig { returns(T::Array[Symbol]) }
|
|
106
|
+
attr_reader :fields
|
|
107
|
+
|
|
108
|
+
# Scored against the record as the new list has it, or nil when the new
|
|
109
|
+
# list does not have it.
|
|
110
|
+
sig { returns(T.nilable(MatchResult)) }
|
|
111
|
+
attr_reader :result
|
|
112
|
+
|
|
113
|
+
# Scored against the record as the old list had it, or nil when the old
|
|
114
|
+
# list did not have it.
|
|
115
|
+
sig { returns(T.nilable(MatchResult)) }
|
|
116
|
+
attr_reader :previous_result
|
|
117
|
+
|
|
118
|
+
# The checksum of the list version this run screened against.
|
|
119
|
+
sig { returns(String) }
|
|
120
|
+
attr_reader :snapshot_id
|
|
121
|
+
|
|
122
|
+
# The checksum of the list version it was compared with.
|
|
123
|
+
sig { returns(String) }
|
|
124
|
+
attr_reader :previous_snapshot_id
|
|
125
|
+
|
|
126
|
+
# Rebuilds an alert from #to_h output, accepting string keys so one
|
|
127
|
+
# survives the round-trip through JSON and back out of whatever a host
|
|
128
|
+
# stored it in.
|
|
129
|
+
sig { params(hash: T.untyped).returns(T.attached_class) }
|
|
130
|
+
def self.from_h(hash)
|
|
131
|
+
attributes = hash.to_h.transform_keys(&:to_sym)
|
|
132
|
+
unknown = attributes.keys - MEMBERS
|
|
133
|
+
raise InvalidArgument, "unknown Alert attribute(s): #{unknown.join(", ")}" if unknown.any?
|
|
134
|
+
|
|
135
|
+
T.unsafe(self).new(**attributes,
|
|
136
|
+
subject: build(Subject, attributes[:subject]),
|
|
137
|
+
result: build(MatchResult, attributes[:result]),
|
|
138
|
+
previous_result: build(MatchResult, attributes[:previous_result]))
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# A value that is already the object passes through, so from_h is safe
|
|
142
|
+
# to call on a half-deserialized hash.
|
|
143
|
+
sig { params(klass: T.untyped, value: T.untyped).returns(T.untyped) }
|
|
144
|
+
def self.build(klass, value) = value.is_a?(Hash) ? klass.from_h(value) : value
|
|
145
|
+
private_class_method :build
|
|
146
|
+
|
|
147
|
+
sig do
|
|
148
|
+
params(subject: T.untyped, change: T.untyped, snapshot_id: T.untyped, previous_snapshot_id: T.untyped,
|
|
149
|
+
result: T.untyped, previous_result: T.untyped, fields: T.untyped).void
|
|
150
|
+
end
|
|
151
|
+
def initialize(subject:, change:, snapshot_id:, previous_snapshot_id:, result: nil, previous_result: nil,
|
|
152
|
+
fields: [])
|
|
153
|
+
@subject = T.let(instance!(:subject, Subject, subject), Subject)
|
|
154
|
+
@change = T.let(change!(change), Symbol)
|
|
155
|
+
@result = T.let(result!(:result, result), T.nilable(MatchResult))
|
|
156
|
+
@previous_result = T.let(result!(:previous_result, previous_result), T.nilable(MatchResult))
|
|
157
|
+
sides!
|
|
158
|
+
@fields = T.let(Array(fields).map(&:to_sym).freeze, T::Array[Symbol])
|
|
159
|
+
@snapshot_id = T.let(string!(:snapshot_id, snapshot_id), String)
|
|
160
|
+
@previous_snapshot_id = T.let(string!(:previous_snapshot_id, previous_snapshot_id), String)
|
|
161
|
+
freeze
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# The caller's own id for the subject, which is what an alert is joined
|
|
165
|
+
# back to a book of business by.
|
|
166
|
+
sig { returns(String) }
|
|
167
|
+
def subject_id = subject.id
|
|
168
|
+
|
|
169
|
+
# The side the alert was raised on: the current one, or the previous one
|
|
170
|
+
# for a delisting, which is the version that actually matched. It is
|
|
171
|
+
# what `#entity` and `#matched_name` read, so an alert describes the
|
|
172
|
+
# record the way it looked when it crossed the threshold.
|
|
173
|
+
#
|
|
174
|
+
# Never nil: an alert with neither side is not a change, and is refused
|
|
175
|
+
# at construction.
|
|
176
|
+
sig { returns(MatchResult) }
|
|
177
|
+
def evidence = T.must(delisted? ? previous_result || result : result || previous_result)
|
|
178
|
+
|
|
179
|
+
# The record this alert is about, as the surviving side has it.
|
|
180
|
+
sig { returns(Entity) }
|
|
181
|
+
def entity = evidence.entity
|
|
182
|
+
|
|
183
|
+
sig { returns(String) }
|
|
184
|
+
def entity_id = entity.id
|
|
185
|
+
|
|
186
|
+
# The spelling that produced the score, as its publisher wrote it. See
|
|
187
|
+
# MatchResult#matched_name.
|
|
188
|
+
sig { returns(Name) }
|
|
189
|
+
def matched_name = evidence.matched_name
|
|
190
|
+
|
|
191
|
+
sig { returns(Symbol) }
|
|
192
|
+
def source = entity.source
|
|
193
|
+
|
|
194
|
+
# What the subject scores against the record now, or nil if the record
|
|
195
|
+
# is no longer on the list.
|
|
196
|
+
sig { returns(T.nilable(Float)) }
|
|
197
|
+
def score = result&.score
|
|
198
|
+
|
|
199
|
+
# What it scored against the record before, or nil if the record was
|
|
200
|
+
# not on the previous list.
|
|
201
|
+
sig { returns(T.nilable(Float)) }
|
|
202
|
+
def previous_score = previous_result&.score
|
|
203
|
+
|
|
204
|
+
# The threshold this subject was screened at, which is half of what
|
|
205
|
+
# makes the change mean anything: the same pair of scores is a new
|
|
206
|
+
# listing at 75 and nothing at all at 95.
|
|
207
|
+
sig { returns(Float) }
|
|
208
|
+
def threshold = evidence.threshold
|
|
209
|
+
|
|
210
|
+
# When the run that produced this alert happened. One instant for a
|
|
211
|
+
# whole run -- see Rescreen.
|
|
212
|
+
sig { returns(Time) }
|
|
213
|
+
def screened_at = evidence.screened_at
|
|
214
|
+
|
|
215
|
+
sig { returns(T::Boolean) }
|
|
216
|
+
def newly_listed? = change == :newly_listed
|
|
217
|
+
|
|
218
|
+
sig { returns(T::Boolean) }
|
|
219
|
+
def delisted? = change == :delisted
|
|
220
|
+
|
|
221
|
+
sig { returns(T::Boolean) }
|
|
222
|
+
def details_changed? = change == :details_changed
|
|
223
|
+
|
|
224
|
+
# The documented shape. Every value is a String, a Float, an Integer, an
|
|
225
|
+
# Array or a Hash of the same, so `JSON.generate(alert.to_h)` needs
|
|
226
|
+
# nothing from this library and `Alert.from_h(JSON.parse(json))` rebuilds
|
|
227
|
+
# exactly this object.
|
|
228
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
229
|
+
def to_h
|
|
230
|
+
{
|
|
231
|
+
subject: subject.to_h,
|
|
232
|
+
change: change,
|
|
233
|
+
fields: fields,
|
|
234
|
+
result: result&.to_h,
|
|
235
|
+
previous_result: previous_result&.to_h,
|
|
236
|
+
snapshot_id: snapshot_id,
|
|
237
|
+
previous_snapshot_id: previous_snapshot_id
|
|
238
|
+
}
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# One line, for the summary a human reads:
|
|
242
|
+
#
|
|
243
|
+
# cust_1 newly listed un_consolidated:6908021 BOSCO TAGANDA 94.1 (was 71.0)
|
|
244
|
+
sig { returns(String) }
|
|
245
|
+
def to_s
|
|
246
|
+
"#{subject_id} #{change.to_s.tr("_", " ")} #{entity_id} #{evidence.matched_name.value} #{movement}"
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
sig { params(other: T.untyped).returns(T::Boolean) }
|
|
250
|
+
def ==(other)
|
|
251
|
+
return false unless other.instance_of?(self.class)
|
|
252
|
+
|
|
253
|
+
to_h == other.to_h
|
|
254
|
+
end
|
|
255
|
+
alias eql? ==
|
|
256
|
+
|
|
257
|
+
sig { returns(Integer) }
|
|
258
|
+
def hash = [self.class, to_h].hash
|
|
259
|
+
|
|
260
|
+
sig { returns(String) }
|
|
261
|
+
def inspect = "#<#{self.class} #{self}>"
|
|
262
|
+
|
|
263
|
+
private
|
|
264
|
+
|
|
265
|
+
# What the score did, which is the sentence an analyst reads first. A
|
|
266
|
+
# side that does not exist prints as the listing event it was.
|
|
267
|
+
sig { returns(String) }
|
|
268
|
+
def movement
|
|
269
|
+
return "#{previous_score} -> delisted" if result.nil?
|
|
270
|
+
return score.to_s if previous_result.nil?
|
|
271
|
+
return "#{score} (unchanged)" if score == previous_score
|
|
272
|
+
|
|
273
|
+
"#{previous_score} -> #{score}"
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
sig { params(member: Symbol, klass: T.untyped, value: T.untyped).returns(T.untyped) }
|
|
277
|
+
def instance!(member, klass, value)
|
|
278
|
+
return value if value.is_a?(klass)
|
|
279
|
+
|
|
280
|
+
raise InvalidArgument, "#{member} must be an #{klass}, got #{value.class}"
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
sig { params(member: Symbol, value: T.untyped).returns(T.nilable(MatchResult)) }
|
|
284
|
+
def result!(member, value) = value.nil? ? nil : instance!(member, MatchResult, value)
|
|
285
|
+
|
|
286
|
+
# An alert with neither side is not a change: nothing was scored, and
|
|
287
|
+
# there is no record for it to be about.
|
|
288
|
+
sig { void }
|
|
289
|
+
def sides!
|
|
290
|
+
return unless result.nil? && previous_result.nil?
|
|
291
|
+
|
|
292
|
+
raise InvalidArgument,
|
|
293
|
+
"an alert needs a result on at least one side of the change -- one that scored against neither " \
|
|
294
|
+
"list version is not a change, and cannot say what it is about"
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
sig { params(value: T.untyped).returns(Symbol) }
|
|
298
|
+
def change!(value)
|
|
299
|
+
symbol = value.to_s.to_sym
|
|
300
|
+
return symbol if CHANGES.include?(symbol)
|
|
301
|
+
|
|
302
|
+
raise InvalidArgument, "unknown change #{value.inspect}, expected one of #{CHANGES.join(", ")}"
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
sig { params(member: Symbol, value: T.untyped).returns(String) }
|
|
306
|
+
def string!(member, value)
|
|
307
|
+
string = value.to_s.strip
|
|
308
|
+
raise InvalidArgument, "#{member} is required -- an alert cites both list versions it compared" if string.empty?
|
|
309
|
+
|
|
310
|
+
-string
|
|
311
|
+
end
|
|
312
|
+
end
|
|
313
|
+
end
|
|
314
|
+
end
|