active_sanction 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +538 -0
  3. data/CODE_OF_CONDUCT.md +74 -0
  4. data/CONTRIBUTING.md +312 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +616 -0
  7. data/SECURITY.md +97 -0
  8. data/active_sanction.gemspec +98 -0
  9. data/docs/adding_a_source.md +1083 -0
  10. data/docs/api_stability.md +357 -0
  11. data/docs/bundle_format.md +407 -0
  12. data/lib/active_sanction/address.rb +170 -0
  13. data/lib/active_sanction/client.rb +341 -0
  14. data/lib/active_sanction/configuration.rb +721 -0
  15. data/lib/active_sanction/countries.txt +292 -0
  16. data/lib/active_sanction/country.rb +164 -0
  17. data/lib/active_sanction/deprecation.rb +168 -0
  18. data/lib/active_sanction/diff/change.rb +208 -0
  19. data/lib/active_sanction/diff.rb +348 -0
  20. data/lib/active_sanction/doctor/checkup.rb +356 -0
  21. data/lib/active_sanction/doctor/diagnosis.rb +245 -0
  22. data/lib/active_sanction/doctor/finding.rb +183 -0
  23. data/lib/active_sanction/doctor/profile.rb +384 -0
  24. data/lib/active_sanction/doctor/report.rb +227 -0
  25. data/lib/active_sanction/doctor.rb +347 -0
  26. data/lib/active_sanction/entity.rb +293 -0
  27. data/lib/active_sanction/error.rb +332 -0
  28. data/lib/active_sanction/fetcher/result.rb +116 -0
  29. data/lib/active_sanction/fetcher.rb +242 -0
  30. data/lib/active_sanction/http_client/errors.rb +94 -0
  31. data/lib/active_sanction/http_client/response.rb +129 -0
  32. data/lib/active_sanction/http_client.rb +324 -0
  33. data/lib/active_sanction/identifier.rb +212 -0
  34. data/lib/active_sanction/index/builder.rb +89 -0
  35. data/lib/active_sanction/index/candidate.rb +63 -0
  36. data/lib/active_sanction/index/entry.rb +66 -0
  37. data/lib/active_sanction/index/features.rb +112 -0
  38. data/lib/active_sanction/index.rb +381 -0
  39. data/lib/active_sanction/match_result.rb +345 -0
  40. data/lib/active_sanction/matcher.rb +388 -0
  41. data/lib/active_sanction/name.rb +183 -0
  42. data/lib/active_sanction/normalizer/cache.rb +82 -0
  43. data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
  44. data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
  45. data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
  46. data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
  47. data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
  48. data/lib/active_sanction/normalizer/dictionary.rb +243 -0
  49. data/lib/active_sanction/normalizer/form.rb +279 -0
  50. data/lib/active_sanction/normalizer.rb +137 -0
  51. data/lib/active_sanction/parsers/column_shape.rb +274 -0
  52. data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
  53. data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
  54. data/lib/active_sanction/parsers/delimited_table.rb +151 -0
  55. data/lib/active_sanction/parsers/format.rb +107 -0
  56. data/lib/active_sanction/parsers/join.rb +149 -0
  57. data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
  58. data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
  59. data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
  60. data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
  61. data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
  62. data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
  63. data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
  64. data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
  65. data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
  66. data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
  67. data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
  68. data/lib/active_sanction/parsers/xml_records.rb +140 -0
  69. data/lib/active_sanction/parsers.rb +94 -0
  70. data/lib/active_sanction/partial_date/parser.rb +124 -0
  71. data/lib/active_sanction/partial_date.rb +303 -0
  72. data/lib/active_sanction/payload_cache/checksum.rb +66 -0
  73. data/lib/active_sanction/payload_cache/entry.rb +304 -0
  74. data/lib/active_sanction/payload_cache.rb +366 -0
  75. data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
  76. data/lib/active_sanction/phonetics.rb +40 -0
  77. data/lib/active_sanction/query.rb +290 -0
  78. data/lib/active_sanction/rescreen/alert.rb +314 -0
  79. data/lib/active_sanction/rescreen.rb +332 -0
  80. data/lib/active_sanction/scorer/adjustments.rb +241 -0
  81. data/lib/active_sanction/scorer/name_score.rb +279 -0
  82. data/lib/active_sanction/scorer/reason.rb +136 -0
  83. data/lib/active_sanction/scorer/result.rb +129 -0
  84. data/lib/active_sanction/scorer/subject.rb +235 -0
  85. data/lib/active_sanction/scorer/weights.rb +306 -0
  86. data/lib/active_sanction/scorer.rb +303 -0
  87. data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
  88. data/lib/active_sanction/similarity/levenshtein.rb +179 -0
  89. data/lib/active_sanction/similarity/token_set.rb +159 -0
  90. data/lib/active_sanction/similarity/token_sort.rb +120 -0
  91. data/lib/active_sanction/similarity.rb +183 -0
  92. data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
  93. data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
  94. data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
  95. data/lib/active_sanction/snapshot/bundle.rb +381 -0
  96. data/lib/active_sanction/snapshot.rb +305 -0
  97. data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
  98. data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
  99. data/lib/active_sanction/sources/australia_dfat.rb +291 -0
  100. data/lib/active_sanction/sources/base.rb +295 -0
  101. data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
  102. data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
  103. data/lib/active_sanction/sources/canada_sema.rb +153 -0
  104. data/lib/active_sanction/sources/definition.rb +360 -0
  105. data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
  106. data/lib/active_sanction/sources/eu_fsf.rb +203 -0
  107. data/lib/active_sanction/sources/ofac/record.rb +251 -0
  108. data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
  109. data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
  110. data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
  111. data/lib/active_sanction/sources/ofac.rb +248 -0
  112. data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
  113. data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
  114. data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
  115. data/lib/active_sanction/sources/remarks.rb +84 -0
  116. data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
  117. data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
  118. data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
  119. data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
  120. data/lib/active_sanction/sources/un_consolidated.rb +140 -0
  121. data/lib/active_sanction/sources.rb +196 -0
  122. data/lib/active_sanction/storage/active_record/reader.rb +124 -0
  123. data/lib/active_sanction/storage/active_record/row.rb +175 -0
  124. data/lib/active_sanction/storage/active_record/writer.rb +141 -0
  125. data/lib/active_sanction/storage/active_record.rb +277 -0
  126. data/lib/active_sanction/storage/base.rb +216 -0
  127. data/lib/active_sanction/storage/file_system.rb +373 -0
  128. data/lib/active_sanction/storage/memory.rb +67 -0
  129. data/lib/active_sanction/storage/meta.rb +188 -0
  130. data/lib/active_sanction/storage.rb +74 -0
  131. data/lib/active_sanction/subject.rb +271 -0
  132. data/lib/active_sanction/sync/report.rb +245 -0
  133. data/lib/active_sanction/sync/result.rb +296 -0
  134. data/lib/active_sanction/sync.rb +390 -0
  135. data/lib/active_sanction/validator_store/file_system.rb +103 -0
  136. data/lib/active_sanction/validator_store/memory.rb +41 -0
  137. data/lib/active_sanction/validator_store.rb +119 -0
  138. data/lib/active_sanction/validators.rb +195 -0
  139. data/lib/active_sanction/version.rb +23 -0
  140. data/lib/active_sanction.rb +331 -0
  141. data/lib/generators/active_sanction/install/install_generator.rb +60 -0
  142. data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
  143. metadata +236 -0
@@ -0,0 +1,40 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ module ActiveSanction
7
+ # What a name sounds like, as a key two spellings of it can share. Stage 3c
8
+ # of the matching pipeline.
9
+ #
10
+ # ActiveSanction::Phonetics::DoubleMetaphone.call("qaddafi") # => ["KTF"]
11
+ # ActiveSanction::Phonetics::DoubleMetaphone.call("gaddafi") # => ["KTF"]
12
+ #
13
+ # ### Why this is not in Similarity
14
+ #
15
+ # Because a key is not a score. Everything in Similarity answers "how close
16
+ # are these two names" with a number between 0 and 1, and the whole of that
17
+ # module's contract -- symmetric, never rounded, `threshold:` as an
18
+ # optimization, a `ceiling` that may never come in under a real score --
19
+ # is about keeping four such numbers comparable enough to blend.
20
+ #
21
+ # This stage answers a different question and gives a different kind of
22
+ # answer: it turns one name into the handful of strings that stand for how
23
+ # it sounds. Two names either share one of those strings or they do not.
24
+ # That is what makes it useful in the two places it is used --
25
+ #
26
+ # - the inverted index (#31) keys on it, so that a query for `GADDAFI`
27
+ # reaches a record spelled `QADHAFI` at all, which no amount of comparing
28
+ # would help with if the record is never fetched; and
29
+ # - the scorer (#32) reads it as one share of the blend it scores a pair on.
30
+ #
31
+ # -- and it is also why it cannot be blended with the other four. A shared
32
+ # key is evidence, at the strength #32 decides. It is never a match on its
33
+ # own: `HSN` is the key for `HUSSEIN`, and equally for `HASSAN`.
34
+ #
35
+ # @api private
36
+ module Phonetics
37
+ end
38
+ end
39
+
40
+ require "active_sanction/phonetics/double_metaphone"
@@ -0,0 +1,290 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/scorer"
7
+ require "active_sanction/sources/definition"
8
+
9
+ module ActiveSanction
10
+ # One screening request: what is known about the subject, and how the
11
+ # search is to be run.
12
+ #
13
+ # query = ActiveSanction::Query.build(
14
+ # name: "Bosco Ntaganda",
15
+ # type: :individual,
16
+ # date_of_birth: "1973",
17
+ # countries: %w[CD],
18
+ # sources: %i[ofac_sdn un_consolidated],
19
+ # threshold: 75,
20
+ # limit: 10
21
+ # )
22
+ #
23
+ # query.subject # => Scorer::Subject, folded once and held
24
+ # query.threshold # => 75.0
25
+ #
26
+ # ### Two kinds of field, and the line between them matters
27
+ #
28
+ # **Evidence** -- the name, the type, the dates of birth, the nationalities,
29
+ # the identifiers -- is what the scorer compares against a record, and it
30
+ # belongs to Scorer::Subject, which this builds and holds. **Search
31
+ # options** -- `sources`, `threshold`, `limit` -- decide which records are
32
+ # looked at and how many come back, and they never touch a comparison.
33
+ #
34
+ # Keeping them apart is what lets a stored decision be re-derived: the
35
+ # evidence says what was screened and the options say what the run was
36
+ # willing to return, and an audit needs both separately. It is also why
37
+ # Subject carries no threshold -- see the note at the end of that class.
38
+ #
39
+ # ### Singular and plural spellings are both accepted
40
+ #
41
+ # `date_of_birth:` and `dates_of_birth:`, `country:`, `countries:` and
42
+ # `nationalities:`, `identifier:` and `identifiers:` all mean the same
43
+ # thing. A caller with one date writes the singular and a caller with three
44
+ # writes the plural, and neither should have to remember which this library
45
+ # prefers. `#to_h` emits the plural, canonical spelling.
46
+ #
47
+ # That resolution happens in `.build` and `.from_h`, which is what every
48
+ # screening call goes through -- `Matcher#screen` builds one of these out of
49
+ # whatever it was handed. `.new` takes the canonical names and nothing else,
50
+ # in the manner of Scorer::Weights: one constructor states the shape and one
51
+ # accepts what a caller wrote.
52
+ #
53
+ # ### Defaults come from configuration, once, here
54
+ #
55
+ # A query with no `threshold:` takes `config.screening_threshold` and one
56
+ # with no `limit:` takes `config.screening_limit`, both read at construction
57
+ # and then fixed. Nothing downstream reads a global: a Matcher screens the
58
+ # numbers on the query it was given, so a configuration changed mid-batch
59
+ # cannot produce a run that is half one threshold and half another.
60
+ #
61
+ # Instances are frozen on construction and compare by value.
62
+ class Query
63
+ extend T::Sig
64
+
65
+ # The canonical spelling of every field, and the shape `#to_h` emits.
66
+ #
67
+ # @api private
68
+ MEMBERS = T.let(
69
+ %i[name type dates_of_birth nationalities identifiers sources threshold limit].freeze,
70
+ T::Array[Symbol]
71
+ )
72
+
73
+ # The spellings a caller may write instead, and what each one means. See
74
+ # the note above.
75
+ #
76
+ # @api private
77
+ ALIASES = T.let(
78
+ {
79
+ date_of_birth: :dates_of_birth, dob: :dates_of_birth, dobs: :dates_of_birth,
80
+ country: :nationalities, countries: :nationalities, nationality: :nationalities,
81
+ identifier: :identifiers, source: :sources
82
+ }.freeze,
83
+ T::Hash[Symbol, Symbol]
84
+ )
85
+
86
+ # The members that are lists. A value that is not one becomes a list of
87
+ # one, which is what makes `identifier: { kind: :passport, value: "AB-1" }`
88
+ # a single identifier: `Array(hash)` reads a Hash as a list of its pairs,
89
+ # and a caller naming one document would get two nonsense identifiers back.
90
+ #
91
+ # @api private
92
+ COLLECTIONS = T.let(%i[dates_of_birth nationalities identifiers sources].freeze, T::Array[Symbol])
93
+
94
+ # The evidence, folded once. Every comparison in a screening run happens
95
+ # against this one object rather than against a name re-folded per
96
+ # candidate.
97
+ sig { returns(Scorer::Subject) }
98
+ attr_reader :subject
99
+
100
+ # Which lists to screen against, or nil for every list the matcher holds.
101
+ # Naming one it does not hold is an error rather than a shorter answer --
102
+ # see Matcher.
103
+ sig { returns(T.nilable(T::Array[Symbol])) }
104
+ attr_reader :sources
105
+
106
+ # 0..100. The lowest score worth reporting, and the number that decides
107
+ # what a screening call costs -- see Scorer.
108
+ sig { returns(Float) }
109
+ attr_reader :threshold
110
+
111
+ # How many results to return, highest score first.
112
+ sig { returns(Integer) }
113
+ attr_reader :limit
114
+
115
+ class << self
116
+ extend T::Sig
117
+
118
+ # Whatever a caller had, as a Query:
119
+ #
120
+ # Query.build("Bosco Ntaganda")
121
+ # Query.build(name: "Bosco Ntaganda", threshold: 80)
122
+ # Query.build(query, limit: 5) # the same query, with one option changed
123
+ #
124
+ # A bare String or Name is a query about that name and nothing else,
125
+ # which is what a batch of names is a list of.
126
+ sig { params(value: T.untyped, overrides: T.untyped).returns(Query) }
127
+ def build(value = nil, **overrides)
128
+ case value
129
+ when Query then overrides.empty? ? value : T.unsafe(value).merge(**overrides)
130
+ when Hash then from_h(normalize(value).merge(normalize(overrides)))
131
+ when nil then from_h(overrides)
132
+ else from_h(normalize(overrides).merge(name: value))
133
+ end
134
+ end
135
+
136
+ # Rebuilds a query from #to_h output, accepting string keys so one
137
+ # stored in an audit record survives the round-trip through JSON.
138
+ sig { params(hash: T.untyped).returns(Query) }
139
+ def from_h(hash)
140
+ attributes = normalize(hash)
141
+ unknown = attributes.keys - MEMBERS
142
+ raise QueryError, "unknown Query attribute(s): #{unknown.join(", ")}" if unknown.any?
143
+
144
+ # `new(**hash)` past a required keyword parameter is one of the few
145
+ # things Sorbet cannot check statically. #initialize validates what
146
+ # arrives, which is where a bad round-trip is caught.
147
+ T.unsafe(self).new(**attributes)
148
+ end
149
+
150
+ private
151
+
152
+ # Symbol keys, with every accepted spelling resolved to its canonical
153
+ # one. A caller that wrote both spellings of the same field is a typo
154
+ # rather than a merge, so it is refused.
155
+ sig { params(hash: T.untyped).returns(T::Hash[Symbol, T.untyped]) }
156
+ def normalize(hash)
157
+ spellings = T.let({}, T::Hash[Symbol, Symbol])
158
+ hash.to_h.each_with_object({}) do |(key, value), attributes|
159
+ written = key.to_s.to_sym
160
+ member = ALIASES.fetch(written, written)
161
+ clash = spellings[member]
162
+ raise QueryError, "#{clash} and #{written} are the same field -- pass one" if conflict?(clash, written)
163
+
164
+ spellings[member] = written
165
+ attributes[member] = wrap(member, value)
166
+ end
167
+ end
168
+
169
+ sig { params(clash: T.nilable(Symbol), written: Symbol).returns(T::Boolean) }
170
+ def conflict?(clash, written) = !clash.nil? && clash != written
171
+
172
+ sig { params(member: Symbol, value: T.untyped).returns(T.untyped) }
173
+ def wrap(member, value)
174
+ return value unless COLLECTIONS.include?(member)
175
+ return value if value.nil? || value.is_a?(Array)
176
+
177
+ [value]
178
+ end
179
+ end
180
+
181
+ # Only `name` is required; see Scorer::Subject on why that is the shape of
182
+ # the problem rather than a convenience.
183
+ sig do
184
+ params(name: T.untyped, type: T.untyped, dates_of_birth: T.untyped, nationalities: T.untyped,
185
+ identifiers: T.untyped, sources: T.untyped, threshold: T.untyped, limit: T.untyped).void
186
+ end
187
+ def initialize(name:, type: nil, dates_of_birth: [], nationalities: [], identifiers: [],
188
+ sources: nil, threshold: nil, limit: nil)
189
+ @subject = T.let(
190
+ Scorer::Subject.new(name: name, type: type, dates_of_birth: dates_of_birth,
191
+ nationalities: nationalities, identifiers: identifiers),
192
+ Scorer::Subject
193
+ )
194
+ @sources = T.let(sources!(sources), T.nilable(T::Array[Symbol]))
195
+ @threshold = T.let(threshold!(threshold), Float)
196
+ @limit = T.let(limit!(limit), Integer)
197
+ freeze
198
+ end
199
+
200
+ # This query with some fields replaced, which is how a batch applies one
201
+ # threshold to a list of names.
202
+ sig { params(overrides: T.untyped).returns(Query) }
203
+ def merge(**overrides) = self.class.build(to_h, **overrides)
204
+
205
+ # The name as the caller wrote it, which is what a report quotes back.
206
+ sig { returns(String) }
207
+ def name = subject.name
208
+
209
+ sig { returns(T.nilable(Symbol)) }
210
+ def type = subject.type
211
+
212
+ sig { returns(T::Array[PartialDate]) }
213
+ def dates_of_birth = subject.dates_of_birth
214
+
215
+ sig { returns(T::Array[String]) }
216
+ def nationalities = subject.nationalities
217
+
218
+ sig { returns(T::Array[Identifier]) }
219
+ def identifiers = subject.identifiers
220
+
221
+ # The folded name every comparison runs against, and what a Matcher hands
222
+ # the index rather than the string it came from.
223
+ sig { returns(Normalizer::Form) }
224
+ def form = subject.form
225
+
226
+ sig { returns(T::Hash[Symbol, T.untyped]) }
227
+ def to_h
228
+ subject.to_h.merge(sources: sources, threshold: threshold, limit: limit)
229
+ end
230
+
231
+ sig { params(other: T.untyped).returns(T::Boolean) }
232
+ def ==(other)
233
+ return false unless other.instance_of?(self.class)
234
+
235
+ to_h == other.to_h
236
+ end
237
+ alias eql? ==
238
+
239
+ sig { returns(Integer) }
240
+ def hash = [self.class, to_h].hash
241
+
242
+ sig { returns(String) }
243
+ def inspect = "#<#{self.class} #{name.inspect} threshold=#{threshold} limit=#{limit}>"
244
+
245
+ private
246
+
247
+ # nil means every list the matcher holds. An empty array does not: a
248
+ # caller that computed its source list and got nothing back is asking to
249
+ # screen against no lists at all, which returns a clean report for
250
+ # everybody, so it is refused rather than quietly read as "all".
251
+ sig { params(value: T.untyped).returns(T.nilable(T::Array[Symbol])) }
252
+ def sources!(value)
253
+ return nil if value.nil?
254
+
255
+ keys = Array(value).map { |key| Sources::Definition.key!(key) }.uniq
256
+ raise QueryError, "sources cannot be empty -- omit it to screen against every list" if keys.empty?
257
+
258
+ keys.freeze
259
+ end
260
+
261
+ # Validated by the scorer's own check, so a threshold on the 0..1 scale is
262
+ # refused here with the message that explains it rather than three layers
263
+ # down.
264
+ sig { params(value: T.untyped).returns(Float) }
265
+ def threshold!(value)
266
+ return Scorer.threshold!(ActiveSanction.config.screening_threshold) if value.nil?
267
+
268
+ number = begin
269
+ Float(value)
270
+ rescue TypeError, ArgumentError
271
+ raise QueryError, "threshold must be a number between 0 and 100, got #{value.inspect}"
272
+ end
273
+ Scorer.threshold!(number)
274
+ end
275
+
276
+ sig { params(value: T.untyped).returns(Integer) }
277
+ def limit!(value)
278
+ return ActiveSanction.config.screening_limit if value.nil?
279
+
280
+ integer = begin
281
+ Integer(value)
282
+ rescue TypeError, ArgumentError
283
+ raise QueryError, "limit must be a whole number of results, got #{value.inspect}"
284
+ end
285
+ raise QueryError, "limit must be at least 1, got #{integer}" unless integer.positive?
286
+
287
+ integer
288
+ end
289
+ end
290
+ end
@@ -0,0 +1,314 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/error"
7
+ require "active_sanction/match_result"
8
+ require "active_sanction/subject"
9
+
10
+ module ActiveSanction
11
+ class Rescreen
12
+ # One subject, one record, and what moved between two list versions.
13
+ #
14
+ # alert.subject_id # => "cust_1"
15
+ # alert.change # => :newly_listed
16
+ # alert.result # => MatchResult, scored against the record as it is now
17
+ # alert.previous_result # => nil -- there was no such record before
18
+ # alert.score # => 94.1
19
+ # alert.previous_score # => nil
20
+ #
21
+ # puts alert
22
+ # # => cust_1 newly listed un_consolidated:6908021 BOSCO TAGANDA 94.1
23
+ #
24
+ # ### The two sides, and why either may be missing
25
+ #
26
+ # An alert is a *change*, so it is two screening results rather than one:
27
+ # `previous_result` is what this subject scored against the record as the
28
+ # old list had it, and `result` is what it scores against the record as
29
+ # the new list has it. A newly listed record has no previous side. A
30
+ # delisted one has no current side. Everything else has both, and the pair
31
+ # is what lets an alert say a subject moved from 71 to 94 rather than
32
+ # merely that it now matches -- which is the difference between an analyst
33
+ # reading a record and an analyst reading a change to one.
34
+ #
35
+ # Both are full MatchResults, each stamped with the checksum of the list
36
+ # version it was scored against, so an alert is defensible the same way a
37
+ # screening decision is: the explanation is on it, and it adds up.
38
+ # `#evidence` is the side the alert was raised on, for a caller that wants
39
+ # the record and does not care which list version described it.
40
+ #
41
+ # The side that did *not* clear is scored again without a cutoff, so its
42
+ # result may sit below the threshold its own query names -- which is
43
+ # exactly what a subject moving into or out of range looks like, and is
44
+ # the whole reason the score is carried rather than only the fact of a
45
+ # match.
46
+ #
47
+ # ### What `change` says, and what it does not
48
+ #
49
+ # It is about **this subject's match**, not about the record's paperwork:
50
+ #
51
+ # - `:newly_listed` -- the subject did not reach the threshold against
52
+ # this record before and does now. Usually because the record is new;
53
+ # also because an existing record gained the alias, the identifier or
54
+ # the date of birth that brought the subject over the line, which is the
55
+ # same event for a compliance team and is why it is not filed
56
+ # separately.
57
+ # - `:delisted` -- it did reach the threshold before and does not now.
58
+ # Usually because the record was withdrawn; also because an amendment
59
+ # moved it out of range. This is the half of a rescreen that a run
60
+ # against new records only would miss, and it is the half that lets a
61
+ # customer back through the door.
62
+ # - `:details_changed` -- it matched before, it matches now, and the
63
+ # record moved underneath it. The score may be identical: a program
64
+ # added or an address corrected changes what a hit *means* without
65
+ # changing what it scores, and deciding that such a change is too small
66
+ # to report would be deciding which sanctions hits a host is willing to
67
+ # miss.
68
+ #
69
+ # Which of those two routes into `:newly_listed` and `:delisted` a given
70
+ # alert took is not guesswork -- `#fields` is empty when the record itself
71
+ # arrived or left, and names the fields that moved when it was amended.
72
+ #
73
+ # ### Both snapshot ids, on every alert
74
+ #
75
+ # A MatchResult cites the one list version it was scored against, and an
76
+ # alert is about two. So it carries both: `previous_snapshot_id` and
77
+ # `snapshot_id` are the checksums the diff was computed over, which is
78
+ # what makes an alert reproducible under audit -- keep the pair and the
79
+ # whole run can be derived again, from lists that can be identified rather
80
+ # than from a copy of an answer nobody can check.
81
+ #
82
+ # Instances are frozen on construction and compare by value.
83
+ class Alert
84
+ extend T::Sig
85
+
86
+ # What happened to this subject's match. See the class comment.
87
+ CHANGES = T.let(%i[newly_listed delisted details_changed].freeze, T::Array[Symbol])
88
+
89
+ # @api private
90
+ MEMBERS = T.let(
91
+ %i[subject change fields result previous_result snapshot_id previous_snapshot_id].freeze,
92
+ T::Array[Symbol]
93
+ )
94
+
95
+ # The book entry this alert is about, as the caller supplied it.
96
+ sig { returns(Subject) }
97
+ attr_reader :subject
98
+
99
+ sig { returns(Symbol) }
100
+ attr_reader :change
101
+
102
+ # The fields of the record that moved, in Entity's member order, or an
103
+ # empty array when the record was added or withdrawn whole. See
104
+ # Diff::Change.
105
+ sig { returns(T::Array[Symbol]) }
106
+ attr_reader :fields
107
+
108
+ # Scored against the record as the new list has it, or nil when the new
109
+ # list does not have it.
110
+ sig { returns(T.nilable(MatchResult)) }
111
+ attr_reader :result
112
+
113
+ # Scored against the record as the old list had it, or nil when the old
114
+ # list did not have it.
115
+ sig { returns(T.nilable(MatchResult)) }
116
+ attr_reader :previous_result
117
+
118
+ # The checksum of the list version this run screened against.
119
+ sig { returns(String) }
120
+ attr_reader :snapshot_id
121
+
122
+ # The checksum of the list version it was compared with.
123
+ sig { returns(String) }
124
+ attr_reader :previous_snapshot_id
125
+
126
+ # Rebuilds an alert from #to_h output, accepting string keys so one
127
+ # survives the round-trip through JSON and back out of whatever a host
128
+ # stored it in.
129
+ sig { params(hash: T.untyped).returns(T.attached_class) }
130
+ def self.from_h(hash)
131
+ attributes = hash.to_h.transform_keys(&:to_sym)
132
+ unknown = attributes.keys - MEMBERS
133
+ raise InvalidArgument, "unknown Alert attribute(s): #{unknown.join(", ")}" if unknown.any?
134
+
135
+ T.unsafe(self).new(**attributes,
136
+ subject: build(Subject, attributes[:subject]),
137
+ result: build(MatchResult, attributes[:result]),
138
+ previous_result: build(MatchResult, attributes[:previous_result]))
139
+ end
140
+
141
+ # A value that is already the object passes through, so from_h is safe
142
+ # to call on a half-deserialized hash.
143
+ sig { params(klass: T.untyped, value: T.untyped).returns(T.untyped) }
144
+ def self.build(klass, value) = value.is_a?(Hash) ? klass.from_h(value) : value
145
+ private_class_method :build
146
+
147
+ sig do
148
+ params(subject: T.untyped, change: T.untyped, snapshot_id: T.untyped, previous_snapshot_id: T.untyped,
149
+ result: T.untyped, previous_result: T.untyped, fields: T.untyped).void
150
+ end
151
+ def initialize(subject:, change:, snapshot_id:, previous_snapshot_id:, result: nil, previous_result: nil,
152
+ fields: [])
153
+ @subject = T.let(instance!(:subject, Subject, subject), Subject)
154
+ @change = T.let(change!(change), Symbol)
155
+ @result = T.let(result!(:result, result), T.nilable(MatchResult))
156
+ @previous_result = T.let(result!(:previous_result, previous_result), T.nilable(MatchResult))
157
+ sides!
158
+ @fields = T.let(Array(fields).map(&:to_sym).freeze, T::Array[Symbol])
159
+ @snapshot_id = T.let(string!(:snapshot_id, snapshot_id), String)
160
+ @previous_snapshot_id = T.let(string!(:previous_snapshot_id, previous_snapshot_id), String)
161
+ freeze
162
+ end
163
+
164
+ # The caller's own id for the subject, which is what an alert is joined
165
+ # back to a book of business by.
166
+ sig { returns(String) }
167
+ def subject_id = subject.id
168
+
169
+ # The side the alert was raised on: the current one, or the previous one
170
+ # for a delisting, which is the version that actually matched. It is
171
+ # what `#entity` and `#matched_name` read, so an alert describes the
172
+ # record the way it looked when it crossed the threshold.
173
+ #
174
+ # Never nil: an alert with neither side is not a change, and is refused
175
+ # at construction.
176
+ sig { returns(MatchResult) }
177
+ def evidence = T.must(delisted? ? previous_result || result : result || previous_result)
178
+
179
+ # The record this alert is about, as the surviving side has it.
180
+ sig { returns(Entity) }
181
+ def entity = evidence.entity
182
+
183
+ sig { returns(String) }
184
+ def entity_id = entity.id
185
+
186
+ # The spelling that produced the score, as its publisher wrote it. See
187
+ # MatchResult#matched_name.
188
+ sig { returns(Name) }
189
+ def matched_name = evidence.matched_name
190
+
191
+ sig { returns(Symbol) }
192
+ def source = entity.source
193
+
194
+ # What the subject scores against the record now, or nil if the record
195
+ # is no longer on the list.
196
+ sig { returns(T.nilable(Float)) }
197
+ def score = result&.score
198
+
199
+ # What it scored against the record before, or nil if the record was
200
+ # not on the previous list.
201
+ sig { returns(T.nilable(Float)) }
202
+ def previous_score = previous_result&.score
203
+
204
+ # The threshold this subject was screened at, which is half of what
205
+ # makes the change mean anything: the same pair of scores is a new
206
+ # listing at 75 and nothing at all at 95.
207
+ sig { returns(Float) }
208
+ def threshold = evidence.threshold
209
+
210
+ # When the run that produced this alert happened. One instant for a
211
+ # whole run -- see Rescreen.
212
+ sig { returns(Time) }
213
+ def screened_at = evidence.screened_at
214
+
215
+ sig { returns(T::Boolean) }
216
+ def newly_listed? = change == :newly_listed
217
+
218
+ sig { returns(T::Boolean) }
219
+ def delisted? = change == :delisted
220
+
221
+ sig { returns(T::Boolean) }
222
+ def details_changed? = change == :details_changed
223
+
224
+ # The documented shape. Every value is a String, a Float, an Integer, an
225
+ # Array or a Hash of the same, so `JSON.generate(alert.to_h)` needs
226
+ # nothing from this library and `Alert.from_h(JSON.parse(json))` rebuilds
227
+ # exactly this object.
228
+ sig { returns(T::Hash[Symbol, T.untyped]) }
229
+ def to_h
230
+ {
231
+ subject: subject.to_h,
232
+ change: change,
233
+ fields: fields,
234
+ result: result&.to_h,
235
+ previous_result: previous_result&.to_h,
236
+ snapshot_id: snapshot_id,
237
+ previous_snapshot_id: previous_snapshot_id
238
+ }
239
+ end
240
+
241
+ # One line, for the summary a human reads:
242
+ #
243
+ # cust_1 newly listed un_consolidated:6908021 BOSCO TAGANDA 94.1 (was 71.0)
244
+ sig { returns(String) }
245
+ def to_s
246
+ "#{subject_id} #{change.to_s.tr("_", " ")} #{entity_id} #{evidence.matched_name.value} #{movement}"
247
+ end
248
+
249
+ sig { params(other: T.untyped).returns(T::Boolean) }
250
+ def ==(other)
251
+ return false unless other.instance_of?(self.class)
252
+
253
+ to_h == other.to_h
254
+ end
255
+ alias eql? ==
256
+
257
+ sig { returns(Integer) }
258
+ def hash = [self.class, to_h].hash
259
+
260
+ sig { returns(String) }
261
+ def inspect = "#<#{self.class} #{self}>"
262
+
263
+ private
264
+
265
+ # What the score did, which is the sentence an analyst reads first. A
266
+ # side that does not exist prints as the listing event it was.
267
+ sig { returns(String) }
268
+ def movement
269
+ return "#{previous_score} -> delisted" if result.nil?
270
+ return score.to_s if previous_result.nil?
271
+ return "#{score} (unchanged)" if score == previous_score
272
+
273
+ "#{previous_score} -> #{score}"
274
+ end
275
+
276
+ sig { params(member: Symbol, klass: T.untyped, value: T.untyped).returns(T.untyped) }
277
+ def instance!(member, klass, value)
278
+ return value if value.is_a?(klass)
279
+
280
+ raise InvalidArgument, "#{member} must be an #{klass}, got #{value.class}"
281
+ end
282
+
283
+ sig { params(member: Symbol, value: T.untyped).returns(T.nilable(MatchResult)) }
284
+ def result!(member, value) = value.nil? ? nil : instance!(member, MatchResult, value)
285
+
286
+ # An alert with neither side is not a change: nothing was scored, and
287
+ # there is no record for it to be about.
288
+ sig { void }
289
+ def sides!
290
+ return unless result.nil? && previous_result.nil?
291
+
292
+ raise InvalidArgument,
293
+ "an alert needs a result on at least one side of the change -- one that scored against neither " \
294
+ "list version is not a change, and cannot say what it is about"
295
+ end
296
+
297
+ sig { params(value: T.untyped).returns(Symbol) }
298
+ def change!(value)
299
+ symbol = value.to_s.to_sym
300
+ return symbol if CHANGES.include?(symbol)
301
+
302
+ raise InvalidArgument, "unknown change #{value.inspect}, expected one of #{CHANGES.join(", ")}"
303
+ end
304
+
305
+ sig { params(member: Symbol, value: T.untyped).returns(String) }
306
+ def string!(member, value)
307
+ string = value.to_s.strip
308
+ raise InvalidArgument, "#{member} is required -- an alert cites both list versions it compared" if string.empty?
309
+
310
+ -string
311
+ end
312
+ end
313
+ end
314
+ end