@dszp/csl-screen 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/LICENSE +21 -0
- package/README.md +126 -0
- package/dist/adapters/autotask.d.ts +16 -0
- package/dist/adapters/autotask.js +11 -0
- package/dist/adapters/onebill.d.ts +17 -0
- package/dist/adapters/onebill.js +14 -0
- package/dist/canonical.d.ts +3 -0
- package/dist/canonical.js +22 -0
- package/dist/decisions.d.ts +28 -0
- package/dist/decisions.js +118 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +8 -0
- package/dist/jaroWinkler.d.ts +2 -0
- package/dist/jaroWinkler.js +41 -0
- package/dist/model.d.ts +86 -0
- package/dist/model.js +1 -0
- package/dist/normalize.d.ts +17 -0
- package/dist/normalize.js +66 -0
- package/dist/render.d.ts +12 -0
- package/dist/render.js +93 -0
- package/dist/screen.d.ts +11 -0
- package/dist/screen.js +181 -0
- package/dist/snapshot.d.ts +59 -0
- package/dist/snapshot.js +130 -0
- package/package.json +79 -0
- package/src/adapters/autotask.ts +23 -0
- package/src/adapters/onebill.ts +29 -0
- package/src/canonical.ts +22 -0
- package/src/decisions.ts +117 -0
- package/src/index.ts +11 -0
- package/src/jaroWinkler.ts +39 -0
- package/src/model.ts +75 -0
- package/src/normalize.ts +68 -0
- package/src/render.ts +142 -0
- package/src/screen.ts +185 -0
- package/src/snapshot.ts +186 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `@dszp/csl-screen` are documented here.
|
|
4
|
+
|
|
5
|
+
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres
|
|
6
|
+
to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.0] — 2026-10-17
|
|
11
|
+
|
|
12
|
+
First release.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- `loadSnapshot` parses the Consolidated Screening List bulk JSON file into a compact snapshot. The
|
|
17
|
+
snapshot records the list's identity: search time, per-list update times, entry count, per-list
|
|
18
|
+
entry counts, and a SHA-256 of the downloaded bytes. Loading fails with a typed
|
|
19
|
+
`CslSnapshotError` when the file is truncated, empty, malformed, shorter than its own `total`, or
|
|
20
|
+
has lost entries since a previous load (`previousEntryCount`, `previousEntriesBySource`).
|
|
21
|
+
- `screen` matches parties against every listed name and alias with four rules: exact (also
|
|
22
|
+
ignoring spaces), contains, within, and similar (Jaro-Winkler capped by token coverage). The
|
|
23
|
+
threshold, stop words and index token length are options. Contains also joins two or three
|
|
24
|
+
adjacent party tokens, so a separator typed inside a listed word does not hide it. A one-word
|
|
25
|
+
listed entity name matches in any position of the party name, except the words in
|
|
26
|
+
`genericTokens` (default `DEFAULT_GENERIC_TOKENS`), which match only as the first token.
|
|
27
|
+
- `makeDecision`, `applyDecisions` and `parseDecisions` record a reviewer's ruling on a flag. A
|
|
28
|
+
clearance holds only while the party's normalised name and a fingerprint of the whole list entry
|
|
29
|
+
are unchanged.
|
|
30
|
+
- `toCsv` and `toMarkdown` render evidence. The CSV guards against spreadsheet formula injection,
|
|
31
|
+
and the Markdown escapes names.
|
|
32
|
+
- `@dszp/csl-screen/autotask` and `@dszp/csl-screen/onebill` map source records to parties.
|
|
33
|
+
- `SnapshotIdentity.contentSha256` and `contentHashOf`: a hash of the entries alone, unchanged by
|
|
34
|
+
`search_performed_at` or a list's `last_imported`, so it changes only when the list's content does.
|
|
35
|
+
|
|
36
|
+
- Names are folded to ASCII for Latin letters that have no accent decomposition (`ß`, `æ`, `œ`, `ø`,
|
|
37
|
+
`ł`, `đ`, `ð`, `þ`, `ı`), and every combining mark and format character (zero-width space, soft
|
|
38
|
+
hyphen, bidi mark, variation selector) is deleted, so an invisible character inside a word does
|
|
39
|
+
not split it. A party name that still has a letter outside the Latin alphabet, or a control,
|
|
40
|
+
private-use or unassigned character, is reported as unscreenable
|
|
41
|
+
(`reason: 'unreadable-characters'`, exported check `hasUnreadableCharacters`), because such a
|
|
42
|
+
character would otherwise split a word and hide a listed match.
|
|
43
|
+
|
|
44
|
+
### Known limitations
|
|
45
|
+
|
|
46
|
+
- A misspelling inside a word (for example a dropped letter) is not caught.
|
|
47
|
+
- A listed one-word name in `genericTokens` matches only as the party's first token. The list is
|
|
48
|
+
an option.
|
|
49
|
+
- A listed word split into four or more pieces inside a longer party name is not caught.
|
|
50
|
+
- Only organisation names are screened, and only against the US list. Names with letters outside the
|
|
51
|
+
Latin alphabet are reported as unscreenable.
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 David Szpunar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# @dszp/csl-screen
|
|
2
|
+
|
|
3
|
+
Screen organisation names against the US Consolidated Screening List (CSL). The library normalises names, matches them against every listed name and alias, and records a reviewer's ruling on each flag so the same flag does not come back until something changes. It has no runtime dependencies and runs in Node 20.19+ and Cloudflare Workers. It is an ES module; `require()` loads it on Node versions with `require(esm)` support (20.19+, 22.12+).
|
|
4
|
+
|
|
5
|
+
This is a screening aid, not legal advice. A clear result does not prove that a party is permitted, and a flag does not prove a violation. A person must review flags.
|
|
6
|
+
|
|
7
|
+
Scope: it screens organisations against the US CSL only. It does not screen against other countries' lists, and it does not transliterate non-Latin names (a name with letters outside the Latin alphabet, or with unreadable characters, cannot be matched safely and is reported as unscreenable).
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
npm install @dszp/csl-screen
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Example
|
|
16
|
+
|
|
17
|
+
The library does no I/O. You fetch the CSL JSON file and your party records, and pass them in.
|
|
18
|
+
|
|
19
|
+
```ts
|
|
20
|
+
import { loadSnapshot, screen, applyDecisions, partyOutcome, toCsv } from '@dszp/csl-screen';
|
|
21
|
+
import { fromAutotaskCompanies } from '@dszp/csl-screen/autotask';
|
|
22
|
+
|
|
23
|
+
// 1. Fetch the CSL bulk JSON file. Use GET: a HEAD request returns 404.
|
|
24
|
+
const res = await fetch('https://data.trade.gov/downloadable_consolidated_screening_list/v1/consolidated.json');
|
|
25
|
+
if (!res.ok) throw new Error(`CSL download failed: HTTP ${res.status}`);
|
|
26
|
+
const bytes = new Uint8Array(await res.arrayBuffer());
|
|
27
|
+
|
|
28
|
+
// 2. Validate it and build the snapshot. Pass last run's counts to refuse a truncated file
|
|
29
|
+
// or one that has lost a list.
|
|
30
|
+
const snapshot = await loadSnapshot(bytes, {
|
|
31
|
+
previousEntryCount: last?.entryCount,
|
|
32
|
+
previousEntriesBySource: last?.entriesBySource,
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
// 3. Turn your records into parties. `include` is required: you choose who is in scope.
|
|
36
|
+
const parties = fromAutotaskCompanies(companies, { include: (c) => c.id !== 0 });
|
|
37
|
+
|
|
38
|
+
// 4. Screen, then apply the rulings reviewers have already made.
|
|
39
|
+
const result = applyDecisions(screen(parties, snapshot), savedDecisions);
|
|
40
|
+
|
|
41
|
+
// 5. Write the evidence.
|
|
42
|
+
const csv = toCsv(result, { runDate: '2026-01-31' });
|
|
43
|
+
// Store result.snapshot (the snapshot identity) as `last` for the next run.
|
|
44
|
+
|
|
45
|
+
// 6. Branch on the outcome.
|
|
46
|
+
console.log(result.counts); // { parties, screened, unscreenable, clear, flagged }
|
|
47
|
+
for (const r of result.parties) {
|
|
48
|
+
if (partyOutcome(r) === 'flagged-new') console.log(r.party.name, r.status === 'flagged' && r.matches);
|
|
49
|
+
}
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`loadSnapshot` throws a `CslSnapshotError` whose `kind` says why: `parse` (not JSON), `shape` (missing `results` or `search_performed_at`, a malformed entry, or `alt_names` or `programs` that is not an array of strings), `total-mismatch`, `sharp-drop` (the entry count fell more than `maxDropRatio`, default 0.1, below `previousEntryCount`) or `source-drop` (a list in `previousEntriesBySource` is missing, or has fewer than half its previous entries). The snapshot identity carries `entriesBySource`, the entry count per list, to pass as `previousEntriesBySource` next time.
|
|
53
|
+
|
|
54
|
+
`partyOutcome` returns `clear`, `unscreenable`, `flagged-new`, `flagged-escalated` or `flagged-cleared`; a party is `flagged-new` while any of its matches still needs review. `toMarkdown(result, { runDate, title, expected })` writes a report. Pass `expected` (the number of parties you meant to screen) and the report starts with an `INCOMPLETE` line if the run has a different count. Pass `sourceRecords` (the number of records you were given before choosing who is in scope) and the header prints `Source records` and, when some were left out, `Excluded by scope`. Pass `preamble` (an array of strings) to print your own lines, such as where the parties and the list came from, as list items under the title. They are escaped like names.
|
|
55
|
+
|
|
56
|
+
To record a ruling on a flag, call `makeDecision(result, { partyId, entryKey, decision, reason, reviewer, decidedAt, runId })`. `decision` is `'cleared'` or `'escalated'`; `reason` and `reviewer` must not be blank. `runId` is optional: pass your identifier for the run whose result the reviewer read, and it is copied into the ruling. Store the returned object and pass it to `applyDecisions` on later runs. When you read stored rulings back from a file or database, run them through `parseDecisions(value)` first: it throws a `TypeError` naming the index and field of the first malformed record. `applyDecisions` also ignores any record whose `decision` is not `cleared` or `escalated`, or whose `reason` is blank, or whose `check` is not `'csl'`.
|
|
57
|
+
|
|
58
|
+
## Adapters
|
|
59
|
+
|
|
60
|
+
Each adapter is a separate entry point and returns `Party[]` (`{ id, name, source, country? }`). You can also build parties by hand.
|
|
61
|
+
|
|
62
|
+
- `@dszp/csl-screen/autotask`: `fromAutotaskCompanies(records, { include, source?, country? })`. The `source` defaults to `'autotask'`.
|
|
63
|
+
- `@dszp/csl-screen/onebill`: `fromOneBillSubscribers(records, { source?, country? })`. The `source` defaults to `'onebill'`. The name is `companyName`, or `accountName` if that is empty. The country is the first address country that is a two-letter code; any other value (such as `United States`) is dropped.
|
|
64
|
+
|
|
65
|
+
A party's `country` only annotates a match, and flags a mismatch with the entry's countries. It never removes a match.
|
|
66
|
+
|
|
67
|
+
## How matching works
|
|
68
|
+
|
|
69
|
+
Names are lowercased, stripped of diacritics (letters with no decomposition fold to ASCII: `ß` is `ss`, `æ` is `ae`, `ø` is `o`, `ł` is `l`), split on anything that is not a letter or digit, and stripped of legal-form words such as `llc`, `inc` and `gmbh` (configurable with `stopWords`). Runs of single-letter tokens join, so `L.L.C.` is `llc` and `D. K. Smith` is `dk smith`. The party is then compared with every listed name and alias that shares a token of at least three characters (`minTokenLength`). A match carries the first rule below that applies:
|
|
70
|
+
|
|
71
|
+
1. **exact**: the normalised names are equal, ignoring spaces ("Rosoboron Export" matches "Rosoboronexport").
|
|
72
|
+
2. **contains**: every token of the listed name appears in the party's name ("Hikvision USA" matches "Hikvision"). Runs of two or three adjacent party tokens also count joined without spaces, so a separator typed inside a listed word does not hide it ("Ros-neft Trading" matches "Rosneft", with reason `contains listed name ignoring spaces`). A listed name of one token counts only when the entry is not an Individual, Vessel or Aircraft, which keeps ship names and personal nicknames from flagging unrelated companies. It counts in any position ("Global Rosneft Services" matches "Rosneft"), except a word in `genericTokens` (default `DEFAULT_GENERIC_TOKENS`: `center`, `focus`, `technology`, `united`), which counts only as the party's first token. Those words are listed one-word names that, measured on real customer names, flagged only unrelated organisations using them as ordinary words. Pass your own `genericTokens` to replace the list; every word on it is a listed name that can be hidden behind another word.
|
|
73
|
+
3. **within**: every token of the party's name appears in the listed name, and the party has a token of four or more characters ("Huawei" matches "Huawei Technologies Co., Ltd.").
|
|
74
|
+
4. **similar**: Jaro-Winkler similarity of the normalised names, capped by how many tokens they share, at or above `threshold` (default 0.9).
|
|
75
|
+
|
|
76
|
+
Combining marks and invisible format characters (zero-width spaces, soft hyphens, bidi marks, variation selectors) are deleted before splitting, so one typed inside a word leaves the word whole.
|
|
77
|
+
|
|
78
|
+
A party name is reported unscreenable (`reason: 'unreadable-characters'`), for a person to screen, when after that fold it still has a letter outside the Latin alphabet, or a control character (other than tab and line breaks), a private-use character or an unassigned code point. Otherwise a look-alike letter from another script, such as a Cyrillic `о` in "Rоsneft", or an unreadable character would split the word and hide a listed match. `hasUnreadableCharacters(name)` runs the same check.
|
|
79
|
+
|
|
80
|
+
A name with no token of three or more characters (for example `Xu Li` or `D Company`) can only match by the exact rule. Such a party is still screened. The `within` rule needs a party token of at least four characters, or `minTokenLength` if that is larger.
|
|
81
|
+
|
|
82
|
+
Known gaps:
|
|
83
|
+
|
|
84
|
+
- A misspelling inside a single token ("Rosoboronexpot") shares no token with the listed name, so it is not found. Finding it needs a character n-gram index, which this library does not have.
|
|
85
|
+
- A listed one-word name in `genericTokens` is found only as the party's first token, so "Acme Center" does not match a listed "Center". Pass a shorter `genericTokens` list (or `[]`) to trade more flags for that coverage.
|
|
86
|
+
- A listed word split into four or more pieces by separators ("Ro s-ne ft Trading") is not joined back, unless the whole party name is that word.
|
|
87
|
+
|
|
88
|
+
## Decisions
|
|
89
|
+
|
|
90
|
+
A ruling is keyed by the party id and the entry key, `source|id`. The CSL `id` alone is not unique, because an entry that sits on two lists shares it.
|
|
91
|
+
|
|
92
|
+
A `cleared` or `escalated` ruling applies only while both of these are unchanged since the reviewer made it:
|
|
93
|
+
|
|
94
|
+
- the party's normalised name, and
|
|
95
|
+
- the fingerprint of the whole CSL entry (a hash of every field of the raw entry).
|
|
96
|
+
|
|
97
|
+
If the party is renamed, or the entry changes in any field, the match returns to `new` and needs a fresh review. The latest ruling for a party and entry wins, comparing the `decidedAt` instants. If two rulings have the same instant, `escalated` wins.
|
|
98
|
+
|
|
99
|
+
Persist rulings yourself (a file, a database, a Worker's storage). The library only creates and applies them. Rulings contain your party names and reviewers, so keep them out of version control for a public repository.
|
|
100
|
+
|
|
101
|
+
## Outputs
|
|
102
|
+
|
|
103
|
+
`toCsv` writes one row per match, and one row for each clear or unscreenable party, with CRLF line endings. A cell that starts with `=`, `+`, `-`, `@`, a tab or a line break gets a leading apostrophe so a spreadsheet does not run it as a formula. `toMarkdown` escapes names before it prints them.
|
|
104
|
+
|
|
105
|
+
The report header lists the parties in the run, how many were screened, and how many were clear, flagged and unscreenable, plus the CSL snapshot (time, entry count, SHA-256) and the matcher version, so a result can be tied to the exact list it was checked against.
|
|
106
|
+
|
|
107
|
+
## Identifying a list version
|
|
108
|
+
|
|
109
|
+
A snapshot carries two hashes. `sha256` covers the downloaded bytes, so it changes whenever the file is republished, including when only `search_performed_at` or a list's `last_imported` moved. `contentSha256` covers the entries alone: it hashes each entry's `key` and `fingerprint`, sorted by key, so it changes only when an entry is added, removed or edited. Compare `contentSha256` to decide whether the list changed. `contentHashOf(entries)` computes the same value from any array of `{ key, fingerprint }`.
|
|
110
|
+
|
|
111
|
+
## Development
|
|
112
|
+
|
|
113
|
+
```sh
|
|
114
|
+
pnpm install
|
|
115
|
+
pnpm test
|
|
116
|
+
pnpm typecheck
|
|
117
|
+
pnpm build
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## Changes
|
|
121
|
+
|
|
122
|
+
See [CHANGELOG.md](CHANGELOG.md).
|
|
123
|
+
|
|
124
|
+
## License
|
|
125
|
+
|
|
126
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Party } from '../model.js';
|
|
2
|
+
/** The fields of an Autotask Companies record the adapter reads; extra fields pass through untouched. */
|
|
3
|
+
export interface AutotaskCompanyLike {
|
|
4
|
+
id: number | string;
|
|
5
|
+
companyName?: string | null;
|
|
6
|
+
[k: string]: unknown;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Maps Autotask companies to screening parties. `include` is required so the caller states which
|
|
10
|
+
* companies are in scope (for example active customers) instead of screening everything by default.
|
|
11
|
+
*/
|
|
12
|
+
export declare function fromAutotaskCompanies<T extends AutotaskCompanyLike>(records: readonly T[], opts: {
|
|
13
|
+
include: (r: T) => boolean;
|
|
14
|
+
source?: string;
|
|
15
|
+
country?: (r: T) => string | undefined;
|
|
16
|
+
}): Party[];
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Maps Autotask companies to screening parties. `include` is required so the caller states which
|
|
3
|
+
* companies are in scope (for example active customers) instead of screening everything by default.
|
|
4
|
+
*/
|
|
5
|
+
export function fromAutotaskCompanies(records, opts) {
|
|
6
|
+
const source = opts.source ?? 'autotask';
|
|
7
|
+
return records.filter(opts.include).map((r) => {
|
|
8
|
+
const country = opts.country?.(r);
|
|
9
|
+
return { id: String(r.id), name: r.companyName ?? '', source, ...(country ? { country } : {}) };
|
|
10
|
+
});
|
|
11
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { Party } from '../model.js';
|
|
2
|
+
/** The fields of a OneBill subscriber record the adapter reads. */
|
|
3
|
+
export interface OneBillSubscriberLike {
|
|
4
|
+
accountNumber: string;
|
|
5
|
+
companyName?: string;
|
|
6
|
+
accountName?: string;
|
|
7
|
+
address?: {
|
|
8
|
+
country?: string;
|
|
9
|
+
}[] | {
|
|
10
|
+
country?: string;
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
/** Maps OneBill subscribers to screening parties. Only a two-letter country code is kept; other values are dropped. */
|
|
14
|
+
export declare function fromOneBillSubscribers<T extends OneBillSubscriberLike>(records: readonly T[], opts?: {
|
|
15
|
+
source?: string;
|
|
16
|
+
country?: (r: T) => string | undefined;
|
|
17
|
+
}): Party[];
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
const ISO2 = /^[A-Za-z]{2}$/;
|
|
2
|
+
function defaultCountry(r) {
|
|
3
|
+
const addrs = Array.isArray(r.address) ? r.address : r.address ? [r.address] : [];
|
|
4
|
+
const hit = addrs.find((a) => typeof a?.country === 'string' && ISO2.test(a.country));
|
|
5
|
+
return hit?.country?.toUpperCase();
|
|
6
|
+
}
|
|
7
|
+
/** Maps OneBill subscribers to screening parties. Only a two-letter country code is kept; other values are dropped. */
|
|
8
|
+
export function fromOneBillSubscribers(records, opts = {}) {
|
|
9
|
+
const source = opts.source ?? 'onebill';
|
|
10
|
+
return records.map((r) => {
|
|
11
|
+
const country = opts.country ? opts.country(r) : defaultCountry(r);
|
|
12
|
+
return { id: r.accountNumber, name: r.companyName?.trim() || r.accountName || '', source, ...(country ? { country } : {}) };
|
|
13
|
+
});
|
|
14
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** JSON.stringify with object keys sorted recursively; arrays keep their order. */
|
|
2
|
+
export function canonicalJson(value) {
|
|
3
|
+
return JSON.stringify(sortKeys(value));
|
|
4
|
+
}
|
|
5
|
+
function sortKeys(value) {
|
|
6
|
+
if (Array.isArray(value))
|
|
7
|
+
return value.map(sortKeys);
|
|
8
|
+
if (value !== null && typeof value === 'object') {
|
|
9
|
+
const src = value;
|
|
10
|
+
// Null prototype: a field named "__proto__" stays an own key instead of setting the prototype and vanishing.
|
|
11
|
+
const out = Object.create(null);
|
|
12
|
+
for (const k of Object.keys(src).sort())
|
|
13
|
+
out[k] = sortKeys(src[k]);
|
|
14
|
+
return out;
|
|
15
|
+
}
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
export async function sha256Hex(data) {
|
|
19
|
+
const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data;
|
|
20
|
+
const digest = await crypto.subtle.digest('SHA-256', bytes);
|
|
21
|
+
return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, '0')).join('');
|
|
22
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Decision, PartyResult, ScreenResult } from './model.js';
|
|
2
|
+
/** Unambiguous: both parts are JSON-encoded, so no separator inside an id can shift the boundary. */
|
|
3
|
+
export declare function decisionKey(partyId: string, entryKey: string): string;
|
|
4
|
+
export interface DecisionInput {
|
|
5
|
+
partyId: string;
|
|
6
|
+
entryKey: string;
|
|
7
|
+
decision: 'cleared' | 'escalated';
|
|
8
|
+
reason: string;
|
|
9
|
+
reviewer: string;
|
|
10
|
+
decidedAt: string;
|
|
11
|
+
/** Identifies the run whose result the reviewer read; copied into the Decision. */
|
|
12
|
+
runId?: string;
|
|
13
|
+
}
|
|
14
|
+
/** Records a ruling bound to the party name and CSL entry content the reviewer actually saw. */
|
|
15
|
+
export declare function makeDecision(result: ScreenResult, input: DecisionInput): Decision;
|
|
16
|
+
/**
|
|
17
|
+
* Returns a new result with each match's `state` and `decision` set from the latest decision for its
|
|
18
|
+
* (party, entry) key. A decision applies only if the party name and CSL entry are unchanged since it was
|
|
19
|
+
* made; otherwise the match is `new` again. Only the latest decision is considered, so a stale one is not
|
|
20
|
+
* bypassed by an older valid one. "Latest" compares parsed instants, so offsets and precision do not matter;
|
|
21
|
+
* a decision whose `decidedAt` does not parse is never latest and never applies. On an exact tie,
|
|
22
|
+
* `escalated` wins over `cleared`, whatever the array order.
|
|
23
|
+
*/
|
|
24
|
+
export declare function applyDecisions(result: ScreenResult, decisions: readonly Decision[]): ScreenResult;
|
|
25
|
+
/** Validates persisted decisions (for example a decisions file read from disk). Throws a TypeError naming the index and field. */
|
|
26
|
+
export declare function parseDecisions(value: unknown): Decision[];
|
|
27
|
+
export type PartyOutcome = 'clear' | 'unscreenable' | 'flagged-new' | 'flagged-escalated' | 'flagged-cleared';
|
|
28
|
+
export declare function partyOutcome(r: PartyResult): PartyOutcome;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/** Unambiguous: both parts are JSON-encoded, so no separator inside an id can shift the boundary. */
|
|
2
|
+
export function decisionKey(partyId, entryKey) {
|
|
3
|
+
return JSON.stringify([partyId, entryKey]);
|
|
4
|
+
}
|
|
5
|
+
const nonEmpty = (v) => typeof v === 'string' && v !== '';
|
|
6
|
+
/** Records a ruling bound to the party name and CSL entry content the reviewer actually saw. */
|
|
7
|
+
export function makeDecision(result, input) {
|
|
8
|
+
const reason = input.reason.trim();
|
|
9
|
+
const reviewer = input.reviewer.trim();
|
|
10
|
+
if (reason === '')
|
|
11
|
+
throw new Error('A decision needs a non-blank reason');
|
|
12
|
+
if (reviewer === '')
|
|
13
|
+
throw new Error('A decision needs a non-blank reviewer');
|
|
14
|
+
const decidedAtMs = Date.parse(input.decidedAt);
|
|
15
|
+
if (!Number.isFinite(decidedAtMs))
|
|
16
|
+
throw new Error(`decidedAt is not a valid timestamp: ${JSON.stringify(input.decidedAt)}`);
|
|
17
|
+
if (input.runId !== undefined && !nonEmpty(input.runId))
|
|
18
|
+
throw new Error('runId, when given, must be a non-empty string');
|
|
19
|
+
const party = result.parties.find((r) => r.party.id === input.partyId);
|
|
20
|
+
const match = party?.status === 'flagged' ? party.matches.find((m) => m.entryKey === input.entryKey) : undefined;
|
|
21
|
+
if (!party || !match)
|
|
22
|
+
throw new Error(`Party ${input.partyId} is not flagged against entry ${input.entryKey}`);
|
|
23
|
+
return {
|
|
24
|
+
check: 'csl',
|
|
25
|
+
partyId: input.partyId,
|
|
26
|
+
entryKey: input.entryKey,
|
|
27
|
+
partyNameKey: party.partyNameKey,
|
|
28
|
+
entryFingerprint: match.entryFingerprint,
|
|
29
|
+
decision: input.decision,
|
|
30
|
+
reason,
|
|
31
|
+
reviewer,
|
|
32
|
+
decidedAt: new Date(decidedAtMs).toISOString(),
|
|
33
|
+
...(input.runId !== undefined ? { runId: input.runId } : {}),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Returns a new result with each match's `state` and `decision` set from the latest decision for its
|
|
38
|
+
* (party, entry) key. A decision applies only if the party name and CSL entry are unchanged since it was
|
|
39
|
+
* made; otherwise the match is `new` again. Only the latest decision is considered, so a stale one is not
|
|
40
|
+
* bypassed by an older valid one. "Latest" compares parsed instants, so offsets and precision do not matter;
|
|
41
|
+
* a decision whose `decidedAt` does not parse is never latest and never applies. On an exact tie,
|
|
42
|
+
* `escalated` wins over `cleared`, whatever the array order.
|
|
43
|
+
*/
|
|
44
|
+
export function applyDecisions(result, decisions) {
|
|
45
|
+
const latest = new Map();
|
|
46
|
+
for (const d of decisions) {
|
|
47
|
+
// Defensive: a record that is not a real ruling must never clear or escalate anything.
|
|
48
|
+
if (d.check !== 'csl')
|
|
49
|
+
continue;
|
|
50
|
+
if (d.decision !== 'cleared' && d.decision !== 'escalated')
|
|
51
|
+
continue;
|
|
52
|
+
if (typeof d.reason !== 'string' || d.reason.trim() === '')
|
|
53
|
+
continue;
|
|
54
|
+
const at = Date.parse(d.decidedAt);
|
|
55
|
+
if (!Number.isFinite(at))
|
|
56
|
+
continue;
|
|
57
|
+
const key = decisionKey(d.partyId, d.entryKey);
|
|
58
|
+
const cur = latest.get(key);
|
|
59
|
+
if (!cur || at > cur.at || (at === cur.at && d.decision === 'escalated'))
|
|
60
|
+
latest.set(key, { d, at });
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
...result,
|
|
64
|
+
parties: result.parties.map((r) => {
|
|
65
|
+
if (r.status !== 'flagged')
|
|
66
|
+
return r;
|
|
67
|
+
return {
|
|
68
|
+
...r,
|
|
69
|
+
matches: r.matches.map((m) => {
|
|
70
|
+
const { decision: _prior, ...rest } = m;
|
|
71
|
+
const d = latest.get(decisionKey(r.party.id, m.entryKey))?.d;
|
|
72
|
+
if (!d || d.partyNameKey !== r.partyNameKey || d.entryFingerprint !== m.entryFingerprint) {
|
|
73
|
+
return { ...rest, state: 'new' };
|
|
74
|
+
}
|
|
75
|
+
return { ...rest, state: d.decision, decision: d };
|
|
76
|
+
}),
|
|
77
|
+
};
|
|
78
|
+
}),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
const nonBlank = (v) => typeof v === 'string' && v.trim() !== '';
|
|
82
|
+
/** Validates persisted decisions (for example a decisions file read from disk). Throws a TypeError naming the index and field. */
|
|
83
|
+
export function parseDecisions(value) {
|
|
84
|
+
if (!Array.isArray(value))
|
|
85
|
+
throw new TypeError('decisions must be an array');
|
|
86
|
+
value.forEach((d, i) => {
|
|
87
|
+
if (d === null || typeof d !== 'object' || Array.isArray(d))
|
|
88
|
+
throw new TypeError(`decisions[${i}] is not an object`);
|
|
89
|
+
const r = d;
|
|
90
|
+
if (r.check !== 'csl')
|
|
91
|
+
throw new TypeError(`decisions[${i}].check must be "csl"`);
|
|
92
|
+
for (const f of ['partyId', 'entryKey', 'partyNameKey', 'entryFingerprint']) {
|
|
93
|
+
if (!nonEmpty(r[f]))
|
|
94
|
+
throw new TypeError(`decisions[${i}].${f} must be a non-empty string`);
|
|
95
|
+
}
|
|
96
|
+
if (r.decision !== 'cleared' && r.decision !== 'escalated')
|
|
97
|
+
throw new TypeError(`decisions[${i}].decision must be "cleared" or "escalated"`);
|
|
98
|
+
if (!nonBlank(r.reason))
|
|
99
|
+
throw new TypeError(`decisions[${i}].reason must not be blank`);
|
|
100
|
+
if (!nonBlank(r.reviewer))
|
|
101
|
+
throw new TypeError(`decisions[${i}].reviewer must not be blank`);
|
|
102
|
+
if (typeof r.decidedAt !== 'string' || !Number.isFinite(Date.parse(r.decidedAt))) {
|
|
103
|
+
throw new TypeError(`decisions[${i}].decidedAt must be a valid timestamp`);
|
|
104
|
+
}
|
|
105
|
+
if (r.runId !== undefined && !nonEmpty(r.runId))
|
|
106
|
+
throw new TypeError(`decisions[${i}].runId must be a non-empty string when present`);
|
|
107
|
+
});
|
|
108
|
+
return value;
|
|
109
|
+
}
|
|
110
|
+
export function partyOutcome(r) {
|
|
111
|
+
if (r.status !== 'flagged')
|
|
112
|
+
return r.status;
|
|
113
|
+
if (r.matches.some((m) => m.state === 'new'))
|
|
114
|
+
return 'flagged-new';
|
|
115
|
+
if (r.matches.some((m) => m.state === 'escalated'))
|
|
116
|
+
return 'flagged-escalated';
|
|
117
|
+
return 'flagged-cleared';
|
|
118
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { DEFAULT_STOP_WORDS, hasUnreadableCharacters, nameTokens, normalizeName } from './normalize.js';
|
|
2
|
+
export { jaroWinkler } from './jaroWinkler.js';
|
|
3
|
+
export { canonicalJson, sha256Hex } from './canonical.js';
|
|
4
|
+
export { loadSnapshot, contentHashOf, CslSnapshotError } from './snapshot.js';
|
|
5
|
+
export type { Snapshot, SlimEntry, SnapshotIdentity, LoadSnapshotOptions } from './snapshot.js';
|
|
6
|
+
export { DEFAULT_GENERIC_TOKENS, screen } from './screen.js';
|
|
7
|
+
export { MATCHER_VERSION } from './model.js';
|
|
8
|
+
export type { Party, MatchState, Match, Decision, PartyResult, ScreenOptions, ScreenResult } from './model.js';
|
|
9
|
+
export { decisionKey, makeDecision, applyDecisions, partyOutcome, parseDecisions } from './decisions.js';
|
|
10
|
+
export type { DecisionInput, PartyOutcome } from './decisions.js';
|
|
11
|
+
export { toCsv, toMarkdown } from './render.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { DEFAULT_STOP_WORDS, hasUnreadableCharacters, nameTokens, normalizeName } from './normalize.js';
|
|
2
|
+
export { jaroWinkler } from './jaroWinkler.js';
|
|
3
|
+
export { canonicalJson, sha256Hex } from './canonical.js';
|
|
4
|
+
export { loadSnapshot, contentHashOf, CslSnapshotError } from './snapshot.js';
|
|
5
|
+
export { DEFAULT_GENERIC_TOKENS, screen } from './screen.js';
|
|
6
|
+
export { MATCHER_VERSION } from './model.js';
|
|
7
|
+
export { decisionKey, makeDecision, applyDecisions, partyOutcome, parseDecisions } from './decisions.js';
|
|
8
|
+
export { toCsv, toMarkdown } from './render.js';
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/** Jaro-Winkler similarity in [0, 1]: match window floor(max/2) - 1, common prefix capped at 4, scale 0.1. */
|
|
2
|
+
export function jaroWinkler(a, b) {
|
|
3
|
+
if (a === b)
|
|
4
|
+
return a.length === 0 ? 0 : 1;
|
|
5
|
+
if (a.length === 0 || b.length === 0)
|
|
6
|
+
return 0;
|
|
7
|
+
const window = Math.max(0, Math.floor(Math.max(a.length, b.length) / 2) - 1);
|
|
8
|
+
const aMatched = new Array(a.length).fill(false);
|
|
9
|
+
const bMatched = new Array(b.length).fill(false);
|
|
10
|
+
let matches = 0;
|
|
11
|
+
for (let i = 0; i < a.length; i++) {
|
|
12
|
+
const lo = Math.max(0, i - window);
|
|
13
|
+
const hi = Math.min(b.length - 1, i + window);
|
|
14
|
+
for (let j = lo; j <= hi; j++) {
|
|
15
|
+
if (!bMatched[j] && a[i] === b[j]) {
|
|
16
|
+
aMatched[i] = true;
|
|
17
|
+
bMatched[j] = true;
|
|
18
|
+
matches++;
|
|
19
|
+
break;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
if (matches === 0)
|
|
24
|
+
return 0;
|
|
25
|
+
let transpositions = 0;
|
|
26
|
+
let k = 0;
|
|
27
|
+
for (let i = 0; i < a.length; i++) {
|
|
28
|
+
if (!aMatched[i])
|
|
29
|
+
continue;
|
|
30
|
+
while (!bMatched[k])
|
|
31
|
+
k++;
|
|
32
|
+
if (a[i] !== b[k])
|
|
33
|
+
transpositions++;
|
|
34
|
+
k++;
|
|
35
|
+
}
|
|
36
|
+
const jaro = (matches / a.length + matches / b.length + (matches - transpositions / 2) / matches) / 3;
|
|
37
|
+
let prefix = 0;
|
|
38
|
+
while (prefix < 4 && prefix < a.length && prefix < b.length && a[prefix] === b[prefix])
|
|
39
|
+
prefix++;
|
|
40
|
+
return jaro + prefix * 0.1 * (1 - jaro);
|
|
41
|
+
}
|
package/dist/model.d.ts
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { SnapshotIdentity } from './snapshot.js';
|
|
2
|
+
export interface Party {
|
|
3
|
+
id: string;
|
|
4
|
+
name: string;
|
|
5
|
+
/** ISO 3166-1 alpha-2 when known. Only annotates matches; never suppresses one. */
|
|
6
|
+
country?: string;
|
|
7
|
+
source: string;
|
|
8
|
+
}
|
|
9
|
+
export type MatchState = 'new' | 'cleared' | 'escalated';
|
|
10
|
+
/** A reviewer's recorded ruling on one party/entry pair, bound to the name and entry content it was made against. */
|
|
11
|
+
export interface Decision {
|
|
12
|
+
check: 'csl';
|
|
13
|
+
partyId: string;
|
|
14
|
+
entryKey: string;
|
|
15
|
+
partyNameKey: string;
|
|
16
|
+
entryFingerprint: string;
|
|
17
|
+
decision: 'cleared' | 'escalated';
|
|
18
|
+
reason: string;
|
|
19
|
+
reviewer: string;
|
|
20
|
+
decidedAt: string;
|
|
21
|
+
/** The screening run whose result the reviewer read when making this ruling. Optional; the caller names it. */
|
|
22
|
+
runId?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface Match {
|
|
25
|
+
/** `${source}|${id}`; unique within a snapshot. */
|
|
26
|
+
entryKey: string;
|
|
27
|
+
/** The raw CSL id, which is not unique on its own. */
|
|
28
|
+
entryId: string;
|
|
29
|
+
entryFingerprint: string;
|
|
30
|
+
matchedName: string;
|
|
31
|
+
listSource: string;
|
|
32
|
+
programs: string[];
|
|
33
|
+
entryType: string | null;
|
|
34
|
+
entryCountries: string[];
|
|
35
|
+
countryMismatch: boolean;
|
|
36
|
+
rule: 'exact' | 'contains' | 'within' | 'similar';
|
|
37
|
+
score: number;
|
|
38
|
+
reason: string;
|
|
39
|
+
state: MatchState;
|
|
40
|
+
decision?: Decision;
|
|
41
|
+
}
|
|
42
|
+
export type PartyResult = {
|
|
43
|
+
party: Party;
|
|
44
|
+
partyNameKey: string;
|
|
45
|
+
} & ({
|
|
46
|
+
status: 'clear';
|
|
47
|
+
} | {
|
|
48
|
+
status: 'unscreenable';
|
|
49
|
+
reason: 'empty-name' | 'unreadable-characters';
|
|
50
|
+
} | {
|
|
51
|
+
status: 'flagged';
|
|
52
|
+
matches: Match[];
|
|
53
|
+
});
|
|
54
|
+
export interface ScreenOptions {
|
|
55
|
+
/** Minimum similarity for the `similar` rule. Default 0.9. */
|
|
56
|
+
threshold?: number;
|
|
57
|
+
/** Replaces DEFAULT_STOP_WORDS. Entries are normalised like names, so 'Inc' or 'L.L.C.' work. */
|
|
58
|
+
stopWords?: readonly string[];
|
|
59
|
+
/** Shortest token that is indexed and used to find candidates. Default 3. */
|
|
60
|
+
minTokenLength?: number;
|
|
61
|
+
/**
|
|
62
|
+
* Replaces DEFAULT_GENERIC_TOKENS: one-word listed entity names that match a party only as its first
|
|
63
|
+
* token. Normalised like names.
|
|
64
|
+
*/
|
|
65
|
+
genericTokens?: readonly string[];
|
|
66
|
+
}
|
|
67
|
+
export declare const MATCHER_VERSION = "2";
|
|
68
|
+
export interface ScreenResult {
|
|
69
|
+
check: 'csl';
|
|
70
|
+
matcher: {
|
|
71
|
+
version: string;
|
|
72
|
+
threshold: number;
|
|
73
|
+
minTokenLength: number;
|
|
74
|
+
stopWords: string[];
|
|
75
|
+
genericTokens: string[];
|
|
76
|
+
};
|
|
77
|
+
snapshot: SnapshotIdentity;
|
|
78
|
+
counts: {
|
|
79
|
+
parties: number;
|
|
80
|
+
screened: number;
|
|
81
|
+
unscreenable: number;
|
|
82
|
+
clear: number;
|
|
83
|
+
flagged: number;
|
|
84
|
+
};
|
|
85
|
+
parties: PartyResult[];
|
|
86
|
+
}
|
package/dist/model.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const MATCHER_VERSION = '2';
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** Legal-form suffixes and filler words that carry no identity for name matching. */
|
|
2
|
+
export declare const DEFAULT_STOP_WORDS: readonly string[];
|
|
3
|
+
/**
|
|
4
|
+
* True when, after the fold, the name still has a letter outside A-Z/a-z, or a control (other than tab,
|
|
5
|
+
* LF and CR), private-use or unassigned character. Such a name cannot be tokenised safely: a look-alike
|
|
6
|
+
* letter from another script (Cyrillic о in "Rоsneft") or an unreadable character would split the word
|
|
7
|
+
* and hide a listed entity, so the caller must screen it by hand.
|
|
8
|
+
*/
|
|
9
|
+
export declare function hasUnreadableCharacters(name: string): boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Lowercase ASCII tokens of a name: diacritics stripped, `&` read as "and",
|
|
12
|
+
* everything outside [a-z0-9] treated as a separator, runs of single-character
|
|
13
|
+
* tokens joined ("L.L.C." is `llc`, "D. K. Smith" is `dk smith`), stop words dropped.
|
|
14
|
+
* Letters outside the Latin alphabet and other unreadable characters act as separators; see `hasUnreadableCharacters`.
|
|
15
|
+
*/
|
|
16
|
+
export declare function nameTokens(name: string, stopWords: ReadonlySet<string>): string[];
|
|
17
|
+
export declare function normalizeName(name: string, stopWords: ReadonlySet<string>): string;
|