@crewhaus/tool-kyc 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,330 @@
1
+ /**
2
+ * The bridge to `@crewhaus/tool-text`, which owns the matching.
3
+ *
4
+ * Jaro-Winkler, the token comparison and the Unicode hygiene are NOT
5
+ * re-implemented here. A second name matcher in this package would be a second
6
+ * answer to "how close are these two names", and the two would disagree
7
+ * exactly where it costs most — a screening tool that scores 0.86 while the
8
+ * text tool scores 0.84 puts a person on or off a review queue depending on
9
+ * which module was asked.
10
+ *
11
+ * They arrive as TOOLS rather than as functions because `tool-text` publishes
12
+ * its matcher as `FuzzyMatch` and its normalizer as `NormalizeText`; its module
13
+ * surface is `RegisteredTool`s plus the diff parser. So this file calls those
14
+ * tools' `execute` and parses the JSON they return, the way
15
+ * `@crewhaus/tool-token` and `@crewhaus/tool-chaincall` call `tool-onchain`'s.
16
+ * That makes these wrappers `async` for work that is pure — a real cost, paid
17
+ * once per name rather than once per character pair, and cheaper than a second
18
+ * Jaro-Winkler.
19
+ *
20
+ * What this file DOES own is the part that is specific to names on a sanctions
21
+ * list: folding diacritics, dropping legal suffixes, and weighting tokens. That
22
+ * is scoring policy, not a matcher.
23
+ */
24
+ import { fuzzyMatch, normalizeText } from "@crewhaus/tool-text";
25
+ /** `FuzzyMatch`'s published schema caps candidates per call and hits per call. */
26
+ const MAX_CANDIDATES_PER_CALL = 10_000;
27
+ export const MAX_HITS_PER_CALL = 100;
28
+ /**
29
+ * `ToolExecuteResult` is a union of a string and a structured content block.
30
+ * Both of these tools return the string arm; anything else means the tool
31
+ * changed shape underneath us, and saying so beats a cast that decodes
32
+ * garbage.
33
+ */
34
+ async function textResult(result, tool) {
35
+ const value = await result;
36
+ if (typeof value !== "string")
37
+ throw new Error(`${tool} did not return a string result`);
38
+ return value;
39
+ }
40
+ /**
41
+ * Legal forms, stripped from an ORGANISATION's name before comparison.
42
+ *
43
+ * "Acme Ltd" and "ACME LIMITED" are the same company, and a token-weighted
44
+ * score that counts "ltd" as a shared token inflates every comparison between
45
+ * two British companies. Dropped suffixes are recorded rather than discarded,
46
+ * because "which tokens did you actually compare" is half of a match basis.
47
+ *
48
+ * The list is deliberately short and covers the forms that are genuinely
49
+ * interchangeable with nothing. It is NOT applied to people: "Inc" is not a
50
+ * legal form in "Ali Inc." when the subject is a person, and more to the point
51
+ * a person's name can legitimately be any word at all.
52
+ */
53
+ export const LEGAL_SUFFIXES = new Set([
54
+ "ab",
55
+ "ag",
56
+ "ao",
57
+ "aps",
58
+ "as",
59
+ "bv",
60
+ "cia",
61
+ "co",
62
+ "company",
63
+ "corp",
64
+ "corporation",
65
+ "eood",
66
+ "gmbh",
67
+ "inc",
68
+ "incorporated",
69
+ "kft",
70
+ "kk",
71
+ "limited",
72
+ "llc",
73
+ "llp",
74
+ "lp",
75
+ "ltd",
76
+ "ltda",
77
+ "nv",
78
+ "ooo",
79
+ "oy",
80
+ "oyj",
81
+ "pao",
82
+ "plc",
83
+ "pt",
84
+ "pte",
85
+ "pty",
86
+ "sa",
87
+ "sarl",
88
+ "sas",
89
+ "spa",
90
+ "srl",
91
+ "zao",
92
+ ]);
93
+ /** Combining marks, i.e. what NFD peels off a letter. */
94
+ const COMBINING_MARKS = /\p{M}+/gu;
95
+ /**
96
+ * The Latin letters NFD cannot help with.
97
+ *
98
+ * NFD + strip-marks folds é to e because the accent is a separate combining
99
+ * character. It does nothing for a letter whose modification is part of the
100
+ * glyph: ø, đ, ł, æ, ß and þ are single code points with no
101
+ * decomposition, so "Ødegård" would keep its ø and never match the "Odegard"
102
+ * spelling the same list publishes two lines further down. Each mapping here is
103
+ * the transliteration the publishers themselves use.
104
+ */
105
+ const UNFOLDABLE_LETTERS = Object.freeze({
106
+ "\u00f8": "o",
107
+ "\u0153": "oe",
108
+ "\u00e6": "ae",
109
+ "\u00df": "ss",
110
+ "\u00fe": "th",
111
+ "\u00f0": "d",
112
+ "\u0111": "d",
113
+ "\u0142": "l",
114
+ "\u0131": "i",
115
+ "\u0127": "h",
116
+ "\u0167": "t",
117
+ "\u014b": "n",
118
+ "\u0259": "e",
119
+ });
120
+ function foldUnfoldable(text) {
121
+ let out = "";
122
+ for (const ch of text)
123
+ out += UNFOLDABLE_LETTERS[ch] ?? ch;
124
+ return out;
125
+ }
126
+ /** Everything that is not a letter or a digit becomes a token break. */
127
+ const NON_ALPHANUMERIC = /[^\p{L}\p{N}]+/gu;
128
+ /**
129
+ * Canonicalize one name for comparison.
130
+ *
131
+ * Step one is `tool-text`'s `NormalizeText` (NFKC, invisibles stripped,
132
+ * lowercased). Step two is the part it deliberately does not do: folding
133
+ * diacritics. That asymmetry is correct in both places — for a diff, "resume"
134
+ * and "résumé" are two different files and folding them would hide a real
135
+ * change; for a name they are one person, and the sanctions lists themselves
136
+ * publish both spellings.
137
+ */
138
+ export async function canonicalizeName(raw, options = {}) {
139
+ const normalized = JSON.parse(await textResult(normalizeText.execute({
140
+ text: raw,
141
+ stripInvisible: true,
142
+ unicode: "NFKC",
143
+ lowercase: true,
144
+ trimTrailingWhitespace: true,
145
+ }), "NormalizeText"));
146
+ const folded = foldUnfoldable(normalized.text.normalize("NFD").replace(COMBINING_MARKS, ""))
147
+ .replace(NON_ALPHANUMERIC, " ")
148
+ .trim();
149
+ const all = folded.split(" ").filter((t) => t.length > 0);
150
+ const dropped = [];
151
+ let tokens = all;
152
+ if (options.dropLegalSuffixes === true) {
153
+ const kept = all.filter((t) => {
154
+ if (!LEGAL_SUFFIXES.has(t))
155
+ return true;
156
+ dropped.push(t);
157
+ return false;
158
+ });
159
+ // A name that is NOTHING but legal forms keeps them. "Limited" on its own
160
+ // is a name, and dropping every token leaves an empty string that matches
161
+ // everything at trigram distance.
162
+ tokens = kept.length > 0 ? kept : all;
163
+ if (kept.length === 0)
164
+ dropped.length = 0;
165
+ }
166
+ return {
167
+ input: raw,
168
+ canonical: tokens.join(" "),
169
+ tokens,
170
+ droppedSuffixes: dropped,
171
+ // `NormalizeText` reports whether it changed anything at all, which for
172
+ // these options means invisibles, a Unicode form or case. Case alone is
173
+ // not interesting, so the raw text is re-checked for the characters that
174
+ // are.
175
+ hadInvisibleCharacters: hasInvisibleCharacters(raw),
176
+ };
177
+ }
178
+ /**
179
+ * Zero-width, bidirectional-control and soft-hyphen characters, by code point.
180
+ *
181
+ * Written as code points rather than as a character-class RANGE because a
182
+ * range spanning the zero-width joiner can split a joined sequence — the same
183
+ * reason `tool-text`'s own normalizer builds its pattern this way. They matter
184
+ * here for a sharper reason than tidiness: a bidi override inside a supplied
185
+ * name is a way to make that name miss a list it should hit.
186
+ */
187
+ const INVISIBLE_CODE_POINTS = new Set([
188
+ 0x00ad, 0x200b, 0x200c, 0x200d, 0x200e, 0x200f, 0x2028, 0x2029, 0x202a, 0x202b, 0x202c, 0x202d,
189
+ 0x202e, 0x2060, 0x2066, 0x2067, 0x2068, 0x2069, 0xfeff,
190
+ ]);
191
+ export function hasInvisibleCharacters(text) {
192
+ for (const ch of text) {
193
+ if (INVISIBLE_CODE_POINTS.has(ch.codePointAt(0) ?? 0))
194
+ return true;
195
+ }
196
+ return false;
197
+ }
198
+ /**
199
+ * Rank `candidates` against `query` through `tool-text`'s `FuzzyMatch`.
200
+ *
201
+ * Candidates are chunked to the size that tool's own schema permits. The
202
+ * schema is not enforced by `execute`, so nothing would stop a 200 000-name
203
+ * list going through in one call — but a published limit that callers honour
204
+ * only when something checks is not a limit, and chunking costs one extra
205
+ * function call per ten thousand names.
206
+ */
207
+ export async function rankNamesCapped(query, candidates, minScore, limit, method = "jaro") {
208
+ const perCall = Math.min(limit, MAX_HITS_PER_CALL);
209
+ const hits = [];
210
+ let capped = false;
211
+ for (let start = 0; start < candidates.length; start += MAX_CANDIDATES_PER_CALL) {
212
+ const chunk = candidates.slice(start, start + MAX_CANDIDATES_PER_CALL);
213
+ if (chunk.length === 0)
214
+ break;
215
+ const out = JSON.parse(await textResult(fuzzyMatch.execute({ query, candidates: chunk, method, minScore, limit: perCall }), "FuzzyMatch"));
216
+ if (out.hits.length >= perCall)
217
+ capped = true;
218
+ for (const hit of out.hits)
219
+ hits.push({ ...hit, index: start + hit.index });
220
+ }
221
+ // Ties break on the ORIGINAL index so two runs over the same list produce
222
+ // the same order. A shortlist whose order depends on chunk boundaries is a
223
+ // diff that changes for no reason.
224
+ hits.sort((a, b) => b.score - a.score || a.index - b.index);
225
+ return { hits: hits.slice(0, limit), capped };
226
+ }
227
+ /** `rankNamesCapped` for the callers that ask for one hit and cannot truncate. */
228
+ export async function rankNames(query, candidates, minScore, limit, method = "jaro") {
229
+ return [...(await rankNamesCapped(query, candidates, minScore, limit, method)).hits];
230
+ }
231
+ /**
232
+ * Every candidate's score against `query`, in the candidates' own order.
233
+ *
234
+ * Chunked at the HITS cap rather than the candidates cap, so every chunk can
235
+ * return all of itself; asking for more hits than a call may give back is how
236
+ * a "score them all" helper silently returns zeros for the tail.
237
+ */
238
+ export async function scoreEach(query, candidates, method = "jaro") {
239
+ const scores = new Array(candidates.length).fill(0);
240
+ for (let start = 0; start < candidates.length; start += MAX_HITS_PER_CALL) {
241
+ const chunk = candidates.slice(start, start + MAX_HITS_PER_CALL);
242
+ if (chunk.length === 0)
243
+ break;
244
+ const out = JSON.parse(await textResult(fuzzyMatch.execute({
245
+ query,
246
+ candidates: chunk,
247
+ method,
248
+ minScore: 0,
249
+ limit: chunk.length,
250
+ }), "FuzzyMatch"));
251
+ for (const hit of out.hits)
252
+ scores[start + hit.index] = hit.score;
253
+ }
254
+ return scores;
255
+ }
256
+ /**
257
+ * Below this, two tokens are not the same word. 0.85 on Jaro-Winkler keeps
258
+ * mohamed/mohammed and abdulaziz/abdelaziz together while keeping hassan and
259
+ * hussein apart — which is the pair the naive threshold gets wrong in both
260
+ * directions.
261
+ */
262
+ export const TOKEN_ALIGNMENT_FLOOR = 0.85;
263
+ /**
264
+ * Align each subject token to its best partner in the entry's tokens, and
265
+ * weight the result by how rare each token is across the list.
266
+ *
267
+ * This is the correction to a flat token-set ratio. On a list where half the
268
+ * entries contain "mohammed", matching that token is worth almost nothing and
269
+ * matching a rare surname is worth almost everything; scoring them equally is
270
+ * what floods an operator with false positives on common given names while
271
+ * still missing the transliteration pairs.
272
+ */
273
+ export async function alignTokens(subjectTokens, entryTokens, weightOf) {
274
+ const alignments = [];
275
+ for (const token of subjectTokens) {
276
+ const weight = weightOf(token);
277
+ if (entryTokens.length === 0) {
278
+ alignments.push({ subjectToken: token, entryToken: null, score: 0, weight });
279
+ continue;
280
+ }
281
+ const [best] = await rankNames(token, entryTokens, 0, 1);
282
+ const aligned = best !== undefined && best.score >= TOKEN_ALIGNMENT_FLOOR;
283
+ alignments.push({
284
+ subjectToken: token,
285
+ entryToken: aligned ? (best?.candidate ?? null) : null,
286
+ score: aligned ? round6(best?.score ?? 0) : 0,
287
+ weight: round6(weight),
288
+ });
289
+ }
290
+ const totalWeight = alignments.reduce((sum, a) => sum + a.weight, 0);
291
+ const earned = alignments.reduce((sum, a) => sum + a.weight * a.score, 0);
292
+ return { alignments, score: totalWeight === 0 ? 0 : round6(earned / totalWeight) };
293
+ }
294
+ /** Six places is past any threshold anybody sets and keeps JSON stable. */
295
+ export function round6(value) {
296
+ return Number(value.toFixed(6));
297
+ }
298
+ /**
299
+ * Inverse document frequency over the list's own names.
300
+ *
301
+ * Computed from the lists being screened against, never from a fixed table:
302
+ * "bank" is a common token on a Russian-sector list and a rare one on a
303
+ * vessel list, and a weight that does not come from the corpus in front of it
304
+ * is a guess.
305
+ */
306
+ export function buildTokenWeights(documents) {
307
+ const documentFrequency = new Map();
308
+ for (const tokens of documents) {
309
+ for (const token of new Set(tokens)) {
310
+ documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
311
+ }
312
+ }
313
+ const total = Math.max(1, documents.length);
314
+ const weights = new Map();
315
+ for (const [token, frequency] of documentFrequency) {
316
+ weights.set(token, Math.log(1 + total / frequency));
317
+ }
318
+ return weights;
319
+ }
320
+ /**
321
+ * The weight for a token the list has never seen.
322
+ *
323
+ * `log(1 + N/1)` — the weight of a token that appears exactly once — because
324
+ * a subject token absent from the whole list is at least as distinctive as the
325
+ * rarest token in it. Treating it as weightless would let an unmatched rare
326
+ * surname cost nothing.
327
+ */
328
+ export function unseenTokenWeight(documentCount) {
329
+ return Math.log(1 + Math.max(1, documentCount));
330
+ }
@@ -0,0 +1,151 @@
1
+ /**
2
+ * The network posture for `@crewhaus/tool-kyc`.
3
+ *
4
+ * Two of this package's three tools read a public register over HTTPS, and
5
+ * every byte they send leaves through `getJson`. As in `@crewhaus/tool-registry`,
6
+ * no caller supplies a URL, a host or an origin: a caller supplies a VAT id,
7
+ * an LEI, a CIK or a ticker, the value is checked against its own grammar, and
8
+ * the URL is built from one of the constants below. There is no allow-list to
9
+ * widen.
10
+ *
11
+ * No request carries a credential. VIES, HMRC's VAT checker, GLEIF and SEC
12
+ * EDGAR are all open-access endpoints; there is nothing to put in an
13
+ * `Authorization` header, so there is nothing to leak at a redirect. If one of
14
+ * them ever answers 401 or 403, that is reported as *could-not-check* with the
15
+ * status in it, never as "not registered" — see `ALWAYS THREE OUTCOMES` in the
16
+ * README.
17
+ *
18
+ * The SSRF gate and the connect-time IP pin live in the default dialler, which
19
+ * is what `_setKycFetch` replaces, so no test in this package resolves a name
20
+ * or opens a socket.
21
+ */
22
+ import { CrewhausError } from "@crewhaus/errors";
23
+ /** A refusal by the origin rule, the redirect rule or the byte cap. */
24
+ export declare class KycNetworkError extends CrewhausError {
25
+ readonly name = "KycNetworkError";
26
+ constructor(message: string);
27
+ }
28
+ /**
29
+ * Every origin this package can dial, and the only ones.
30
+ *
31
+ * `www.sec.gov` is here as well as `data.sec.gov` because EDGAR's
32
+ * ticker→CIK map is published on the website host while the submissions
33
+ * documents are on the data host — one company lookup by ticker legitimately
34
+ * touches both.
35
+ */
36
+ export declare const ORIGINS: Readonly<{
37
+ /** VIES, the EU's proxy to the member states' own VAT systems. */
38
+ vies: "https://ec.europa.eu";
39
+ /** HMRC's VAT check-a-number API. VIES has not covered GB since 2021. */
40
+ hmrc: "https://api.service.hmrc.gov.uk";
41
+ /** GLEIF, the LEI issuer of record. */
42
+ gleif: "https://api.gleif.org";
43
+ /** EDGAR submissions. */
44
+ edgarData: "https://data.sec.gov";
45
+ /** EDGAR's ticker→CIK map. */
46
+ edgarFiles: "https://www.sec.gov";
47
+ }>;
48
+ /**
49
+ * SEC asks every automated reader for a User-Agent that identifies the caller
50
+ * and says where to complain, and answers 403 without one. The others do not
51
+ * care, so one header for all four keeps the request path identical.
52
+ */
53
+ export declare const USER_AGENT = "crewhaus-tool-kyc (+https://github.com/crewhaus/factory)";
54
+ export declare const DEFAULT_TIMEOUT_MS = 20000;
55
+ export declare const MAX_TIMEOUT_MS = 120000;
56
+ export declare const MAX_REDIRECTS = 3;
57
+ /**
58
+ * 8 MB. An EDGAR submissions document for a frequent filer genuinely runs to
59
+ * megabytes and the ticker map is over a megabyte on its own; this is memory,
60
+ * not context, since only a handful of fields is ever returned.
61
+ */
62
+ export declare const DEFAULT_MAX_BYTES: number;
63
+ /** Compact JSON — every byte returned is a byte in somebody's context window. */
64
+ export declare const json: (value: unknown) => string;
65
+ export type KycFetch = (req: Request) => Promise<Response>;
66
+ /**
67
+ * Test-only injection point, the convention every networked package here
68
+ * follows. `undefined` restores the guarded dialler; a suite that sets it must
69
+ * restore it, or the next file in the same bun process inherits the stub.
70
+ */
71
+ export declare function _setKycFetch(fn: KycFetch | undefined): void;
72
+ /**
73
+ * Why a read did not produce a document.
74
+ *
75
+ * These are kept apart because they are different answers about the SUBJECT,
76
+ * not just different plumbing failures. `notFound` is genuinely about the
77
+ * thing asked for — no LEI record with that id — and every other kind means
78
+ * *this tool could not find out*, which is the distinction the whole package
79
+ * exists to preserve.
80
+ */
81
+ export type FetchFailureKind = "notFound" | "rateLimited" | "unauthorized" | "status" | "tooLarge" | "malformed" | "refused" | "transport";
82
+ /** The kinds that mean "could not check", i.e. everything except `notFound`. */
83
+ export declare function isUnavailable(kind: FetchFailureKind): boolean;
84
+ export type JsonFetch = {
85
+ readonly ok: true;
86
+ readonly value: unknown;
87
+ readonly status: number;
88
+ readonly bytes: number;
89
+ } | {
90
+ readonly ok: false;
91
+ readonly kind: FetchFailureKind;
92
+ readonly status?: number;
93
+ readonly message: string;
94
+ /** Seconds the server asked us to wait, when it said. */
95
+ readonly retryAfter?: string;
96
+ };
97
+ export type GetJsonOptions = {
98
+ readonly signal?: AbortSignal;
99
+ readonly accept?: string;
100
+ readonly maxBytes?: number;
101
+ /**
102
+ * A JSON body, which makes the call a POST.
103
+ *
104
+ * Exactly one endpoint needs it: VIES issues a consultation number — the
105
+ * receipt an auditor asks for — only on the two-party call, and that call is
106
+ * a POST. Nothing here sends a body anywhere else, and a POST is never
107
+ * redirected (see below).
108
+ */
109
+ readonly postJson?: unknown;
110
+ };
111
+ /**
112
+ * GET a JSON document from one of the registers above.
113
+ *
114
+ * Failures are RETURNED rather than thrown: a caller reading four registers
115
+ * wants a row saying what went wrong with one of them, not to lose the other
116
+ * three. The one exception is a URL whose origin is not ours, which is a
117
+ * defect in this package and throws.
118
+ */
119
+ export declare function getJson(rawUrl: string, options?: GetJsonOptions): Promise<JsonFetch>;
120
+ /** Scheme, host and path — never the query, which can carry a subject's name. */
121
+ export declare function safeLabel(url: URL | string): string;
122
+ export type CappedBody = {
123
+ readonly text: string;
124
+ readonly bytes: number;
125
+ readonly truncated: boolean;
126
+ };
127
+ /**
128
+ * Drain a body with a hard byte cap, cancelling the stream the moment the cap
129
+ * is passed. The cap bounds MEMORY, not just what is kept.
130
+ */
131
+ export declare function readCapped(res: Response, maxBytes: number): Promise<CappedBody>;
132
+ export type Deadline = {
133
+ readonly signal: AbortSignal;
134
+ /** Clear the timer. Always call it, or the process keeps a handle alive. */
135
+ cancel(): void;
136
+ };
137
+ /**
138
+ * A deadline that also honours the runtime's own cancellation. Every tool here
139
+ * opens one before its first byte and cancels it in a `finally`: VIES in
140
+ * particular can hang for a member state that is half up, and a tool that
141
+ * hangs forever is worse than one that reports `unavailable`.
142
+ */
143
+ export declare function startDeadline(ms: number, outer?: AbortSignal): Deadline;
144
+ /**
145
+ * An instant, as an ISO-8601 string WITH an offset.
146
+ *
147
+ * Offset-less strings are refused: per ECMAScript they mean local time, so the
148
+ * same evidence record would carry a different day on two machines — and the
149
+ * day a screening ran is the part of the record that matters later.
150
+ */
151
+ export declare function parseInstant(value: string, field: string): number;