@dszp/csl-screen 0.1.0

package/src/adapters/onebill.ts

@@ -0,0 +1,29 @@
+import type { Party } from '../model.js';
+
+/** The fields of a OneBill subscriber record the adapter reads. */
+export interface OneBillSubscriberLike {
+  accountNumber: string;
+  companyName?: string;
+  accountName?: string;
+  address?: { country?: string }[] | { country?: string };
+}
+
+const ISO2 = /^[A-Za-z]{2}$/;
+
+function defaultCountry(r: OneBillSubscriberLike): string | undefined {
+  const addrs = Array.isArray(r.address) ? r.address : r.address ? [r.address] : [];
+  const hit = addrs.find((a) => typeof a?.country === 'string' && ISO2.test(a.country));
+  return hit?.country?.toUpperCase();
+}
+
+/** Maps OneBill subscribers to screening parties. Only a two-letter country code is kept; other values are dropped. */
+export function fromOneBillSubscribers<T extends OneBillSubscriberLike>(
+  records: readonly T[],
+  opts: { source?: string; country?: (r: T) => string | undefined } = {},
+): Party[] {
+  const source = opts.source ?? 'onebill';
+  return records.map((r) => {
+    const country = opts.country ? opts.country(r) : defaultCountry(r);
+    return { id: r.accountNumber, name: r.companyName?.trim() || r.accountName || '', source, ...(country ? { country } : {}) };
+  });
+}

package/src/canonical.ts

@@ -0,0 +1,22 @@
+/** JSON.stringify with object keys sorted recursively; arrays keep their order. */
+export function canonicalJson(value: unknown): string {
+  return JSON.stringify(sortKeys(value));
+}
+
+function sortKeys(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(sortKeys);
+  if (value !== null && typeof value === 'object') {
+    const src = value as Record<string, unknown>;
+    // Null prototype: a field named "__proto__" stays an own key instead of setting the prototype and vanishing.
+    const out: Record<string, unknown> = Object.create(null);
+    for (const k of Object.keys(src).sort()) out[k] = sortKeys(src[k]);
+    return out;
+  }
+  return value;
+}
+
+export async function sha256Hex(data: Uint8Array | string): Promise<string> {
+  const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data;
+  const digest = await crypto.subtle.digest('SHA-256', bytes as BufferSource);
+  return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, '0')).join('');
+}

package/src/decisions.ts

@@ -0,0 +1,117 @@
+import type { Decision, Match, PartyResult, ScreenResult } from './model.js';
+
+/** Unambiguous: both parts are JSON-encoded, so no separator inside an id can shift the boundary. */
+export function decisionKey(partyId: string, entryKey: string): string {
+  return JSON.stringify([partyId, entryKey]);
+}
+
+export interface DecisionInput {
+  partyId: string;
+  entryKey: string;
+  decision: 'cleared' | 'escalated';
+  reason: string;
+  reviewer: string;
+  decidedAt: string;
+  /** Identifies the run whose result the reviewer read; copied into the Decision. */
+  runId?: string;
+}
+
+const nonEmpty = (v: unknown): v is string => typeof v === 'string' && v !== '';
+
+/** Records a ruling bound to the party name and CSL entry content the reviewer actually saw. */
+export function makeDecision(result: ScreenResult, input: DecisionInput): Decision {
+  const reason = input.reason.trim();
+  const reviewer = input.reviewer.trim();
+  if (reason === '') throw new Error('A decision needs a non-blank reason');
+  if (reviewer === '') throw new Error('A decision needs a non-blank reviewer');
+  const decidedAtMs = Date.parse(input.decidedAt);
+  if (!Number.isFinite(decidedAtMs)) throw new Error(`decidedAt is not a valid timestamp: ${JSON.stringify(input.decidedAt)}`);
+  if (input.runId !== undefined && !nonEmpty(input.runId)) throw new Error('runId, when given, must be a non-empty string');
+  const party = result.parties.find((r) => r.party.id === input.partyId);
+  const match = party?.status === 'flagged' ? party.matches.find((m) => m.entryKey === input.entryKey) : undefined;
+  if (!party || !match) throw new Error(`Party ${input.partyId} is not flagged against entry ${input.entryKey}`);
+  return {
+    check: 'csl',
+    partyId: input.partyId,
+    entryKey: input.entryKey,
+    partyNameKey: party.partyNameKey,
+    entryFingerprint: match.entryFingerprint,
+    decision: input.decision,
+    reason,
+    reviewer,
+    decidedAt: new Date(decidedAtMs).toISOString(),
+    ...(input.runId !== undefined ? { runId: input.runId } : {}),
+  };
+}
+
+/**
+ * Returns a new result with each match's `state` and `decision` set from the latest decision for its
+ * (party, entry) key. A decision applies only if the party name and CSL entry are unchanged since it was
+ * made; otherwise the match is `new` again. Only the latest decision is considered, so a stale one is not
+ * bypassed by an older valid one. "Latest" compares parsed instants, so offsets and precision do not matter;
+ * a decision whose `decidedAt` does not parse is never latest and never applies. On an exact tie,
+ * `escalated` wins over `cleared`, whatever the array order.
+ */
+export function applyDecisions(result: ScreenResult, decisions: readonly Decision[]): ScreenResult {
+  const latest = new Map<string, { d: Decision; at: number }>();
+  for (const d of decisions) {
+    // Defensive: a record that is not a real ruling must never clear or escalate anything.
+    if (d.check !== 'csl') continue;
+    if (d.decision !== 'cleared' && d.decision !== 'escalated') continue;
+    if (typeof d.reason !== 'string' || d.reason.trim() === '') continue;
+    const at = Date.parse(d.decidedAt);
+    if (!Number.isFinite(at)) continue;
+    const key = decisionKey(d.partyId, d.entryKey);
+    const cur = latest.get(key);
+    if (!cur || at > cur.at || (at === cur.at && d.decision === 'escalated')) latest.set(key, { d, at });
+  }
+  return {
+    ...result,
+    parties: result.parties.map((r): PartyResult => {
+      if (r.status !== 'flagged') return r;
+      return {
+        ...r,
+        matches: r.matches.map((m): Match => {
+          const { decision: _prior, ...rest } = m;
+          const d = latest.get(decisionKey(r.party.id, m.entryKey))?.d;
+          if (!d || d.partyNameKey !== r.partyNameKey || d.entryFingerprint !== m.entryFingerprint) {
+            return { ...rest, state: 'new' };
+          }
+          return { ...rest, state: d.decision, decision: d };
+        }),
+      };
+    }),
+  };
+}
+
+const nonBlank = (v: unknown): v is string => typeof v === 'string' && v.trim() !== '';
+
+/** Validates persisted decisions (for example a decisions file read from disk). Throws a TypeError naming the index and field. */
+export function parseDecisions(value: unknown): Decision[] {
+  if (!Array.isArray(value)) throw new TypeError('decisions must be an array');
+  value.forEach((d: unknown, i) => {
+    if (d === null || typeof d !== 'object' || Array.isArray(d)) throw new TypeError(`decisions[${i}] is not an object`);
+    const r = d as Record<string, unknown>;
+    if (r.check !== 'csl') throw new TypeError(`decisions[${i}].check must be "csl"`);
+    for (const f of ['partyId', 'entryKey', 'partyNameKey', 'entryFingerprint']) {
+      if (!nonEmpty(r[f])) throw new TypeError(`decisions[${i}].${f} must be a non-empty string`);
+    }
+    if (r.decision !== 'cleared' && r.decision !== 'escalated') throw new TypeError(`decisions[${i}].decision must be "cleared" or "escalated"`);
+    if (!nonBlank(r.reason)) throw new TypeError(`decisions[${i}].reason must not be blank`);
+    if (!nonBlank(r.reviewer)) throw new TypeError(`decisions[${i}].reviewer must not be blank`);
+    if (typeof r.decidedAt !== 'string' || !Number.isFinite(Date.parse(r.decidedAt))) {
+      throw new TypeError(`decisions[${i}].decidedAt must be a valid timestamp`);
+    }
+    if (r.runId !== undefined && !nonEmpty(r.runId)) throw new TypeError(`decisions[${i}].runId must be a non-empty string when present`);
+  });
+  return value as Decision[];
+}
+
+export type PartyOutcome = 'clear' | 'unscreenable' | 'flagged-new' | 'flagged-escalated' | 'flagged-cleared';
+
+export function partyOutcome(r: PartyResult): PartyOutcome {
+  if (r.status !== 'flagged') return r.status;
+  if (r.matches.some((m) => m.state === 'new')) return 'flagged-new';
+  if (r.matches.some((m) => m.state === 'escalated')) return 'flagged-escalated';
+  return 'flagged-cleared';
+}

package/src/index.ts

@@ -0,0 +1,11 @@
+export { DEFAULT_STOP_WORDS, hasUnreadableCharacters, nameTokens, normalizeName } from './normalize.js';
+export { jaroWinkler } from './jaroWinkler.js';
+export { canonicalJson, sha256Hex } from './canonical.js';
+export { loadSnapshot, contentHashOf, CslSnapshotError } from './snapshot.js';
+export type { Snapshot, SlimEntry, SnapshotIdentity, LoadSnapshotOptions } from './snapshot.js';
+export { DEFAULT_GENERIC_TOKENS, screen } from './screen.js';
+export { MATCHER_VERSION } from './model.js';
+export type { Party, MatchState, Match, Decision, PartyResult, ScreenOptions, ScreenResult } from './model.js';
+export { decisionKey, makeDecision, applyDecisions, partyOutcome, parseDecisions } from './decisions.js';
+export type { DecisionInput, PartyOutcome } from './decisions.js';
+export { toCsv, toMarkdown } from './render.js';

package/src/jaroWinkler.ts

@@ -0,0 +1,39 @@
+/** Jaro-Winkler similarity in [0, 1]: match window floor(max/2) - 1, common prefix capped at 4, scale 0.1. */
+export function jaroWinkler(a: string, b: string): number {
+  if (a === b) return a.length === 0 ? 0 : 1;
+  if (a.length === 0 || b.length === 0) return 0;
+
+  const window = Math.max(0, Math.floor(Math.max(a.length, b.length) / 2) - 1);
+  const aMatched: boolean[] = new Array(a.length).fill(false);
+  const bMatched: boolean[] = new Array(b.length).fill(false);
+
+  let matches = 0;
+  for (let i = 0; i < a.length; i++) {
+    const lo = Math.max(0, i - window);
+    const hi = Math.min(b.length - 1, i + window);
+    for (let j = lo; j <= hi; j++) {
+      if (!bMatched[j] && a[i] === b[j]) {
+        aMatched[i] = true;
+        bMatched[j] = true;
+        matches++;
+        break;
+      }
+    }
+  }
+  if (matches === 0) return 0;
+
+  let transpositions = 0;
+  let k = 0;
+  for (let i = 0; i < a.length; i++) {
+    if (!aMatched[i]) continue;
+    while (!bMatched[k]) k++;
+    if (a[i] !== b[k]) transpositions++;
+    k++;
+  }
+
+  const jaro = (matches / a.length + matches / b.length + (matches - transpositions / 2) / matches) / 3;
+
+  let prefix = 0;
+  while (prefix < 4 && prefix < a.length && prefix < b.length && a[prefix] === b[prefix]) prefix++;
+  return jaro + prefix * 0.1 * (1 - jaro);
+}

package/src/model.ts

@@ -0,0 +1,75 @@
+import type { SnapshotIdentity } from './snapshot.js';
+
+export interface Party {
+  id: string;
+  name: string;
+  /** ISO 3166-1 alpha-2 when known. Only annotates matches; never suppresses one. */
+  country?: string;
+  source: string;
+}
+
+export type MatchState = 'new' | 'cleared' | 'escalated';
+
+/** A reviewer's recorded ruling on one party/entry pair, bound to the name and entry content it was made against. */
+export interface Decision {
+  check: 'csl';
+  partyId: string;
+  entryKey: string;
+  partyNameKey: string;
+  entryFingerprint: string;
+  decision: 'cleared' | 'escalated';
+  reason: string;
+  reviewer: string;
+  decidedAt: string;
+  /** The screening run whose result the reviewer read when making this ruling. Optional; the caller names it. */
+  runId?: string;
+}
+
+export interface Match {
+  /** `${source}|${id}`; unique within a snapshot. */
+  entryKey: string;
+  /** The raw CSL id, which is not unique on its own. */
+  entryId: string;
+  entryFingerprint: string;
+  matchedName: string;
+  listSource: string;
+  programs: string[];
+  entryType: string | null;
+  entryCountries: string[];
+  countryMismatch: boolean;
+  rule: 'exact' | 'contains' | 'within' | 'similar';
+  score: number;
+  reason: string;
+  state: MatchState;
+  decision?: Decision;
+}
+
+export type PartyResult = { party: Party; partyNameKey: string } & (
+  | { status: 'clear' }
+  | { status: 'unscreenable'; reason: 'empty-name' | 'unreadable-characters' }
+  | { status: 'flagged'; matches: Match[] }
+);
+
+export interface ScreenOptions {
+  /** Minimum similarity for the `similar` rule. Default 0.9. */
+  threshold?: number;
+  /** Replaces DEFAULT_STOP_WORDS. Entries are normalised like names, so 'Inc' or 'L.L.C.' work. */
+  stopWords?: readonly string[];
+  /** Shortest token that is indexed and used to find candidates. Default 3. */
+  minTokenLength?: number;
+  /**
+   * Replaces DEFAULT_GENERIC_TOKENS: one-word listed entity names that match a party only as its first
+   * token. Normalised like names.
+   */
+  genericTokens?: readonly string[];
+}
+
+export const MATCHER_VERSION = '2';
+
+export interface ScreenResult {
+  check: 'csl';
+  matcher: { version: string; threshold: number; minTokenLength: number; stopWords: string[]; genericTokens: string[] };
+  snapshot: SnapshotIdentity;
+  counts: { parties: number; screened: number; unscreenable: number; clear: number; flagged: number };
+  parties: PartyResult[];
+}

package/src/normalize.ts

@@ -0,0 +1,68 @@
+/** Legal-form suffixes and filler words that carry no identity for name matching. */
+export const DEFAULT_STOP_WORDS: readonly string[] = [
+  'llc', 'inc', 'incorporated', 'co', 'corp', 'corporation', 'company',
+  'ltd', 'limited', 'plc', 'pc', 'pllc', 'lp', 'llp', 'the', 'and', 'of',
+  'sa', 'ag', 'gmbh', 'srl', 'bv', 'nv', 'oao', 'ooo', 'zao', 'jsc', 'pjsc',
+];
+
+/** Latin letters with no NFKD decomposition, and their ASCII spelling. */
+const LATIN_FOLD: Readonly<Record<string, string>> = {
+  'ß': 'ss', 'ẞ': 'SS', 'æ': 'ae', 'Æ': 'AE', 'œ': 'oe', 'Œ': 'OE', 'ø': 'o', 'Ø': 'O',
+  'ł': 'l', 'Ł': 'L', 'đ': 'd', 'Đ': 'D', 'ð': 'd', 'Ð': 'D', 'þ': 'th', 'Þ': 'TH', 'ı': 'i',
+};
+const LATIN_FOLD_RE = new RegExp(`[${Object.keys(LATIN_FOLD).join('')}]`, 'g');
+
+/**
+ * NFKD, every combining mark (\p{M}) and format character (\p{Cf}) deleted, then the Latin fold:
+ * the form every name is compared in. Deleting rather than separating means an invisible character
+ * typed inside a word (a zero-width space, soft hyphen, bidi mark or variation selector in "Ros\u200Bneft")
+ * leaves the word whole instead of splitting it into fragments that match nothing.
+ */
+function foldLatin(name: string): string {
+  return name
+    .normalize('NFKD')
+    .replace(/[\p{M}\p{Cf}]/gu, '')
+    .replace(LATIN_FOLD_RE, (c) => LATIN_FOLD[c]!);
+}
+
+/**
+ * True when, after the fold, the name still has a letter outside A-Z/a-z, or a control (other than tab,
+ * LF and CR), private-use or unassigned character. Such a name cannot be tokenised safely: a look-alike
+ * letter from another script (Cyrillic о in "Rоsneft") or an unreadable character would split the word
+ * and hide a listed entity, so the caller must screen it by hand.
+ */
+export function hasUnreadableCharacters(name: string): boolean {
+  return /[^\P{L}A-Za-z]|[^\P{Cc}\t\n\r]|[\p{Co}\p{Cn}]/u.test(foldLatin(name));
+}
+
+/**
+ * Lowercase ASCII tokens of a name: diacritics stripped, `&` read as "and",
+ * everything outside [a-z0-9] treated as a separator, runs of single-character
+ * tokens joined ("L.L.C." is `llc`, "D. K. Smith" is `dk smith`), stop words dropped.
+ * Letters outside the Latin alphabet and other unreadable characters act as separators; see `hasUnreadableCharacters`.
+ */
+export function nameTokens(name: string, stopWords: ReadonlySet<string>): string[] {
+  const raw = foldLatin(name)
+    .toLowerCase()
+    .replace(/&/g, ' and ')
+    .replace(/[^a-z0-9]+/g, ' ')
+    .split(' ')
+    .filter((t) => t !== '');
+  // Join before dropping stop words, so initials ("L.L.C.") become the word they spell.
+  const joined: string[] = [];
+  let run = '';
+  for (const t of raw) {
+    if (t.length === 1) run += t;
+    else {
+      if (run) joined.push(run);
+      run = '';
+      joined.push(t);
+    }
+  }
+  if (run) joined.push(run);
+  return joined.filter((t) => !stopWords.has(t));
+}
+
+export function normalizeName(name: string, stopWords: ReadonlySet<string>): string {
+  return nameTokens(name, stopWords).join(' ');
+}

package/src/render.ts

@@ -0,0 +1,142 @@
+import { partyOutcome } from './decisions.js';
+import type { Match, PartyResult, ScreenResult } from './model.js';
+
+const COLUMNS = [
+  'party_id', 'party_name', 'source', 'result', 'matched_name', 'list', 'entry_key', 'rule', 'score',
+  'reason', 'entry_countries', 'party_country', 'decision', 'decided_by', 'decided_at', 'run_date',
+] as const;
+
+/** A leading = + - @ (or tab/CR/LF) makes a spreadsheet read the cell as a formula; a leading apostrophe stops that. */
+function guardFormula(s: string): string {
+  return /^[=+\-@\t\r\n]/.test(s) ? `'${s}` : s;
+}
+
+/** Guard first, then RFC 4180 quoting, so the apostrophe sits inside the quotes. */
+function csvCell(s: string): string {
+  const g = guardFormula(s);
+  return /[",\r\n]/.test(g) ? `"${g.replace(/"/g, '""')}"` : g;
+}
+
+export function toCsv(result: ScreenResult, opts: { runDate: string }): string {
+  const lines: string[] = [COLUMNS.join(',')];
+  const row = (cells: string[]) => lines.push(cells.map(csvCell).join(','));
+  for (const r of result.parties) {
+    const outcome = partyOutcome(r);
+    const head = [r.party.id, r.party.name, r.party.source, outcome];
+    if (r.status !== 'flagged') {
+      row([...head, '', '', '', '', '', '', '', '', '', '', opts.runDate]);
+      continue;
+    }
+    for (const m of r.matches) {
+      row([
+        ...head, m.matchedName, m.listSource, m.entryKey, m.rule, m.score.toFixed(2), m.reason,
+        m.entryCountries.join(';'), r.party.country ?? '',
+        m.decision?.decision ?? '', m.decision?.reviewer ?? '', m.decision?.decidedAt ?? '', opts.runDate,
+      ]);
+    }
+  }
+  return lines.join('\r\n') + '\r\n';
+}
+
+/**
+ * Names come from client records and the public list, so they are untrusted. Line breaks become spaces
+ * (a table row or list item must stay on one line), then every character Markdown or inline HTML could
+ * act on is backslash-escaped. `&` is included so an entity such as `<` stays literal text.
+ */
+function md(s: string): string {
+  return s
+    .replace(/\r\n|\r|\n/g, ' ')
+    .replace(/[\\|`*_~<>[\]&]/g, '\\$&')
+    .replace(/^(\s*)#/, '$1\\#');
+}
+
+const UNSCREENABLE_REASON = {
+  'empty-name': 'name is empty after normalisation',
+  'unreadable-characters': 'name contains non-Latin or unreadable characters; screen it by hand',
+} as const;
+
+const list = (xs: readonly string[]) => (xs.length ? md(xs.join(', ')) : 'none');
+
+function matchLine(r: Extract<PartyResult, { status: 'flagged' }>, m: Match): string {
+  const p = r.party;
+  const d = m.decision;
+  const decision = d ? `${d.decision} by ${md(d.reviewer)} at ${md(d.decidedAt)}: ${md(d.reason)}` : 'none';
+  const partyCountry = p.country ? md(p.country) : 'not provided';
+  const mismatch = m.countryMismatch ? ' (country mismatch)' : '';
+  return [
+    `- **${md(p.name)}** (id ${md(p.id)}, source ${md(p.source)}) matched **${md(m.matchedName)}**`,
+    `  - List: ${md(m.listSource)}; programs: ${list(m.programs)}; entry type: ${m.entryType ? md(m.entryType) : 'unknown'}`,
+    `  - Countries: entry ${list(m.entryCountries)}; party ${partyCountry}${mismatch}`,
+    `  - Rule: ${m.rule}, score ${m.score.toFixed(2)}; reason: ${md(m.reason)}`,
+    `  - Decision: ${decision}`,
+  ].join('\n');
+}
+
+export function toMarkdown(
+  result: ScreenResult,
+  opts: {
+    runDate: string;
+    title?: string;
+    expected?: number;
+    sourceRecords?: number;
+    /** Caller lines (for example where the parties and the list came from), printed as list items under the title. */
+    preamble?: readonly string[];
+  },
+): string {
+  const { counts, snapshot: s, matcher } = result;
+  const out: string[] = [];
+  if (opts.expected !== undefined && opts.expected !== counts.parties) {
+    out.push(`INCOMPLETE: ${counts.parties} parties in run of expected ${opts.expected}`, '');
+  }
+  out.push(`# ${md(opts.title || 'Restricted-party screening report')}`, '');
+  if (opts.preamble?.length) out.push(...opts.preamble.map((l) => `- ${md(l)}`), '');
+  out.push(`Run date: ${md(opts.runDate)}`, '');
+  const excluded = opts.sourceRecords !== undefined ? opts.sourceRecords - counts.parties : 0;
+  out.push(
+    ...(opts.sourceRecords !== undefined ? [`- Source records: ${opts.sourceRecords}`] : []),
+    opts.expected !== undefined
+      ? `- Parties in run: ${counts.parties} of ${opts.expected} expected`
+      : `- Parties in run: ${counts.parties}`,
+    ...(excluded > 0 ? [`- Excluded by scope: ${excluded}`] : []),
+    `- Screened: ${counts.screened}`,
+    `- Clear: ${counts.clear}`,
+    `- Flagged: ${counts.flagged}`,
+    `- Unscreenable: ${counts.unscreenable}`,
+    '',
+    '## Consolidated Screening List snapshot',
+    '',
+    `- Search performed at: ${md(s.searchPerformedAt || 'unknown')}`,
+    `- Entries: ${s.entryCount}`,
+    `- SHA-256: ${s.sha256}`,
+    '- Sources:',
+    ...s.sources.map((x) => `  - ${md(x.source)}: last updated ${md(x.sourceLastUpdated ?? 'unknown')}`),
+    '',
+    '## Matcher',
+    '',
+    `- Version ${md(matcher.version)}`,
+    `- Threshold: ${matcher.threshold}`,
+    '',
+  );
+
+  const flagged = result.parties.filter((r): r is Extract<PartyResult, { status: 'flagged' }> => r.status === 'flagged');
+  const section = (title: string, state: Match['state']) => {
+    const lines = flagged.flatMap((r) => r.matches.filter((m) => m.state === state).map((m) => matchLine(r, m)));
+    if (lines.length) out.push(`## ${title}`, '', ...lines, '');
+  };
+  section('New flags', 'new');
+  section('Escalated (open)', 'escalated');
+  const unscreenable = result.parties.filter((r): r is Extract<PartyResult, { status: 'unscreenable' }> => r.status === 'unscreenable');
+  if (unscreenable.length) {
+    out.push(
+      '## Unscreenable parties',
+      '',
+      ...unscreenable.map(
+        (r) =>
+          `- ${r.party.name.trim() ? md(r.party.name) : '(blank name)'} (id ${md(r.party.id)}, source ${md(r.party.source)}): ${UNSCREENABLE_REASON[r.reason]}`,
+      ),
+      '',
+    );
+  }
+  section('Cleared (still matching)', 'cleared');
+  return out.join('\n');
+}