@dszp/csl-screen 0.1.0

package/src/screen.ts

@@ -0,0 +1,185 @@
+import { jaroWinkler } from './jaroWinkler.js';
+import { MATCHER_VERSION } from './model.js';
+import type { Match, Party, PartyResult, ScreenOptions, ScreenResult } from './model.js';
+import { DEFAULT_STOP_WORDS, hasUnreadableCharacters, nameTokens } from './normalize.js';
+import type { SlimEntry, Snapshot } from './snapshot.js';
+
+interface IndexedName {
+  entry: SlimEntry;
+  name: string;
+  key: string;
+  tokens: Set<string>;
+  /** Has a token >= minTokenLength. Names without one match only by `exact` (with or without spaces). */
+  indexable: boolean;
+}
+
+const RULE_RANK = { exact: 3, contains: 2, within: 1, similar: 0 } as const;
+type Rule = keyof typeof RULE_RANK;
+
+/** A one-token listed name on these entry types is too generic (a ship, a nickname) to match inside a longer party name. */
+const NON_ENTITY_TYPES: ReadonlySet<string> = new Set(['Individual', 'Vessel', 'Aircraft']);
+/**
+ * One-word listed entity names (after stop words are dropped) too generic to flag anywhere but a party
+ * name's first token. Each was measured on real customer names: as a later token it flagged only
+ * unrelated organisations using it as an ordinary word.
+ * A word joins this list only from such a measurement, because each one is a name that can be hidden
+ * behind another word.
+ */
+export const DEFAULT_GENERIC_TOKENS: readonly string[] = ['center', 'focus', 'technology', 'united'];
+
+/** Longest run of adjacent party tokens joined when looking for a listed word split by separators. */
+const MAX_RUN = 3;
+const spaceless = (key: string) => key.replace(/ /g, '');
+const push = <K, V>(m: Map<K, V[]>, k: K, v: V) => {
+  const bucket = m.get(k);
+  if (bucket) bucket.push(v);
+  else m.set(k, [v]);
+};
+
+export function screen(parties: Party[], snapshot: Snapshot, options: ScreenOptions = {}): ScreenResult {
+  const threshold = options.threshold ?? 0.9;
+  const minTokenLength = options.minTokenLength ?? 3;
+  if (!(typeof threshold === 'number' && threshold > 0 && threshold <= 1)) {
+    throw new TypeError(`threshold must be a number in (0, 1], got ${String(threshold)}`);
+  }
+  if (!Number.isInteger(minTokenLength) || minTokenLength < 1) {
+    throw new TypeError(`minTokenLength must be an integer >= 1, got ${String(minTokenLength)}`);
+  }
+  const withinMinLength = Math.max(4, minTokenLength);
+  // Caller stop words go through the same normalisation as names, so 'Inc' and 'L.L.C.' behave.
+  const noStops: ReadonlySet<string> = new Set();
+  const stopWords = new Set((options.stopWords ?? DEFAULT_STOP_WORDS).flatMap((w) => nameTokens(w, noStops)));
+  const genericTokens = new Set((options.genericTokens ?? DEFAULT_GENERIC_TOKENS).flatMap((w) => nameTokens(w, noStops)));
+
+  const seenIds = new Set<string>();
+  for (const party of parties) {
+    if (seenIds.has(party.id)) throw new Error(`Duplicate party id: ${party.id}`);
+    seenIds.add(party.id);
+  }
+
+  const index = new Map<string, IndexedName[]>();
+  const bySpaceless = new Map<string, IndexedName[]>();
+  let longestSpaceless = 0;
+
+  for (const entry of snapshot.entries) {
+    for (const name of [entry.name, ...entry.altNames]) {
+      const tokens = nameTokens(name, stopWords);
+      if (tokens.length === 0) continue;
+      const indexed: IndexedName = { entry, name, key: tokens.join(' '), tokens: new Set(tokens), indexable: tokens.some((t) => t.length >= minTokenLength) };
+      push(bySpaceless, spaceless(indexed.key), indexed);
+      longestSpaceless = Math.max(longestSpaceless, spaceless(indexed.key).length);
+      for (const token of indexed.tokens) if (token.length >= minTokenLength) push(index, token, indexed);
+    }
+  }
+
+  const results: PartyResult[] = parties.map((party): PartyResult => {
+    const tokens = nameTokens(party.name, stopWords);
+    const partyNameKey = tokens.join(' ');
+    const ptoks = new Set(tokens);
+    // A non-Latin letter or unreadable character would split a word into fragments (or vanish), so a listed name could be missed.
+    if (hasUnreadableCharacters(party.name)) return { party, partyNameKey, status: 'unscreenable', reason: 'unreadable-characters' };
+    // An empty key cannot be screened; it must not read as clear.
+    if (partyNameKey === '') return { party, partyNameKey, status: 'unscreenable', reason: 'empty-name' };
+    // `within` needs a party token long enough to be distinctive: at least 4, and never shorter than an indexed token.
+    const hasLongToken = tokens.some((t) => t.length >= withinMinLength);
+
+    // Each party token, and each run of two or three adjacent tokens joined without spaces, with the position
+    // it starts at. A run catches a separator inserted inside a listed word ("Ros-neft Trading" holds
+    // "rosneft"). At most MAX_RUN units per position keeps the work linear in the party name's length; a run
+    // longer than any listed name cannot match and is not looked up.
+    const units = new Map<string, number>();
+    for (let i = 0; i < tokens.length; i++) {
+      let run = '';
+      for (let j = i; j < tokens.length && j - i < MAX_RUN; j++) {
+        run += tokens[j];
+        if (run.length > longestSpaceless && j > i) break;
+        if (!units.has(run)) units.set(run, i);
+      }
+    }
+
+    const candidates = new Set<IndexedName>();
+    for (const u of units.keys()) {
+      if (u.length >= minTokenLength) for (const c of index.get(u) ?? []) candidates.add(c);
+      for (const c of bySpaceless.get(u) ?? []) candidates.add(c);
+    }
+    for (const c of bySpaceless.get(spaceless(partyNameKey)) ?? []) candidates.add(c);
+
+    const best = new Map<string, { rule: Rule; score: number; name: IndexedName; reason: string | null }>();
+    for (const cand of candidates) {
+      let shared = 0;
+      for (const t of cand.tokens) if (ptoks.has(t)) shared++;
+      const cover = shared / Math.max(ptoks.size, cand.tokens.size);
+      const score = Math.min(jaroWinkler(partyNameKey, cand.key), 0.5 + cover / 2);
+      // Contained when every listed token is a party token, or a run of them, or the whole listed name is one run.
+      const candSpaceless = spaceless(cand.key);
+      const inUnits = [...cand.tokens].every((t) => units.has(t)) || units.has(candSpaceless);
+      // A one-token name on an entity counts in any position, except a generic word, which counts only first.
+      const at = units.get(candSpaceless);
+      const containedAlone =
+        cand.tokens.size > 1 ||
+        (!NON_ENTITY_TYPES.has(cand.entry.type ?? '') && at !== undefined && (at === 0 || !genericTokens.has(candSpaceless)));
+      let rule: Rule | null = null;
+      let reason: string | null = null;
+      if (partyNameKey === cand.key) rule = 'exact';
+      else if (spaceless(partyNameKey) === spaceless(cand.key)) {
+        rule = 'exact';
+        reason = 'exact ignoring spaces';
+      } else if (!cand.indexable) continue;
+      else if (shared === cand.tokens.size && containedAlone) rule = 'contains';
+      else if (inUnits && containedAlone) {
+        rule = 'contains';
+        reason = 'contains listed name ignoring spaces';
+      }
+      else if (shared === ptoks.size && hasLongToken) rule = 'within';
+      else if (score >= threshold) rule = 'similar';
+      if (rule === null) continue;
+      const cur = best.get(cand.entry.key);
+      if (!cur || RULE_RANK[rule] > RULE_RANK[cur.rule] || (rule === cur.rule && score > cur.score)) {
+        best.set(cand.entry.key, { rule, score, name: cand, reason });
+      }
+    }
+
+    if (best.size === 0) return { party, partyNameKey, status: 'clear' };
+
+    const country = party.country?.toUpperCase();
+    const matches: Match[] = [...best.values()].map(({ rule, score, name, reason }) => ({
+      entryKey: name.entry.key,
+      entryId: name.entry.id,
+      entryFingerprint: name.entry.fingerprint,
+      matchedName: name.name,
+      listSource: name.entry.source,
+      programs: name.entry.programs,
+      entryType: name.entry.type,
+      entryCountries: name.entry.countries,
+      countryMismatch: !!country && name.entry.countries.length > 0 && !name.entry.countries.includes(country),
+      rule,
+      // An exact rule is a full match whatever the token-cover heuristic says (spaceless matches cover fewer tokens).
+      score: rule === 'exact' ? 1 : score,
+      reason:
+        reason ??
+        (rule === 'exact' ? 'exact after normalisation'
+        : rule === 'contains' ? 'contains listed name'
+        : rule === 'within' ? 'listed name contains party name'
+        : `similar ${score.toFixed(2)}`),
+      state: 'new',
+    }));
+    matches.sort((a, b) => RULE_RANK[b.rule] - RULE_RANK[a.rule] || b.score - a.score || (a.entryKey < b.entryKey ? -1 : a.entryKey > b.entryKey ? 1 : 0));
+    return { party, partyNameKey, status: 'flagged', matches };
+  });
+
+  const count = (s: PartyResult['status']) => results.filter((r) => r.status === s).length;
+  const unscreenable = count('unscreenable');
+  return {
+    check: 'csl',
+    matcher: { version: MATCHER_VERSION, threshold, minTokenLength, stopWords: [...stopWords].sort(), genericTokens: [...genericTokens].sort() },
+    snapshot: snapshot.identity,
+    counts: {
+      parties: results.length,
+      screened: results.length - unscreenable,
+      unscreenable,
+      clear: count('clear'),
+      flagged: count('flagged'),
+    },
+    parties: results,
+  };
+}

package/src/snapshot.ts

@@ -0,0 +1,186 @@
+import { canonicalJson, sha256Hex } from './canonical.js';
+
+export interface SnapshotIdentity {
+  searchPerformedAt: string;
+  sources: { source: string; sourceLastUpdated: string | null; lastImported: string | null }[];
+  entryCount: number;
+  /** Entries per CSL `source` (list name), keys sorted. Feeds the next run's `previousEntriesBySource`. */
+  entriesBySource: Record<string, number>;
+  /** SHA-256 of the downloaded bytes. Changes on every republish, even when no entry changed. */
+  sha256: string;
+  /** Hash of the entries alone (see `contentHashOf`). Unchanged by `search_performed_at` and per-list `last_imported`. */
+  contentSha256: string;
+}
+
+export interface SlimEntry {
+  /** `${source}|${id}`. The CSL `id` alone is not unique: an entry on two lists shares it. */
+  key: string;
+  id: string;
+  name: string;
+  altNames: string[];
+  source: string;
+  type: string | null;
+  countries: string[];
+  programs: string[];
+  sourceListUrl: string | null;
+  /** SHA-256 of the canonical JSON of the whole raw entry; changes when any field changes. */
+  fingerprint: string;
+}
+
+export interface Snapshot {
+  identity: SnapshotIdentity;
+  entries: SlimEntry[];
+}
+
+export class CslSnapshotError extends Error {
+  constructor(
+    public kind: 'parse' | 'shape' | 'total-mismatch' | 'sharp-drop' | 'source-drop',
+    message: string,
+  ) {
+    super(message);
+    this.name = 'CslSnapshotError';
+  }
+}
+
+export interface LoadSnapshotOptions {
+  /** Entry count of the snapshot being replaced; enables the sharp-drop check. */
+  previousEntryCount?: number;
+  /** Largest tolerated fractional shrink against previousEntryCount. Default 0.1. */
+  maxDropRatio?: number;
+  /**
+   * `entriesBySource` of the snapshot being replaced; enables the source-drop check. A source in this map that is
+   * missing from the new file, or has fewer than half its previous entries, fails the load. Catches the loss of
+   * one small list, which the whole-file sharp-drop check cannot see.
+   */
+  previousEntriesBySource?: Record<string, number>;
+}
+
+type Obj = Record<string, unknown>;
+const isObj = (v: unknown): v is Obj => v !== null && typeof v === 'object' && !Array.isArray(v);
+const strOrNull = (v: unknown): string | null => (typeof v === 'string' ? v : null);
+const MIN_SOURCE_RATIO = 0.5;
+
+/** Null or absent is an empty list; anything else must be an array of strings, so a format change fails loudly. */
+function strings(v: unknown, where: string): string[] {
+  if (v === null || v === undefined) return [];
+  if (!Array.isArray(v) || !v.every((x) => typeof x === 'string')) {
+    throw new CslSnapshotError('shape', `${where} must be an array of strings, null or absent`);
+  }
+  return v;
+}
+
+/**
+ * SHA-256 hex over the lines `${key}\t${fingerprint}\n`, sorted by key in byte order (plain `<` on strings, not
+ * `localeCompare`, so the result does not depend on the runtime's locale), concatenated. Entry order in the file
+ * does not matter; any change to an entry's fingerprint, or to the set of keys, does.
+ */
+export async function contentHashOf(entries: readonly { key: string; fingerprint: string }[]): Promise<string> {
+  const sorted = [...entries].sort((x, y) => (x.key < y.key ? -1 : x.key > y.key ? 1 : 0));
+  return sha256Hex(sorted.map((e) => `${e.key}\t${e.fingerprint}\n`).join(''));
+}
+
+export async function loadSnapshot(
+  input: Uint8Array | ArrayBuffer | string,
+  opts: LoadSnapshotOptions = {},
+): Promise<Snapshot> {
+  const bytes =
+    typeof input === 'string' ? new TextEncoder().encode(input)
+    : input instanceof Uint8Array ? input
+    : new Uint8Array(input);
+  const sha256 = await sha256Hex(bytes);
+
+  let doc: unknown;
+  try {
+    doc = JSON.parse(new TextDecoder().decode(bytes));
+  } catch (e) {
+    throw new CslSnapshotError('parse', `CSL file is not valid JSON: ${(e as Error).message}`);
+  }
+  if (!isObj(doc) || !Array.isArray(doc.results)) {
+    throw new CslSnapshotError('shape', 'CSL file has no "results" array');
+  }
+  const raw = doc.results;
+  if (raw.length === 0) {
+    throw new CslSnapshotError('shape', 'CSL file has an empty "results" array');
+  }
+  if (typeof doc.search_performed_at !== 'string') {
+    throw new CslSnapshotError('shape', 'CSL file has no "search_performed_at" string');
+  }
+  const searchPerformedAt = doc.search_performed_at;
+  if (typeof doc.total !== 'number' || raw.length !== doc.total) {
+    throw new CslSnapshotError('total-mismatch', `results has ${raw.length} entries but total is ${String(doc.total)}`);
+  }
+  const previous = opts.previousEntryCount;
+  const maxDrop = opts.maxDropRatio ?? 0.1;
+  // NaN or a ratio of 1 or more would silently disable the check.
+  if (!(typeof maxDrop === 'number' && maxDrop >= 0 && maxDrop < 1)) {
+    throw new TypeError(`maxDropRatio must be a number in [0, 1), got ${String(maxDrop)}`);
+  }
+  if (previous !== undefined && previous > 0 && raw.length < previous * (1 - maxDrop)) {
+    throw new CslSnapshotError('sharp-drop', `entry count fell from ${previous} to ${raw.length}, beyond the ${maxDrop} allowed drop`);
+  }
+
+  const entries: SlimEntry[] = [];
+  const seenKeys = new Set<string>();
+  for (const [i, r] of raw.entries()) {
+    if (!isObj(r) || typeof r.id !== 'string' || typeof r.name !== 'string' || typeof r.source !== 'string') {
+      throw new CslSnapshotError('shape', `results[${i}] lacks a string id, name or source`);
+    }
+    const countries = new Set<string>();
+    const addCountry = (c: unknown) => {
+      if (typeof c === 'string' && c) countries.add(c.toUpperCase());
+    };
+    addCountry(r.country);
+    if (Array.isArray(r.addresses)) for (const a of r.addresses) if (isObj(a)) addCountry(a.country);
+    const key = `${r.source}|${r.id}`;
+    if (seenKeys.has(key)) {
+      throw new CslSnapshotError('shape', `duplicate entry key ${key} at results[${i}]`);
+    }
+    seenKeys.add(key);
+    entries.push({
+      key,
+      id: r.id,
+      name: r.name,
+      altNames: strings(r.alt_names, `results[${i}].alt_names`),
+      source: r.source,
+      type: strOrNull(r.type),
+      countries: [...countries],
+      programs: strings(r.programs, `results[${i}].programs`),
+      sourceListUrl: strOrNull(r.source_list_url),
+      fingerprint: await sha256Hex(canonicalJson(r)),
+    });
+  }
+
+  // Null-prototype maps: a source named "constructor" or "__proto__" is an ordinary key.
+  const counted: Record<string, number> = Object.create(null);
+  for (const e of entries) counted[e.source] = (counted[e.source] ?? 0) + 1;
+  // The published identity is a plain object (callers may use its prototype methods); fromEntries still makes
+  // "__proto__" an own data property. Lookups below stay on the null-prototype map.
+  const entriesBySource: Record<string, number> = Object.fromEntries(Object.keys(counted).sort().map((k) => [k, counted[k]!]));
+  for (const [source, before] of Object.entries(opts.previousEntriesBySource ?? {})) {
+    if (typeof before !== 'number' || !Number.isFinite(before) || before < 0) {
+      throw new TypeError(`previousEntriesBySource["${source}"] must be a non-negative number, got ${JSON.stringify(before)}`);
+    }
+    const now = counted[source] ?? 0;
+    if (now < before * MIN_SOURCE_RATIO) {
+      throw new CslSnapshotError(
+        'source-drop',
+        `source "${source}" fell from ${before} to ${now} entries${now === 0 ? ' (missing from the file)' : ''}, below ${MIN_SOURCE_RATIO * 100}% of the previous count`,
+      );
+    }
+  }
+
+  const sourcesUsed = Array.isArray(doc.sources_used) ? doc.sources_used : [];
+  const identity: SnapshotIdentity = {
+    searchPerformedAt,
+    sources: sourcesUsed.filter(isObj).map((s) => ({
+      source: typeof s.source === 'string' ? s.source : '',
+      sourceLastUpdated: strOrNull(s.source_last_updated),
+      lastImported: strOrNull(s.last_imported),
+    })),
+    entryCount: entries.length,
+    entriesBySource,
+    sha256,
+    contentSha256: await contentHashOf(entries),
+  };
+  return { identity, entries };
+}