goldenmatch 0.1.0

package/src/core/sensitivity.ts

@@ -0,0 +1,196 @@
+/**
+ * sensitivity.ts — Parameter sweep engine for GoldenMatch.
+ * Edge-safe: no Node.js imports, pure TypeScript only.
+ *
+ * Ports goldenmatch/core/sensitivity.py.
+ */
+
+import type { Row, GoldenMatchConfig } from "./types.js";
+import { runDedupePipeline } from "./pipeline.js";
+import { compareClusters } from "./compare-clusters.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface SweepParam {
+  /** Dot-path into the config, e.g. "threshold", "blocking.maxBlockSize". */
+  readonly path: string;
+  readonly values: readonly unknown[];
+}
+
+export interface SweepPoint {
+  readonly params: Readonly<Record<string, unknown>>;
+  readonly stats: Readonly<Record<string, number>>;
+  readonly twi?: number;
+  readonly error?: string;
+}
+
+export interface SensitivityResult {
+  readonly baseline: SweepPoint;
+  readonly points: readonly SweepPoint[];
+  readonly stable: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Dot-path config override
+// ---------------------------------------------------------------------------
+
+/** Set a nested property by dot-path, returning a new object (shallow-cloned chain). */
+function setPath(
+  root: Record<string, unknown>,
+  path: string,
+  value: unknown,
+): Record<string, unknown> {
+  // Simple dot path; array indices via [n] not supported in this edge-safe port
+  const parts = path.split(".").filter((p) => p.length > 0);
+  if (parts.length === 0) return root;
+  const clone: Record<string, unknown> = { ...root };
+  let cursor: Record<string, unknown> = clone;
+  for (let i = 0; i < parts.length - 1; i++) {
+    const key = parts[i]!;
+    const child = cursor[key];
+    const childObj =
+      child !== null && typeof child === "object" && !Array.isArray(child)
+        ? { ...(child as Record<string, unknown>) }
+        : {};
+    cursor[key] = childObj;
+    cursor = childObj;
+  }
+  cursor[parts[parts.length - 1]!] = value;
+  return clone;
+}
+
+// ---------------------------------------------------------------------------
+// Stats extraction
+// ---------------------------------------------------------------------------
+
+function statsFrom(result: ReturnType<typeof runDedupePipeline>): Record<string, number> {
+  return {
+    totalRecords: result.stats.totalRecords,
+    totalClusters: result.stats.totalClusters,
+    matchedRecords: result.stats.matchedRecords,
+    uniqueRecords: result.stats.uniqueRecords,
+    matchRate: result.stats.matchRate,
+    scoredPairs: result.scoredPairs.length,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Cartesian product of sweep values
+// ---------------------------------------------------------------------------
+
+function cartesianPoints(
+  params: readonly SweepParam[],
+): Readonly<Record<string, unknown>>[] {
+  if (params.length === 0) return [];
+  let acc: Record<string, unknown>[] = [{}];
+  for (const p of params) {
+    const next: Record<string, unknown>[] = [];
+    for (const base of acc) {
+      for (const v of p.values) {
+        next.push({ ...base, [p.path]: v });
+      }
+    }
+    acc = next;
+  }
+  return acc;
+}
+
+// ---------------------------------------------------------------------------
+// runSensitivity
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a parameter sweep.
+ *
+ * Each point in the Cartesian product of `params` is applied to
+ * `baselineConfig`, the dedupe pipeline runs, and the resulting clusters are
+ * compared against the baseline via CCMS. A `stable` flag is set when every
+ * point's TWI is within 0.05 of 1.0.
+ *
+ * Per-point errors are caught and stored on the point so that partial
+ * results are preserved.
+ */
+export function runSensitivity(
+  rows: readonly Row[],
+  baselineConfig: GoldenMatchConfig,
+  params: readonly SweepParam[],
+): SensitivityResult {
+  // Baseline run
+  const baselineRun = runDedupePipeline(rows, baselineConfig);
+  const baseline: SweepPoint = {
+    params: {},
+    stats: statsFrom(baselineRun),
+    twi: 1.0,
+  };
+
+  const points: SweepPoint[] = [];
+  const combos = cartesianPoints(params);
+
+  let stable = true;
+  for (const combo of combos) {
+    let cfg: GoldenMatchConfig = baselineConfig;
+    for (const [path, value] of Object.entries(combo)) {
+      cfg = setPath(
+        cfg as Record<string, unknown>,
+        path,
+        value,
+      ) as GoldenMatchConfig;
+    }
+
+    try {
+      const runResult = runDedupePipeline(rows, cfg);
+      let twi: number | undefined;
+      try {
+        twi = compareClusters(baselineRun.clusters, runResult.clusters).twi;
+      } catch (err) {
+        // eslint-disable-next-line no-console
+        console.warn(
+          `TWI comparison failed for sweep point ${JSON.stringify(combo)}: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        );
+        twi = undefined;
+      }
+      if (twi === undefined || Math.abs(1 - twi) > 0.05) stable = false;
+      points.push({
+        params: combo,
+        stats: statsFrom(runResult),
+        ...(twi !== undefined ? { twi } : {}),
+      });
+    } catch (err) {
+      stable = false;
+      points.push({
+        params: combo,
+        stats: {},
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
+  return { baseline, points, stable };
+}
+
+// ---------------------------------------------------------------------------
+// stabilityReport
+// ---------------------------------------------------------------------------
+
+/** Render a human-readable stability report for a sensitivity result. */
+export function stabilityReport(result: SensitivityResult): string {
+  const lines: string[] = [];
+  lines.push("Sensitivity sweep:");
+  lines.push(`  Baseline: ${JSON.stringify(result.baseline.stats)}`);
+  lines.push(`  Points:   ${result.points.length}`);
+  lines.push(`  Stable:   ${result.stable ? "yes" : "no"}`);
+  for (const p of result.points) {
+    const twiStr = p.twi !== undefined ? p.twi.toFixed(4) : "n/a";
+    const errStr = p.error !== undefined ? ` error=${p.error}` : "";
+    lines.push(
+      `  - params=${JSON.stringify(p.params)} twi=${twiStr} clusters=${
+        p.stats["totalClusters"] ?? "?"
+      }${errStr}`,
+    );
+  }
+  return lines.join("\n");
+}

package/src/core/standardize.ts

@@ -0,0 +1,279 @@
+/**
+ * standardize.ts — Data standardization for GoldenMatch-JS.
+ * Edge-safe: no `node:` imports, pure TypeScript only.
+ *
+ * Ports standardization from goldenmatch/core/standardize.py.
+ * These are data cleaning transforms applied to columns before matching.
+ */
+
+import type { Row } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Address abbreviations (USPS standard)
+// ---------------------------------------------------------------------------
+
+/** Map of full word (lowercase) to USPS abbreviation. */
+const ADDRESS_ABBREVIATIONS: Readonly<Record<string, string>> = {
+  street: "St",
+  avenue: "Ave",
+  boulevard: "Blvd",
+  drive: "Dr",
+  lane: "Ln",
+  road: "Rd",
+  court: "Ct",
+  place: "Pl",
+  circle: "Cir",
+  terrace: "Ter",
+  highway: "Hwy",
+  parkway: "Pkwy",
+  expressway: "Expy",
+  freeway: "Fwy",
+  trail: "Trl",
+  way: "Way",
+  north: "N",
+  south: "S",
+  east: "E",
+  west: "W",
+  northeast: "NE",
+  northwest: "NW",
+  southeast: "SE",
+  southwest: "SW",
+  apartment: "Apt",
+  suite: "Ste",
+  building: "Bldg",
+  floor: "Fl",
+  room: "Rm",
+  unit: "Unit",
+  department: "Dept",
+  "post office box": "PO Box",
+  "p.o. box": "PO Box",
+  "po box": "PO Box",
+};
+
+// ---------------------------------------------------------------------------
+// Individual standardizer functions
+// ---------------------------------------------------------------------------
+
+/**
+ * Standardize email: lowercase, strip, validate basic structure.
+ * Returns null for invalid emails.
+ */
+function stdEmail(value: string): string | null {
+  const v = value.trim().toLowerCase();
+  if (!v || !v.includes("@")) return null;
+  const domain = v.split("@").pop();
+  if (!domain || !domain.includes(".")) return null;
+  return v;
+}
+
+/**
+ * Standardize name to proper case (Title Case).
+ * Handles hyphenated names: mary-jane -> Mary-Jane.
+ */
+function stdNameProper(value: string): string | null {
+  const v = value.trim();
+  if (!v) return null;
+  // Collapse whitespace
+  const collapsed = v.replace(/\s+/g, " ");
+  // Title-case each whitespace-separated word; within a word handle hyphens
+  const titleWord = (word: string): string => {
+    if (!word) return "";
+    const hyphenParts = word.split("-");
+    return hyphenParts
+      .map((p) => {
+        if (!p) return "";
+        return p.charAt(0).toUpperCase() + p.slice(1).toLowerCase();
+      })
+      .join("-");
+  };
+  return collapsed.split(" ").map(titleWord).join(" ");
+}
+
+/**
+ * Standardize name to UPPER CASE.
+ */
+function stdNameUpper(value: string): string | null {
+  const v = value.trim().replace(/\s+/g, " ").toUpperCase();
+  return v || null;
+}
+
+/**
+ * Standardize name to lower case.
+ */
+function stdNameLower(value: string): string | null {
+  const v = value.trim().replace(/\s+/g, " ");
+  return v ? v.toLowerCase() : null;
+}
+
+/**
+ * Standardize phone: digits only, strip US country code if 11 digits starting with 1.
+ * Returns null if fewer than 7 digits.
+ */
+function stdPhone(value: string): string | null {
+  let digits = value.replace(/\D/g, "");
+  if (!digits) return null;
+  // Strip US country code
+  if (digits.length === 11 && digits.startsWith("1")) {
+    digits = digits.slice(1);
+  }
+  // Must be at least 7 digits
+  if (digits.length < 7) return null;
+  return digits;
+}
+
+/**
+ * Standardize ZIP code to first 5 digits, zero-padded.
+ */
+function stdZip5(value: string): string | null {
+  // Take part before hyphen or space
+  const first = value.split("-")[0]!.split(" ")[0]!;
+  const digits = first.replace(/\D/g, "");
+  if (!digits) return null;
+  return digits.slice(0, 5).padStart(5, "0");
+}
+
+/**
+ * Title-case a single word.
+ */
+function titleCase(word: string): string {
+  if (!word) return word;
+  return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
+}
+
+/**
+ * Standardize address: title case, USPS abbreviations, normalize whitespace.
+ */
+function stdAddress(value: string): string | null {
+  let v = value.trim();
+  if (!v) return null;
+  // Normalize whitespace
+  v = v.replace(/\s+/g, " ");
+  const words = v.split(" ");
+  const result: string[] = [];
+  let i = 0;
+  while (i < words.length) {
+    // Check two-word phrases first (e.g. "post office")
+    if (i + 1 < words.length) {
+      const twoWord = `${words[i]} ${words[i + 1]}`.toLowerCase();
+      if (twoWord in ADDRESS_ABBREVIATIONS) {
+        result.push(ADDRESS_ABBREVIATIONS[twoWord]!);
+        i += 2;
+        continue;
+      }
+    }
+    // Strip trailing punctuation for lookup
+    const wordLower = words[i]!.toLowerCase().replace(/[.,]+$/, "");
+    if (wordLower in ADDRESS_ABBREVIATIONS) {
+      result.push(ADDRESS_ABBREVIATIONS[wordLower]!);
+    } else {
+      result.push(titleCase(words[i]!));
+    }
+    i += 1;
+  }
+  return result.join(" ");
+}
+
+/**
+ * Standardize state to uppercase, strip.
+ */
+function stdState(value: string): string | null {
+  const v = value.trim().toUpperCase();
+  return v || null;
+}
+
+/**
+ * Strip whitespace, normalize to null if empty.
+ */
+function stdStrip(value: string): string | null {
+  const v = value.trim();
+  return v || null;
+}
+
+/**
+ * Collapse multiple spaces to one, strip.
+ */
+function stdTrimWhitespace(value: string): string | null {
+  const v = value.replace(/\s+/g, " ").trim();
+  return v || null;
+}
+
+// ---------------------------------------------------------------------------
+// Standardizer registry
+// ---------------------------------------------------------------------------
+
+/** Map of standardizer name to function. */
+const STANDARDIZERS: Readonly<Record<string, (value: string) => string | null>> = {
+  email: stdEmail,
+  name_proper: stdNameProper,
+  name_upper: stdNameUpper,
+  name_lower: stdNameLower,
+  phone: stdPhone,
+  zip5: stdZip5,
+  address: stdAddress,
+  state: stdState,
+  strip: stdStrip,
+  trim_whitespace: stdTrimWhitespace,
+};
+
+// ---------------------------------------------------------------------------
+// applyStandardizer — dispatch to the correct standardizer
+// ---------------------------------------------------------------------------
+
+/**
+ * Apply a named standardizer to a string value.
+ *
+ * @throws Error if the standardizer name is not recognized.
+ */
+export function applyStandardizer(value: string, name: string): string {
+  const fn = STANDARDIZERS[name];
+  if (!fn) {
+    const available = Object.keys(STANDARDIZERS).sort().join(", ");
+    throw new Error(
+      `Unknown standardizer: "${name}". Available: ${available}`,
+    );
+  }
+  const result = fn(value);
+  // Standardizers may return null for invalid data; treat as empty string
+  // so downstream pipeline can decide how to handle it.
+  return result ?? "";
+}
+
+// ---------------------------------------------------------------------------
+// applyStandardization — apply rules to all rows
+// ---------------------------------------------------------------------------
+
+/**
+ * Apply standardization rules to rows.
+ *
+ * `rules` maps column names to arrays of standardizer names that are
+ * applied in sequence. For example:
+ *
+ * ```ts
+ * applyStandardization(rows, {
+ *   email: ["email"],
+ *   first_name: ["strip", "name_proper"],
+ *   phone: ["phone"],
+ * });
+ * ```
+ *
+ * Returns new row objects (does not mutate originals).
+ * Null/undefined column values are skipped (left as-is).
+ */
+export function applyStandardization(
+  rows: readonly Row[],
+  rules: Readonly<Record<string, readonly string[]>>,
+): Row[] {
+  return rows.map((row) => {
+    const newRow: Record<string, unknown> = { ...row };
+    for (const [column, standardizers] of Object.entries(rules)) {
+      const val = row[column];
+      if (val === null || val === undefined) continue;
+      let str = String(val);
+      for (const stdName of standardizers) {
+        str = applyStandardizer(str, stdName);
+      }
+      newRow[column] = str;
+    }
+    return newRow as Row;
+  });
+}

package/src/core/streaming.ts

@@ -0,0 +1,128 @@
+/**
+ * streaming.ts — Incremental single-record match + cluster updates.
+ * Edge-safe: no Node.js imports, pure TypeScript only.
+ *
+ * Ports goldenmatch/core/streaming.py.
+ */
+
+import type { Row, MatchkeyConfig, ClusterInfo } from "./types.js";
+import { addToCluster } from "./cluster.js";
+import { matchOne } from "./match-one.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface StreamAddResult {
+  readonly rowId: number;
+  readonly matchedIds: readonly number[];
+  readonly clusterId: number;
+}
+
+export interface StreamProcessorConfig {
+  readonly matchkey: MatchkeyConfig;
+  readonly threshold: number;
+  readonly maxClusterSize?: number;
+}
+
+export interface StreamSnapshot {
+  readonly clusters: ReadonlyMap<number, ClusterInfo>;
+  readonly rows: readonly Row[];
+}
+
+// ---------------------------------------------------------------------------
+// StreamProcessor
+// ---------------------------------------------------------------------------
+
+/**
+ * Incremental record processor.
+ *
+ * On each `add()` the new row is matched against all previously seen rows
+ * using `matchOne`, then folded into the cluster map via `addToCluster`.
+ */
+export class StreamProcessor {
+  private readonly clusters = new Map<number, ClusterInfo>();
+  private readonly rowsById = new Map<number, Row>();
+  private readonly order: number[] = [];
+  private nextId = 0;
+
+  constructor(private readonly config: StreamProcessorConfig) {}
+
+  /** Add a new record and return match + cluster info. */
+  add(row: Row): StreamAddResult {
+    const rowId = (row["__row_id__"] as number | undefined) ?? this.nextId;
+    if (typeof row["__row_id__"] !== "number") {
+      // Attach row_id if missing
+      row = { ...row, __row_id__: rowId };
+    }
+    if (rowId >= this.nextId) {
+      this.nextId = rowId + 1;
+    }
+
+    // Matchkey with threshold override (exact variant has no threshold).
+    const base = this.config.matchkey;
+    const mk: MatchkeyConfig =
+      base.type === "exact"
+        ? base
+        : { ...base, threshold: this.config.threshold };
+
+    // Build snapshot of existing rows (exclude self if duplicate id)
+    const existing: Row[] = [];
+    for (const id of this.order) {
+      if (id === rowId) continue;
+      const r = this.rowsById.get(id);
+      if (r !== undefined) existing.push(r);
+    }
+
+    const hits = matchOne(row, existing, mk);
+    const matchPairs: [number, number][] = hits.map((h) => [h.rowId, h.score]);
+
+    addToCluster(
+      rowId,
+      matchPairs,
+      this.clusters,
+      this.config.maxClusterSize ?? 100,
+    );
+
+    // Register the row
+    if (!this.rowsById.has(rowId)) {
+      this.order.push(rowId);
+    }
+    this.rowsById.set(rowId, row);
+
+    // Find the cluster id the record landed in
+    let landedCid = -1;
+    for (const [cid, info] of this.clusters) {
+      if (info.members.includes(rowId)) {
+        landedCid = cid;
+        break;
+      }
+    }
+
+    return {
+      rowId,
+      matchedIds: hits.map((h) => h.rowId),
+      clusterId: landedCid,
+    };
+  }
+
+  /** Number of records ingested. */
+  get size(): number {
+    return this.rowsById.size;
+  }
+
+  /** Snapshot of current cluster state + rows. */
+  snapshot(): StreamSnapshot {
+    const rows: Row[] = [];
+    for (const id of this.order) {
+      const r = this.rowsById.get(id);
+      if (r !== undefined) rows.push(r);
+    }
+    // Clone clusters to decouple callers from internal map
+    const frozen = new Map<number, ClusterInfo>();
+    for (const [cid, info] of this.clusters) {
+      frozen.set(cid, info);
+    }
+    return { clusters: frozen, rows };
+  }
+}