goldenmatch 0.1.0

package/src/core/api.ts

@@ -0,0 +1,220 @@
+/**
+ * api.ts — High-level API functions wrapping the pipeline.
+ * Edge-safe: no `node:` imports, pure TypeScript only.
+ *
+ * Ports goldenmatch/_api.py convenience functions.
+ */
+
+import type {
+  Row,
+  GoldenMatchConfig,
+  DedupeResult,
+  MatchResult,
+  MatchkeyConfig,
+  MatchkeyField,
+  BlockingKeyConfig,
+} from "./types.js";
+import {
+  makeConfig,
+  makeMatchkeyConfig,
+  makeMatchkeyField,
+  makeBlockingConfig,
+} from "./types.js";
+import { runDedupePipeline, runMatchPipeline } from "./pipeline.js";
+import { scoreField, scorePair, asString } from "./scorer.js";
+import { applyTransforms } from "./transforms.js";
+
+// ---------------------------------------------------------------------------
+// Options
+// ---------------------------------------------------------------------------
+
+export interface DedupeOptions {
+  /** Full config object -- takes precedence over shorthand options. */
+  readonly config?: GoldenMatchConfig;
+  /** Columns for exact matching (creates one exact matchkey per column). */
+  readonly exact?: readonly string[];
+  /** Columns for fuzzy matching with per-field thresholds. */
+  readonly fuzzy?: Readonly<Record<string, number>>;
+  /** Blocking key columns (lowercase transform applied). */
+  readonly blocking?: readonly string[];
+  /** Overall fuzzy threshold (default 0.85). */
+  readonly threshold?: number;
+  /** Enable LLM scorer for borderline pairs. Requires OPENAI_API_KEY or ANTHROPIC_API_KEY in env. */
+  readonly llmScorer?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Build config from shorthand options
+// ---------------------------------------------------------------------------
+
+function buildConfigFromOptions(options?: DedupeOptions): GoldenMatchConfig {
+  if (options?.config) return options.config;
+
+  const matchkeys: MatchkeyConfig[] = [];
+  const threshold = options?.threshold ?? 0.85;
+
+  // Exact matchkeys: one per column
+  if (options?.exact) {
+    for (const col of options.exact) {
+      matchkeys.push(
+        makeMatchkeyConfig({
+          name: `exact_${col}`,
+          type: "exact",
+          fields: [
+            makeMatchkeyField({
+              field: col,
+              transforms: ["lowercase", "strip"],
+              scorer: "exact",
+            }),
+          ],
+        }),
+      );
+    }
+  }
+
+  // Fuzzy matchkey: all fuzzy columns combined into one weighted matchkey
+  if (options?.fuzzy) {
+    const fuzzyEntries = Object.entries(options.fuzzy);
+    if (fuzzyEntries.length > 0) {
+      const fields: MatchkeyField[] = fuzzyEntries.map(([col, weight]) =>
+        makeMatchkeyField({
+          field: col,
+          transforms: ["lowercase", "strip"],
+          scorer: "jaro_winkler",
+          weight,
+        }),
+      );
+      matchkeys.push(
+        makeMatchkeyConfig({
+          name: "fuzzy_combined",
+          type: "weighted",
+          fields,
+          threshold,
+        }),
+      );
+    }
+  }
+
+  // Blocking config
+  let blocking = makeBlockingConfig();
+  if (options?.blocking && options.blocking.length > 0) {
+    const keys: BlockingKeyConfig[] = options.blocking.map((col) => ({
+      fields: [col],
+      transforms: ["lowercase", "strip"],
+    }));
+    blocking = makeBlockingConfig({ keys });
+  }
+
+  const partial: Partial<GoldenMatchConfig> = {
+    blocking,
+    threshold,
+  };
+  if (matchkeys.length > 0) {
+    (partial as Record<string, unknown>).matchkeys = matchkeys;
+  }
+  if (options?.llmScorer) {
+    (partial as Record<string, unknown>).llmScorer = {
+      enabled: true,
+      autoThreshold: 0.9,
+      candidateLo: 0.6,
+      candidateHi: 0.9,
+      batchSize: 10,
+      maxWorkers: 4,
+      mode: "pairwise",
+    };
+  }
+  return makeConfig(partial);
+}
+
+// ---------------------------------------------------------------------------
+// Public API: dedupe
+// ---------------------------------------------------------------------------
+
+/**
+ * Deduplicate an array of row objects.
+ *
+ * Shorthand usage:
+ * ```ts
+ * const result = dedupe(rows, {
+ *   exact: ["email"],
+ *   fuzzy: { name: 0.85, address: 0.7 },
+ *   blocking: ["zip"],
+ *   threshold: 0.85,
+ * });
+ * ```
+ *
+ * Or provide a full config:
+ * ```ts
+ * const result = dedupe(rows, { config: myConfig });
+ * ```
+ */
+export function dedupe(
+  rows: readonly Row[],
+  options?: DedupeOptions,
+): DedupeResult {
+  const config = buildConfigFromOptions(options);
+  return runDedupePipeline(rows, config);
+}
+
+// ---------------------------------------------------------------------------
+// Public API: match
+// ---------------------------------------------------------------------------
+
+/**
+ * Match target rows against reference rows.
+ *
+ * Same options as `dedupe()`. Returns matched/unmatched target rows.
+ */
+export function match(
+  target: readonly Row[],
+  reference: readonly Row[],
+  options?: DedupeOptions,
+): MatchResult {
+  const config = buildConfigFromOptions(options);
+  return runMatchPipeline(target, reference, config);
+}
+
+// ---------------------------------------------------------------------------
+// Public API: scoreStrings
+// ---------------------------------------------------------------------------
+
+/**
+ * Score two strings using the specified scorer algorithm.
+ *
+ * @param a - First string.
+ * @param b - Second string.
+ * @param scorer - Scorer name (default: "jaro_winkler").
+ *   Valid scorers: exact, jaro_winkler, levenshtein, token_sort,
+ *   soundex_match, dice, jaccard, ensemble.
+ * @returns Similarity score between 0.0 and 1.0.
+ */
+export function scoreStrings(
+  a: string,
+  b: string,
+  scorer: string = "jaro_winkler",
+): number {
+  const result = scoreField(a, b, scorer);
+  return result ?? 0.0;
+}
+
+// ---------------------------------------------------------------------------
+// Public API: scorePairRecord
+// ---------------------------------------------------------------------------
+
+/**
+ * Score a pair of row objects across specified fields using weighted
+ * aggregation.
+ *
+ * @param rowA - First row.
+ * @param rowB - Second row.
+ * @param fields - Field configs specifying which fields to compare,
+ *   transforms to apply, scorer to use, and weight.
+ * @returns Weighted similarity score between 0.0 and 1.0.
+ */
+export function scorePairRecord(
+  rowA: Row,
+  rowB: Row,
+  fields: readonly MatchkeyField[],
+): number {
+  return scorePair(rowA, rowB, fields);
+}

package/src/core/autoconfig.ts

@@ -0,0 +1,363 @@
+/**
+ * autoconfig.ts — Auto-generate a GoldenMatch config from sample data.
+ * Edge-safe: no `node:` imports.
+ *
+ * Ports goldenmatch/core/autoconfig.py. Profiles the rows, classifies
+ * columns, and builds exact/weighted matchkeys + blocking config.
+ */
+
+import type {
+  Row,
+  GoldenMatchConfig,
+  MatchkeyConfig,
+  MatchkeyField,
+  BlockingKeyConfig,
+  BlockingConfig,
+} from "./types.js";
+import {
+  makeConfig,
+  makeMatchkeyConfig,
+  makeMatchkeyField,
+  makeBlockingConfig,
+  makeGoldenRulesConfig,
+} from "./types.js";
+import { profileRows, type ColumnProfile, type DatasetProfile } from "./profiler.js";
+
+// ---------------------------------------------------------------------------
+// Options
+// ---------------------------------------------------------------------------
+
+export interface AutoconfigOptions {
+  readonly llmProvider?: string;
+  readonly llmAuto?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Name-based classification patterns (authoritative over data profiling for
+// some signals — matches Python's _DATE_PATTERNS / _GEO_PATTERNS behavior).
+// ---------------------------------------------------------------------------
+
+const EMAIL_NAME_PATTERNS = [/email/i, /e_mail/i, /e-mail/i];
+const PHONE_NAME_PATTERNS = [/phone/i, /tel(?!e)/i, /mobile/i, /cell/i];
+const NAME_NAME_PATTERNS = [/name/i, /first/i, /last/i, /full_name/i, /surname/i];
+const ZIP_NAME_PATTERNS = [/zip/i, /postal/i, /postcode/i];
+const GEO_NAME_PATTERNS = [
+  /^city/i,
+  /city_desc/i,
+  /^state/i,
+  /state_cd/i,
+  /county/i,
+  /country/i,
+  /^region/i,
+  /province/i,
+];
+const DATE_NAME_PATTERNS = [
+  /date/i,
+  /created/i,
+  /modified/i,
+  /updated/i,
+  /_at$/i,
+  /birth/i,
+  /dob/i,
+];
+const ID_NAME_PATTERNS = [/^id$/i, /_id$/i, /uuid/i, /guid/i];
+
+// Re-exported for consumers that wanted the spec-level constants.
+export const EMAIL_PATTERNS = EMAIL_NAME_PATTERNS;
+export const PHONE_PATTERNS = PHONE_NAME_PATTERNS;
+export const NAME_PATTERNS = NAME_NAME_PATTERNS;
+export const ZIP_PATTERNS = ZIP_NAME_PATTERNS;
+export const GEO_PATTERNS = GEO_NAME_PATTERNS;
+export const DATE_PATTERNS = DATE_NAME_PATTERNS;
+export const ID_PATTERNS = ID_NAME_PATTERNS;
+
+function nameMatches(name: string, patterns: readonly RegExp[]): boolean {
+  return patterns.some((re) => re.test(name));
+}
+
+// ---------------------------------------------------------------------------
+// Column classification (authoritative: date > geo > name heuristics)
+// ---------------------------------------------------------------------------
+
+type ClassifiedKind =
+  | "email"
+  | "phone"
+  | "zip"
+  | "geo"
+  | "date"
+  | "name"
+  | "id"
+  | "numeric"
+  | "text";
+
+function classifyColumn(profile: ColumnProfile): ClassifiedKind {
+  const name = profile.name;
+
+  // Date is checked first so that date-like columns never get misclassified
+  // as phones by the profiler's value heuristic.
+  if (nameMatches(name, DATE_NAME_PATTERNS)) return "date";
+  if (profile.inferredType === "date") return "date";
+
+  if (nameMatches(name, GEO_NAME_PATTERNS)) return "geo";
+  if (profile.inferredType === "geo") return "geo";
+
+  if (nameMatches(name, EMAIL_NAME_PATTERNS) || profile.inferredType === "email") {
+    return "email";
+  }
+  if (nameMatches(name, PHONE_NAME_PATTERNS) || profile.inferredType === "phone") {
+    return "phone";
+  }
+  if (nameMatches(name, ZIP_NAME_PATTERNS) || profile.inferredType === "zip") {
+    return "zip";
+  }
+  if (nameMatches(name, NAME_NAME_PATTERNS) || profile.inferredType === "name") {
+    return "name";
+  }
+  if (nameMatches(name, ID_NAME_PATTERNS) || profile.inferredType === "id") {
+    return "id";
+  }
+  if (profile.inferredType === "numeric") return "numeric";
+  return "text";
+}
+
+// ---------------------------------------------------------------------------
+// Heuristic builders
+// ---------------------------------------------------------------------------
+
+function buildExactMatchkeys(
+  profiles: readonly ColumnProfile[],
+): MatchkeyConfig[] {
+  const out: MatchkeyConfig[] = [];
+  for (const p of profiles) {
+    const kind = classifyColumn(p);
+    // zip/geo are blocking signals, NOT identity claims.
+    if (kind === "zip" || kind === "geo" || kind === "date" || kind === "text") {
+      continue;
+    }
+
+    // Skip sparse & near-constant columns
+    if (p.nullRate > 0.4) continue;
+    if (p.cardinalityRatio < 0.01) continue;
+
+    // Only identifier-like columns get exact matchkeys with >=0.5 cardinality.
+    const isIdentifier =
+      kind === "email" || kind === "phone" || kind === "id";
+    if (!isIdentifier) continue;
+    if (p.cardinalityRatio < 0.5) continue;
+
+    const transforms: string[] =
+      kind === "email"
+        ? ["lowercase", "strip"]
+        : kind === "phone"
+          ? ["digits_only"]
+          : ["strip"];
+
+    out.push(
+      makeMatchkeyConfig({
+        name: `exact_${p.name}`,
+        type: "exact",
+        fields: [
+          makeMatchkeyField({
+            field: p.name,
+            transforms,
+            scorer: "exact",
+            weight: 1.0,
+          }),
+        ],
+        threshold: 1.0,
+      }),
+    );
+  }
+  return out;
+}
+
+function buildWeightedMatchkey(
+  profiles: readonly ColumnProfile[],
+): MatchkeyConfig | null {
+  const fields: MatchkeyField[] = [];
+
+  for (const p of profiles) {
+    const kind = classifyColumn(p);
+    if (p.nullRate > 0.5) continue;
+
+    if (kind === "name") {
+      fields.push(
+        makeMatchkeyField({
+          field: p.name,
+          transforms: ["lowercase", "strip", "normalize_whitespace"],
+          scorer: "jaro_winkler",
+          weight: 0.6,
+        }),
+      );
+    } else if (kind === "email") {
+      fields.push(
+        makeMatchkeyField({
+          field: p.name,
+          transforms: ["lowercase", "strip"],
+          scorer: "jaro_winkler",
+          weight: 0.3,
+        }),
+      );
+    } else if (kind === "phone") {
+      fields.push(
+        makeMatchkeyField({
+          field: p.name,
+          transforms: ["digits_only"],
+          scorer: "exact",
+          weight: 0.25,
+        }),
+      );
+    } else if (kind === "zip") {
+      fields.push(
+        makeMatchkeyField({
+          field: p.name,
+          transforms: ["digits_only"],
+          scorer: "exact",
+          weight: 0.15,
+        }),
+      );
+    } else if (kind === "geo") {
+      fields.push(
+        makeMatchkeyField({
+          field: p.name,
+          transforms: ["lowercase", "strip"],
+          scorer: "exact",
+          weight: 0.1,
+        }),
+      );
+    } else if (kind === "text" && p.avgLength >= 10) {
+      // Long free-text columns: token_sort to catch reordering
+      fields.push(
+        makeMatchkeyField({
+          field: p.name,
+          transforms: ["lowercase", "strip", "token_sort"],
+          scorer: "token_sort",
+          weight: 0.2,
+        }),
+      );
+    }
+  }
+
+  if (fields.length === 0) return null;
+
+  return makeMatchkeyConfig({
+    name: "weighted_identity",
+    type: "weighted",
+    fields,
+    threshold: 0.85,
+    rerank: false,
+  });
+}
+
+function buildBlocking(profiles: readonly ColumnProfile[]): BlockingConfig {
+  const keys: BlockingKeyConfig[] = [];
+
+  // Prefer zip > geo > first-letter of name
+  for (const p of profiles) {
+    const kind = classifyColumn(p);
+    if (kind !== "zip") continue;
+    if (p.nullRate > 0.2) continue;
+    if (p.cardinalityRatio >= 0.95) continue;
+    keys.push({
+      fields: [p.name],
+      transforms: ["digits_only", "substring:0:5"],
+    });
+    break;
+  }
+
+  if (keys.length === 0) {
+    for (const p of profiles) {
+      const kind = classifyColumn(p);
+      if (kind !== "geo") continue;
+      if (p.nullRate > 0.2) continue;
+      if (p.cardinalityRatio >= 0.95) continue;
+      keys.push({
+        fields: [p.name],
+        transforms: ["lowercase", "strip"],
+      });
+      break;
+    }
+  }
+
+  if (keys.length === 0) {
+    for (const p of profiles) {
+      const kind = classifyColumn(p);
+      if (kind !== "name") continue;
+      if (p.nullRate > 0.2) continue;
+      if (p.cardinalityRatio >= 0.95) continue;
+      keys.push({
+        fields: [p.name],
+        transforms: ["lowercase", "strip", "substring:0:1"],
+      });
+      break;
+    }
+  }
+
+  // Last resort: first non-null column that isn't near-unique or sparse
+  if (keys.length === 0) {
+    for (const p of profiles) {
+      if (p.nullRate > 0.2) continue;
+      if (p.cardinalityRatio >= 0.95) continue;
+      if (p.cardinalityRatio < 0.01) continue;
+      keys.push({
+        fields: [p.name],
+        transforms: ["lowercase", "strip"],
+      });
+      break;
+    }
+  }
+
+  return makeBlockingConfig({
+    strategy: "static",
+    keys,
+    maxBlockSize: 1000,
+    skipOversized: true,
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Public entry points
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a GoldenMatchConfig by profiling the provided rows.
+ *
+ * Mirrors goldenmatch.core.autoconfig.auto_configure_df. Does not apply
+ * standardization rules directly — callers can merge them onto the result.
+ */
+export function autoConfigureRows(
+  rows: readonly Row[],
+  options?: AutoconfigOptions,
+): GoldenMatchConfig {
+  const profile: DatasetProfile = profileRows(rows);
+  const profiles = profile.columns;
+
+  const exactKeys = buildExactMatchkeys(profiles);
+  const weighted = buildWeightedMatchkey(profiles);
+  const matchkeys: MatchkeyConfig[] = [...exactKeys];
+  if (weighted) matchkeys.push(weighted);
+
+  const blocking = buildBlocking(profiles);
+  const goldenRules = makeGoldenRulesConfig({ defaultStrategy: "most_complete" });
+
+  const config = makeConfig({
+    matchkeys,
+    blocking,
+    goldenRules,
+    threshold: 0.85,
+    ...(options?.llmAuto !== undefined ? { llmAuto: options.llmAuto } : {}),
+  });
+
+  return config;
+}
+
+/**
+ * Convenience alias for API parity with the Python function that starts
+ * from "files" (which, in edge-safe land, means pre-loaded row arrays).
+ */
+export function autoConfigure(
+  rows: readonly Row[],
+  options?: AutoconfigOptions,
+): GoldenMatchConfig {
+  return autoConfigureRows(rows, options);
+}

package/src/core/autofix.ts

@@ -0,0 +1,102 @@
+/**
+ * autofix.ts — Lightweight row auto-fix utilities.
+ * Edge-safe: no Node.js imports, pure TypeScript only.
+ *
+ * Ports goldenmatch/core/autofix.py. Trims whitespace, nulls empty strings,
+ * and converts common "no value" tokens to null.
+ */
+
+import type { Row } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface AutoFixLog {
+  readonly fixType: string;
+  readonly column: string;
+  readonly affectedRows: number;
+}
+
+export interface AutoFixResult {
+  readonly rows: Row[];
+  readonly log: AutoFixLog[];
+}
+
+// ---------------------------------------------------------------------------
+// Tokens treated as null
+// ---------------------------------------------------------------------------
+
+const NULL_TOKENS: ReadonlySet<string> = new Set([
+  "n/a",
+  "na",
+  "none",
+  "null",
+  "nil",
+  "unknown",
+  "unk",
+  "-",
+  "--",
+  "?",
+]);
+
+function isNullToken(s: string): boolean {
+  const lower = s.trim().toLowerCase();
+  if (lower.length === 0) return true;
+  return NULL_TOKENS.has(lower);
+}
+
+// ---------------------------------------------------------------------------
+// autoFixRows
+// ---------------------------------------------------------------------------
+
+/**
+ * Apply conservative fixes row-by-row:
+ * - trim string values
+ * - convert empty strings and common "no value" tokens to null
+ *
+ * Internal columns (prefix `__`) are preserved unchanged.
+ */
+export function autoFixRows(rows: readonly Row[]): AutoFixResult {
+  const out: Row[] = [];
+  const trimCounts = new Map<string, number>();
+  const nullCounts = new Map<string, number>();
+
+  for (const row of rows) {
+    const fixed: Record<string, unknown> = {};
+    let changed = false;
+    for (const [key, value] of Object.entries(row)) {
+      if (key.startsWith("__")) {
+        fixed[key] = value;
+        continue;
+      }
+      if (typeof value === "string") {
+        const trimmed = value.trim();
+        if (trimmed !== value) {
+          trimCounts.set(key, (trimCounts.get(key) ?? 0) + 1);
+          changed = true;
+        }
+        if (isNullToken(trimmed)) {
+          fixed[key] = null;
+          nullCounts.set(key, (nullCounts.get(key) ?? 0) + 1);
+          changed = true;
+        } else {
+          fixed[key] = trimmed;
+        }
+      } else {
+        fixed[key] = value;
+      }
+    }
+    out.push(changed ? (fixed as Row) : row);
+  }
+
+  const log: AutoFixLog[] = [];
+  for (const [col, n] of trimCounts) {
+    log.push({ fixType: "trim_whitespace", column: col, affectedRows: n });
+  }
+  for (const [col, n] of nullCounts) {
+    log.push({ fixType: "null_empty_or_token", column: col, affectedRows: n });
+  }
+
+  return { rows: out, log };
+}