goldenmatch 0.1.0

package/src/core/profiler.ts

@@ -0,0 +1,212 @@
+/**
+ * profiler.ts — Lightweight per-column data profiler.
+ * Edge-safe: no `node:` imports.
+ *
+ * Ports parts of goldenmatch/core/profiler.py that autoconfig relies on.
+ */
+
+import type { Row } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type ColumnType =
+  | "email"
+  | "phone"
+  | "zip"
+  | "date"
+  | "name"
+  | "geo"
+  | "id"
+  | "numeric"
+  | "text";
+
+export interface ColumnProfile {
+  readonly name: string;
+  readonly nullRate: number;
+  readonly nullCount: number;
+  readonly totalCount: number;
+  readonly distinctCount: number;
+  readonly cardinalityRatio: number;
+  readonly inferredType: ColumnType;
+  readonly avgLength: number;
+  readonly maxLength: number;
+  readonly sampleValues: readonly string[];
+}
+
+export interface DatasetProfile {
+  readonly rowCount: number;
+  readonly columns: readonly ColumnProfile[];
+  readonly byName: Readonly<Record<string, ColumnProfile>>;
+}
+
+// ---------------------------------------------------------------------------
+// Regex heuristics
+// ---------------------------------------------------------------------------
+
+const EMAIL_VALUE_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
+const PHONE_STRIP_RE = /[()\-+.\s]/g;
+const DATE_VALUE_RES: readonly RegExp[] = [
+  /^\d{1,2}[/\-]\d{1,2}[/\-]\d{2,4}$/,
+  /^\d{4}[/\-]\d{1,2}[/\-]\d{1,2}$/,
+  /^\d{1,2}\s[A-Za-z]+\s\d{2,4}$/,
+];
+const ZIP_VALUE_RE = /^\d{5}(-?\d{4})?$/;
+const NAME_VALUE_RE = /^[A-Za-z][A-Za-z \-']{0,28}[A-Za-z]$|^[A-Za-z]{2,3}$/;
+
+// ---------------------------------------------------------------------------
+// Per-column profiling
+// ---------------------------------------------------------------------------
+
+function toStringOrNull(value: unknown): string | null {
+  if (value === null || value === undefined) return null;
+  if (typeof value === "string") {
+    const t = value.trim();
+    return t.length === 0 ? null : t;
+  }
+  return String(value);
+}
+
+function guessType(values: readonly string[], columnName: string): ColumnType {
+  if (values.length === 0) return "text";
+  const n = values.length;
+  const lname = columnName.toLowerCase();
+
+  // Email: >60% look like addresses
+  const emailCount = values.reduce(
+    (acc, v) => acc + (EMAIL_VALUE_RE.test(v) ? 1 : 0),
+    0,
+  );
+  if (emailCount / n > 0.6) return "email";
+
+  // Phone
+  let phoneCount = 0;
+  for (const v of values) {
+    const stripped = v.replace(PHONE_STRIP_RE, "");
+    if (/^\d+$/.test(stripped) && stripped.length >= 7 && stripped.length <= 15) {
+      phoneCount++;
+    }
+  }
+  if (phoneCount / n > 0.6) return "phone";
+
+  // Zip: 5 or 9 digits (with optional dash)
+  const zipCount = values.reduce(
+    (acc, v) => acc + (ZIP_VALUE_RE.test(v) ? 1 : 0),
+    0,
+  );
+  if (zipCount / n > 0.6) return "zip";
+
+  // Date
+  let dateCount = 0;
+  for (const v of values) {
+    if (DATE_VALUE_RES.some((re) => re.test(v))) dateCount++;
+  }
+  if (dateCount / n > 0.6) return "date";
+
+  // Geographic columns by name + short text values
+  if (/^(city|state|county|country|region|province)/i.test(lname)) return "geo";
+  if (/city_desc|state_cd|country_code|state_code/i.test(lname)) return "geo";
+
+  // Identifier columns by name
+  if (/^id$|_id$|uuid|guid/i.test(lname)) return "id";
+
+  // Name: >60% match alpha-name pattern
+  const nameCount = values.reduce(
+    (acc, v) => acc + (NAME_VALUE_RE.test(v) ? 1 : 0),
+    0,
+  );
+  if (nameCount / n > 0.6) return "name";
+
+  // Numeric
+  let numericCount = 0;
+  for (const v of values) {
+    if (/^-?\d+(\.\d+)?$/.test(v)) numericCount++;
+  }
+  if (numericCount / n > 0.8) return "numeric";
+
+  return "text";
+}
+
+function profileColumn(name: string, rawValues: readonly unknown[]): ColumnProfile {
+  const totalCount = rawValues.length;
+  let nullCount = 0;
+  const nonNull: string[] = [];
+  for (const v of rawValues) {
+    const s = toStringOrNull(v);
+    if (s === null) nullCount++;
+    else nonNull.push(s);
+  }
+
+  const distinct = new Set(nonNull);
+  const distinctCount = distinct.size;
+  const cardinalityRatio = totalCount > 0 ? distinctCount / totalCount : 0;
+
+  let totalLen = 0;
+  let maxLen = 0;
+  for (const v of nonNull) {
+    totalLen += v.length;
+    if (v.length > maxLen) maxLen = v.length;
+  }
+  const avgLength = nonNull.length > 0 ? totalLen / nonNull.length : 0;
+  const nullRate = totalCount > 0 ? nullCount / totalCount : 0;
+
+  // Sample values (first 5 unique)
+  const sampleValues: string[] = [];
+  for (const v of distinct) {
+    sampleValues.push(v);
+    if (sampleValues.length >= 5) break;
+  }
+
+  // Subsample for type guessing for performance
+  const sampleForType = nonNull.length > 500 ? nonNull.slice(0, 500) : nonNull;
+  const inferredType = guessType(sampleForType, name);
+
+  return {
+    name,
+    nullRate,
+    nullCount,
+    totalCount,
+    distinctCount,
+    cardinalityRatio,
+    inferredType,
+    avgLength,
+    maxLength: maxLen,
+    sampleValues,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/** Profile all columns of a row array. */
+export function profileRows(rows: readonly Row[]): DatasetProfile {
+  if (rows.length === 0) {
+    return { rowCount: 0, columns: [], byName: {} };
+  }
+
+  // Collect column names from all rows (not just first)
+  const colSet = new Set<string>();
+  for (const r of rows) {
+    for (const k of Object.keys(r)) {
+      if (!k.startsWith("__")) colSet.add(k);
+    }
+  }
+  const columns = [...colSet];
+
+  const profiles: ColumnProfile[] = [];
+  const byName: Record<string, ColumnProfile> = {};
+  for (const col of columns) {
+    const values = rows.map((r) => r[col]);
+    const profile = profileColumn(col, values);
+    profiles.push(profile);
+    byName[col] = profile;
+  }
+
+  return {
+    rowCount: rows.length,
+    columns: profiles,
+    byName,
+  };
+}

package/src/core/quality.ts

@@ -0,0 +1,197 @@
+/**
+ * quality.ts — Lightweight quality scan stub.
+ * Edge-safe: no Node.js imports, pure TypeScript only.
+ *
+ * Ports a subset of goldenmatch/core/quality.py. The Python version
+ * integrates with GoldenCheck; this port only provides the interface and a
+ * handful of basic heuristics that are safe to run client-side.
+ */
+
+import type { Row, QualityConfig } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type QualitySeverity = "info" | "warn" | "error";
+
+export interface QualityFinding {
+  readonly column: string;
+  readonly issue: string;
+  readonly severity: QualitySeverity;
+  readonly affectedRows: number;
+  readonly sampleValues: readonly unknown[];
+}
+
+export interface QualityRunResult {
+  readonly rows: readonly Row[];
+  readonly findings: readonly QualityFinding[];
+}
+
+// ---------------------------------------------------------------------------
+// Pattern detectors
+// ---------------------------------------------------------------------------
+
+const EMAIL_RE = /^[^@\s]+@[^@\s]+\.[^@\s]+$/;
+const DIGITS_RE = /^\d+$/;
+const DATE_PATTERNS: readonly RegExp[] = [
+  /^\d{4}-\d{2}-\d{2}$/, // ISO
+  /^\d{1,2}\/\d{1,2}\/\d{2,4}$/, // US
+  /^\d{1,2}-\d{1,2}-\d{2,4}$/,
+  /^\d{8}$/, // yyyymmdd
+];
+
+function collectColumns(rows: readonly Row[]): string[] {
+  const cols = new Set<string>();
+  for (const row of rows) {
+    for (const key of Object.keys(row)) {
+      if (!key.startsWith("__")) cols.add(key);
+    }
+  }
+  return [...cols];
+}
+
+function asStr(v: unknown): string | null {
+  if (v === null || v === undefined) return null;
+  if (typeof v === "string") return v;
+  if (typeof v === "number" || typeof v === "boolean") return String(v);
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// scanQuality
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a few cheap heuristics across the dataset: high null rate, low
+ * cardinality, inconsistent date format, obviously malformed emails.
+ */
+export function scanQuality(
+  rows: readonly Row[],
+  _config?: QualityConfig,
+): QualityFinding[] {
+  const findings: QualityFinding[] = [];
+  if (rows.length === 0) return findings;
+
+  const total = rows.length;
+  const columns = collectColumns(rows);
+
+  for (const col of columns) {
+    let nullCount = 0;
+    let emailLike = 0;
+    let malformedEmail = 0;
+    let dateLike = 0;
+    const dateFormatsSeen = new Set<number>();
+    const nonNullSamples: unknown[] = [];
+    const distinct = new Set<string>();
+
+    for (const row of rows) {
+      const raw = row[col];
+      if (raw === null || raw === undefined || raw === "") {
+        nullCount++;
+        continue;
+      }
+      if (nonNullSamples.length < 5) nonNullSamples.push(raw);
+      const s = asStr(raw);
+      if (s !== null) {
+        distinct.add(s);
+        // Email heuristics
+        if (s.includes("@")) {
+          emailLike++;
+          if (!EMAIL_RE.test(s)) malformedEmail++;
+        }
+        // Date format tracking
+        for (let i = 0; i < DATE_PATTERNS.length; i++) {
+          if (DATE_PATTERNS[i]!.test(s)) {
+            dateFormatsSeen.add(i);
+            dateLike++;
+            break;
+          }
+        }
+      }
+    }
+
+    const nullRate = nullCount / total;
+    if (nullRate > 0.5) {
+      findings.push({
+        column: col,
+        issue: `High null rate: ${(nullRate * 100).toFixed(1)}%`,
+        severity: nullRate > 0.9 ? "error" : "warn",
+        affectedRows: nullCount,
+        sampleValues: [],
+      });
+    }
+
+    const nonNull = total - nullCount;
+    if (nonNull > 0) {
+      const cardinalityRatio = distinct.size / nonNull;
+      if (cardinalityRatio < 0.001 && distinct.size <= 1) {
+        findings.push({
+          column: col,
+          issue: "Constant column (single distinct non-null value)",
+          severity: "info",
+          affectedRows: nonNull,
+          sampleValues: nonNullSamples,
+        });
+      }
+    }
+
+    if (emailLike > 0 && malformedEmail > 0) {
+      findings.push({
+        column: col,
+        issue: `Malformed email values (${malformedEmail} of ${emailLike})`,
+        severity: "warn",
+        affectedRows: malformedEmail,
+        sampleValues: nonNullSamples,
+      });
+    }
+
+    if (dateLike > 0 && dateFormatsSeen.size > 1) {
+      findings.push({
+        column: col,
+        issue: `Inconsistent date formats (${dateFormatsSeen.size} distinct patterns)`,
+        severity: "warn",
+        affectedRows: dateLike,
+        sampleValues: nonNullSamples,
+      });
+    }
+
+    // Numeric-looking string column
+    if (nonNull > 0 && distinct.size > 0) {
+      let digitCount = 0;
+      for (const v of distinct) {
+        if (DIGITS_RE.test(v)) digitCount++;
+      }
+      if (digitCount === distinct.size && distinct.size > 1) {
+        // Entire column is numeric strings — informational.
+        findings.push({
+          column: col,
+          issue: "Column contains only numeric strings (consider typing)",
+          severity: "info",
+          affectedRows: nonNull,
+          sampleValues: nonNullSamples,
+        });
+      }
+    }
+  }
+
+  return findings;
+}
+
+// ---------------------------------------------------------------------------
+// runQualityCheck
+// ---------------------------------------------------------------------------
+
+/**
+ * Pass-through runner: produce findings, echo rows unchanged.
+ *
+ * Mirrors `_scan_only` / `run_quality_check` from the Python module: no
+ * GoldenCheck, no row rewrites, just reportable findings.
+ */
+export function runQualityCheck(
+  rows: readonly Row[],
+  config?: QualityConfig,
+): QualityRunResult {
+  const findings = scanQuality(rows, config);
+  return { rows, findings };
+}

package/src/core/review-queue.ts

@@ -0,0 +1,177 @@
+/**
+ * review-queue.ts — Human-in-the-loop pair gating.
+ * Edge-safe: no Node.js imports, pure TypeScript only.
+ *
+ * Ports goldenmatch/core/review_queue.py. Default gates: >=0.95 auto-approve,
+ * <0.75 auto-reject, everything in between needs review.
+ */
+
+import type { ScoredPair } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type ReviewStatus = "pending" | "approved" | "rejected";
+
+export interface ReviewItem {
+  readonly pairId: string;
+  readonly idA: number;
+  readonly idB: number;
+  readonly score: number;
+  readonly status: ReviewStatus;
+  readonly createdAt: number;
+}
+
+export interface GatedResult {
+  readonly autoApproved: readonly ScoredPair[];
+  readonly needsReview: readonly ReviewItem[];
+  readonly rejected: readonly ScoredPair[];
+}
+
+export interface GateOptions {
+  readonly approveAbove?: number;
+  readonly rejectBelow?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function canonicalIds(a: number, b: number): [number, number] {
+  return a < b ? [a, b] : [b, a];
+}
+
+function pairIdFor(a: number, b: number): string {
+  const [lo, hi] = canonicalIds(a, b);
+  return `${lo}:${hi}`;
+}
+
+function now(): number {
+  // Date.now is edge-safe (no node imports).
+  return Date.now();
+}
+
+// ---------------------------------------------------------------------------
+// gatePairs
+// ---------------------------------------------------------------------------
+
+/**
+ * Split pairs into auto-approved, needs-review, and rejected buckets.
+ *
+ * Defaults: approveAbove=0.95, rejectBelow=0.75.
+ */
+export function gatePairs(
+  pairs: readonly ScoredPair[],
+  options?: GateOptions,
+): GatedResult {
+  const approveAbove = options?.approveAbove ?? 0.95;
+  const rejectBelow = options?.rejectBelow ?? 0.75;
+
+  const autoApproved: ScoredPair[] = [];
+  const needsReview: ReviewItem[] = [];
+  const rejected: ScoredPair[] = [];
+  const t = now();
+
+  for (const p of pairs) {
+    if (p.score >= approveAbove) {
+      autoApproved.push(p);
+    } else if (p.score < rejectBelow) {
+      rejected.push(p);
+    } else {
+      const [lo, hi] = canonicalIds(p.idA, p.idB);
+      needsReview.push({
+        pairId: `${lo}:${hi}`,
+        idA: lo,
+        idB: hi,
+        score: p.score,
+        status: "pending",
+        createdAt: t,
+      });
+    }
+  }
+
+  return { autoApproved, needsReview, rejected };
+}
+
+// ---------------------------------------------------------------------------
+// ReviewQueue
+// ---------------------------------------------------------------------------
+
+/**
+ * In-memory review queue for human adjudication of borderline pairs.
+ */
+export class ReviewQueue {
+  private readonly items = new Map<string, ReviewItem>();
+
+  /** Add a pair as a pending review item (idempotent by canonical pair id). */
+  add(pair: ScoredPair): void {
+    const [lo, hi] = canonicalIds(pair.idA, pair.idB);
+    const pairId = `${lo}:${hi}`;
+    if (this.items.has(pairId)) return;
+    this.items.set(pairId, {
+      pairId,
+      idA: lo,
+      idB: hi,
+      score: pair.score,
+      status: "pending",
+      createdAt: now(),
+    });
+  }
+
+  /** Get an item by canonical pair id ("minId:maxId"). */
+  get(pairId: string): ReviewItem | undefined {
+    return this.items.get(pairId);
+  }
+
+  /** Mark a pair approved. No-op if unknown. */
+  approve(pairId: string): void {
+    const item = this.items.get(pairId);
+    if (item === undefined) return;
+    this.items.set(pairId, { ...item, status: "approved" });
+  }
+
+  /** Mark a pair rejected. No-op if unknown. */
+  reject(pairId: string): void {
+    const item = this.items.get(pairId);
+    if (item === undefined) return;
+    this.items.set(pairId, { ...item, status: "rejected" });
+  }
+
+  /** All pending items. */
+  pending(): ReviewItem[] {
+    const out: ReviewItem[] = [];
+    for (const item of this.items.values()) {
+      if (item.status === "pending") out.push(item);
+    }
+    return out;
+  }
+
+  /** All approved items. */
+  approved(): ReviewItem[] {
+    const out: ReviewItem[] = [];
+    for (const item of this.items.values()) {
+      if (item.status === "approved") out.push(item);
+    }
+    return out;
+  }
+
+  /** All rejected items. */
+  rejected(): ReviewItem[] {
+    const out: ReviewItem[] = [];
+    for (const item of this.items.values()) {
+      if (item.status === "rejected") out.push(item);
+    }
+    return out;
+  }
+
+  /** Current queue size. */
+  size(): number {
+    return this.items.size;
+  }
+
+  /** Canonical pair id helper ("minId:maxId"). */
+  static pairIdFor(a: number, b: number): string {
+    return pairIdFor(a, b);
+  }
+}