goldenmatch 0.1.0

package/src/core/memory/corrections.ts

@@ -0,0 +1,179 @@
+/**
+ * memory/corrections.ts — Apply stored corrections to scored pairs.
+ * Edge-safe: no `node:` imports.
+ *
+ * Ports goldenmatch/core/memory/corrections.py. A correction is only
+ * applied if both rows still hash to the values seen when the correction
+ * was recorded (dual-hash staleness detection).
+ */
+
+import type { Row, ScoredPair } from "../types.js";
+import { makeScoredPair } from "../types.js";
+import type { Correction, MemoryStore } from "./store.js";
+
+// ---------------------------------------------------------------------------
+// Row hashing
+// ---------------------------------------------------------------------------
+
+/**
+ * Deterministic FNV-1a 32-bit hash. Matches store-side hashing so
+ * corrections can survive serialization/round-trips.
+ */
+function hashString(s: string): string {
+  let h = 2166136261;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return (h >>> 0).toString(16);
+}
+
+/** Hash of a row across its non-internal fields (sorted, stringified). */
+export function hashRow(row: Row): string {
+  const keys = Object.keys(row)
+    .filter((k) => !k.startsWith("__"))
+    .sort();
+  const parts: string[] = [];
+  for (const k of keys) {
+    const v = row[k];
+    const s = v === null || v === undefined ? "\u0000null" : String(v);
+    parts.push(`${k}=${s}`);
+  }
+  return hashString(parts.join("|"));
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function pairKey(a: number, b: number): string {
+  return a < b ? `${a}|${b}` : `${b}|${a}`;
+}
+
+function getRowId(row: Row): number | null {
+  const raw = row["__row_id__"];
+  if (typeof raw === "number") return raw;
+  if (typeof raw === "string") {
+    const n = Number(raw);
+    return Number.isFinite(n) ? n : null;
+  }
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Stored-correction metadata
+// ---------------------------------------------------------------------------
+
+export interface StoredRowHashes {
+  readonly rowIdAHash: string;
+  readonly rowIdBHash: string;
+}
+
+/**
+ * A caller can either provide a per-correction hash map (populated at
+ * collection time) or ask applyCorrections to compute current hashes alone
+ * — in which case staleness detection is a no-op (hashes always match).
+ */
+export interface ApplyCorrectionsOptions {
+  readonly originalHashes?: ReadonlyMap<string, StoredRowHashes>;
+  /** When a correction matches, clamp pair score to this value. Default 1.0 for match, 0.0 for no_match. */
+  readonly matchScore?: number;
+  readonly noMatchScore?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Apply corrections
+// ---------------------------------------------------------------------------
+
+/**
+ * Apply user corrections stored in `store` to a list of scored pairs.
+ *
+ * For each correction:
+ *   - Find the pair (idA,idB) in the scored_pairs list.
+ *   - If caller supplied original hashes, compare them against a fresh
+ *     hash of the current row. Mismatch => stale, skip.
+ *   - Otherwise apply the verdict:
+ *       "match"    -> score clamped to matchScore (default 1.0)
+ *       "no_match" -> score clamped to noMatchScore (default 0.0)
+ *
+ * Returns the modified pairs plus counts of applied / stale corrections.
+ */
+export function applyCorrections(
+  pairs: readonly ScoredPair[],
+  rows: readonly Row[],
+  store: MemoryStore,
+  options?: ApplyCorrectionsOptions,
+): { pairs: readonly ScoredPair[]; applied: number; stale: number } {
+  const matchScore = options?.matchScore ?? 1.0;
+  const noMatchScore = options?.noMatchScore ?? 0.0;
+
+  // Build index: rowId -> Row for current-state hashing.
+  const rowById = new Map<number, Row>();
+  for (const r of rows) {
+    const id = getRowId(r);
+    if (id !== null) rowById.set(id, r);
+  }
+
+  // Index corrections by canonical pair key.
+  const byPair = new Map<string, Correction>();
+  for (const c of store.list()) {
+    const key = pairKey(c.rowIdA, c.rowIdB);
+    const existing = byPair.get(key);
+    // Keep the highest-trust correction per pair (most recent on tie).
+    if (
+      existing === undefined ||
+      c.trust > existing.trust ||
+      (c.trust === existing.trust && c.timestamp > existing.timestamp)
+    ) {
+      byPair.set(key, c);
+    }
+  }
+
+  let applied = 0;
+  let stale = 0;
+  const out: ScoredPair[] = [];
+
+  for (const pair of pairs) {
+    const key = pairKey(pair.idA, pair.idB);
+    const correction = byPair.get(key);
+    if (!correction) {
+      out.push(pair);
+      continue;
+    }
+
+    // Dual-hash staleness check (if caller populated `originalHashes`).
+    if (options?.originalHashes) {
+      const stored = options.originalHashes.get(key);
+      if (stored) {
+        const rowA = rowById.get(pair.idA);
+        const rowB = rowById.get(pair.idB);
+        if (!rowA || !rowB) {
+          stale++;
+          out.push(pair);
+          continue;
+        }
+        const currentA = hashRow(rowA);
+        const currentB = hashRow(rowB);
+        const match =
+          (currentA === stored.rowIdAHash && currentB === stored.rowIdBHash) ||
+          (currentA === stored.rowIdBHash && currentB === stored.rowIdAHash);
+        if (!match) {
+          stale++;
+          out.push(pair);
+          continue;
+        }
+      }
+    }
+
+    applied++;
+    out.push(
+      makeScoredPair(
+        pair.idA,
+        pair.idB,
+        correction.verdict === "match" ? matchScore : noMatchScore,
+      ),
+    );
+  }
+
+  return { pairs: out, applied, stale };
+}

package/src/core/memory/learner.ts

@@ -0,0 +1,218 @@
+/**
+ * memory/learner.ts — Threshold tuning & weight learning from corrections.
+ * Edge-safe: no `node:` imports.
+ *
+ * Ports goldenmatch/core/memory/learner.py. Given ≥10 corrections, sweep
+ * thresholds and pick the one maximizing F1 on the correction set. Given
+ * ≥50 corrections with per-field subscores, fit a simple logistic-
+ * regression-like weight update.
+ */
+
+import type { LearningConfig, MatchkeyConfig } from "../types.js";
+import type { Correction } from "./store.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface LearnedParams {
+  readonly threshold?: number;
+  readonly fieldWeights?: Readonly<Record<string, number>>;
+  readonly correctionCount: number;
+}
+
+/**
+ * Per-correction subscores. When present, keys correspond to matchkey field
+ * names and values are in [0,1] representing each field's contribution.
+ * The learner uses these only when ≥ weightsMinCorrections samples include
+ * them.
+ */
+export interface CorrectionSubscores {
+  readonly pairKey: string; // "minId|maxId"
+  readonly subscores: Readonly<Record<string, number>>;
+}
+
+// ---------------------------------------------------------------------------
+// Learner
+// ---------------------------------------------------------------------------
+
+const DEFAULT_LEARNING_CONFIG: LearningConfig = {
+  thresholdMinCorrections: 10,
+  weightsMinCorrections: 50,
+};
+
+export class MemoryLearner {
+  constructor(
+    private readonly config: LearningConfig = DEFAULT_LEARNING_CONFIG,
+  ) {}
+
+  /**
+   * Tune threshold and (optionally) field weights from corrections.
+   *
+   * Threshold tuning: sweep 0.5..0.95 in 0.05 steps, compute F1 using each
+   * correction's stored `score` vs its verdict. Returns the threshold with
+   * the best F1 (ties break toward higher threshold for precision).
+   *
+   * Field weights: requires subscores. Fits a tiny gradient update that
+   * nudges weights toward better discrimination of match / no_match.
+   */
+  learn(
+    corrections: readonly Correction[],
+    baseline: MatchkeyConfig,
+    subscores?: readonly CorrectionSubscores[],
+  ): LearnedParams {
+    const result: {
+      threshold?: number;
+      fieldWeights?: Record<string, number>;
+      correctionCount: number;
+    } = { correctionCount: corrections.length };
+
+    if (corrections.length >= this.config.thresholdMinCorrections) {
+      const tuned = tuneThreshold(corrections);
+      if (tuned !== null) result.threshold = tuned;
+    }
+
+    if (
+      subscores &&
+      corrections.length >= this.config.weightsMinCorrections &&
+      subscores.length >= this.config.weightsMinCorrections
+    ) {
+      const learnedWeights = tuneWeights(corrections, subscores, baseline);
+      if (learnedWeights) result.fieldWeights = learnedWeights;
+    }
+
+    return result;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Threshold tuning
+// ---------------------------------------------------------------------------
+
+/**
+ * Sweep thresholds in [0.5, 0.95] step 0.05 and pick one maximizing F1.
+ * Returns null if corrections cannot produce a meaningful F1 (e.g. all
+ * same verdict).
+ */
+function tuneThreshold(corrections: readonly Correction[]): number | null {
+  const positives = corrections.filter((c) => c.verdict === "match");
+  const negatives = corrections.filter((c) => c.verdict === "no_match");
+  if (positives.length === 0 || negatives.length === 0) return null;
+
+  let bestThreshold = 0.85;
+  let bestF1 = -1;
+
+  for (let t = 0.5; t <= 0.95 + 1e-9; t += 0.05) {
+    let tp = 0;
+    let fp = 0;
+    let fn = 0;
+    for (const c of corrections) {
+      const predicted = c.score >= t;
+      if (c.verdict === "match") {
+        if (predicted) tp++;
+        else fn++;
+      } else {
+        if (predicted) fp++;
+      }
+    }
+    const precision = tp + fp === 0 ? 0 : tp / (tp + fp);
+    const recall = tp + fn === 0 ? 0 : tp / (tp + fn);
+    const f1 =
+      precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
+    if (f1 > bestF1 || (f1 === bestF1 && t > bestThreshold)) {
+      bestF1 = f1;
+      bestThreshold = t;
+    }
+  }
+
+  return Number(bestThreshold.toFixed(3));
+}
+
+// ---------------------------------------------------------------------------
+// Weight tuning (simple gradient pass)
+// ---------------------------------------------------------------------------
+
+function sigmoid(x: number): number {
+  if (x >= 0) {
+    const ex = Math.exp(-x);
+    return 1 / (1 + ex);
+  }
+  const ex = Math.exp(x);
+  return ex / (1 + ex);
+}
+
+function tuneWeights(
+  corrections: readonly Correction[],
+  subscores: readonly CorrectionSubscores[],
+  baseline: MatchkeyConfig,
+): Record<string, number> | null {
+  const subByPair = new Map<string, Record<string, number>>();
+  for (const s of subscores) {
+    subByPair.set(s.pairKey, { ...s.subscores });
+  }
+
+  // Collect field list from baseline matchkey.
+  const fields = baseline.fields.map((f) => f.field);
+  if (fields.length === 0) return null;
+
+  // Initialize weights from baseline.
+  const weights = new Map<string, number>();
+  for (const f of baseline.fields) weights.set(f.field, f.weight);
+
+  // Build training set: for each correction we find its subscores.
+  type Sample = { y: number; x: Record<string, number> };
+  const samples: Sample[] = [];
+  for (const c of corrections) {
+    const [a, b] = c.rowIdA < c.rowIdB ? [c.rowIdA, c.rowIdB] : [c.rowIdB, c.rowIdA];
+    const key = `${a}|${b}`;
+    const sub = subByPair.get(key);
+    if (!sub) continue;
+    samples.push({
+      y: c.verdict === "match" ? 1 : 0,
+      x: sub,
+    });
+  }
+  if (samples.length < 10) return null;
+
+  const learningRate = 0.1;
+  const iterations = 50;
+  for (let iter = 0; iter < iterations; iter++) {
+    const grad = new Map<string, number>();
+    for (const f of fields) grad.set(f, 0);
+
+    for (const sample of samples) {
+      let z = 0;
+      for (const f of fields) {
+        const w = weights.get(f) ?? 0;
+        const x = sample.x[f] ?? 0;
+        z += w * x;
+      }
+      const pred = sigmoid(z);
+      const err = pred - sample.y;
+      for (const f of fields) {
+        const x = sample.x[f] ?? 0;
+        grad.set(f, (grad.get(f) ?? 0) + err * x);
+      }
+    }
+
+    for (const f of fields) {
+      const g = (grad.get(f) ?? 0) / samples.length;
+      const w = weights.get(f) ?? 0;
+      weights.set(f, w - learningRate * g);
+    }
+  }
+
+  // Re-normalize weights so they sum to 1 (matchkey weights must average
+  // out to the original budget; keep same total).
+  const originalTotal = baseline.fields.reduce((acc, f) => acc + f.weight, 0);
+  const newTotal = fields.reduce((acc, f) => acc + Math.max(0, weights.get(f) ?? 0), 0);
+  if (newTotal <= 0) return null;
+  const scale = originalTotal / newTotal;
+
+  const out: Record<string, number> = {};
+  for (const f of fields) {
+    const w = Math.max(0, weights.get(f) ?? 0) * scale;
+    out[f] = Number(w.toFixed(4));
+  }
+  return out;
+}

package/src/core/memory/store.ts

@@ -0,0 +1,114 @@
+/**
+ * memory/store.ts — Learning Memory store (in-memory backend).
+ * Edge-safe: no `node:` imports.
+ *
+ * Ports goldenmatch/core/memory/store.py. SQLite / Postgres backends are
+ * deferred (they require host-specific drivers); the in-memory backend
+ * keeps all corrections in a plain array with trust-based upsert.
+ */
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface Correction {
+  readonly rowIdA: number;
+  readonly rowIdB: number;
+  readonly verdict: "match" | "no_match";
+  readonly feature: string;
+  readonly score: number;
+  readonly timestamp: number;
+  readonly trust: number;
+  readonly source: string;
+}
+
+export interface MemoryStoreConfig {
+  readonly backend: "memory" | "sqlite" | "postgres";
+  readonly path?: string;
+  readonly trustDefault?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function pairFeatureKey(c: Correction): string {
+  const [a, b] = c.rowIdA < c.rowIdB ? [c.rowIdA, c.rowIdB] : [c.rowIdB, c.rowIdA];
+  return `${a}|${b}|${c.feature}`;
+}
+
+// ---------------------------------------------------------------------------
+// MemoryStore
+// ---------------------------------------------------------------------------
+
+export class MemoryStore {
+  private corrections: Correction[] = [];
+
+  constructor(private readonly config: MemoryStoreConfig = { backend: "memory" }) {
+    if (config.backend !== "memory") {
+      // SQLite/Postgres backends intentionally unsupported in edge-safe code.
+      // Callers that need persistence should swap in a host-specific wrapper.
+      // We don't throw here to keep the class usable for tests.
+    }
+  }
+
+  /** Append a correction unconditionally. */
+  add(correction: Correction): void {
+    this.corrections.push(correction);
+  }
+
+  /** Append many corrections unconditionally. */
+  addBatch(corrections: readonly Correction[]): void {
+    for (const c of corrections) this.corrections.push(c);
+  }
+
+  /** All corrections, in insertion order. */
+  list(): readonly Correction[] {
+    return this.corrections;
+  }
+
+  /** Corrections whose verdict is "match". */
+  listMatches(): readonly Correction[] {
+    return this.corrections.filter((c) => c.verdict === "match");
+  }
+
+  /** Corrections whose verdict is "no_match". */
+  listNonMatches(): readonly Correction[] {
+    return this.corrections.filter((c) => c.verdict === "no_match");
+  }
+
+  count(): number {
+    return this.corrections.length;
+  }
+
+  clear(): void {
+    this.corrections = [];
+  }
+
+  /**
+   * Trust-based upsert: if a correction for the same (pair, feature) already
+   * exists, keep whichever has higher trust. Ties break toward the more recent
+   * correction.
+   */
+  upsert(correction: Correction): void {
+    const key = pairFeatureKey(correction);
+    for (let i = 0; i < this.corrections.length; i++) {
+      const existing = this.corrections[i]!;
+      if (pairFeatureKey(existing) !== key) continue;
+
+      const newer = correction.timestamp >= existing.timestamp;
+      const higherTrust = correction.trust > existing.trust;
+      const equalTrustButNewer = correction.trust === existing.trust && newer;
+      if (higherTrust || equalTrustButNewer) {
+        this.corrections[i] = correction;
+      }
+      return;
+    }
+    this.corrections.push(correction);
+  }
+
+  /** Return the effective config (for debugging). */
+  getConfig(): MemoryStoreConfig {
+    return this.config;
+  }
+}