@gamaze/hicortex 0.23.1 → 0.24.0

package/dist/eval/dups.d.ts

@@ -1,100 +0,0 @@
-/**
- * D1 — duplicate-rate audit (#191 mechanical baseline).
- *
- * For each memory, finds its top-10 nearest neighbors by vector distance and
- * keeps every pair whose cosine similarity clears the lowest of three report
- * thresholds (0.90 / 0.92 / 0.95). Clusters are built per threshold with
- * union-find; the headline number at each threshold is "excess" — how many
- * rows would disappear if every cluster were merged down to one memory.
- *
- * Duplicate pairs are additionally attributed to either the #189 recovery
- * re-ingest (a retried capture segment produced a second row for content
- * already stored) or organic near-duplication (independent sessions that
- * happened to cover the same ground), using two signals: a shared base
- * `source_session` (the `#<segment>` suffix stripped), or ingestion runs that
- * differ while `created_at` is near-identical (a re-ingest preserves the
- * original session date but lands in a later ingestion run).
- *
- * The clustering primitives (union-find, KNN edge building, metadata-mismatch
- * check) live in `../cluster.js` — extracted (#100/#191) so `hicortex dedup`
- * reuses the exact same math instead of re-implementing it. Re-exported below
- * for backward compatibility with existing importers of this module.
- */
-import type Database from "better-sqlite3";
-import { UnionFind, clusterEdges, clusterExcess, clusterMetadataMismatch, type Edge, type ClusterMetadataMismatch } from "../cluster.js";
-export { UnionFind, clusterEdges, clusterExcess, clusterMetadataMismatch, type ClusterMetadataMismatch };
-/** @deprecated import `Edge` from `../cluster.js` instead. */
-export type DupEdge = Edge;
-/** Cosine thresholds the report evaluates, low to high. */
-export declare const DUP_THRESHOLDS: readonly [0.9, 0.92, 0.95];
-export interface DupMemoryRow {
-    id: string;
-    content: string;
-    created_at: string;
-    ingested_at: string;
-    source_session: string | null;
-    project: string | null;
-    privacy: string;
-    source_agent: string;
-}
-/**
- * Partition memories into ingestion "runs" by `ingested_at`: sorted
- * ascending, a gap greater than `gapHours` starts a new run. Returns a map
- * of memory id -> run id (0-based, monotonically increasing).
- */
-export declare function sessionizeByIngestedAt(rows: Array<{
-    id: string;
-    ingested_at: string;
-}>, gapHours?: number): Map<string, number>;
-/** Strip the `#<segment>` suffix from a `source_session` value. Null-safe. */
-export declare function baseSessionId(sourceSession: string | null): string | null;
-/** Whether two ISO timestamps fall within `toleranceDays` of each other. */
-export declare function nearIdenticalDate(a: string, b: string, toleranceDays?: number): boolean;
-export type PairAttribution = "recovery_reingest" | "organic";
-/**
- * Classify a duplicate pair as a #189 recovery re-ingest or organic overlap.
- * Recovery re-ingest when either: both sides share the same base
- * `source_session`, or they landed in different ingestion runs but
- * `created_at` is near-identical (the re-ingest preserves the original
- * session date while landing in a later run).
- */
-export declare function attributePair(a: DupMemoryRow, b: DupMemoryRow, runOf: Map<string, number>): PairAttribution;
-export interface DupThresholdResult {
-    threshold: number;
-    clusterCount: number;
-    excess: number;
-    /** Cluster sizes, largest first — a quick histogram without dumping content. */
-    clusterSizes: number[];
-}
-export interface DupClusterDump {
-    size: number;
-    members: Array<{
-        id: string;
-        created_at: string;
-        preview: string;
-    }>;
-    attribution: {
-        recoveryReingest: number;
-        organic: number;
-    };
-    metadataMismatch: ClusterMetadataMismatch;
-}
-export interface DupReport {
-    totalMemories: number;
-    knnK: number;
-    thresholds: DupThresholdResult[];
-    /** Attribution over every pair at/above the lowest threshold (broadest view). */
-    pairAttribution: {
-        recoveryReingest: number;
-        organic: number;
-        totalPairs: number;
-    };
-    /** Top clusters at the lowest threshold, largest first, for manual review. */
-    topClusters: DupClusterDump[];
-}
-/**
- * Run the full D1 duplicate audit against an open (readonly) snapshot
- * connection. Pure aside from the DB reads — safe against a readonly
- * connection, no writes attempted.
- */
-export declare function runDupAudit(db: Database.Database, topN?: number): DupReport;

package/dist/eval/dups.js

@@ -1,174 +0,0 @@
-"use strict";
-/**
- * D1 — duplicate-rate audit (#191 mechanical baseline).
- *
- * For each memory, finds its top-10 nearest neighbors by vector distance and
- * keeps every pair whose cosine similarity clears the lowest of three report
- * thresholds (0.90 / 0.92 / 0.95). Clusters are built per threshold with
- * union-find; the headline number at each threshold is "excess" — how many
- * rows would disappear if every cluster were merged down to one memory.
- *
- * Duplicate pairs are additionally attributed to either the #189 recovery
- * re-ingest (a retried capture segment produced a second row for content
- * already stored) or organic near-duplication (independent sessions that
- * happened to cover the same ground), using two signals: a shared base
- * `source_session` (the `#<segment>` suffix stripped), or ingestion runs that
- * differ while `created_at` is near-identical (a re-ingest preserves the
- * original session date but lands in a later ingestion run).
- *
- * The clustering primitives (union-find, KNN edge building, metadata-mismatch
- * check) live in `../cluster.js` — extracted (#100/#191) so `hicortex dedup`
- * reuses the exact same math instead of re-implementing it. Re-exported below
- * for backward compatibility with existing importers of this module.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DUP_THRESHOLDS = exports.clusterMetadataMismatch = exports.clusterExcess = exports.clusterEdges = exports.UnionFind = void 0;
-exports.sessionizeByIngestedAt = sessionizeByIngestedAt;
-exports.baseSessionId = baseSessionId;
-exports.nearIdenticalDate = nearIdenticalDate;
-exports.attributePair = attributePair;
-exports.runDupAudit = runDupAudit;
-const cluster_js_1 = require("../cluster.js");
-Object.defineProperty(exports, "UnionFind", { enumerable: true, get: function () { return cluster_js_1.UnionFind; } });
-Object.defineProperty(exports, "clusterEdges", { enumerable: true, get: function () { return cluster_js_1.clusterEdges; } });
-Object.defineProperty(exports, "clusterExcess", { enumerable: true, get: function () { return cluster_js_1.clusterExcess; } });
-Object.defineProperty(exports, "clusterMetadataMismatch", { enumerable: true, get: function () { return cluster_js_1.clusterMetadataMismatch; } });
-/** Cosine thresholds the report evaluates, low to high. */
-exports.DUP_THRESHOLDS = [0.9, 0.92, 0.95];
-/** Neighbors requested per memory (excluding the memory itself). */
-const KNN_K = 10;
-/** Ingestion-run gap: a pause longer than this starts a new "run". */
-const DEFAULT_RUN_GAP_HOURS = 1;
-/** created_at proximity treated as "near-identical" for cross-run pairs. */
-const DEFAULT_SAME_DATE_TOLERANCE_DAYS = 1;
-// ---------------------------------------------------------------------------
-// Sessionization + attribution (pure — unit tested)
-// ---------------------------------------------------------------------------
-/**
- * Partition memories into ingestion "runs" by `ingested_at`: sorted
- * ascending, a gap greater than `gapHours` starts a new run. Returns a map
- * of memory id -> run id (0-based, monotonically increasing).
- */
-function sessionizeByIngestedAt(rows, gapHours = DEFAULT_RUN_GAP_HOURS) {
-    const sorted = [...rows].sort((a, b) => a.ingested_at.localeCompare(b.ingested_at));
-    const gapMs = gapHours * 3_600_000;
-    const runOf = new Map();
-    let runId = -1;
-    let prevTime = null;
-    for (const r of sorted) {
-        const t = Date.parse(r.ingested_at);
-        const valid = Number.isFinite(t);
-        if (prevTime === null || !valid || t - prevTime > gapMs) {
-            runId++;
-        }
-        runOf.set(r.id, runId);
-        if (valid)
-            prevTime = t;
-    }
-    return runOf;
-}
-/** Strip the `#<segment>` suffix from a `source_session` value. Null-safe. */
-function baseSessionId(sourceSession) {
-    if (!sourceSession)
-        return null;
-    const idx = sourceSession.indexOf("#");
-    return idx === -1 ? sourceSession : sourceSession.slice(0, idx);
-}
-/** Whether two ISO timestamps fall within `toleranceDays` of each other. */
-function nearIdenticalDate(a, b, toleranceDays = DEFAULT_SAME_DATE_TOLERANCE_DAYS) {
-    const ta = Date.parse(a);
-    const tb = Date.parse(b);
-    if (!Number.isFinite(ta) || !Number.isFinite(tb))
-        return false;
-    return Math.abs(ta - tb) / 86_400_000 <= toleranceDays;
-}
-/**
- * Classify a duplicate pair as a #189 recovery re-ingest or organic overlap.
- * Recovery re-ingest when either: both sides share the same base
- * `source_session`, or they landed in different ingestion runs but
- * `created_at` is near-identical (the re-ingest preserves the original
- * session date while landing in a later run).
- */
-function attributePair(a, b, runOf) {
-    const baseA = baseSessionId(a.source_session);
-    const baseB = baseSessionId(b.source_session);
-    if (baseA !== null && baseA === baseB)
-        return "recovery_reingest";
-    const runA = runOf.get(a.id);
-    const runB = runOf.get(b.id);
-    if (runA !== undefined &&
-        runB !== undefined &&
-        runA !== runB &&
-        nearIdenticalDate(a.created_at, b.created_at)) {
-        return "recovery_reingest";
-    }
-    return "organic";
-}
-/**
- * Run the full D1 duplicate audit against an open (readonly) snapshot
- * connection. Pure aside from the DB reads — safe against a readonly
- * connection, no writes attempted.
- */
-function runDupAudit(db, topN = 15) {
-    const rows = db
-        .prepare(`SELECT id, content, created_at, ingested_at, source_session, project, privacy, source_agent
-       FROM memories`)
-        .all();
-    const byId = new Map(rows.map((r) => [r.id, r]));
-    const runOf = sessionizeByIngestedAt(rows.map((r) => ({ id: r.id, ingested_at: r.ingested_at })));
-    const lowestThreshold = Math.min(...exports.DUP_THRESHOLDS);
-    const edges = (0, cluster_js_1.buildKnnEdges)(db, { k: KNN_K, minCosine: lowestThreshold });
-    const thresholds = exports.DUP_THRESHOLDS.map((threshold) => {
-        const clusters = (0, cluster_js_1.clusterEdges)(edges, threshold);
-        return {
-            threshold,
-            clusterCount: clusters.length,
-            excess: (0, cluster_js_1.clusterExcess)(clusters),
-            clusterSizes: clusters.map((c) => c.length).sort((a, b) => b - a),
-        };
-    });
-    let recoveryReingest = 0;
-    let organic = 0;
-    for (const e of edges) {
-        const a = byId.get(e.a);
-        const b = byId.get(e.b);
-        if (!a || !b)
-            continue;
-        if (attributePair(a, b, runOf) === "recovery_reingest")
-            recoveryReingest++;
-        else
-            organic++;
-    }
-    const broadestClusters = (0, cluster_js_1.clusterEdges)(edges, lowestThreshold).sort((a, b) => b.length - a.length);
-    const topClusters = broadestClusters.slice(0, topN).map((memberIds) => {
-        const members = memberIds.map((id) => byId.get(id)).filter((m) => !!m);
-        let recovery = 0;
-        let organicCount = 0;
-        for (let i = 0; i < members.length; i++) {
-            for (let j = i + 1; j < members.length; j++) {
-                if (attributePair(members[i], members[j], runOf) === "recovery_reingest")
-                    recovery++;
-                else
-                    organicCount++;
-            }
-        }
-        const sortedMembers = [...members].sort((a, b) => a.created_at.localeCompare(b.created_at));
-        return {
-            size: members.length,
-            members: sortedMembers.map((m) => ({
-                id: m.id,
-                created_at: m.created_at,
-                preview: m.content.slice(0, 80),
-            })),
-            attribution: { recoveryReingest: recovery, organic: organicCount },
-            metadataMismatch: (0, cluster_js_1.clusterMetadataMismatch)(members),
-        };
-    });
-    return {
-        totalMemories: rows.length,
-        knnK: KNN_K,
-        thresholds,
-        pairAttribution: { recoveryReingest, organic, totalPairs: edges.length },
-        topClusters,
-    };
-}

package/dist/eval/eval-clock.d.ts

@@ -1,32 +0,0 @@
-/**
- * #458 — the shared pinned-clock helper for eval harnesses.
- *
- * The recall evals score against wall-clock time (recency + decay terms in
- * computeScore/effectiveStrength), so a pre-photo and a post-run hours apart
- * drift even on identical code — PR D evidenced mode-OFF units moving with no
- * changed code in the path. The fix is a harness-visible pin: every eval that
- * calls retrieve() (or the decay/adoption stats) accepts a `--now <ISO>` flag
- * and threads the SAME instant into the retrieve() `now` seam (retrieval.ts —
- * the injectable-clock idiom of run-deadline.ts), making before/after runs
- * wall-clock-independent. Default stays the live clock — pinning is opt-in,
- * for comparability.
- *
- * Fail-explicit by design (the repo convention): an invalid pin THROWS with a
- * clear message rather than silently falling back to the live clock — a run
- * that believed it was pinned but wasn't would silently reintroduce the drift
- * class this helper exists to remove.
- */
-/**
- * Parse a `--now` flag value into the pinned instant.
- *
- * @param raw the raw flag value. undefined or empty/whitespace → null (the
- *        live clock — the flag was not passed). Anything else must parse as a
- *        Date; garbage or an invalid instant (e.g. month 13) throws.
- * @returns the pinned Date, or null for the live clock.
- */
-export declare function parsePinnedNow(raw: string | undefined): Date | null;
-/**
- * Human-readable clock mode for report headers / photo JSON: "pinned <ISO>"
- * or "live" — so artifacts are self-describing about which clock produced them.
- */
-export declare function clockLabel(now: Date | null): string;

package/dist/eval/eval-clock.js

@@ -1,47 +0,0 @@
-"use strict";
-/**
- * #458 — the shared pinned-clock helper for eval harnesses.
- *
- * The recall evals score against wall-clock time (recency + decay terms in
- * computeScore/effectiveStrength), so a pre-photo and a post-run hours apart
- * drift even on identical code — PR D evidenced mode-OFF units moving with no
- * changed code in the path. The fix is a harness-visible pin: every eval that
- * calls retrieve() (or the decay/adoption stats) accepts a `--now <ISO>` flag
- * and threads the SAME instant into the retrieve() `now` seam (retrieval.ts —
- * the injectable-clock idiom of run-deadline.ts), making before/after runs
- * wall-clock-independent. Default stays the live clock — pinning is opt-in,
- * for comparability.
- *
- * Fail-explicit by design (the repo convention): an invalid pin THROWS with a
- * clear message rather than silently falling back to the live clock — a run
- * that believed it was pinned but wasn't would silently reintroduce the drift
- * class this helper exists to remove.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parsePinnedNow = parsePinnedNow;
-exports.clockLabel = clockLabel;
-/**
- * Parse a `--now` flag value into the pinned instant.
- *
- * @param raw the raw flag value. undefined or empty/whitespace → null (the
- *        live clock — the flag was not passed). Anything else must parse as a
- *        Date; garbage or an invalid instant (e.g. month 13) throws.
- * @returns the pinned Date, or null for the live clock.
- */
-function parsePinnedNow(raw) {
-    if (raw === undefined || raw.trim() === "")
-        return null;
-    const parsed = new Date(raw);
-    if (Number.isNaN(parsed.getTime())) {
-        throw new Error(`eval clock: invalid --now value "${raw}" — pass a full ISO 8601 instant ` +
-            `(e.g. 2026-09-17T09:00:00.000Z) or omit the flag to run on the live clock`);
-    }
-    return parsed;
-}
-/**
- * Human-readable clock mode for report headers / photo JSON: "pinned <ISO>"
- * or "live" — so artifacts are self-describing about which clock produced them.
- */
-function clockLabel(now) {
-    return now === null ? "live" : `pinned ${now.toISOString()}`;
-}

package/dist/eval/eval-db.d.ts

@@ -1,25 +0,0 @@
-/**
- * Read-only snapshot access for the #191 mechanical audit baseline.
- *
- * The eval NEVER touches a live database — it runs against a checkpointed
- * copy (`data/audit-<date>/snapshot.db`, gitignored). This module opens that
- * copy in better-sqlite3's `readonly` mode and loads the sqlite-vec
- * extension the same way `db.ts#initDb` does, WITHOUT calling `initDb()`
- * itself: `initDb` runs schema migrations, which write to the file. A
- * snapshot is assumed to already be at the current schema version (verified
- * by `assertReadonly`, which also proves no migration silently ran).
- */
-import Database from "better-sqlite3";
-/**
- * Open a DB snapshot for read-only analysis.
- *
- * Throws if the path does not exist, or if a write attempt against the
- * returned connection would (surprisingly) succeed — the second check is a
- * belt-and-suspenders guard against a future better-sqlite3/OS combination
- * where `readonly: true` is silently ignored (e.g. a non-standard
- * filesystem), so a bug can never turn the audit into a mutation of
- * production data.
- */
-export declare function openSnapshot(dbPath: string): Database.Database;
-/** Convert a sqlite-vec embedding BLOB (as read back from `memory_vectors`) to a Float32Array. */
-export declare function blobToEmbedding(blob: Buffer): Float32Array;

package/dist/eval/eval-db.js

@@ -1,67 +0,0 @@
-"use strict";
-/**
- * Read-only snapshot access for the #191 mechanical audit baseline.
- *
- * The eval NEVER touches a live database — it runs against a checkpointed
- * copy (`data/audit-<date>/snapshot.db`, gitignored). This module opens that
- * copy in better-sqlite3's `readonly` mode and loads the sqlite-vec
- * extension the same way `db.ts#initDb` does, WITHOUT calling `initDb()`
- * itself: `initDb` runs schema migrations, which write to the file. A
- * snapshot is assumed to already be at the current schema version (verified
- * by `assertReadonly`, which also proves no migration silently ran).
- */
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.openSnapshot = openSnapshot;
-exports.blobToEmbedding = blobToEmbedding;
-const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
-const node_fs_1 = require("node:fs");
-/**
- * Open a DB snapshot for read-only analysis.
- *
- * Throws if the path does not exist, or if a write attempt against the
- * returned connection would (surprisingly) succeed — the second check is a
- * belt-and-suspenders guard against a future better-sqlite3/OS combination
- * where `readonly: true` is silently ignored (e.g. a non-standard
- * filesystem), so a bug can never turn the audit into a mutation of
- * production data.
- */
-function openSnapshot(dbPath) {
-    if (!(0, node_fs_1.existsSync)(dbPath)) {
-        throw new Error(`Snapshot DB not found at ${dbPath}`);
-    }
-    const db = new better_sqlite3_1.default(dbPath, { readonly: true });
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const sqliteVec = require("sqlite-vec");
-    sqliteVec.load(db);
-    assertReadonly(db);
-    return db;
-}
-/**
- * Verify the connection truly refuses writes. Uses a table guaranteed to
- * exist in any migrated Hicortex DB (`schema_version`) and a no-op-shaped
- * statement (touches a version number that cannot exist) so that even if the
- * guard somehow failed open, the blast radius is a single junk row rather
- * than corruption of real data.
- */
-function assertReadonly(db) {
-    let wroteSuccessfully = false;
-    try {
-        db.prepare("INSERT INTO schema_version (version, name, applied_at) VALUES (-1, '__eval_readonly_probe__', '')").run();
-        wroteSuccessfully = true;
-    }
-    catch {
-        // Expected: SQLITE_READONLY. The connection is safe to use.
-    }
-    if (wroteSuccessfully) {
-        throw new Error("Snapshot DB accepted a write — refusing to run the eval against a " +
-            "connection that is not truly read-only. Check the better-sqlite3 " +
-            "readonly option and filesystem permissions.");
-    }
-}
-/** Convert a sqlite-vec embedding BLOB (as read back from `memory_vectors`) to a Float32Array. */
-function blobToEmbedding(blob) {
-    return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
-}

package/dist/eval/graph-eval.d.ts

@@ -1,89 +0,0 @@
-/**
- * D6 — link-graph health audit (#191 mechanical baseline).
- *
- * Of the ~6.5k links: how many are meaningful vs near-duplicate noise, does
- * the stored `memory_links.strength` still match a fresh cosine recompute
- * (drift), and how do all these stats look before vs after the
- * `relinkCursor` watermark (the resumable `hicortex relink` migration to the
- * corrected cosine formula, #145).
- */
-import type Database from "better-sqlite3";
-/**
- * Deterministic sample of `rows` (#460): a seeded partial Fisher–Yates
- * shuffles the LAST `size` slots from the back, and that shuffled tail is
- * the sample. Rows must already be in a stable order (the caller sorts by
- * PK) so the draw does not depend on SQLite's scan order.
- */
-export declare function seededSample<T>(rows: T[], size: number, seed: number): T[];
-export interface LinkRow {
-    source_id: string;
-    target_id: string;
-    relationship: string;
-    strength: number;
-}
-/** Cosine similarity between two L2-normalized embeddings (dot product). */
-export declare function cosineBetween(a: Float32Array, b: Float32Array): number;
-export declare function byRelationshipCounts(links: LinkRow[]): Record<string, number>;
-/**
- * Partition links by whether their SOURCE memory's rowid has been covered by
- * the `hicortex relink` watermark. `relinkCursor` is the last fully
- * committed rowid (relink.ts) — relink iterates memories by rowid and
- * discovers/refreshes links FROM each one, so a link's source rowid <=
- * cursor means a relink pass has already run for that source (current
- * formula); null cursor (never run) puts everything in "notYetRelinked".
- */
-export declare function partitionByRelinkCursor(links: LinkRow[], sourceRowid: Map<string, number>, relinkCursor: number | null): {
-    relinked: LinkRow[];
-    notYetRelinked: LinkRow[];
-};
-export interface HubEntry {
-    id: string;
-    degree: number;
-    project: string | null;
-    domain: string | null;
-    preview: string;
-}
-export interface DegreeReport {
-    memoriesWithLinks: number;
-    totalMemories: number;
-    degreeHistogram: Record<string, number>;
-    topHubs: HubEntry[];
-}
-export declare function runDegreeAudit(db: Database.Database, topN?: number): DegreeReport;
-export interface DriftReport {
-    sampleSize: number;
-    driftHistogram: Record<string, number>;
-    meanAbsDrift: number;
-    maxAbsDrift: number;
-    skippedMissingEmbedding: number;
-}
-/**
- * Recompute cosine for a link sample and compare to stored `strength` (post
- * migration-5 rescale, should track closely). The sample is deterministic
- * (#460): all link rows in stable PK order, drawn by a seeded Fisher–Yates —
- * two runs on the same build + snapshot produce an identical report, so
- * before/after comparisons can require full-output identity.
- */
-export declare function runDriftSample(db: Database.Database, embeddings: Map<string, Float32Array>, sampleSize?: number, seed?: number): DriftReport;
-export interface PartitionStats {
-    linkCount: number;
-    byRelationship: Record<string, number>;
-    dupNoiseLinks: number;
-    dupNoiseShare: number;
-}
-export interface GraphReport {
-    totalLinks: number;
-    byRelationship: Record<string, number>;
-    degree: DegreeReport;
-    drift: DriftReport;
-    dupNoise: {
-        threshold: number;
-        count: number;
-        share: number;
-        measured: number;
-    };
-    relinkCursor: number | null;
-    relinkedPartition: PartitionStats;
-    notYetRelinkedPartition: PartitionStats;
-}
-export declare function runGraphAudit(db: Database.Database, stateDir?: string): GraphReport;