@gamaze/hicortex 0.20.4 → 0.20.5

package/dist/consolidate.js

@@ -66,6 +66,8 @@ const state_js_1 = require("./state.js");
 const domain_classify_js_1 = require("./domain-classify.js");
 const schema_prototypes_js_1 = require("./schema-prototypes.js");
 const nofit_js_1 = require("./nofit.js");
+const reconsolidation_js_1 = require("./reconsolidation.js");
+const dedup_js_1 = require("./dedup.js");
 // Default config constants (matching Python config.py)
 /**
  * Default ceiling on total LLM calls across all classify-tier consolidation
@@ -1065,7 +1067,7 @@ function parseSupersessionReply(reply) {
  */
 async function classifySupersession(llm, oldContent, newContent) {
     try {
-        const r = await llm.completeClassify(buildSupersessionPrompt(oldContent, newContent), 32);
+        const r = await llm.completeClassify(buildSupersessionPrompt(oldContent, newContent));
         return { verdict: parseSupersessionReply(r.text), usage: r.usage };
     }
     catch {
@@ -1360,6 +1362,66 @@ function stageMemoryCapEviction(db, dryRun, cap) {
         `memories (corpus was ${count}, cap ${cap}).`);
     return { cap, evicted: victims.length };
 }
+/**
+ * Minimal resolution-stage report for a SKIPPED (quiet-night) consolidation
+ * run (#392): every stage field zero except `merges` — the deterministic
+ * zone's own report — and its derived deterministic band snapshot. Keeps the
+ * one-report surface intact (the zone is the only resolution work a quiet
+ * night does) while telemetry's "skipped = zero LLM work" stays true. Knob
+ * validation mirrors the stage's own (invalid → defaults).
+ */
+async function skippedRunResolutionReport(db, dryRun, stateDir, options = {}) {
+    const validNumber = (v, fallback, ok) => {
+        const n = Number(v);
+        return Number.isFinite(n) && ok(n) ? n : fallback;
+    };
+    const autoMergeThreshold = validNumber(options.autoMergeThreshold, dedup_js_1.DEFAULT_DEDUP_MERGE_THRESHOLD, (n) => n > 0 && n <= 1);
+    const maxMerges = validNumber(options.maxMerges, dedup_js_1.DEFAULT_DEDUP_NIGHTLY_MAX_MERGES, (n) => n >= 0);
+    const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
+        stateDir,
+        threshold: autoMergeThreshold,
+        maxMerges,
+        dryRun,
+        acquireLock: options.acquireLock,
+    });
+    const bandStats = {};
+    if (merges.max_merges > 0) {
+        bandStats[`>=${autoMergeThreshold}`] = {
+            pairs: merges.losers_merged,
+            merge: merges.losers_merged,
+            corrects: 0,
+            supersedes: 0,
+            none: 0,
+            merge_below_gate: 0,
+            conf_sum: merges.losers_merged,
+            ...(merges.skipped_metadata_mismatch > 0
+                ? { metadata_skipped: merges.skipped_metadata_mismatch }
+                : {}),
+        };
+    }
+    return {
+        scanned: 0,
+        pairs_evaluated: 0,
+        rewritten: 0,
+        absorbed: 0,
+        kept_linked: 0,
+        marked_superseded: 0,
+        marked_retracted: 0,
+        below_gate: 0,
+        contract_failed: 0,
+        skipped_infra: 0,
+        skipped_idempotent: 0,
+        explicit_verified: 0,
+        explicit_divergent: 0,
+        cursor: (0, state_js_1.loadState)(stateDir).reconsolidationCursor ?? 0,
+        merges,
+        merge_pairs_applied: 0,
+        merge_below_gate: 0,
+        skipped_above_ceiling: 0,
+        skipped_metadata_mismatch: 0,
+        band_stats: bandStats,
+    };
+}
 async function runConsolidation(db, llm, embedFn, dryRun = false, skipReflection = false, stateDir, domainOptions, supersessionOptions, 
 /** Total LLM-call ceiling across classify-tier stages (#241). The caller
  *  reads `consolidateMaxLlmCalls` from config and passes it; unset → the
@@ -1368,7 +1430,13 @@ budgetMaxCalls,
 /** Soft cap on the corpus (#245). Nightly.ts reads `memorySoftCap` from
  *  config and passes it; unset → `DEFAULT_MEMORY_SOFT_CAP` (10000). `0`
  *  disables eviction (indefinite growth). */
-memorySoftCap) {
+memorySoftCap, 
+/** Reconsolidation-stage knobs (#384), threaded from config by nightly.ts
+ *  (correctionMinSimilarity / correctionRewriteMinConfidence) exactly like
+ *  supersessionOptions above; unset fields → the stage's defaults.
+ *  Appended AFTER the pre-#384 params so every existing positional caller
+ *  (tests, hosted nightly) keeps its argument meaning. */
+reconsolidationOptions) {
     const start = new Date();
     const report = {
         started_at: start.toISOString(),
@@ -1407,6 +1475,13 @@ memorySoftCap) {
     // but the cap stage is pure DB: cheap, idempotent when under cap).
     report.stages.memory_cap = stageMemoryCapEviction(db, dryRun, memorySoftCap ?? exports.DEFAULT_MEMORY_SOFT_CAP);
     if (skip) {
+        // #392: the deterministic merge zone is LLM-free, so a quiet night (zero
+        // new memories → this skip) still drains a pre-existing duplicate
+        // backlog — the memory_cap precedent. Results ride the ONE resolution
+        // stage report (telemetry's "skipped = zero LLM work" stays true), and
+        // the zone never runs twice: the main path runs it INSIDE the stage, this
+        // skip path returns before that.
+        report.stages.reconsolidation = await skippedRunResolutionReport(db, dryRun, stateDir, reconsolidationOptions);
         report.status = "skipped";
         report.completed_at = new Date().toISOString();
         return report;
@@ -1455,6 +1530,11 @@ memorySoftCap) {
         report.stages.hub_boost = stageHubBoost(db, dryRun);
         // Stage 3.7: Supersession Detection (#191 Phase B)
         report.stages.supersession = await stageSupersession(db, llm, budget, embedFn, dryRun, stateDir, supersessionOptions);
+        // Stage 3.8: Reconsolidation (#384) — resolve corrections: rewrite
+        // fact-shaped targets in place (absorbing transition-only triggers),
+        // mark everything else. Rides the same shared budget under its own stage
+        // label + cursor (supersession-stage pattern).
+        report.stages.reconsolidation = await (0, reconsolidation_js_1.stageReconsolidation)(db, llm, budget, embedFn, dryRun, stateDir, reconsolidationOptions);
         // Stage 4: Decay & Prune
         report.stages.decay_prune = stageDecayPrune(db, dryRun);
         // (Memory cap eviction moved before the precheck skip — see above.)

package/dist/db.js

@@ -511,6 +511,42 @@ const MIGRATIONS = [
             db.exec("UPDATE memories SET memory_type = 'learnings'  WHERE memory_type = 'lesson'");
         },
     },
+    {
+        version: 14,
+        name: "reconsolidation_status_history",
+        up: (db) => {
+            // #384 reconsolidation. `memories.status` is the code-defined state
+            // vocabulary (NULL/active default; 'superseded'/'retracted' demote in
+            // ranking; 'corrected' = rewritten, never demotes; 'absorbed' =
+            // invisible to recall — no vector, no FTS row). NULL default with NO
+            // backfill: legacy supersessions stay link-driven (findSupersededIds
+            // already demotes them); a status is only ever written by the
+            // reconsolidation stage, an explicit ingest mark, or a rollback.
+            // `memory_history` records CONTENT REWRITES ONLY (before/after content,
+            // prior status, trigger dispositions) — one structure serving audit AND
+            // `hicortex history --rollback`. Marks need no history row: they are
+            // auditable via links + status. Idempotent: hasColumn + IF NOT EXISTS.
+            if (!hasColumn(db, "memories", "status")) {
+                db.exec("ALTER TABLE memories ADD COLUMN status TEXT");
+            }
+            db.exec(`
+        CREATE TABLE IF NOT EXISTS memory_history (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          memory_id TEXT NOT NULL,
+          old_content TEXT NOT NULL,
+          new_content TEXT NOT NULL,
+          prev_status TEXT,
+          new_status TEXT,
+          triggers_json TEXT,
+          evidence_id TEXT,
+          confidence REAL,
+          cause TEXT NOT NULL,
+          created_at TEXT NOT NULL
+        )
+      `);
+            db.exec("CREATE INDEX IF NOT EXISTS idx_memory_history_memory ON memory_history(memory_id)");
+        },
+    },
 ];
 /**
  * Run all pending migrations against the database.

package/dist/dedup.d.ts

@@ -1,52 +1,101 @@
 /**
- * `hicortex dedup` — cluster + merge near-duplicate memories (issue #100).
+ * `hicortex dedup` + the nightly deterministic merge zone (issues #100, #392).
  *
  * Corpus-quality companion to `hicortex relink`/`classify-domains`: instead of
- * discovering NEW structure, this command collapses memories that are
- * near-identical (top-10 KNN cosine >= dedupMergeThreshold, default 0.92,
- * union-find clustered — same math as the #191 D1 duplicate-rate audit; see
- * cluster.ts). Default is a DRY RUN: report only, zero writes. `--apply`
- * executes the merge.
+ * discovering NEW structure, this collapses memories that are near-identical
+ * (top-10 KNN cosine >= threshold, default 0.92, union-find clustered — same
+ * math as the #191 D1 duplicate-rate audit; see cluster.ts). Two surfaces,
+ * ONE core:
  *
- * Per cluster:
+ *  - `runDedup` — the manual CLI. Default is a DRY RUN: report only, zero
+ *    writes. `--apply` executes the merge. Threshold resolution:
+ *    `--threshold` > config `dedupAutoMergeThreshold` > legacy config
+ *    `dedupMergeThreshold` > 0.92.
+ *  - `runDeterministicMergeZone` (#392) — the nightly's LLM-free merge zone:
+ *    pairs at/above the ceiling merge deterministically, ZERO LLM calls, under
+ *    its own pacing cap (`dedupNightlyMaxMerges`). Called from the
+ *    reconsolidation stage (and from the quiet-night skip path in
+ *    consolidate.ts) so one stage report covers all resolution work.
+ *
+ * Per cluster (shared `planDedup`/`mergeCluster` core — no forks):
  *   - Canonical = highest access_count (tie: oldest created_at, then
  *     lexicographically smallest id — fully deterministic for audit).
  *   - Losers' links are re-pointed onto the canonical (a link that would
  *     become a self-link, or one whose (canonical, target) ordered pair
  *     ALREADY holds an edge, is skipped rather than overwritten — see
- *     planLinkRepoints for why `relationship` cannot be part of that guard).
+ *     planLinkRepoints for why `relationship` cannot be part of that guard);
+ *     the losers' own link rows are then deleted (previously cascade-deleted
+ *     with the row).
  *   - canonical.access_count/shown_count = summed across the cluster;
  *     last_accessed = max; base_strength = max.
  *   - Tags are UNIONED onto the canonical (weights NULL — the next nightly's
  *     reconsolidation pass recomputes weights and the derived primary from
- *     the merged tag set).
- *   - A `dedup_log` row is written per loser BEFORE it is deleted — audit
- *     trail AND the safety net /distill consults (mcp-server.ts) so a
- *     deleted loser's `source_session` marker still blocks a re-ingest.
- *   - Losers are deleted via storage.deleteMemory (cascades links/tags/
- *     vectors/FTS).
+ *     the merged tag set); the losers' tag rows are cleared and their domain
+ *     set NULL — a loser must not count in moduleIndex/tag recomputes.
+ *   - A `dedup_log` row is written per loser — audit trail AND the safety net
+ *     /distill consults (mcp-server.ts) so an absorbed loser's
+ *     `source_session` marker still blocks a re-ingest.
+ *   - Losers are ABSORBED (storage.absorbMemory), not deleted (#392): status
+ *     'absorbed', vector + FTS rows dropped, plain row retained — invisible
+ *     to recall, fetchable by id as evidence. Same vocabulary as the
+ *     reconsolidation rewrite path. Merges are NOT history-rollback-able —
+ *     dedup_log (loser_id → canonical_id) + the retained loser row is the
+ *     record.
  *
- * A cluster whose members disagree on project, privacy, or source_agent is
- * SKIPPED entirely and listed for manual review — no --force in this release.
+ * A cluster whose members disagree on project or source_agent is SKIPPED
+ * entirely and listed for manual review — no --force in this release.
  *
- * Safety rails on --apply:
+ * Safety rails when applying (CLI and zone alike):
  *   - A full DB backup (SQLite backup API) is taken FIRST, to
- *     ~/.hicortex/backups/pre-dedup-<ISO>.db. Abort (no merges attempted) if
- *     the backup fails.
+ *     <state>/backups/pre-dedup-<ISO>.db, pruned to `backupRetention` newest
+ *     (pattern-scoped: full `hicortex-*.tar.gz` artifacts keep their own
+ *     count). The CLI aborts (no merges attempted) if the backup fails; the
+ *     nightly zone is fail-soft (backup_failed flag, zero merges).
  *   - The existing single-flight capture lock (capture.ts) is held for the
  *     duration of the merge so a concurrent nightly/capture run can't race
- *     the dedup_log bookkeeping the merge relies on.
+ *     the dedup_log bookkeeping the merge relies on. The CLI fails fast on a
+ *     busy lock; the zone reports lock_busy and merges nothing.
  *
  * Server-mode only (needs the local DB), like relink/classify-domains.
  */
 import type Database from "better-sqlite3";
 import { type ClusterMetadataMismatch } from "./cluster.js";
 import { acquireCaptureLock } from "./capture.js";
+import type { DeterministicMergeZoneReport } from "./types.js";
 /**
  * Default merge threshold. Measured on the #191 mechanical audit corpus:
  * 89 clusters / 110 excess rows at 0.92 (data/audit-20260729/eval-report.md).
+ * #392: also the default `dedupAutoMergeThreshold` — the deterministic/LLM
+ * boundary of the unified resolution pass.
  */
 export declare const DEFAULT_DEDUP_MERGE_THRESHOLD = 0.92;
+/**
+ * Default pacing cap on merge OPERATIONS per nightly run (#392): the zone's
+ * clusters and the stage's judged pair merges count against ONE cap. Bounds a
+ * misbehaving-distiller burst; a large backlog drains over a few nights.
+ * `0` disables the merge machinery entirely.
+ */
+export declare const DEFAULT_DEDUP_NIGHTLY_MAX_MERGES = 250;
+/**
+ * Row shape read from `memories` for merge decisions — a superset of the
+ * fields the Memory type declares (shown_count isn't on that interface yet).
+ * `status` rides along so judged-pair merges can defensively drop rows that
+ * were absorbed between verdict and apply.
+ */
+interface DedupMemberRow {
+    id: string;
+    content: string;
+    access_count: number;
+    shown_count: number | null;
+    last_accessed: string | null;
+    base_strength: number;
+    created_at: string;
+    project: string | null;
+    privacy: string | null;
+    source_agent: string;
+    source_session: string | null;
+    status: string | null;
+}
 export interface DedupClusterPlan {
     size: number;
     canonicalId: string;
@@ -94,8 +143,8 @@ export interface DedupReport {
     linksSkippedExisting: number;
     /** --apply only: clusters actually merged. */
     merged?: number;
-    /** --apply only: loser rows deleted. */
-    losersDeleted?: number;
+    /** --apply only: loser rows absorbed (hidden from recall, kept as evidence). */
+    losersAbsorbed?: number;
     /** --apply only: clusters that errored mid-merge (rolled back; left for a re-run). */
     failedClusters?: number;
     /** --apply only: path to the pre-merge backup. */
@@ -104,7 +153,7 @@ export interface DedupReport {
 export interface DedupOptions {
     /** Execute the merge. Default false = dry run (report only, zero writes). */
     apply?: boolean;
-    /** Override config.dedupMergeThreshold for one run. */
+    /** Override the configured threshold for one run. */
     threshold?: number;
     /** DB path override (tests / manual snapshot verification). Defaults to resolveDbPath(). */
     dbPath?: string;
@@ -116,7 +165,7 @@ export interface DedupOptions {
     acquireLock?: typeof acquireCaptureLock;
     /**
      * Test-only failure injection: called once per cluster merge, after the
-     * link/tag/counter writes but before the audit-log + delete step. Throwing
+     * link/tag/counter writes but before the audit-log + absorb step. Throwing
      * here proves a mid-merge error rolls the WHOLE cluster's writes back
      * (better-sqlite3 transaction semantics) rather than leaving a half-merged
      * cluster. Never set in production.
@@ -135,10 +184,93 @@ export interface LinkRepointPlan {
     skippedSelfLink: number;
     skippedExisting: number;
 }
+/** One cluster's execution plan from `planDedup` — canonical, losers, members. */
+export interface DedupMergePlan {
+    canonical: DedupMemberRow;
+    losers: DedupMemberRow[];
+    /** All member rows, oldest first (CLI preview lines derive from this). */
+    membersOldestFirst: DedupMemberRow[];
+}
+export interface PlanDedupResult {
+    /** Every cluster found at the threshold (mergeable + mismatch-skipped). */
+    clusterCount: number;
+    /** Clusters that passed the metadata rails, in discovery order. */
+    mergePlans: DedupMergePlan[];
+    mismatchSkipped: DedupMismatchCluster[];
+}
+/**
+ * Discovery + merge planning at a cosine threshold (read-only — no writes).
+ * The ONE clustering core shared by the manual CLI (`runDedup`) and the
+ * nightly deterministic merge zone (`runDeterministicMergeZone`): KNN edges
+ * (k=10) → union-find clusters → member load → metadata-rail classification →
+ * canonical pick. Never forked.
+ */
+export declare function planDedup(db: Database.Database, threshold: number): PlanDedupResult;
+export type MergeMemoryIdsResult = {
+    ok: true;
+    canonicalId: string;
+    loserIds: string[];
+    linksRepointed: number;
+} | {
+    ok: false;
+    reason: "metadata_mismatch" | "no_members";
+};
+/**
+ * Merge an explicit set of memories (the judged-pair phase of #392: the
+ * reconsolidation stage queues verdict-confirmed pairs and applies them
+ * through THIS function so the merge math stays single-definition). Loads the
+ * LIVE rows at apply time — members that vanished or were absorbed between
+ * verdict and apply are dropped defensively; a metadata disagreement refuses
+ * the merge (both memories stay live). One transaction for the whole set.
+ */
+export declare function mergeMemoryIds(db: Database.Database, ids: string[]): MergeMemoryIdsResult;
+/**
+ * Take a pre-merge DB backup to <stateDir>/backups/pre-dedup-<ISO>.db and
+ * prune older pre-dedup backups to `backupRetention` (config, default 7;
+ * pattern-scoped so full `hicortex-*.tar.gz` artifacts keep their own,
+ * independent retention count). THROWS on failure — the callers own the
+ * policy: the CLI aborts, the nightly zone is fail-soft. Returns the path.
+ */
+export declare function takePreDedupBackup(db: Database.Database, stateDir: string, config?: Record<string, unknown> | null): Promise<string>;
+export interface DeterministicMergeZoneOptions {
+    /** State dir (lock + backup + band-stats persistence). Defaults to ~/.hicortex. */
+    stateDir?: string;
+    /** Cosine ceiling; validated (0,1] → DEFAULT_DEDUP_MERGE_THRESHOLD. */
+    threshold?: number;
+    /**
+     * Pacing cap on merge operations this run; validated >= 0 →
+     * DEFAULT_DEDUP_NIGHTLY_MAX_MERGES. `0` disables the machinery entirely
+     * (discovery skipped, zeroed report).
+     */
+    maxMerges?: number;
+    /** Discovery + bounded preview only — zero writes, no lock, no backup. */
+    dryRun?: boolean;
+    /** Config override (backupRetention) — defaults to reading stateDir/config.json. */
+    config?: Record<string, unknown> | null;
+    /** Capture-lock acquirer override (tests). Defaults to the real capture.ts lock. */
+    acquireLock?: typeof acquireCaptureLock;
+}
+/**
+ * The >= dedupAutoMergeThreshold band of the unified resolution pass (#392):
+ * planDedup discovery + per-cluster mergeCluster — LLM-free, budget-free, so
+ * an LLM-less night still drains duplicates. Its own short capture-lock
+ * window and pre-merge backup; fail-soft on a busy lock (lock_busy) and on a
+ * backup failure (backup_failed) — zero merges either way, never a throw.
+ *
+ * Also persists the deterministic band's cumulative statistics to state.json
+ * `resolutionBandStats` (label `>=threshold`; losers count as merge verdicts
+ * at confidence 1.0, mismatch clusters as metadata_skipped) — skipped
+ * entirely on dry-run. Called from the reconsolidation stage (main path) and
+ * from runConsolidation's quiet-night skip path — exactly one of the two per
+ * run.
+ */
+export declare function runDeterministicMergeZone(db: Database.Database, opts?: DeterministicMergeZoneOptions): Promise<DeterministicMergeZoneReport>;
 /**
  * Run `hicortex dedup`. Dry run by default (options.apply falsy) — discovery
  * + merge planning only, zero writes. `options.apply` executes: backup, then
- * one transaction per cluster.
+ * one transaction per cluster. Fails fast (throws) on a busy capture lock or
+ * a failed backup — a deliberate manual command should be retried by the
+ * operator, not silently deferred (the nightly zone is the fail-soft twin).
  */
 export declare function runDedup(options?: DedupOptions): Promise<DedupReport>;
 /** Escape SQL LIKE wildcards — session ids (e.g. Hermes) can contain "_"/"%". */