@gamaze/hicortex 0.20.5 → 0.20.7

package/dist/consolidate.js

@@ -1402,6 +1402,8 @@ async function skippedRunResolutionReport(db, dryRun, stateDir, options = {}) {
     return {
         scanned: 0,
         pairs_evaluated: 0,
+        pairs_discovered: 0, // #394: the scan doesn't run on a quiet night — nothing discovered
+        pairs_discovered_unlinked: 0,
         rewritten: 0,
         absorbed: 0,
         kept_linked: 0,

package/dist/nightly.js

@@ -748,6 +748,12 @@ async function runNightly(options = {}) {
                             autoMergeThreshold: (savedConfig?.dedupAutoMergeThreshold ??
                                 savedConfig?.dedupMergeThreshold),
                             maxMerges: savedConfig?.dedupNightlyMaxMerges,
+                            // #401 runtime bounds: the wall-clock deadline (default 120
+                            // min, 0 disables) and the per-run classify-call ceiling
+                            // (default 600, 0 disables). Same posture — the stage
+                            // validates and falls back on invalid/absent values.
+                            maxMinutes: savedConfig?.reconsolidationMaxMinutes,
+                            maxCalls: savedConfig?.reconsolidationMaxCalls,
                         });
                         console.log(`[hicortex] Consolidation ${report.status} in ${report.elapsed_seconds}s` +
                             (report.stages.reflection ? ` (${report.stages.reflection.lessons_generated} lessons)` : ""));

package/dist/reconsolidation.d.ts

@@ -57,6 +57,24 @@ export declare const DEFAULT_CORRECTION_MIN_SIMILARITY = 0.75;
  * weak rewrite is corruption.
  */
 export declare const DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = 0.8;
+/**
+ * Default wall-clock bound for the stage, in minutes (#401). Checked at the
+ * top of the candidate scan loop (and before each rewrite contract call); on
+ * expiry the scan breaks cleanly at the last fully-considered candidate and
+ * the next run resumes from the persisted cursor. 120 sits safely under any
+ * sane process-level nightly timeout. 0 disables the bound. Invalid →
+ * default.
+ */
+export declare const DEFAULT_RECONSOLIDATION_MAX_MINUTES = 120;
+/**
+ * Default per-run classify-call ceiling for the stage (#401) — the
+ * supersessionMaxCalls pattern with a NON-ZERO default ON PURPOSE: that
+ * knob's 0=unlimited default is what let the first full-corpus pass grow
+ * unbounded. Counts EVERY classify-tier call the stage makes (mark
+ * verifications, pair verdicts, rewrite contracts). 0 disables the cap.
+ * Invalid → default.
+ */
+export declare const DEFAULT_RECONSOLIDATION_MAX_CALLS = 600;
 /** Head of the old content quoted in the provenance footer. */
 export declare const FOOTER_HEAD_MAX_CHARS = 160;
 /** The code-defined status vocabulary (see module doc). Not user-configurable. */
@@ -91,6 +109,24 @@ export interface ReconsolidationOptions {
      * pair merges against ONE cap. Invalid → default.
      */
     maxMerges?: number;
+    /**
+     * reconsolidationMaxMinutes (config; default 120; 0 disables) — wall-clock
+     * deadline for the stage (#401), measured from stage start. Checked at the
+     * top of the candidate scan loop and before each rewrite contract call; on
+     * expiry the scan breaks cleanly — the cursor already points at the last
+     * fully-considered candidate, so the run ends consistent and the next
+     * nightly resumes from it. Invalid → default.
+     */
+    maxMinutes?: number;
+    /**
+     * reconsolidationMaxCalls (config; default 600; 0 disables) — per-run
+     * ceiling on classify-tier calls for the stage (#401), the
+     * supersessionMaxCalls pattern with a NON-ZERO default (the 0=unlimited
+     * default there is what removed the last per-stage bound). Exhaustion
+     * mid-neighbor-loop or mid-rewrite-phase stops/defers cleanly at the
+     * current candidate boundary. Invalid → default.
+     */
+    maxCalls?: number;
     /**
      * Capture-lock acquirer override (tests) — the deterministic zone and the
      * judged-merge phase each hold a short lock window. Defaults to the real
@@ -98,7 +134,7 @@ export interface ReconsolidationOptions {
      */
     acquireLock?: typeof acquireCaptureLock;
 }
-/** The 14-field stage report (typed once, in ConsolidationReport). */
+/** The stage report (typed once, in ConsolidationReport — field list there). */
 export type ReconsolidationStageResult = NonNullable<ConsolidationReport["stages"]["reconsolidation"]>;
 /**
  * True when a memory is REWRITE-ELIGIBLE — a fact-shaped target. The fork is
@@ -282,7 +318,10 @@ export declare function bandForCosine(bands: ResolutionBand[], cosine: number):
  * silently dropped by the cursor passing it).
  *
  * Dry-run: the zone's discovery + the free idempotency check only — zero LLM
- * calls, zero writes, no cursor or band-stats persistence.
+ * calls, zero writes, no cursor or band-stats persistence. Gate discovery is
+ * reported on every run (pairs_discovered / pairs_discovered_unlinked, #394) —
+ * on a dry-run they are the sizing numbers (pairs_evaluated stays 0: no calls
+ * are ever made).
  */
 export declare function stageReconsolidation(db: Database.Database, llm: LlmClient, budget: StageBudget, embedFn: EmbedFn, dryRun: boolean, stateDir: string | undefined, options?: ReconsolidationOptions): Promise<ReconsolidationStageResult>;
 export interface RollbackResult {

package/dist/reconsolidation.js

@@ -71,7 +71,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = exports.DEFAULT_CORRECTION_MIN_SIMILARITY = exports.RECONSOLIDATION_STAGE_LABEL = void 0;
+exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.DEFAULT_RECONSOLIDATION_MAX_CALLS = exports.DEFAULT_RECONSOLIDATION_MAX_MINUTES = exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = exports.DEFAULT_CORRECTION_MIN_SIMILARITY = exports.RECONSOLIDATION_STAGE_LABEL = void 0;
 exports.isFactShapedTarget = isFactShapedTarget;
 exports.buildCorrectionVerdictPrompt = buildCorrectionVerdictPrompt;
 exports.parseCorrectionVerdict = parseCorrectionVerdict;
@@ -115,6 +115,24 @@ exports.DEFAULT_CORRECTION_MIN_SIMILARITY = 0.75;
  * weak rewrite is corruption.
  */
 exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = 0.8;
+/**
+ * Default wall-clock bound for the stage, in minutes (#401). Checked at the
+ * top of the candidate scan loop (and before each rewrite contract call); on
+ * expiry the scan breaks cleanly at the last fully-considered candidate and
+ * the next run resumes from the persisted cursor. 120 sits safely under any
+ * sane process-level nightly timeout. 0 disables the bound. Invalid →
+ * default.
+ */
+exports.DEFAULT_RECONSOLIDATION_MAX_MINUTES = 120;
+/**
+ * Default per-run classify-call ceiling for the stage (#401) — the
+ * supersessionMaxCalls pattern with a NON-ZERO default ON PURPOSE: that
+ * knob's 0=unlimited default is what let the first full-corpus pass grow
+ * unbounded. Counts EVERY classify-tier call the stage makes (mark
+ * verifications, pair verdicts, rewrite contracts). 0 disables the cap.
+ * Invalid → default.
+ */
+exports.DEFAULT_RECONSOLIDATION_MAX_CALLS = 600;
 /** Neighbor pool size before older/similarity filtering narrows to top 5 (supersession mirror). */
 const CORRECTION_NEIGHBOR_POOL = 15;
 /** Older-neighbor pairs kept per candidate after filtering (supersession mirror). */
@@ -501,7 +519,10 @@ async function classifyPair(llm, oldContent, newContent) {
  * silently dropped by the cursor passing it).
  *
  * Dry-run: the zone's discovery + the free idempotency check only — zero LLM
- * calls, zero writes, no cursor or band-stats persistence.
+ * calls, zero writes, no cursor or band-stats persistence. Gate discovery is
+ * reported on every run (pairs_discovered / pairs_discovered_unlinked, #394) —
+ * on a dry-run they are the sizing numbers (pairs_evaluated stays 0: no calls
+ * are ever made).
  */
 async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir, options = {}) {
     // Config values pass through `unknown`-typed JSON — validate, never trust.
@@ -513,6 +534,13 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
     const rewriteMinConfidence = validNumber(options.rewriteMinConfidence, exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE, (n) => n > 0 && n <= 1);
     const autoMergeThreshold = validNumber(options.autoMergeThreshold, dedup_js_1.DEFAULT_DEDUP_MERGE_THRESHOLD, (n) => n > 0 && n <= 1);
     const maxMerges = validNumber(options.maxMerges, dedup_js_1.DEFAULT_DEDUP_NIGHTLY_MAX_MERGES, (n) => n >= 0);
+    const maxMinutes = validNumber(options.maxMinutes, exports.DEFAULT_RECONSOLIDATION_MAX_MINUTES, (n) => n >= 0);
+    const maxCalls = validNumber(options.maxCalls, exports.DEFAULT_RECONSOLIDATION_MAX_CALLS, (n) => n >= 0);
+    // ---- #401 runtime bounds. The wall-clock deadline is measured from stage
+    // start (the deterministic zone's runtime counts against it — the binding
+    // constraint must be THIS knob, never the process-level backstop).
+    const deadlineAt = maxMinutes > 0 ? Date.now() + Math.round(maxMinutes * 60_000) : Infinity;
+    const deadlineHit = () => Date.now() >= deadlineAt;
     // ---- #392 phase 0: the deterministic merge zone (pairs >= the ceiling),
     // LLM-free and budget-free — an LLM-less night still drains duplicates. Its
     // own short lock window, pre-merge backup, and pacing cap; fail-soft, never
@@ -548,6 +576,8 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
         .all(startCursor);
     let scanned = 0;
     let pairsEvaluated = 0;
+    let pairsDiscovered = 0; // #394: gate discovery — counted before any skip/judgment
+    let pairsDiscoveredUnlinked = 0;
     let rewritten = 0;
     let absorbed = 0;
     let keptLinked = 0;
@@ -576,6 +606,40 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
         storage.addLink(db, oldId, newId, relationship, strength);
         linksCreatedThisRun.add(`${oldId}|${newId}`);
     };
+    // ---- #401: mid-scan cursor persistence. Called at EVERY scan-loop exit
+    // path (deadline, call/budget cap, mark-verify budget stop, discovery
+    // failure) AND after every fully-considered candidate, so a killed run
+    // loses at most the candidate in flight. The end-of-stage updateState
+    // below stays the authoritative final write (it also applies the
+    // pendingMinRowid hold — that variable is only ever set AFTER the scan
+    // loop, so it is null at every call site here). updateState is
+    // load→mutate→temp-rename atomic.
+    let callsUsed = 0;
+    let deadlineStopped = false;
+    let callCapStopped = false;
+    // #402 follow-up (reviewer note 1): the hard-kill orphan floor. Queued
+    // merges and open rewrite groups are applied only in the POST-scan
+    // phases — until then their verdicts exist only in memory, and a
+    // SIGKILL/OOM between two persists would strand them BEHIND the persisted
+    // cursor (the next run would skip them forever). This tracks the smallest
+    // candidate rowid contributing to queued-but-unapplied work;
+    // persistCursor clamps every checkpoint below it so a resumed run
+    // re-detects the pairs (dup-over-loss). Deliberately SEPARATE from the
+    // post-loop pendingMinRowid hold above — different lifetime, different
+    // writers.
+    let scanPendingMinRowid = null;
+    const notePendingRowid = (rowid) => {
+        scanPendingMinRowid =
+            scanPendingMinRowid === null ? rowid : Math.min(scanPendingMinRowid, rowid);
+    };
+    const persistCursor = () => {
+        if (dryRun)
+            return;
+        const checkpoint = scanPendingMinRowid !== null ? Math.min(cursor, scanPendingMinRowid - 1) : cursor;
+        (0, state_js_1.updateState)((s) => {
+            s.reconsolidationCursor = checkpoint;
+        }, stateDir);
+    };
     const groups = new Map();
     const addTrigger = (target, trigger, confidence, cosine, explicit) => {
         let group = groups.get(target.id);
@@ -592,19 +656,43 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
                 candidateRowid: trigger.__rowid,
                 explicit,
             });
+            notePendingRowid(trigger.__rowid); // orphan floor — group unapplied until the rewrite phase
         }
     };
     for (const candidate of rows) {
-        if (!dryRun && budget.exhausted)
+        // #401: runtime bounds first — exit cleanly at the last fully-considered
+        // candidate boundary (cursor = the previous candidate's rowid here).
+        if (!dryRun && deadlineHit()) {
+            deadlineStopped = true;
+            persistCursor();
+            break;
+        }
+        if (!dryRun && budget.exhausted) {
+            persistCursor();
+            break;
+        }
+        // #401: the per-stage call cap stops the scan at the candidate boundary
+        // (the in-loop check below is the mid-candidate backstop — supersession
+        // mirrors both).
+        if (!dryRun && maxCalls > 0 && callsUsed >= maxCalls) {
+            callCapStopped = true;
+            persistCursor();
             break;
+        }
         scanned++;
         // ---- AC7: verify incoming explicit marks (corrected_by/superseded_by
         // links targeting this candidate) before they can join a rewrite group.
+        // #401: only OPERATOR marks are verified — applyExplicitMark writes
+        // strength 1.0, while every stage-created link carries a measured cosine
+        // strength < 1 (markLink sites + supersession). Without the filter, every
+        // prior night's stage output re-entered verification: a self-sustaining
+        // backlog that re-litigated settled verdicts forever.
         if (!dryRun) {
             const incoming = db
                 .prepare(`SELECT source_id, relationship FROM memory_links
            WHERE target_id = ? AND source_id != ?
-             AND relationship IN ('corrected_by', 'superseded_by')`)
+             AND relationship IN ('corrected_by', 'superseded_by')
+             AND strength >= 1.0`)
                 .all(candidate.id, candidate.id);
             let markBudgetStop = false;
             for (const mark of incoming) {
@@ -621,6 +709,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
                 }
                 const { verdict, usage } = await classifyPair(llm, target.content, candidate.content);
                 budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
+                callsUsed++; // #401
                 pairsEvaluated++;
                 if (!verdict) {
                     skippedInfra++; // mark retained; the neighborhood is revisited via newer candidacies
@@ -639,8 +728,10 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
                         `(verdict ${verdict.action}, confidence ${verdict.confidence.toFixed(2)}) — mark retained`);
                 }
             }
-            if (markBudgetStop)
+            if (markBudgetStop) {
+                persistCursor(); // #401: this candidate's remaining marks re-verify next run
                 break;
+            }
         }
         // ---- AC2: detection pairs against older KNN neighbors.
         let neighbors;
@@ -650,13 +741,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
         catch (err) {
             console.warn(`[hicortex] reconsolidation: discovery failed for ${candidate.id.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
             cursor = candidate.__rowid;
+            persistCursor(); // #401: every exit path persists
             continue;
         }
         for (const neighbor of neighbors) {
+            pairsDiscovered++; // every neighbor passed the floor gate
             if (alreadyResolutionLinked(db, neighbor.id, candidate.id)) {
                 skippedIdempotent++;
                 continue;
             }
+            pairsDiscoveredUnlinked++; // still unlinked — the actionable candidate
             // #392: pairs at/above the ceiling belong to the deterministic zone —
             // counted here, never LLM-judged (the zone merges them or holds them
             // for its cap; re-detection is structural, not cursor-based).
@@ -667,10 +761,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
             }
             if (dryRun)
                 continue; // preview only — no LLM call, no write
-            if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL))
+            // #401: the per-stage call cap rides the same boundary as the budget —
+            // supersession-stage pattern (consolidate.ts stageSupersession).
+            if ((maxCalls > 0 && callsUsed >= maxCalls) || !budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
+                callCapStopped = maxCalls > 0 && callsUsed >= maxCalls;
+                persistCursor(); // cursor still points at the last fully-considered candidate
                 break;
+            }
             const { verdict, usage } = await classifyPair(llm, neighbor.content, candidate.content);
             budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
+            callsUsed++; // #401
             pairsEvaluated++;
             if (!verdict) {
                 skippedInfra++;
@@ -693,6 +793,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
                 }
                 else {
                     queuedMerges.push({ oldId: neighbor.id, newId: candidate.id, candidateRowid: candidate.__rowid });
+                    notePendingRowid(candidate.__rowid); // orphan floor — merge unapplied until the merge phase
                 }
                 continue;
             }
@@ -726,6 +827,20 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
             // verdict "none" → nothing to do
         }
         cursor = candidate.__rowid;
+        // #402 follow-up (reviewer note 1): persist after EVERY fully-considered
+        // candidate — the 50-candidate batch left a kill window that could
+        // strand several candidates of scan progress. updateState is an atomic
+        // temp-rename of a small file and the loop cadence is seconds per
+        // candidate; the cost is negligible.
+        persistCursor();
+    }
+    if (deadlineStopped) {
+        console.log(`[hicortex] Reconsolidation: wall-clock deadline reached (reconsolidationMaxMinutes) — ` +
+            `scan stopped at cursor ${cursor}; the next run resumes from there`);
+    }
+    else if (callCapStopped) {
+        console.log(`[hicortex] Reconsolidation: per-run call cap reached (reconsolidationMaxCalls) — ` +
+            `scan stopped at cursor ${cursor}; the next run resumes from there`);
     }
     // ---- #392 judged-merge phase: apply the queued pair merges through the
     // dedup core (mergeMemoryIds — same canonical pick, link re-points,
@@ -838,6 +953,19 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
     };
     if (!dryRun && groups.size > 0) {
         for (const group of groups.values()) {
+            // #401: the bounds stop the rewrite phase too — a group whose rewrite
+            // call was never made is left untouched and holds the cursor (never
+            // partially applied), exactly like the budget-exhausted path below.
+            if (deadlineHit()) {
+                deadlineStopped = true;
+                deferFrom(group.targetId);
+                break;
+            }
+            if (maxCalls > 0 && callsUsed >= maxCalls) {
+                callCapStopped = true;
+                deferFrom(group.targetId);
+                break;
+            }
             if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
                 deferFrom(group.targetId);
                 break;
@@ -849,6 +977,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
                 const r = await llm.completeClassify(buildRewritePrompt(group.target.content, triggersArg));
                 contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
                 budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
+                callsUsed++; // #401: rewrite contracts count toward the stage call cap
             }
             catch {
                 infraError = true;
@@ -957,6 +1086,8 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
     for (const [label, stat] of runBands)
         bandStats[label] = stat;
     if (!dryRun) {
+        // #401: the authoritative FINAL cursor write — the mid-scan persists
+        // above are checkpoints; this one also applies the pendingMinRowid hold.
         (0, state_js_1.updateState)((s) => {
             s.reconsolidationCursor = cursor;
             // Cumulative judged-band accumulation (#392) — the zone already
@@ -984,6 +1115,8 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
     return {
         scanned,
         pairs_evaluated: pairsEvaluated,
+        pairs_discovered: pairsDiscovered,
+        pairs_discovered_unlinked: pairsDiscoveredUnlinked,
         rewritten,
         absorbed,
         kept_linked: keptLinked,

package/dist/types.d.ts

@@ -224,6 +224,16 @@ export interface ConsolidationReport {
             scanned: number;
             /** Pairs actually sent to the verdict LLM (detection + explicit-mark verification). */
             pairs_evaluated: number;
+            /**
+             * #394: pairs the similarity floor discovered this run (KNN neighbors
+             * at/above correctionMinSimilarity), counted before any skip or
+             * judgment — the only sizing number a dry-run can show, where
+             * pairs_evaluated is always 0.
+             */
+            pairs_discovered: number;
+            /** #394: discovered pairs with no resolution link yet — the actionable
+             * candidates (deterministic-zone work + would-be verdict calls). */
+            pairs_discovered_unlinked: number;
             /** Targets rewritten in place this run (one history row each). */
             rewritten: number;
             /** Triggers absorbed (invisible to recall: vector + FTS dropped). */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gamaze/hicortex",
-  "version": "0.20.5",
+  "version": "0.20.7",
   "description": "Persistent agent identity for AI agents \u2014 a hand-edited identity layer, nightly-distilled experience, and lessons injected every session, shared across your whole fleet. Works with Hermes, OpenClaw, Claude Code, Pi, and opencode.",
   "main": "dist/index.js",
   "bin": {