@gamaze/hicortex 0.20.9 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +8 -0
  2. package/assets/dashboard.html +4174 -835
  3. package/dist/calibration.d.ts +119 -0
  4. package/dist/calibration.js +149 -1
  5. package/dist/capture-health.d.ts +87 -0
  6. package/dist/capture-health.js +106 -0
  7. package/dist/capture-pause.d.ts +86 -0
  8. package/dist/capture-pause.js +127 -0
  9. package/dist/capture.d.ts +9 -0
  10. package/dist/capture.js +2 -1
  11. package/dist/cli.js +36 -0
  12. package/dist/consolidate.d.ts +35 -0
  13. package/dist/consolidate.js +85 -9
  14. package/dist/dashboard.d.ts +322 -3
  15. package/dist/dashboard.js +592 -7
  16. package/dist/db.js +105 -0
  17. package/dist/eval/importance-eval.d.ts +85 -0
  18. package/dist/eval/importance-eval.js +286 -0
  19. package/dist/eval/planted-fixtures.d.ts +1 -1
  20. package/dist/eval/ranking-battery.d.ts +78 -0
  21. package/dist/eval/ranking-battery.js +181 -0
  22. package/dist/eval/ranking-eval.d.ts +41 -0
  23. package/dist/eval/ranking-eval.js +391 -0
  24. package/dist/eval/ranking-fixtures.d.ts +77 -0
  25. package/dist/eval/ranking-fixtures.js +226 -0
  26. package/dist/identity-store.d.ts +21 -0
  27. package/dist/identity-store.js +49 -0
  28. package/dist/init.d.ts +14 -0
  29. package/dist/init.js +32 -0
  30. package/dist/mcp-server.d.ts +12 -0
  31. package/dist/mcp-server.js +184 -3
  32. package/dist/nightly.d.ts +9 -1
  33. package/dist/nightly.js +59 -7
  34. package/dist/prompts.d.ts +10 -0
  35. package/dist/prompts.js +28 -5
  36. package/dist/reconsolidation.d.ts +59 -30
  37. package/dist/reconsolidation.js +526 -296
  38. package/dist/rescore-importance.d.ts +80 -0
  39. package/dist/rescore-importance.js +236 -0
  40. package/dist/retrieval.d.ts +12 -0
  41. package/dist/retrieval.js +30 -1
  42. package/dist/stages.d.ts +37 -0
  43. package/dist/stages.js +51 -0
  44. package/dist/state.d.ts +32 -6
  45. package/dist/storage.d.ts +34 -2
  46. package/dist/storage.js +63 -6
  47. package/dist/types.d.ts +48 -0
  48. package/package.json +3 -1
@@ -43,13 +43,32 @@
43
43
  * NOTHING else: no status change, no rewrite, no merge queue; both records
44
44
  * stay live so the consumer sees both truths. Both merge paths (the zone's
45
45
  * planDedup and the judged mergeMemoryIds) refuse to blend a conflicts-linked
46
- * pair, counted as conflict_skipped. The zone therefore runs AFTER the
47
- * rewrite phase — with the zone first, a >=0.92 conflict pair was blended
46
+ * pair, counted as conflict_skipped. The zone therefore runs AFTER the scan —
47
+ * with the zone first, a >=0.92 conflict pair was blended
48
48
  * before the judge ever saw it (the planted-eval harm: canonical=older, the
49
49
  * newer truth erased); running it last means verdicts/marks/binds land first
50
50
  * and the zone merges only what no verdict claimed — a conflicts bind set by
51
51
  * this run's scan guards the SAME run's zone.
52
52
  *
53
+ * #439 apply-on-confirm — confirmed merges and rewrite groups apply at the
54
+ * candidate BOUNDARY (the end of the scan iteration that confirmed them), not
55
+ * in post-scan phases. The old end-of-run batch was a completion assumption
56
+ * written when nightlies finished in an hour; under #405 budget pressure it
57
+ * became a days-long queue where confirmed work never landed and every night
58
+ * re-paid the judgment cost (cursor held below un-applied groups, pairs
59
+ * re-detected, re-judged). Now each judged-merge pair applies via
60
+ * mergeMemoryIds in its OWN transaction at confirmation time, each rewrite
61
+ * group via its own rewrite call + applyRewriteGroup transaction; the cursor
62
+ * advances per APPLIED candidate, so a deferral holds it below exactly ONE
63
+ * candidate's pairs. One pre-merge backup per run (lazy, before the first
64
+ * application); the capture lock is taken per boundary batch with a same-run
65
+ * retry list + a final drain. A shared trigger IS the current candidate, so
66
+ * the multi-target keep rule resolves across the boundary's groups (any keep
67
+ * keeps). A target corrected by two different candidates takes two sequential
68
+ * rewrites — the second composes the already-corrected story — instead of one
69
+ * grouped call (the ONE-call grouping was a cost optimization, not a
70
+ * correctness invariant; accepted semantics change).
71
+ *
53
72
  * Status vocabulary (code-defined, extensible — deliberately NOT config):
54
73
  * NULL/'active' default | 'superseded' + 'retracted' demote in ranking |
55
74
  * 'corrected' = rewritten, never demotes (demoting it would bury the
@@ -601,16 +620,18 @@ async function classifyPair(llm, oldContent, newContent) {
601
620
  }
602
621
  }
603
622
  /**
604
- * Nightly reconsolidation stage (#384, #392 — THE unified resolution stage).
623
+ * Nightly reconsolidation stage (#384, #392 — THE unified resolution stage;
624
+ * #439 apply-on-confirm).
605
625
  *
606
626
  * Phase order (#393 guard-C): the deterministic merge zone (pairs >= the
607
- * ceiling) runs LAST — after the scan, the judged-merge phase, and the
608
- * rewrite phase. Judgment outranks the deterministic sweep: verdicts, marks,
609
- * and binds land first and the zone merges only what no verdict claimed. With
610
- * the zone first, a >=0.92 genuine-conflict pair was blended before the judge
611
- * ever saw it (the planted-eval harm); running it last means a `conflicts`
612
- * bind set by this run's scan guards the SAME run's zone. Zone internals
613
- * (lock, backup, deadline, persistBand, fail-soft) are unchanged.
627
+ * ceiling) runs LAST — after the scan (which now includes every judged-merge
628
+ * application and rewrite, #439). Judgment outranks the deterministic sweep:
629
+ * verdicts, marks, and binds land first and the zone merges only what no
630
+ * verdict claimed. With the zone first, a >=0.92 genuine-conflict pair was
631
+ * blended before the judge ever saw it (the planted-eval harm); running it
632
+ * last means a `conflicts` bind set by this run's scan guards the SAME run's
633
+ * zone. Zone internals (lock, backup, deadline, persistBand, fail-soft) are
634
+ * unchanged.
614
635
  *
615
636
  * Scan: every memory with rowid > reconsolidationCursor (no shape filter;
616
637
  * absorbed candidates are skipped — invisible memories are not re-judged).
@@ -621,25 +642,33 @@ async function classifyPair(llm, oldContent, newContent) {
621
642
  * judged) plus the scout's FTS hits for correction-shaped memories (same
622
643
  * verdict loop, NO similarity gate; guard-C: a scout hit whose KNN twin sits
623
644
  * at/above the ceiling is re-tagged scout so the pair IS judged instead of
624
- * being left for the zone to blend). Confirmed
625
- * `corrects` pairs above the confidence gate on fact-shaped targets group by
626
- * target into ONE rewrite call each (AC3); confirmed `merge` pairs queue for
627
- * the merge phase; a `conflicts` verdict writes the conflicts link and
628
- * nothing else (both live); everything else is mark-only.
645
+ * being left for the zone to blend). Confirmed `corrects` pairs above the
646
+ * confidence gate on fact-shaped targets group by target; a `conflicts`
647
+ * verdict writes the conflicts link and nothing else (both live); everything
648
+ * else is mark-only.
629
649
  *
630
- * Merge phase (#392): queued pairs merge through the dedup core under one
631
- * lock/backup window. A pair that cannot apply keeps both memories and holds
632
- * the cursor; a conflicts-linked or metadata-mismatched refusal keeps both
633
- * and advances (the verdict was rendered).
650
+ * #439 BOUNDARY apply: at the END of each candidate iteration everything it
651
+ * confirmed applies IMMEDIATELY — merges first (each judged-merge pair via
652
+ * mergeMemoryIds in its own transaction, under the boundary's short lock
653
+ * window; ONE lazy pre-merge backup per run), then the iteration's rewrite
654
+ * groups (one rewrite LLM call + one applyRewriteGroup transaction each;
655
+ * dispositions resolved ACROSS the boundary's groups — the multi-target keep
656
+ * rule: a trigger absorbed only if every group's contract says absorb). A
657
+ * busy capture lock pushes the boundary's merges onto a same-run retry list
658
+ * (retried at the next boundary and once in a final drain after the scan);
659
+ * a deadline, a backup failure, or a rewrite-call refusal/infra error defers
660
+ * the remaining work and holds the cursor.
634
661
  *
635
- * Cursor discipline mirrors stageSupersession: the cursor advances past a
636
- * candidate once its neighbor set has been considered, regardless of infra
637
- * skips — EXCEPT when rewrite groups or confirmed merges could not be applied
638
- * (budget exhausted / rewrite-call infra error / merge cap or lock): the
639
- * cursor then holds BELOW the earliest candidate contributing to the
640
- * un-applied work, so those pairs are re-detected next run (dup-over-loss —
641
- * an un-marked, un-rewritten, un-merged confirmed resolution must never be
642
- * silently dropped by the cursor passing it).
662
+ * Cursor discipline: the cursor advances past a candidate only when its
663
+ * iteration's confirmed work has LANDED (or was refused-with-verdict-rendered:
664
+ * metadata mismatch, conflict-linked, mark-only fallback). A deferral holds
665
+ * the cursor BELOW the current candidate — bounded to ONE candidate's pairs,
666
+ * re-detected and re-judged next run (dup-over-loss — a confirmed resolution
667
+ * must never be silently dropped by the cursor passing it). The separate
668
+ * scan high-water (state.reconsolidationScannedRowid) records the max
669
+ * candidate rowid ENTERED and is never held back, so the report can split
670
+ * verdict calls into pairs_reevaluated (at/below the prior high-water) vs
671
+ * pairs_new — the convergence measurement.
643
672
  *
644
673
  * Dry-run: the zone's discovery + the free idempotency check only — zero LLM
645
674
  * calls, zero writes, no cursor or band-stats persistence. Gate discovery is
@@ -678,6 +707,14 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
678
707
  runBands.set(band.label, stat);
679
708
  };
680
709
  const startCursor = (0, state_js_1.loadState)(stateDir).reconsolidationCursor ?? 0;
710
+ // #439 convergence measurement: the scan high-water is the max candidate
711
+ // rowid any run has ENTERED — never held back by un-applied work. This
712
+ // run's re-judged/new split keys on the PREVIOUS run's persisted value: a
713
+ // verdict on a candidate at/below it re-judges pairs a prior run already
714
+ // judged but could not apply (the cursor held below them, so they
715
+ // re-detect). Once the backlog drains, pairs_reevaluated reads 0.
716
+ const prevScannedRowid = (0, state_js_1.loadState)(stateDir).reconsolidationScannedRowid ?? startCursor;
717
+ let scannedRowidHighwater = startCursor;
681
718
  // NO shape filter (AC2) — unlike stageSupersession. Absorbed rows are
682
719
  // excluded: they are invisible to recall and must not re-enter judgment.
683
720
  const rows = db
@@ -714,12 +751,29 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
714
751
  let scoutScanned = 0;
715
752
  let scoutCorrectionShaped = 0;
716
753
  let scoutCandidatesFound = 0;
754
+ // #439 observability: the re-judged/new verdict split (keyed on the prior
755
+ // run's scan high-water) + the scan-stability guard's skip count + the
756
+ // confirmed-merge deferral count (still un-applied at run end).
757
+ let pairsReevaluated = 0;
758
+ let pairsNew = 0;
759
+ let skippedAbsorbed = 0;
760
+ let mergePairsDeferred = 0;
717
761
  let cursor = startCursor;
718
- const queuedMerges = [];
719
- // #392 cursor-hold anchor, shared by the merge phase and the rewrite phase:
720
- // un-applied work holds the cursor BELOW the earliest contributing
721
- // candidate so the pairs are re-detected next run (dup-over-loss).
722
- let pendingMinRowid = null;
762
+ // Lock-busy survivors: boundary merges that could not take the capture
763
+ // lock, plus (fix round, #440 review finding 1) deadline/backup-dropped
764
+ // tails re-queued at their boundary instead of discarded. Same-run only —
765
+ // retried (in full) at the next boundary and once in the final drain after
766
+ // the scan. While the list is non-empty, every persisted checkpoint clamps
767
+ // below the earliest contributing candidate (pendingRetryFloor — the kill
768
+ // window cannot strand them); still un-applied at run end, the drain holds
769
+ // the cursor below that same floor (dup-over-loss).
770
+ const retryMerges = [];
771
+ // ONE pre-merge backup per run (#439): takePreDedupBackup is a full SQLite
772
+ // copy, so a per-boundary backup would be hundreds of full-DB copies on a
773
+ // backlog night. Taken LAZILY, immediately before the first judged-merge
774
+ // application; remembered for the rest of the run (the zone takes its own,
775
+ // independent backup, as before).
776
+ let mergeWindowBackedUp = false;
723
777
  // Links created by THIS stage in THIS run — lets the explicit-mark pass
724
778
  // distinguish operator marks (pre-existing) from stage output.
725
779
  const linksCreatedThisRun = new Set();
@@ -727,40 +781,50 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
727
781
  storage.addLink(db, oldId, newId, relationship, strength);
728
782
  linksCreatedThisRun.add(`${oldId}|${newId}`);
729
783
  };
730
- // ---- #401: mid-scan cursor persistence. Called at EVERY scan-loop exit
731
- // path (deadline, budget cap, mark-verify budget stop, discovery
732
- // failure) AND after every fully-considered candidate, so a killed run
733
- // loses at most the candidate in flight. The end-of-stage updateState
734
- // below stays the authoritative final write (it also applies the
735
- // pendingMinRowid hold — that variable is only ever set AFTER the scan
736
- // loop, so it is null at every call site here). updateState is
737
- // load→mutate→temp-rename atomic.
784
+ // ---- #401/#439 mid-scan cursor persistence. Called at EVERY scan-loop
785
+ // exit path AND at the end of every candidate iteration — AFTER that
786
+ // iteration's boundary apply, so the persisted cursor only ever advances
787
+ // past candidates whose confirmed work has landed; a killed run loses at
788
+ // most the candidate in flight. The ONE exception is lock-busy retry
789
+ // survivors: they intentionally ride the retry list while the scan
790
+ // continues (the lock may clear this run), so while any are pending every
791
+ // persisted checkpoint CLAMPS below their earliest contributor — a SIGKILL
792
+ // in the window between a busy boundary and the pair landing must never
793
+ // strand a confirmed merge behind the cursor (the #402 orphan-floor
794
+ // discipline, re-scoped to the retry list; the clamp lifts automatically
795
+ // once a later boundary or the final drain applies them). Also persists
796
+ // the scan high-water (never held back). updateState is load→mutate→
797
+ // temp-rename atomic.
738
798
  let deadlineStopped = false;
739
- // #402 follow-up (reviewer note 1): the hard-kill orphan floor. Queued
740
- // merges and open rewrite groups are applied only in the POST-scan
741
- // phases — until then their verdicts exist only in memory, and a
742
- // SIGKILL/OOM between two persists would strand them BEHIND the persisted
743
- // cursor (the next run would skip them forever). This tracks the smallest
744
- // candidate rowid contributing to queued-but-unapplied work;
745
- // persistCursor clamps every checkpoint below it so a resumed run
746
- // re-detects the pairs (dup-over-loss). Deliberately SEPARATE from the
747
- // post-loop pendingMinRowid hold above — different lifetime, different
748
- // writers.
749
- let scanPendingMinRowid = null;
750
- const notePendingRowid = (rowid) => {
751
- scanPendingMinRowid =
752
- scanPendingMinRowid === null ? rowid : Math.min(scanPendingMinRowid, rowid);
799
+ // #439: once an iteration's confirmed work deferred (deadline at the
800
+ // boundary, backup failure, rewrite refusal/infra error), the cursor never
801
+ // advances again this run — a later iteration must not push it past the
802
+ // held candidate's rowid.
803
+ let cursorHold = false;
804
+ // Fix round (#440 review, finding 1): the floor below the earliest
805
+ // candidate contributing to a still-un-applied retry merge. Applied at
806
+ // every persist so the kill-with-pending-retry window cannot strand them.
807
+ const pendingRetryFloor = () => retryMerges.length > 0
808
+ ? Math.min(...retryMerges.map((p) => p.candidateRowid)) - 1
809
+ : null;
810
+ const clampedCursor = () => {
811
+ const floor = pendingRetryFloor();
812
+ return floor !== null ? Math.min(cursor, floor) : cursor;
753
813
  };
754
814
  const persistCursor = () => {
755
815
  if (dryRun)
756
816
  return;
757
- const checkpoint = scanPendingMinRowid !== null ? Math.min(cursor, scanPendingMinRowid - 1) : cursor;
758
817
  (0, state_js_1.updateState)((s) => {
759
- s.reconsolidationCursor = checkpoint;
818
+ s.reconsolidationCursor = clampedCursor();
819
+ s.reconsolidationScannedRowid = Math.max(scannedRowidHighwater, s.reconsolidationScannedRowid ?? 0);
760
820
  }, stateDir);
761
821
  };
762
- const groups = new Map();
763
- const addTrigger = (target, trigger, confidence, cosine, explicit) => {
822
+ // #439: rewrite groups live ONLY inside the candidate iteration that
823
+ // formed them (its boundary applies or degrades them, then they are
824
+ // discarded). Every trigger is the current candidate — a target corrected
825
+ // by two different candidates takes two sequential rewrites instead of
826
+ // the old one grouped call.
827
+ const addTrigger = (groups, target, trigger, confidence, cosine, explicit) => {
764
828
  let group = groups.get(target.id);
765
829
  if (!group) {
766
830
  group = { targetId: target.id, target, triggers: [] };
@@ -775,10 +839,9 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
775
839
  candidateRowid: trigger.__rowid,
776
840
  explicit,
777
841
  });
778
- notePendingRowid(trigger.__rowid); // orphan floor — group unapplied until the rewrite phase
779
842
  }
780
843
  };
781
- for (const candidate of rows) {
844
+ for (const snapshotted of rows) {
782
845
  // #401: runtime bounds first — exit cleanly at the last fully-considered
783
846
  // candidate boundary (cursor = the previous candidate's rowid here).
784
847
  if (!dryRun && deadlineHit()) {
@@ -790,7 +853,50 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
790
853
  persistCursor();
791
854
  break;
792
855
  }
856
+ // ---- #439 scan-stability guard: `rows` is ONE snapshot fetched at stage
857
+ // start with status != 'absorbed'; boundary applies (merge losers,
858
+ // rewrite triggers absorbed) can mark FUTURE rows of that snapshot
859
+ // absorbed after the filter ran. Without this re-read such a candidate
860
+ // would be scouted/judged on stale content and its KNN would silently
861
+ // re-embed it (its vector row is gone). Absorbed → skip (counted,
862
+ // cursor passes it); otherwise the LIVE row's content/status drives the
863
+ // rest of the iteration (refreshes content rewritten by an earlier
864
+ // boundary when created_at and rowid order diverge).
865
+ const live = db
866
+ .prepare(`SELECT rowid AS __rowid, * FROM memories WHERE rowid = ?`)
867
+ .get(snapshotted.__rowid);
868
+ if (!live) {
869
+ // Vanished entirely (deleted out from under the scan) — defensive;
870
+ // nothing to judge, the cursor passes it (never past an active hold).
871
+ scannedRowidHighwater = Math.max(scannedRowidHighwater, snapshotted.__rowid);
872
+ if (!cursorHold)
873
+ cursor = snapshotted.__rowid;
874
+ persistCursor();
875
+ continue;
876
+ }
877
+ if (live.status === "absorbed") {
878
+ skippedAbsorbed++;
879
+ scannedRowidHighwater = Math.max(scannedRowidHighwater, live.__rowid);
880
+ if (!cursorHold)
881
+ cursor = live.__rowid;
882
+ persistCursor();
883
+ continue;
884
+ }
885
+ const candidate = live;
793
886
  scanned++;
887
+ // The high-water advances as candidates are ENTERED — even when the
888
+ // iteration's work later defers (it is the SCAN mark, never held back).
889
+ scannedRowidHighwater = Math.max(scannedRowidHighwater, candidate.__rowid);
890
+ // #439 per-iteration confirmed work, applied at the boundary below.
891
+ const iterMerges = [];
892
+ const iterGroups = new Map();
893
+ // This iteration's confirmed work could not land (deadline at the
894
+ // boundary, backup failure, rewrite refusal/infra error): the cursor
895
+ // holds below this candidate.
896
+ let boundaryHold = false;
897
+ // Stop the scan AFTER the boundary (budget stop / rewrite-infra
898
+ // deferral / backup failure): further verdicts could not land anyway.
899
+ let stopScan = false;
794
900
  // ---- AC7: verify incoming explicit marks (corrected_by/superseded_by
795
901
  // links targeting this candidate) before they can join a rewrite group.
796
902
  // #401: only OPERATOR marks are verified — applyExplicitMark writes
@@ -821,12 +927,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
821
927
  const { verdict, usage } = await classifyPair(llm, target.content, candidate.content);
822
928
  budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
823
929
  pairsEvaluated++;
930
+ if (candidate.__rowid <= prevScannedRowid)
931
+ pairsReevaluated++;
932
+ else
933
+ pairsNew++;
824
934
  if (!verdict) {
825
935
  skippedInfra++; // mark retained; the neighborhood is revisited via newer candidacies
826
936
  continue;
827
937
  }
828
938
  if (verdict.action === "corrects" && verdict.confidence >= rewriteMinConfidence && isFactShapedTarget(target)) {
829
- addTrigger(target, candidate, verdict.confidence, null, true);
939
+ addTrigger(iterGroups, target, candidate, verdict.confidence, null, true);
830
940
  explicitVerified++;
831
941
  }
832
942
  else {
@@ -839,8 +949,12 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
839
949
  }
840
950
  }
841
951
  if (markBudgetStop) {
842
- persistCursor(); // #401: this candidate's remaining marks re-verify next run
843
- break;
952
+ // #401: this candidate's remaining marks re-verify next run — the
953
+ // cursor HOLDS below it (boundaryHold), and #439 still runs the
954
+ // boundary: marks confirmed before the stop apply, exactly as they
955
+ // did when the rewrite phase was post-scan.
956
+ stopScan = true;
957
+ boundaryHold = true;
844
958
  }
845
959
  }
846
960
  // ---- #393 B: scout shape call — ONE classify-tier call per candidate,
@@ -852,27 +966,31 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
852
966
  // discipline. Dry-run skips the call entirely: it is LLM work, and
853
967
  // dry-runs make zero LLM calls (the scout counters read 0 there).
854
968
  let scoutReferences = null;
855
- if (!dryRun) {
969
+ if (!dryRun && !stopScan) {
856
970
  if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
857
- persistCursor(); // shape call refused — this candidate re-scouts next run
858
- break;
971
+ // Shape call refused — this candidate re-scouts next run: the cursor
972
+ // HOLDS below it (the candidate was entered but not considered).
973
+ stopScan = true;
974
+ boundaryHold = true;
859
975
  }
860
- scoutScanned++;
861
- try {
862
- const r = await llm.complete(buildScoutShapePrompt(candidate.content));
863
- budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
864
- const shape = parseScoutShape(r.text);
865
- if (!shape) {
866
- skippedInfra++;
976
+ else {
977
+ scoutScanned++;
978
+ try {
979
+ const r = await llm.complete(buildScoutShapePrompt(candidate.content));
980
+ budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
981
+ const shape = parseScoutShape(r.text);
982
+ if (!shape) {
983
+ skippedInfra++;
984
+ }
985
+ else if (shape.correction) {
986
+ scoutCorrectionShaped++;
987
+ scoutReferences = shape.references;
988
+ }
867
989
  }
868
- else if (shape.correction) {
869
- scoutCorrectionShaped++;
870
- scoutReferences = shape.references;
990
+ catch {
991
+ skippedInfra++;
871
992
  }
872
993
  }
873
- catch {
874
- skippedInfra++;
875
- }
876
994
  }
877
995
  // ---- AC2: detection pairs against older neighbors — TWO sources (#393 B):
878
996
  // the KNN similarity source (floor-gated, the pre-B baseline) and, for
@@ -880,22 +998,26 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
880
998
  // claim's terms → corpus search; NO similarity gate — cosine is a ranker,
881
999
  // never a blocker). Merged + deduped by neighbor id: a pair found by both
882
1000
  // sources is judged ONCE, as similarity (with band attribution).
883
- let neighbors;
884
- try {
885
- neighbors = await findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarity);
886
- }
887
- catch (err) {
888
- console.warn(`[hicortex] reconsolidation: discovery failed for ${candidate.id.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
889
- cursor = candidate.__rowid;
890
- persistCursor(); // #401: every exit path persists
891
- continue;
1001
+ let discoveryFailed = false;
1002
+ let neighbors = [];
1003
+ if (!stopScan) {
1004
+ try {
1005
+ neighbors = await findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarity);
1006
+ }
1007
+ catch (err) {
1008
+ console.warn(`[hicortex] reconsolidation: discovery failed for ${candidate.id.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1009
+ // #439: skip this candidate's neighbor judgments but still run the
1010
+ // boundary below — marks confirmed earlier this iteration apply
1011
+ // (they did when the phases were post-scan).
1012
+ discoveryFailed = true;
1013
+ }
892
1014
  }
893
1015
  const neighborEntries = neighbors.map((n) => ({
894
1016
  mem: n,
895
1017
  cosine: (0, retrieval_js_1.l2ToCosine)(n.distance),
896
1018
  source: "similarity",
897
1019
  }));
898
- if (scoutReferences !== null) {
1020
+ if (!discoveryFailed && scoutReferences !== null) {
899
1021
  const candidateVec = storage.getStoredEmbedding(db, candidate.id);
900
1022
  const knnById = new Map(neighborEntries.map((e) => [e.mem.id, e]));
901
1023
  for (const entry of findScoutNeighbors(db, candidate, scoutReferences, candidateVec)) {
@@ -943,12 +1065,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
943
1065
  continue; // preview only — no LLM call, no write
944
1066
  // #405: the ONE run budget's refusal is the only call cap.
945
1067
  if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
946
- persistCursor(); // cursor still points at the last fully-considered candidate
1068
+ stopScan = true; // the boundary below still applies what this candidate already confirmed
947
1069
  break;
948
1070
  }
949
1071
  const { verdict, usage } = await classifyPair(llm, entry.mem.content, candidate.content);
950
1072
  budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
951
1073
  pairsEvaluated++;
1074
+ if (candidate.__rowid <= prevScannedRowid)
1075
+ pairsReevaluated++;
1076
+ else
1077
+ pairsNew++;
952
1078
  if (!verdict) {
953
1079
  skippedInfra++;
954
1080
  continue;
@@ -958,8 +1084,8 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
958
1084
  // pairs reach them through a different, cosine-blind door.
959
1085
  if (entry.source === "similarity")
960
1086
  recordBand(pairCosine, verdict.action, verdict.confidence);
961
- // #392: a merge verdict is queued for the merge phase (below) — no
962
- // link, no write here. Below the confidence gate BOTH memories stay
1087
+ // #392/#439: a merge verdict queues for THIS iteration's boundary —
1088
+ // no link, no write here. Below the confidence gate BOTH memories stay
963
1089
  // live: a weak mark is recoverable, and there is nothing to mark for a
964
1090
  // duplicate — keeping both is the recoverable outcome.
965
1091
  if (verdict.action === "merge") {
@@ -975,8 +1101,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
975
1101
  }
976
1102
  }
977
1103
  else {
978
- queuedMerges.push({ oldId: entry.mem.id, newId: candidate.id, candidateRowid: candidate.__rowid });
979
- notePendingRowid(candidate.__rowid); // orphan floor — merge unapplied until the merge phase
1104
+ iterMerges.push({ oldId: entry.mem.id, newId: candidate.id, candidateRowid: candidate.__rowid });
980
1105
  }
981
1106
  continue;
982
1107
  }
@@ -1017,73 +1142,302 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1017
1142
  markedRetracted++;
1018
1143
  continue;
1019
1144
  }
1020
- addTrigger(entry.mem, candidate, verdict.confidence, cosine, false);
1145
+ addTrigger(iterGroups, entry.mem, candidate, verdict.confidence, cosine, false);
1021
1146
  }
1022
1147
  // verdict "none" → nothing to do
1023
1148
  }
1024
- cursor = candidate.__rowid;
1025
- // #402 follow-up (reviewer note 1): persist after EVERY fully-considered
1026
- // candidate — the 50-candidate batch left a kill window that could
1027
- // strand several candidates of scan progress. updateState is an atomic
1028
- // temp-rename of a small file and the loop cadence is seconds per
1029
- // candidate; the cost is negligible.
1149
+ // ---- #439 BOUNDARY: apply everything this candidate confirmed, NOW —
1150
+ // merges first, then rewrite groups (the order the old post-scan phases
1151
+ // used; preserves the existing tolerance where a merge loser that is
1152
+ // also a rewrite trigger stays absorbed while the rewrite still composes
1153
+ // its content). Each application is its own transaction
1154
+ // (mergeMemoryIds / applyRewriteGroup — group-internal atomicity
1155
+ // preserved); a busy capture lock defers merges to the same-run retry
1156
+ // list, everything else defers by holding the cursor below this
1157
+ // candidate (bounded to ONE candidate's pairs).
1158
+ if (!dryRun) {
1159
+ // Earlier lock-busy survivors retry FIRST (oldest verdicts land
1160
+ // first), ahead of this candidate's fresh confirmations.
1161
+ const mergeBatch = [...retryMerges.splice(0, retryMerges.length), ...iterMerges];
1162
+ if (mergeBatch.length > 0) {
1163
+ if (deadlineHit()) {
1164
+ deadlineStopped = true;
1165
+ // Fix round (#440 review, finding 1): never DROP the batch — it can
1166
+ // begin with lock-busy survivors contributed by EARLIER candidates
1167
+ // that the cursor has already passed. Re-queue for the final drain;
1168
+ // its hold-below-earliest-contributor (and the persist clamp) keeps
1169
+ // every un-applied pair re-detectable next run.
1170
+ retryMerges.push(...mergeBatch);
1171
+ boundaryHold = true;
1172
+ console.log(`[hicortex] Reconsolidation: ${mergeBatch.length} confirmed merge(s) re-queued for the final drain — run deadline reached`);
1173
+ }
1174
+ else {
1175
+ const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
1176
+ const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
1177
+ if (!release) {
1178
+ // Busy capture run — the batch rides the same-run retry list:
1179
+ // retried at the next boundary and once in the final drain. Not
1180
+ // a cursor hold yet (the lock may clear this run).
1181
+ retryMerges.push(...mergeBatch);
1182
+ console.warn(`[hicortex] Reconsolidation: capture lock busy — ${mergeBatch.length} confirmed merge(s) deferred to a retry this run`);
1183
+ }
1184
+ else {
1185
+ try {
1186
+ let backupOk = true;
1187
+ if (!mergeWindowBackedUp) {
1188
+ try {
1189
+ await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
1190
+ mergeWindowBackedUp = true;
1191
+ }
1192
+ catch (err) {
1193
+ backupOk = false;
1194
+ console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
1195
+ `(${err instanceof Error ? err.message : String(err)}) — ${mergeBatch.length} merge(s) deferred`);
1196
+ }
1197
+ }
1198
+ if (!backupOk) {
1199
+ // Verdicts that cannot land must not keep being paid: stop
1200
+ // the scan. The batch re-queues for the final drain — a
1201
+ // TRANSIENT backup failure can still recover there; a
1202
+ // persistent one ends with the drain holding the cursor
1203
+ // below the earliest contributor (never just this candidate,
1204
+ // when retry survivors ride the batch).
1205
+ retryMerges.push(...mergeBatch);
1206
+ boundaryHold = true;
1207
+ stopScan = true;
1208
+ }
1209
+ else {
1210
+ for (let i = 0; i < mergeBatch.length; i++) {
1211
+ const pair = mergeBatch[i];
1212
+ // #405: the deadline stop-check between local merge
1213
+ // transactions — a safe boundary; deferred pairs hold the
1214
+ // cursor below this candidate and retry next run.
1215
+ if (deadlineHit()) {
1216
+ deadlineStopped = true;
1217
+ // Fix round (#440 review, finding 1): the un-applied tail
1218
+ // re-queues (it can contain earlier candidates' retry
1219
+ // survivors) — the final drain applies or holds it.
1220
+ retryMerges.push(...mergeBatch.slice(i));
1221
+ boundaryHold = true;
1222
+ console.log(`[hicortex] Reconsolidation: ${mergeBatch.length - i} confirmed merge(s) re-queued for the final drain — run deadline reached`);
1223
+ break;
1224
+ }
1225
+ const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
1226
+ if (result.ok) {
1227
+ mergePairsApplied++;
1228
+ console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
1229
+ `into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
1230
+ }
1231
+ else if (result.reason === "metadata_mismatch") {
1232
+ skippedMetadataMismatch++;
1233
+ console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
1234
+ `skipped (metadata mismatch) — both kept`);
1235
+ }
1236
+ else if (result.reason === "conflict_linked") {
1237
+ // #393 guard-C: the pair is conflicts-linked (operator-planted
1238
+ // or a prior verdict) — never blended; the cursor advances,
1239
+ // this verdict was rendered.
1240
+ conflictSkippedJudged++;
1241
+ console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
1242
+ `skipped (conflict-flagged) — both kept`);
1243
+ }
1244
+ // "no_members": a member vanished/was absorbed since the
1245
+ // verdict — nothing to merge, nothing to hold; the cursor
1246
+ // advances past it.
1247
+ }
1248
+ }
1249
+ }
1250
+ finally {
1251
+ release();
1252
+ }
1253
+ }
1254
+ }
1255
+ }
1256
+ if (iterGroups.size > 0) {
1257
+ // One rewrite call per group — the group's triggers are all THIS
1258
+ // candidate (#439: a target corrected by two different candidates
1259
+ // takes two sequential rewrites, one per boundary; the second call
1260
+ // composes the already-corrected story). A call that was never made
1261
+ // (budget/infra/deadline) defers the group — untouched, never
1262
+ // partially applied — and holds the cursor below this candidate.
1263
+ const contracts = new Map(); // null = contract failed
1264
+ let rewritesDeferred = false;
1265
+ for (const group of iterGroups.values()) {
1266
+ if (deadlineHit()) {
1267
+ deadlineStopped = true;
1268
+ rewritesDeferred = true;
1269
+ break;
1270
+ }
1271
+ if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
1272
+ rewritesDeferred = true;
1273
+ stopScan = true;
1274
+ break;
1275
+ }
1276
+ const triggersArg = group.triggers.map((t) => ({ id: t.id, content: t.memory.content }));
1277
+ let contract = null;
1278
+ let infraError = false;
1279
+ try {
1280
+ const r = await llm.complete(buildRewritePrompt(group.target.content, triggersArg));
1281
+ contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
1282
+ budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
1283
+ }
1284
+ catch {
1285
+ infraError = true;
1286
+ }
1287
+ if (infraError) {
1288
+ skippedInfra++;
1289
+ rewritesDeferred = true; // group NOT marked, NOT rewritten — retried next run
1290
+ stopScan = true; // verdicts past this hold could not advance the cursor anyway
1291
+ break;
1292
+ }
1293
+ contracts.set(group.targetId, contract);
1294
+ if (!contract)
1295
+ contractFailed++;
1296
+ }
1297
+ if (rewritesDeferred) {
1298
+ boundaryHold = true;
1299
+ }
1300
+ else {
1301
+ // Multi-target keep rule WITHIN the boundary: the shared trigger is
1302
+ // the current candidate, so every group it touches resolves here —
1303
+ // absorbed only if EVERY contract says absorb (any keep keeps).
1304
+ const finalOutcome = new Map();
1305
+ for (const contract of contracts.values()) {
1306
+ if (!contract)
1307
+ continue;
1308
+ for (const t of contract.triggers) {
1309
+ if (t.disposition === "keep" || finalOutcome.get(t.id) === "keep")
1310
+ finalOutcome.set(t.id, "keep");
1311
+ else
1312
+ finalOutcome.set(t.id, "absorb");
1313
+ }
1314
+ }
1315
+ // Counted from APPLIED groups only (a deferred group's dispositions
1316
+ // never took effect); a trigger in several applied groups counts once.
1317
+ const appliedOutcome = new Map();
1318
+ for (const group of iterGroups.values()) {
1319
+ const contract = contracts.get(group.targetId);
1320
+ if (contract === undefined)
1321
+ continue; // pending group — untouched this run
1322
+ if (contract === null) {
1323
+ // Failed rewrite contract → whole group mark-only, never a partial
1324
+ // apply. Content untouched, NO trigger absorbed.
1325
+ try {
1326
+ applyMarkOnlyGroup(db, group);
1327
+ }
1328
+ catch (err) {
1329
+ console.warn(`[hicortex] reconsolidation: mark-only fallback failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1330
+ skippedInfra++;
1331
+ boundaryHold = true; // retried next run
1332
+ continue;
1333
+ }
1334
+ markedRetracted++;
1335
+ console.log(`[hicortex] Reconsolidation: rewrite contract failed for ${group.targetId.slice(0, 8)} — group degraded to mark-only`);
1336
+ continue;
1337
+ }
1338
+ let applied = false;
1339
+ try {
1340
+ applied = await applyRewriteGroup(db, group, contract, finalOutcome, embedFn);
1341
+ }
1342
+ catch (err) {
1343
+ console.warn(`[hicortex] reconsolidation: rewrite apply failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1344
+ }
1345
+ if (!applied) {
1346
+ skippedInfra++; // defensive absorbed-target guard, or an apply error — retry next run
1347
+ boundaryHold = true;
1348
+ continue;
1349
+ }
1350
+ rewritten++;
1351
+ for (const t of contract.triggers) {
1352
+ const outcome = finalOutcome.get(t.id) ?? "keep";
1353
+ if (outcome === "keep" || appliedOutcome.get(t.id) === "keep")
1354
+ appliedOutcome.set(t.id, "keep");
1355
+ else
1356
+ appliedOutcome.set(t.id, "absorb");
1357
+ }
1358
+ }
1359
+ for (const outcome of appliedOutcome.values()) {
1360
+ if (outcome === "absorb")
1361
+ absorbed++;
1362
+ else
1363
+ keptLinked++;
1364
+ }
1365
+ }
1366
+ }
1367
+ }
1368
+ // #439 cursor advance: past this candidate ONLY when its confirmed work
1369
+ // landed (or was refused-with-verdict-rendered). Once anything deferred,
1370
+ // the latch holds the cursor below that candidate for the rest of the
1371
+ // run — a later iteration must never advance past an earlier hold.
1372
+ if (boundaryHold)
1373
+ cursorHold = true;
1374
+ if (!cursorHold)
1375
+ cursor = candidate.__rowid;
1376
+ // #402/#439: persist after EVERY candidate — AFTER the boundary apply,
1377
+ // so the checkpoint only ever crosses candidates whose work landed. A
1378
+ // SIGKILL between persists re-detects at most the in-flight candidate.
1030
1379
  persistCursor();
1380
+ if (stopScan || deadlineStopped)
1381
+ break;
1031
1382
  }
1032
1383
  if (deadlineStopped) {
1033
1384
  console.log(`[hicortex] Reconsolidation: run deadline reached (nightlyTimeBudgetMinutes) — ` +
1034
1385
  `scan stopped at cursor ${cursor}; the next run resumes from there`);
1035
1386
  }
1036
- // ---- #392 judged-merge phase: apply the queued pair merges through the
1037
- // dedup core (mergeMemoryIds — same canonical pick, link re-points,
1038
- // dedup_log, absorb). One short lock/backup window for the whole batch, one
1039
- // transaction per pair. #405: the dedupNightlyMaxMerges cap is gone — the
1040
- // run deadline bounds the merge loop (a stop-check between local
1041
- // transactions; the deferred pairs hold the cursor below their candidates).
1042
- // A pair that cannot apply (deadline, busy lock, failed backup) keeps BOTH
1043
- // memories live and holds the cursor below its candidate — a confirmed
1044
- // merge is never silently dropped by the cursor passing it (dup-over-loss).
1045
- // A metadata-rail refusal is different: the verdict WAS rendered, both
1046
- // memories stay live, the cursor advances.
1047
- let mergePairsDeferred = 0;
1048
- if (!dryRun && queuedMerges.length > 0) {
1049
- const holdQueued = (from) => {
1050
- for (let i = from; i < queuedMerges.length; i++) {
1051
- pendingMinRowid =
1052
- pendingMinRowid === null
1053
- ? queuedMerges[i].candidateRowid
1054
- : Math.min(pendingMinRowid, queuedMerges[i].candidateRowid);
1055
- }
1387
+ // ---- #439 final drain: lock-busy merge survivors get ONE more attempt
1388
+ // right after the scan (the retry list is same-run only — everything else
1389
+ // applied at its boundary). Still busy (or the deadline/backup refuses) →
1390
+ // the pairs stay un-applied, counted, and the cursor holds below the
1391
+ // earliest contributing candidate (dup-over-loss; logged).
1392
+ if (!dryRun && retryMerges.length > 0) {
1393
+ const batch = retryMerges.splice(0, retryMerges.length);
1394
+ // Hold below the earliest contributor of the UN-APPLIED tail only (fix
1395
+ // round, minor review note: the old whole-batch min over-held past pairs
1396
+ // that had just applied in the same loop).
1397
+ const holdBelow = (fromIndex) => {
1398
+ cursor = Math.min(cursor, Math.min(...batch.slice(fromIndex).map((p) => p.candidateRowid)) - 1);
1056
1399
  };
1057
- {
1400
+ if (deadlineHit()) {
1401
+ deadlineStopped = true;
1402
+ mergePairsDeferred += batch.length;
1403
+ holdBelow(0);
1404
+ console.log(`[hicortex] Reconsolidation: ${batch.length} confirmed merge(s) deferred — run deadline reached`);
1405
+ }
1406
+ else {
1058
1407
  const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
1059
1408
  const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
1060
1409
  if (!release) {
1061
- mergePairsDeferred = queuedMerges.length;
1062
- holdQueued(0); // a busy capture run defers the batch — fail-soft
1063
- console.warn(`[hicortex] Reconsolidation: capture lock busy — ${mergePairsDeferred} confirmed merge(s) deferred to next run`);
1410
+ mergePairsDeferred += batch.length;
1411
+ holdBelow(0);
1412
+ console.warn(`[hicortex] Reconsolidation: capture lock busy at the final drain — ` +
1413
+ `${batch.length} confirmed merge(s) deferred to next run`);
1064
1414
  }
1065
1415
  else {
1066
1416
  try {
1067
1417
  let backupOk = true;
1068
- try {
1069
- await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
1418
+ if (!mergeWindowBackedUp) {
1419
+ try {
1420
+ await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
1421
+ mergeWindowBackedUp = true;
1422
+ }
1423
+ catch (err) {
1424
+ backupOk = false;
1425
+ console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
1426
+ `(${err instanceof Error ? err.message : String(err)}) — ${batch.length} merge(s) deferred`);
1427
+ }
1070
1428
  }
1071
- catch (err) {
1072
- backupOk = false;
1073
- console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
1074
- `(${err instanceof Error ? err.message : String(err)}) — ${queuedMerges.length} merge(s) deferred`);
1429
+ if (!backupOk) {
1430
+ mergePairsDeferred += batch.length;
1431
+ holdBelow(0);
1075
1432
  }
1076
- if (backupOk) {
1077
- for (let i = 0; i < queuedMerges.length; i++) {
1078
- const pair = queuedMerges[i];
1079
- // #405: the deadline stop-check between local merge
1080
- // transactions — a safe boundary; deferred pairs hold the
1081
- // cursor below their candidates and retry next run.
1433
+ else {
1434
+ for (let i = 0; i < batch.length; i++) {
1435
+ const pair = batch[i];
1082
1436
  if (deadlineHit()) {
1083
1437
  deadlineStopped = true;
1084
- mergePairsDeferred = queuedMerges.length - i;
1085
- holdQueued(i);
1086
- console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred — run deadline reached`);
1438
+ mergePairsDeferred += batch.length - i;
1439
+ holdBelow(i);
1440
+ console.log(`[hicortex] Reconsolidation: ${batch.length - i} confirmed merge(s) deferred — run deadline reached`);
1087
1441
  break;
1088
1442
  }
1089
1443
  const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
@@ -1098,22 +1452,14 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1098
1452
  `skipped (metadata mismatch) — both kept`);
1099
1453
  }
1100
1454
  else if (result.reason === "conflict_linked") {
1101
- // #393 guard-C: the pair is conflicts-linked (operator-planted
1102
- // or a prior verdict) — never blended; the cursor advances,
1103
- // this verdict was rendered.
1104
1455
  conflictSkippedJudged++;
1105
1456
  console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
1106
1457
  `skipped (conflict-flagged) — both kept`);
1107
1458
  }
1108
1459
  // "no_members": a member vanished/was absorbed since the
1109
- // verdict — nothing to merge, nothing to hold; the cursor
1110
- // advances past it.
1460
+ // verdict — nothing to merge, nothing to hold.
1111
1461
  }
1112
1462
  }
1113
- else {
1114
- mergePairsDeferred = queuedMerges.length;
1115
- holdQueued(0);
1116
- }
1117
1463
  }
1118
1464
  finally {
1119
1465
  release();
@@ -1121,141 +1467,19 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1121
1467
  }
1122
1468
  }
1123
1469
  }
1124
- // ---- Rewrite phase (AC3/AC4/AC5). Three sub-phases so the multi-target
1125
- // keep rule can be honored: (R1) collect contracts, (R2) resolve every
1126
- // trigger's FINAL disposition across all groups, (R3) apply one transaction
1127
- // per group. A group whose rewrite call was never made (budget/infra) is
1128
- // left untouched and holds the cursor — never partially applied.
1129
- const contracts = new Map(); // null = contract failed
1130
- // pendingMinRowid (min candidate rowid among un-applied work) is declared
1131
- // above — shared with the merge phase's holdQueued.
1132
- const deferFrom = (fromTargetId) => {
1133
- let seen = false;
1134
- for (const group of groups.values()) {
1135
- if (!seen && group.targetId !== fromTargetId)
1136
- continue;
1137
- seen = true;
1138
- for (const t of group.triggers) {
1139
- pendingMinRowid = pendingMinRowid === null ? t.candidateRowid : Math.min(pendingMinRowid, t.candidateRowid);
1140
- }
1141
- }
1142
- };
1143
- if (!dryRun && groups.size > 0) {
1144
- for (const group of groups.values()) {
1145
- // #401: the bounds stop the rewrite phase too — a group whose rewrite
1146
- // call was never made is left untouched and holds the cursor (never
1147
- // partially applied), exactly like the budget-exhausted path below.
1148
- if (deadlineHit()) {
1149
- deadlineStopped = true;
1150
- deferFrom(group.targetId);
1151
- break;
1152
- }
1153
- if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
1154
- deferFrom(group.targetId);
1155
- break;
1156
- }
1157
- const triggersArg = group.triggers.map((t) => ({ id: t.id, content: t.memory.content }));
1158
- let contract = null;
1159
- let infraError = false;
1160
- try {
1161
- const r = await llm.complete(buildRewritePrompt(group.target.content, triggersArg));
1162
- contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
1163
- budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
1164
- }
1165
- catch {
1166
- infraError = true;
1167
- }
1168
- if (infraError) {
1169
- skippedInfra++;
1170
- deferFrom(group.targetId); // group NOT marked, NOT rewritten — retried next run
1171
- break;
1172
- }
1173
- contracts.set(group.targetId, contract);
1174
- if (!contract)
1175
- contractFailed++;
1176
- }
1177
- // R2: final per-trigger disposition — a trigger in multiple groups is
1178
- // absorbed only if EVERY disposition says absorb (any keep keeps it).
1179
- const finalOutcome = new Map();
1180
- for (const contract of contracts.values()) {
1181
- if (!contract)
1182
- continue;
1183
- for (const t of contract.triggers) {
1184
- if (t.disposition === "keep" || finalOutcome.get(t.id) === "keep")
1185
- finalOutcome.set(t.id, "keep");
1186
- else
1187
- finalOutcome.set(t.id, "absorb");
1188
- }
1189
- }
1190
- // R3: apply (one transaction per group). An apply that fails mid-flight
1191
- // (embed error, DB error) writes NOTHING (the transaction never ran) —
1192
- // the group is deferred like a pending one so it retries next run.
1193
- const appliedOutcome = new Map();
1194
- const deferGroup = (group) => {
1195
- for (const t of group.triggers) {
1196
- pendingMinRowid = pendingMinRowid === null ? t.candidateRowid : Math.min(pendingMinRowid, t.candidateRowid);
1197
- }
1198
- };
1199
- for (const group of groups.values()) {
1200
- const contract = contracts.get(group.targetId);
1201
- if (contract === undefined)
1202
- continue; // pending group — untouched this run
1203
- if (contract === null) {
1204
- // Failed rewrite contract → whole group mark-only, never a partial
1205
- // apply. Content untouched, NO trigger absorbed.
1206
- try {
1207
- applyMarkOnlyGroup(db, group);
1208
- }
1209
- catch (err) {
1210
- console.warn(`[hicortex] reconsolidation: mark-only fallback failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1211
- skippedInfra++;
1212
- deferGroup(group);
1213
- continue;
1214
- }
1215
- markedRetracted++;
1216
- console.log(`[hicortex] Reconsolidation: rewrite contract failed for ${group.targetId.slice(0, 8)} — group degraded to mark-only`);
1217
- continue;
1218
- }
1219
- let applied = false;
1220
- try {
1221
- applied = await applyRewriteGroup(db, group, contract, finalOutcome, embedFn);
1222
- }
1223
- catch (err) {
1224
- console.warn(`[hicortex] reconsolidation: rewrite apply failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1225
- }
1226
- if (!applied) {
1227
- skippedInfra++; // defensive absorbed-target guard, or an apply error — retry next run
1228
- deferGroup(group);
1229
- continue;
1230
- }
1231
- rewritten++;
1232
- // Counted from APPLIED groups only (a deferred group's dispositions
1233
- // never took effect); a trigger in several applied groups counts once.
1234
- for (const t of contract.triggers) {
1235
- const outcome = finalOutcome.get(t.id) ?? "keep";
1236
- if (outcome === "keep" || appliedOutcome.get(t.id) === "keep")
1237
- appliedOutcome.set(t.id, "keep");
1238
- else
1239
- appliedOutcome.set(t.id, "absorb");
1240
- }
1241
- }
1242
- for (const outcome of appliedOutcome.values()) {
1243
- if (outcome === "absorb")
1244
- absorbed++;
1245
- else
1246
- keptLinked++;
1247
- }
1248
- }
1249
1470
  // ---- #393 guard-C zone reorder: the deterministic merge zone (pairs >=
1250
- // the ceiling) runs LAST — after the scan, the judged-merge phase, and the
1251
- // rewrite phase. Judgment outranks the deterministic sweep: verdicts,
1252
- // marks, and binds land first, and the zone merges only what no verdict
1253
- // claimed. With the zone first, a >=0.92 genuine-conflict pair was blended
1254
- // before the judge ever saw it (canonical = oldest, the newer truth erased
1255
- // — the planted-eval harm); running it last means a `conflicts` bind set by
1256
- // THIS run's scan guards the SAME run's zone. LLM-free and budget-free — an
1257
- // LLM-less night still drains duplicates. Its own short lock window,
1258
- // pre-merge backup, and #405 deadline stop-check; fail-soft, never a throw.
1471
+ // the ceiling) runs LAST — after the scan (which includes every #439
1472
+ // boundary apply: judged merges + rewrites). Judgment outranks the
1473
+ // deterministic sweep: verdicts, marks, and binds land first, and the zone
1474
+ // merges only what no verdict claimed. With the zone first, a >=0.92
1475
+ // genuine-conflict pair was blended before the judge ever saw it
1476
+ // (canonical = oldest, the newer truth erased — the planted-eval harm);
1477
+ // running it last means a `conflicts` bind set by THIS run's scan guards
1478
+ // the SAME run's zone. LLM-free and budget-free — an LLM-less night still
1479
+ // drains duplicates (a deadline-deferred cluster re-detects next run at
1480
+ // zero token cost — content-based discovery, no cursor involvement). Its
1481
+ // own short lock window, pre-merge backup, and #405 deadline stop-check;
1482
+ // fail-soft, never a throw.
1259
1483
  const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
1260
1484
  stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
1261
1485
  threshold: autoMergeThreshold,
@@ -1263,12 +1487,6 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1263
1487
  acquireLock: options.acquireLock,
1264
1488
  deadline,
1265
1489
  });
1266
- // Cursor hold: un-applied work (rewrite groups, confirmed merges) holds the
1267
- // cursor BELOW its earliest contributing candidate so the pairs are
1268
- // re-detected next run.
1269
- if (pendingMinRowid !== null) {
1270
- cursor = Math.min(cursor, pendingMinRowid - 1);
1271
- }
1272
1490
  // Report snapshot: the deterministic band (from the zone's own numbers —
1273
1491
  // losers are merge verdicts at confidence 1.0; the zone persists the
1274
1492
  // cumulative copy itself) plus this run's judged bands.
@@ -1289,10 +1507,15 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1289
1507
  for (const [label, stat] of runBands)
1290
1508
  bandStats[label] = stat;
1291
1509
  if (!dryRun) {
1292
- // #401: the authoritative FINAL cursor write — the mid-scan persists
1293
- // above are checkpoints; this one also applies the pendingMinRowid hold.
1510
+ // #401/#439: the authoritative FINAL write — the mid-scan persists above
1511
+ // are checkpoints; this one applies the final-drain cursor hold (already
1512
+ // folded into `cursor`) and the scan high-water. The retry floor is
1513
+ // applied defensively too: the drain splices retryMerges empty on every
1514
+ // path, but a non-empty list here would mean a confirmed merge stranded
1515
+ // behind the cursor — clamp, never write past un-applied work.
1294
1516
  (0, state_js_1.updateState)((s) => {
1295
- s.reconsolidationCursor = cursor;
1517
+ s.reconsolidationCursor = clampedCursor();
1518
+ s.reconsolidationScannedRowid = Math.max(scannedRowidHighwater, s.reconsolidationScannedRowid ?? 0);
1296
1519
  // Cumulative judged-band accumulation (#392) — the zone already
1297
1520
  // persisted the deterministic band under its own label.
1298
1521
  if (runBands.size > 0) {
@@ -1306,13 +1529,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1306
1529
  }
1307
1530
  }, stateDir);
1308
1531
  }
1309
- if (rows.length > 0 || groups.size > 0 || mergePairsApplied > 0 || mergeBelowGate > 0) {
1310
- console.log(`[hicortex] Reconsolidation: ${scanned} scanned, ${pairsEvaluated} pairs evaluated, ` +
1532
+ if (rows.length > 0 || mergePairsApplied > 0 || mergeBelowGate > 0) {
1533
+ console.log(`[hicortex] Reconsolidation: ${scanned} scanned, ${pairsEvaluated} pairs evaluated ` +
1534
+ `(${pairsReevaluated} re-judged / ${pairsNew} new), ` +
1311
1535
  `${rewritten} rewritten (${absorbed} triggers absorbed, ${keptLinked} kept), ` +
1312
- `${mergePairsApplied} pair(s) merged, ${markedSuperseded} superseded, ` +
1536
+ `${mergePairsApplied} pair(s) merged (${mergePairsDeferred} deferred), ` +
1537
+ `${markedSuperseded} superseded, ` +
1313
1538
  `${markedRetracted} retracted (${belowGate} below gate, ${mergeBelowGate} merge below gate, ` +
1314
1539
  `${contractFailed} contract failed), ${skippedInfra} infra-skipped, ${skippedIdempotent} ` +
1315
- `already-linked, ${skippedAboveCeiling} above ceiling, ${explicitVerified} explicit verified, ` +
1540
+ `already-linked, ${skippedAbsorbed} absorbed-skip, ${skippedAboveCeiling} above ceiling, ` +
1541
+ `${explicitVerified} explicit verified, ` +
1316
1542
  `${explicitDivergent} explicit divergent, scout ${scoutScanned} scanned / ` +
1317
1543
  `${scoutCorrectionShaped} correction-shaped / ${scoutCandidatesFound} candidate pair(s), ` +
1318
1544
  `${conflictFlagged} conflict-flagged, ${conflictSkippedJudged + merges.skipped_conflict} conflict-skipped ` +
@@ -1335,6 +1561,10 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1335
1561
  explicit_verified: explicitVerified,
1336
1562
  explicit_divergent: explicitDivergent,
1337
1563
  cursor,
1564
+ pairs_reevaluated: pairsReevaluated,
1565
+ pairs_new: pairsNew,
1566
+ skipped_absorbed: skippedAbsorbed,
1567
+ merge_pairs_deferred: mergePairsDeferred,
1338
1568
  merges,
1339
1569
  merge_pairs_applied: mergePairsApplied,
1340
1570
  merge_below_gate: mergeBelowGate,