@gamaze/hicortex 0.20.9 → 0.20.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/dashboard.html +3989 -836
- package/dist/calibration.d.ts +119 -0
- package/dist/calibration.js +149 -1
- package/dist/capture-health.d.ts +87 -0
- package/dist/capture-health.js +106 -0
- package/dist/capture-pause.d.ts +86 -0
- package/dist/capture-pause.js +127 -0
- package/dist/capture.d.ts +9 -0
- package/dist/capture.js +2 -1
- package/dist/cli.js +36 -0
- package/dist/consolidate.d.ts +35 -0
- package/dist/consolidate.js +85 -9
- package/dist/dashboard.d.ts +322 -3
- package/dist/dashboard.js +592 -7
- package/dist/db.js +105 -0
- package/dist/eval/importance-eval.d.ts +85 -0
- package/dist/eval/importance-eval.js +286 -0
- package/dist/eval/planted-fixtures.d.ts +1 -1
- package/dist/eval/ranking-battery.d.ts +78 -0
- package/dist/eval/ranking-battery.js +181 -0
- package/dist/eval/ranking-eval.d.ts +41 -0
- package/dist/eval/ranking-eval.js +391 -0
- package/dist/eval/ranking-fixtures.d.ts +77 -0
- package/dist/eval/ranking-fixtures.js +226 -0
- package/dist/identity-store.d.ts +21 -0
- package/dist/identity-store.js +49 -0
- package/dist/init.d.ts +14 -0
- package/dist/init.js +32 -0
- package/dist/mcp-server.d.ts +12 -0
- package/dist/mcp-server.js +184 -3
- package/dist/nightly.d.ts +9 -1
- package/dist/nightly.js +59 -7
- package/dist/prompts.d.ts +10 -0
- package/dist/prompts.js +28 -5
- package/dist/reconsolidation.d.ts +59 -30
- package/dist/reconsolidation.js +526 -296
- package/dist/rescore-importance.d.ts +80 -0
- package/dist/rescore-importance.js +236 -0
- package/dist/retrieval.d.ts +12 -0
- package/dist/retrieval.js +30 -1
- package/dist/stages.d.ts +37 -0
- package/dist/stages.js +51 -0
- package/dist/state.d.ts +32 -6
- package/dist/storage.d.ts +34 -2
- package/dist/storage.js +63 -6
- package/dist/types.d.ts +48 -0
- package/package.json +3 -1
package/dist/reconsolidation.js
CHANGED
|
@@ -43,13 +43,32 @@
|
|
|
43
43
|
* NOTHING else: no status change, no rewrite, no merge queue; both records
|
|
44
44
|
* stay live so the consumer sees both truths. Both merge paths (the zone's
|
|
45
45
|
* planDedup and the judged mergeMemoryIds) refuse to blend a conflicts-linked
|
|
46
|
-
* pair, counted as conflict_skipped. The zone therefore runs AFTER the
|
|
47
|
-
*
|
|
46
|
+
* pair, counted as conflict_skipped. The zone therefore runs AFTER the scan —
|
|
47
|
+
* with the zone first, a >=0.92 conflict pair was blended
|
|
48
48
|
* before the judge ever saw it (the planted-eval harm: canonical=older, the
|
|
49
49
|
* newer truth erased); running it last means verdicts/marks/binds land first
|
|
50
50
|
* and the zone merges only what no verdict claimed — a conflicts bind set by
|
|
51
51
|
* this run's scan guards the SAME run's zone.
|
|
52
52
|
*
|
|
53
|
+
* #439 apply-on-confirm — confirmed merges and rewrite groups apply at the
|
|
54
|
+
* candidate BOUNDARY (the end of the scan iteration that confirmed them), not
|
|
55
|
+
* in post-scan phases. The old end-of-run batch was a completion assumption
|
|
56
|
+
* written when nightlies finished in an hour; under #405 budget pressure it
|
|
57
|
+
* became a days-long queue where confirmed work never landed and every night
|
|
58
|
+
* re-paid the judgment cost (cursor held below un-applied groups, pairs
|
|
59
|
+
* re-detected, re-judged). Now each judged-merge pair applies via
|
|
60
|
+
* mergeMemoryIds in its OWN transaction at confirmation time, each rewrite
|
|
61
|
+
* group via its own rewrite call + applyRewriteGroup transaction; the cursor
|
|
62
|
+
* advances per APPLIED candidate, so a deferral holds it below exactly ONE
|
|
63
|
+
* candidate's pairs. One pre-merge backup per run (lazy, before the first
|
|
64
|
+
* application); the capture lock is taken per boundary batch with a same-run
|
|
65
|
+
* retry list + a final drain. A shared trigger IS the current candidate, so
|
|
66
|
+
* the multi-target keep rule resolves across the boundary's groups (any keep
|
|
67
|
+
* keeps). A target corrected by two different candidates takes two sequential
|
|
68
|
+
* rewrites — the second composes the already-corrected story — instead of one
|
|
69
|
+
* grouped call (the ONE-call grouping was a cost optimization, not a
|
|
70
|
+
* correctness invariant; accepted semantics change).
|
|
71
|
+
*
|
|
53
72
|
* Status vocabulary (code-defined, extensible — deliberately NOT config):
|
|
54
73
|
* NULL/'active' default | 'superseded' + 'retracted' demote in ranking |
|
|
55
74
|
* 'corrected' = rewritten, never demotes (demoting it would bury the
|
|
@@ -601,16 +620,18 @@ async function classifyPair(llm, oldContent, newContent) {
|
|
|
601
620
|
}
|
|
602
621
|
}
|
|
603
622
|
/**
|
|
604
|
-
* Nightly reconsolidation stage (#384, #392 — THE unified resolution stage
|
|
623
|
+
* Nightly reconsolidation stage (#384, #392 — THE unified resolution stage;
|
|
624
|
+
* #439 apply-on-confirm).
|
|
605
625
|
*
|
|
606
626
|
* Phase order (#393 guard-C): the deterministic merge zone (pairs >= the
|
|
607
|
-
* ceiling) runs LAST — after the scan
|
|
608
|
-
* rewrite
|
|
609
|
-
* and binds land first and the zone merges only what no
|
|
610
|
-
* the zone first, a >=0.92 genuine-conflict pair was
|
|
611
|
-
* ever saw it (the planted-eval harm); running it
|
|
612
|
-
* bind set by this run's scan guards the SAME run's
|
|
613
|
-
* (lock, backup, deadline, persistBand, fail-soft) are
|
|
627
|
+
* ceiling) runs LAST — after the scan (which now includes every judged-merge
|
|
628
|
+
* application and rewrite, #439). Judgment outranks the deterministic sweep:
|
|
629
|
+
* verdicts, marks, and binds land first and the zone merges only what no
|
|
630
|
+
* verdict claimed. With the zone first, a >=0.92 genuine-conflict pair was
|
|
631
|
+
* blended before the judge ever saw it (the planted-eval harm); running it
|
|
632
|
+
* last means a `conflicts` bind set by this run's scan guards the SAME run's
|
|
633
|
+
* zone. Zone internals (lock, backup, deadline, persistBand, fail-soft) are
|
|
634
|
+
* unchanged.
|
|
614
635
|
*
|
|
615
636
|
* Scan: every memory with rowid > reconsolidationCursor (no shape filter;
|
|
616
637
|
* absorbed candidates are skipped — invisible memories are not re-judged).
|
|
@@ -621,25 +642,33 @@ async function classifyPair(llm, oldContent, newContent) {
|
|
|
621
642
|
* judged) plus the scout's FTS hits for correction-shaped memories (same
|
|
622
643
|
* verdict loop, NO similarity gate; guard-C: a scout hit whose KNN twin sits
|
|
623
644
|
* at/above the ceiling is re-tagged scout so the pair IS judged instead of
|
|
624
|
-
* being left for the zone to blend). Confirmed
|
|
625
|
-
*
|
|
626
|
-
*
|
|
627
|
-
*
|
|
628
|
-
* nothing else (both live); everything else is mark-only.
|
|
645
|
+
* being left for the zone to blend). Confirmed `corrects` pairs above the
|
|
646
|
+
* confidence gate on fact-shaped targets group by target; a `conflicts`
|
|
647
|
+
* verdict writes the conflicts link and nothing else (both live); everything
|
|
648
|
+
* else is mark-only.
|
|
629
649
|
*
|
|
630
|
-
*
|
|
631
|
-
*
|
|
632
|
-
*
|
|
633
|
-
*
|
|
650
|
+
* #439 BOUNDARY apply: at the END of each candidate iteration everything it
|
|
651
|
+
* confirmed applies IMMEDIATELY — merges first (each judged-merge pair via
|
|
652
|
+
* mergeMemoryIds in its own transaction, under the boundary's short lock
|
|
653
|
+
* window; ONE lazy pre-merge backup per run), then the iteration's rewrite
|
|
654
|
+
* groups (one rewrite LLM call + one applyRewriteGroup transaction each;
|
|
655
|
+
* dispositions resolved ACROSS the boundary's groups — the multi-target keep
|
|
656
|
+
* rule: a trigger absorbed only if every group's contract says absorb). A
|
|
657
|
+
* busy capture lock pushes the boundary's merges onto a same-run retry list
|
|
658
|
+
* (retried at the next boundary and once in a final drain after the scan);
|
|
659
|
+
* a deadline, a backup failure, or a rewrite-call refusal/infra error defers
|
|
660
|
+
* the remaining work and holds the cursor.
|
|
634
661
|
*
|
|
635
|
-
* Cursor discipline
|
|
636
|
-
*
|
|
637
|
-
*
|
|
638
|
-
*
|
|
639
|
-
*
|
|
640
|
-
*
|
|
641
|
-
*
|
|
642
|
-
*
|
|
662
|
+
* Cursor discipline: the cursor advances past a candidate only when its
|
|
663
|
+
* iteration's confirmed work has LANDED (or was refused-with-verdict-rendered:
|
|
664
|
+
* metadata mismatch, conflict-linked, mark-only fallback). A deferral holds
|
|
665
|
+
* the cursor BELOW the current candidate — bounded to ONE candidate's pairs,
|
|
666
|
+
* re-detected and re-judged next run (dup-over-loss — a confirmed resolution
|
|
667
|
+
* must never be silently dropped by the cursor passing it). The separate
|
|
668
|
+
* scan high-water (state.reconsolidationScannedRowid) records the max
|
|
669
|
+
* candidate rowid ENTERED and is never held back, so the report can split
|
|
670
|
+
* verdict calls into pairs_reevaluated (at/below the prior high-water) vs
|
|
671
|
+
* pairs_new — the convergence measurement.
|
|
643
672
|
*
|
|
644
673
|
* Dry-run: the zone's discovery + the free idempotency check only — zero LLM
|
|
645
674
|
* calls, zero writes, no cursor or band-stats persistence. Gate discovery is
|
|
@@ -678,6 +707,14 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
678
707
|
runBands.set(band.label, stat);
|
|
679
708
|
};
|
|
680
709
|
const startCursor = (0, state_js_1.loadState)(stateDir).reconsolidationCursor ?? 0;
|
|
710
|
+
// #439 convergence measurement: the scan high-water is the max candidate
|
|
711
|
+
// rowid any run has ENTERED — never held back by un-applied work. This
|
|
712
|
+
// run's re-judged/new split keys on the PREVIOUS run's persisted value: a
|
|
713
|
+
// verdict on a candidate at/below it re-judges pairs a prior run already
|
|
714
|
+
// judged but could not apply (the cursor held below them, so they
|
|
715
|
+
// re-detect). Once the backlog drains, pairs_reevaluated reads 0.
|
|
716
|
+
const prevScannedRowid = (0, state_js_1.loadState)(stateDir).reconsolidationScannedRowid ?? startCursor;
|
|
717
|
+
let scannedRowidHighwater = startCursor;
|
|
681
718
|
// NO shape filter (AC2) — unlike stageSupersession. Absorbed rows are
|
|
682
719
|
// excluded: they are invisible to recall and must not re-enter judgment.
|
|
683
720
|
const rows = db
|
|
@@ -714,12 +751,29 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
714
751
|
let scoutScanned = 0;
|
|
715
752
|
let scoutCorrectionShaped = 0;
|
|
716
753
|
let scoutCandidatesFound = 0;
|
|
754
|
+
// #439 observability: the re-judged/new verdict split (keyed on the prior
|
|
755
|
+
// run's scan high-water) + the scan-stability guard's skip count + the
|
|
756
|
+
// confirmed-merge deferral count (still un-applied at run end).
|
|
757
|
+
let pairsReevaluated = 0;
|
|
758
|
+
let pairsNew = 0;
|
|
759
|
+
let skippedAbsorbed = 0;
|
|
760
|
+
let mergePairsDeferred = 0;
|
|
717
761
|
let cursor = startCursor;
|
|
718
|
-
|
|
719
|
-
//
|
|
720
|
-
//
|
|
721
|
-
//
|
|
722
|
-
|
|
762
|
+
// Lock-busy survivors: boundary merges that could not take the capture
|
|
763
|
+
// lock, plus (fix round, #440 review finding 1) deadline/backup-dropped
|
|
764
|
+
// tails re-queued at their boundary instead of discarded. Same-run only —
|
|
765
|
+
// retried (in full) at the next boundary and once in the final drain after
|
|
766
|
+
// the scan. While the list is non-empty, every persisted checkpoint clamps
|
|
767
|
+
// below the earliest contributing candidate (pendingRetryFloor — the kill
|
|
768
|
+
// window cannot strand them); still un-applied at run end, the drain holds
|
|
769
|
+
// the cursor below that same floor (dup-over-loss).
|
|
770
|
+
const retryMerges = [];
|
|
771
|
+
// ONE pre-merge backup per run (#439): takePreDedupBackup is a full SQLite
|
|
772
|
+
// copy, so a per-boundary backup would be hundreds of full-DB copies on a
|
|
773
|
+
// backlog night. Taken LAZILY, immediately before the first judged-merge
|
|
774
|
+
// application; remembered for the rest of the run (the zone takes its own,
|
|
775
|
+
// independent backup, as before).
|
|
776
|
+
let mergeWindowBackedUp = false;
|
|
723
777
|
// Links created by THIS stage in THIS run — lets the explicit-mark pass
|
|
724
778
|
// distinguish operator marks (pre-existing) from stage output.
|
|
725
779
|
const linksCreatedThisRun = new Set();
|
|
@@ -727,40 +781,50 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
727
781
|
storage.addLink(db, oldId, newId, relationship, strength);
|
|
728
782
|
linksCreatedThisRun.add(`${oldId}|${newId}`);
|
|
729
783
|
};
|
|
730
|
-
// ---- #401
|
|
731
|
-
// path
|
|
732
|
-
//
|
|
733
|
-
//
|
|
734
|
-
//
|
|
735
|
-
//
|
|
736
|
-
//
|
|
737
|
-
//
|
|
784
|
+
// ---- #401/#439 mid-scan cursor persistence. Called at EVERY scan-loop
|
|
785
|
+
// exit path AND at the end of every candidate iteration — AFTER that
|
|
786
|
+
// iteration's boundary apply, so the persisted cursor only ever advances
|
|
787
|
+
// past candidates whose confirmed work has landed; a killed run loses at
|
|
788
|
+
// most the candidate in flight. The ONE exception is lock-busy retry
|
|
789
|
+
// survivors: they intentionally ride the retry list while the scan
|
|
790
|
+
// continues (the lock may clear this run), so while any are pending every
|
|
791
|
+
// persisted checkpoint CLAMPS below their earliest contributor — a SIGKILL
|
|
792
|
+
// in the window between a busy boundary and the pair landing must never
|
|
793
|
+
// strand a confirmed merge behind the cursor (the #402 orphan-floor
|
|
794
|
+
// discipline, re-scoped to the retry list; the clamp lifts automatically
|
|
795
|
+
// once a later boundary or the final drain applies them). Also persists
|
|
796
|
+
// the scan high-water (never held back). updateState is load→mutate→
|
|
797
|
+
// temp-rename atomic.
|
|
738
798
|
let deadlineStopped = false;
|
|
739
|
-
// #
|
|
740
|
-
//
|
|
741
|
-
//
|
|
742
|
-
//
|
|
743
|
-
|
|
744
|
-
//
|
|
745
|
-
//
|
|
746
|
-
//
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
const
|
|
751
|
-
|
|
752
|
-
|
|
799
|
+
// #439: once an iteration's confirmed work deferred (deadline at the
|
|
800
|
+
// boundary, backup failure, rewrite refusal/infra error), the cursor never
|
|
801
|
+
// advances again this run — a later iteration must not push it past the
|
|
802
|
+
// held candidate's rowid.
|
|
803
|
+
let cursorHold = false;
|
|
804
|
+
// Fix round (#440 review, finding 1): the floor below the earliest
|
|
805
|
+
// candidate contributing to a still-un-applied retry merge. Applied at
|
|
806
|
+
// every persist so the kill-with-pending-retry window cannot strand them.
|
|
807
|
+
const pendingRetryFloor = () => retryMerges.length > 0
|
|
808
|
+
? Math.min(...retryMerges.map((p) => p.candidateRowid)) - 1
|
|
809
|
+
: null;
|
|
810
|
+
const clampedCursor = () => {
|
|
811
|
+
const floor = pendingRetryFloor();
|
|
812
|
+
return floor !== null ? Math.min(cursor, floor) : cursor;
|
|
753
813
|
};
|
|
754
814
|
const persistCursor = () => {
|
|
755
815
|
if (dryRun)
|
|
756
816
|
return;
|
|
757
|
-
const checkpoint = scanPendingMinRowid !== null ? Math.min(cursor, scanPendingMinRowid - 1) : cursor;
|
|
758
817
|
(0, state_js_1.updateState)((s) => {
|
|
759
|
-
s.reconsolidationCursor =
|
|
818
|
+
s.reconsolidationCursor = clampedCursor();
|
|
819
|
+
s.reconsolidationScannedRowid = Math.max(scannedRowidHighwater, s.reconsolidationScannedRowid ?? 0);
|
|
760
820
|
}, stateDir);
|
|
761
821
|
};
|
|
762
|
-
|
|
763
|
-
|
|
822
|
+
// #439: rewrite groups live ONLY inside the candidate iteration that
|
|
823
|
+
// formed them (its boundary applies or degrades them, then they are
|
|
824
|
+
// discarded). Every trigger is the current candidate — a target corrected
|
|
825
|
+
// by two different candidates takes two sequential rewrites instead of
|
|
826
|
+
// the old one grouped call.
|
|
827
|
+
const addTrigger = (groups, target, trigger, confidence, cosine, explicit) => {
|
|
764
828
|
let group = groups.get(target.id);
|
|
765
829
|
if (!group) {
|
|
766
830
|
group = { targetId: target.id, target, triggers: [] };
|
|
@@ -775,10 +839,9 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
775
839
|
candidateRowid: trigger.__rowid,
|
|
776
840
|
explicit,
|
|
777
841
|
});
|
|
778
|
-
notePendingRowid(trigger.__rowid); // orphan floor — group unapplied until the rewrite phase
|
|
779
842
|
}
|
|
780
843
|
};
|
|
781
|
-
for (const
|
|
844
|
+
for (const snapshotted of rows) {
|
|
782
845
|
// #401: runtime bounds first — exit cleanly at the last fully-considered
|
|
783
846
|
// candidate boundary (cursor = the previous candidate's rowid here).
|
|
784
847
|
if (!dryRun && deadlineHit()) {
|
|
@@ -790,7 +853,50 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
790
853
|
persistCursor();
|
|
791
854
|
break;
|
|
792
855
|
}
|
|
856
|
+
// ---- #439 scan-stability guard: `rows` is ONE snapshot fetched at stage
|
|
857
|
+
// start with status != 'absorbed'; boundary applies (merge losers,
|
|
858
|
+
// rewrite triggers absorbed) can mark FUTURE rows of that snapshot
|
|
859
|
+
// absorbed after the filter ran. Without this re-read such a candidate
|
|
860
|
+
// would be scouted/judged on stale content and its KNN would silently
|
|
861
|
+
// re-embed it (its vector row is gone). Absorbed → skip (counted,
|
|
862
|
+
// cursor passes it); otherwise the LIVE row's content/status drives the
|
|
863
|
+
// rest of the iteration (refreshes content rewritten by an earlier
|
|
864
|
+
// boundary when created_at and rowid order diverge).
|
|
865
|
+
const live = db
|
|
866
|
+
.prepare(`SELECT rowid AS __rowid, * FROM memories WHERE rowid = ?`)
|
|
867
|
+
.get(snapshotted.__rowid);
|
|
868
|
+
if (!live) {
|
|
869
|
+
// Vanished entirely (deleted out from under the scan) — defensive;
|
|
870
|
+
// nothing to judge, the cursor passes it (never past an active hold).
|
|
871
|
+
scannedRowidHighwater = Math.max(scannedRowidHighwater, snapshotted.__rowid);
|
|
872
|
+
if (!cursorHold)
|
|
873
|
+
cursor = snapshotted.__rowid;
|
|
874
|
+
persistCursor();
|
|
875
|
+
continue;
|
|
876
|
+
}
|
|
877
|
+
if (live.status === "absorbed") {
|
|
878
|
+
skippedAbsorbed++;
|
|
879
|
+
scannedRowidHighwater = Math.max(scannedRowidHighwater, live.__rowid);
|
|
880
|
+
if (!cursorHold)
|
|
881
|
+
cursor = live.__rowid;
|
|
882
|
+
persistCursor();
|
|
883
|
+
continue;
|
|
884
|
+
}
|
|
885
|
+
const candidate = live;
|
|
793
886
|
scanned++;
|
|
887
|
+
// The high-water advances as candidates are ENTERED — even when the
|
|
888
|
+
// iteration's work later defers (it is the SCAN mark, never held back).
|
|
889
|
+
scannedRowidHighwater = Math.max(scannedRowidHighwater, candidate.__rowid);
|
|
890
|
+
// #439 per-iteration confirmed work, applied at the boundary below.
|
|
891
|
+
const iterMerges = [];
|
|
892
|
+
const iterGroups = new Map();
|
|
893
|
+
// This iteration's confirmed work could not land (deadline at the
|
|
894
|
+
// boundary, backup failure, rewrite refusal/infra error): the cursor
|
|
895
|
+
// holds below this candidate.
|
|
896
|
+
let boundaryHold = false;
|
|
897
|
+
// Stop the scan AFTER the boundary (budget stop / rewrite-infra
|
|
898
|
+
// deferral / backup failure): further verdicts could not land anyway.
|
|
899
|
+
let stopScan = false;
|
|
794
900
|
// ---- AC7: verify incoming explicit marks (corrected_by/superseded_by
|
|
795
901
|
// links targeting this candidate) before they can join a rewrite group.
|
|
796
902
|
// #401: only OPERATOR marks are verified — applyExplicitMark writes
|
|
@@ -821,12 +927,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
821
927
|
const { verdict, usage } = await classifyPair(llm, target.content, candidate.content);
|
|
822
928
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
|
|
823
929
|
pairsEvaluated++;
|
|
930
|
+
if (candidate.__rowid <= prevScannedRowid)
|
|
931
|
+
pairsReevaluated++;
|
|
932
|
+
else
|
|
933
|
+
pairsNew++;
|
|
824
934
|
if (!verdict) {
|
|
825
935
|
skippedInfra++; // mark retained; the neighborhood is revisited via newer candidacies
|
|
826
936
|
continue;
|
|
827
937
|
}
|
|
828
938
|
if (verdict.action === "corrects" && verdict.confidence >= rewriteMinConfidence && isFactShapedTarget(target)) {
|
|
829
|
-
addTrigger(target, candidate, verdict.confidence, null, true);
|
|
939
|
+
addTrigger(iterGroups, target, candidate, verdict.confidence, null, true);
|
|
830
940
|
explicitVerified++;
|
|
831
941
|
}
|
|
832
942
|
else {
|
|
@@ -839,8 +949,12 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
839
949
|
}
|
|
840
950
|
}
|
|
841
951
|
if (markBudgetStop) {
|
|
842
|
-
|
|
843
|
-
|
|
952
|
+
// #401: this candidate's remaining marks re-verify next run — the
|
|
953
|
+
// cursor HOLDS below it (boundaryHold), and #439 still runs the
|
|
954
|
+
// boundary: marks confirmed before the stop apply, exactly as they
|
|
955
|
+
// did when the rewrite phase was post-scan.
|
|
956
|
+
stopScan = true;
|
|
957
|
+
boundaryHold = true;
|
|
844
958
|
}
|
|
845
959
|
}
|
|
846
960
|
// ---- #393 B: scout shape call — ONE classify-tier call per candidate,
|
|
@@ -852,27 +966,31 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
852
966
|
// discipline. Dry-run skips the call entirely: it is LLM work, and
|
|
853
967
|
// dry-runs make zero LLM calls (the scout counters read 0 there).
|
|
854
968
|
let scoutReferences = null;
|
|
855
|
-
if (!dryRun) {
|
|
969
|
+
if (!dryRun && !stopScan) {
|
|
856
970
|
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
857
|
-
|
|
858
|
-
|
|
971
|
+
// Shape call refused — this candidate re-scouts next run: the cursor
|
|
972
|
+
// HOLDS below it (the candidate was entered but not considered).
|
|
973
|
+
stopScan = true;
|
|
974
|
+
boundaryHold = true;
|
|
859
975
|
}
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
976
|
+
else {
|
|
977
|
+
scoutScanned++;
|
|
978
|
+
try {
|
|
979
|
+
const r = await llm.complete(buildScoutShapePrompt(candidate.content));
|
|
980
|
+
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
981
|
+
const shape = parseScoutShape(r.text);
|
|
982
|
+
if (!shape) {
|
|
983
|
+
skippedInfra++;
|
|
984
|
+
}
|
|
985
|
+
else if (shape.correction) {
|
|
986
|
+
scoutCorrectionShaped++;
|
|
987
|
+
scoutReferences = shape.references;
|
|
988
|
+
}
|
|
867
989
|
}
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
scoutReferences = shape.references;
|
|
990
|
+
catch {
|
|
991
|
+
skippedInfra++;
|
|
871
992
|
}
|
|
872
993
|
}
|
|
873
|
-
catch {
|
|
874
|
-
skippedInfra++;
|
|
875
|
-
}
|
|
876
994
|
}
|
|
877
995
|
// ---- AC2: detection pairs against older neighbors — TWO sources (#393 B):
|
|
878
996
|
// the KNN similarity source (floor-gated, the pre-B baseline) and, for
|
|
@@ -880,22 +998,26 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
880
998
|
// claim's terms → corpus search; NO similarity gate — cosine is a ranker,
|
|
881
999
|
// never a blocker). Merged + deduped by neighbor id: a pair found by both
|
|
882
1000
|
// sources is judged ONCE, as similarity (with band attribution).
|
|
883
|
-
let
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
1001
|
+
let discoveryFailed = false;
|
|
1002
|
+
let neighbors = [];
|
|
1003
|
+
if (!stopScan) {
|
|
1004
|
+
try {
|
|
1005
|
+
neighbors = await findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarity);
|
|
1006
|
+
}
|
|
1007
|
+
catch (err) {
|
|
1008
|
+
console.warn(`[hicortex] reconsolidation: discovery failed for ${candidate.id.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1009
|
+
// #439: skip this candidate's neighbor judgments but still run the
|
|
1010
|
+
// boundary below — marks confirmed earlier this iteration apply
|
|
1011
|
+
// (they did when the phases were post-scan).
|
|
1012
|
+
discoveryFailed = true;
|
|
1013
|
+
}
|
|
892
1014
|
}
|
|
893
1015
|
const neighborEntries = neighbors.map((n) => ({
|
|
894
1016
|
mem: n,
|
|
895
1017
|
cosine: (0, retrieval_js_1.l2ToCosine)(n.distance),
|
|
896
1018
|
source: "similarity",
|
|
897
1019
|
}));
|
|
898
|
-
if (scoutReferences !== null) {
|
|
1020
|
+
if (!discoveryFailed && scoutReferences !== null) {
|
|
899
1021
|
const candidateVec = storage.getStoredEmbedding(db, candidate.id);
|
|
900
1022
|
const knnById = new Map(neighborEntries.map((e) => [e.mem.id, e]));
|
|
901
1023
|
for (const entry of findScoutNeighbors(db, candidate, scoutReferences, candidateVec)) {
|
|
@@ -943,12 +1065,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
943
1065
|
continue; // preview only — no LLM call, no write
|
|
944
1066
|
// #405: the ONE run budget's refusal is the only call cap.
|
|
945
1067
|
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
946
|
-
|
|
1068
|
+
stopScan = true; // the boundary below still applies what this candidate already confirmed
|
|
947
1069
|
break;
|
|
948
1070
|
}
|
|
949
1071
|
const { verdict, usage } = await classifyPair(llm, entry.mem.content, candidate.content);
|
|
950
1072
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
|
|
951
1073
|
pairsEvaluated++;
|
|
1074
|
+
if (candidate.__rowid <= prevScannedRowid)
|
|
1075
|
+
pairsReevaluated++;
|
|
1076
|
+
else
|
|
1077
|
+
pairsNew++;
|
|
952
1078
|
if (!verdict) {
|
|
953
1079
|
skippedInfra++;
|
|
954
1080
|
continue;
|
|
@@ -958,8 +1084,8 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
958
1084
|
// pairs reach them through a different, cosine-blind door.
|
|
959
1085
|
if (entry.source === "similarity")
|
|
960
1086
|
recordBand(pairCosine, verdict.action, verdict.confidence);
|
|
961
|
-
// #392: a merge verdict
|
|
962
|
-
// link, no write here. Below the confidence gate BOTH memories stay
|
|
1087
|
+
// #392/#439: a merge verdict queues for THIS iteration's boundary —
|
|
1088
|
+
// no link, no write here. Below the confidence gate BOTH memories stay
|
|
963
1089
|
// live: a weak mark is recoverable, and there is nothing to mark for a
|
|
964
1090
|
// duplicate — keeping both is the recoverable outcome.
|
|
965
1091
|
if (verdict.action === "merge") {
|
|
@@ -975,8 +1101,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
975
1101
|
}
|
|
976
1102
|
}
|
|
977
1103
|
else {
|
|
978
|
-
|
|
979
|
-
notePendingRowid(candidate.__rowid); // orphan floor — merge unapplied until the merge phase
|
|
1104
|
+
iterMerges.push({ oldId: entry.mem.id, newId: candidate.id, candidateRowid: candidate.__rowid });
|
|
980
1105
|
}
|
|
981
1106
|
continue;
|
|
982
1107
|
}
|
|
@@ -1017,73 +1142,302 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1017
1142
|
markedRetracted++;
|
|
1018
1143
|
continue;
|
|
1019
1144
|
}
|
|
1020
|
-
addTrigger(entry.mem, candidate, verdict.confidence, cosine, false);
|
|
1145
|
+
addTrigger(iterGroups, entry.mem, candidate, verdict.confidence, cosine, false);
|
|
1021
1146
|
}
|
|
1022
1147
|
// verdict "none" → nothing to do
|
|
1023
1148
|
}
|
|
1024
|
-
|
|
1025
|
-
//
|
|
1026
|
-
//
|
|
1027
|
-
//
|
|
1028
|
-
//
|
|
1029
|
-
//
|
|
1149
|
+
// ---- #439 BOUNDARY: apply everything this candidate confirmed, NOW —
|
|
1150
|
+
// merges first, then rewrite groups (the order the old post-scan phases
|
|
1151
|
+
// used; preserves the existing tolerance where a merge loser that is
|
|
1152
|
+
// also a rewrite trigger stays absorbed while the rewrite still composes
|
|
1153
|
+
// its content). Each application is its own transaction
|
|
1154
|
+
// (mergeMemoryIds / applyRewriteGroup — group-internal atomicity
|
|
1155
|
+
// preserved); a busy capture lock defers merges to the same-run retry
|
|
1156
|
+
// list, everything else defers by holding the cursor below this
|
|
1157
|
+
// candidate (bounded to ONE candidate's pairs).
|
|
1158
|
+
if (!dryRun) {
|
|
1159
|
+
// Earlier lock-busy survivors retry FIRST (oldest verdicts land
|
|
1160
|
+
// first), ahead of this candidate's fresh confirmations.
|
|
1161
|
+
const mergeBatch = [...retryMerges.splice(0, retryMerges.length), ...iterMerges];
|
|
1162
|
+
if (mergeBatch.length > 0) {
|
|
1163
|
+
if (deadlineHit()) {
|
|
1164
|
+
deadlineStopped = true;
|
|
1165
|
+
// Fix round (#440 review, finding 1): never DROP the batch — it can
|
|
1166
|
+
// begin with lock-busy survivors contributed by EARLIER candidates
|
|
1167
|
+
// that the cursor has already passed. Re-queue for the final drain;
|
|
1168
|
+
// its hold-below-earliest-contributor (and the persist clamp) keeps
|
|
1169
|
+
// every un-applied pair re-detectable next run.
|
|
1170
|
+
retryMerges.push(...mergeBatch);
|
|
1171
|
+
boundaryHold = true;
|
|
1172
|
+
console.log(`[hicortex] Reconsolidation: ${mergeBatch.length} confirmed merge(s) re-queued for the final drain — run deadline reached`);
|
|
1173
|
+
}
|
|
1174
|
+
else {
|
|
1175
|
+
const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
|
|
1176
|
+
const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
|
|
1177
|
+
if (!release) {
|
|
1178
|
+
// Busy capture run — the batch rides the same-run retry list:
|
|
1179
|
+
// retried at the next boundary and once in the final drain. Not
|
|
1180
|
+
// a cursor hold yet (the lock may clear this run).
|
|
1181
|
+
retryMerges.push(...mergeBatch);
|
|
1182
|
+
console.warn(`[hicortex] Reconsolidation: capture lock busy — ${mergeBatch.length} confirmed merge(s) deferred to a retry this run`);
|
|
1183
|
+
}
|
|
1184
|
+
else {
|
|
1185
|
+
try {
|
|
1186
|
+
let backupOk = true;
|
|
1187
|
+
if (!mergeWindowBackedUp) {
|
|
1188
|
+
try {
|
|
1189
|
+
await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
|
|
1190
|
+
mergeWindowBackedUp = true;
|
|
1191
|
+
}
|
|
1192
|
+
catch (err) {
|
|
1193
|
+
backupOk = false;
|
|
1194
|
+
console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
|
|
1195
|
+
`(${err instanceof Error ? err.message : String(err)}) — ${mergeBatch.length} merge(s) deferred`);
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
if (!backupOk) {
|
|
1199
|
+
// Verdicts that cannot land must not keep being paid: stop
|
|
1200
|
+
// the scan. The batch re-queues for the final drain — a
|
|
1201
|
+
// TRANSIENT backup failure can still recover there; a
|
|
1202
|
+
// persistent one ends with the drain holding the cursor
|
|
1203
|
+
// below the earliest contributor (never just this candidate,
|
|
1204
|
+
// when retry survivors ride the batch).
|
|
1205
|
+
retryMerges.push(...mergeBatch);
|
|
1206
|
+
boundaryHold = true;
|
|
1207
|
+
stopScan = true;
|
|
1208
|
+
}
|
|
1209
|
+
else {
|
|
1210
|
+
for (let i = 0; i < mergeBatch.length; i++) {
|
|
1211
|
+
const pair = mergeBatch[i];
|
|
1212
|
+
// #405: the deadline stop-check between local merge
|
|
1213
|
+
// transactions — a safe boundary; deferred pairs hold the
|
|
1214
|
+
// cursor below this candidate and retry next run.
|
|
1215
|
+
if (deadlineHit()) {
|
|
1216
|
+
deadlineStopped = true;
|
|
1217
|
+
// Fix round (#440 review, finding 1): the un-applied tail
|
|
1218
|
+
// re-queues (it can contain earlier candidates' retry
|
|
1219
|
+
// survivors) — the final drain applies or holds it.
|
|
1220
|
+
retryMerges.push(...mergeBatch.slice(i));
|
|
1221
|
+
boundaryHold = true;
|
|
1222
|
+
console.log(`[hicortex] Reconsolidation: ${mergeBatch.length - i} confirmed merge(s) re-queued for the final drain — run deadline reached`);
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
|
|
1226
|
+
if (result.ok) {
|
|
1227
|
+
mergePairsApplied++;
|
|
1228
|
+
console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1229
|
+
`into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
|
|
1230
|
+
}
|
|
1231
|
+
else if (result.reason === "metadata_mismatch") {
|
|
1232
|
+
skippedMetadataMismatch++;
|
|
1233
|
+
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1234
|
+
`skipped (metadata mismatch) — both kept`);
|
|
1235
|
+
}
|
|
1236
|
+
else if (result.reason === "conflict_linked") {
|
|
1237
|
+
// #393 guard-C: the pair is conflicts-linked (operator-planted
|
|
1238
|
+
// or a prior verdict) — never blended; the cursor advances,
|
|
1239
|
+
// this verdict was rendered.
|
|
1240
|
+
conflictSkippedJudged++;
|
|
1241
|
+
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1242
|
+
`skipped (conflict-flagged) — both kept`);
|
|
1243
|
+
}
|
|
1244
|
+
// "no_members": a member vanished/was absorbed since the
|
|
1245
|
+
// verdict — nothing to merge, nothing to hold; the cursor
|
|
1246
|
+
// advances past it.
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
finally {
|
|
1251
|
+
release();
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
if (iterGroups.size > 0) {
|
|
1257
|
+
// One rewrite call per group — the group's triggers are all THIS
|
|
1258
|
+
// candidate (#439: a target corrected by two different candidates
|
|
1259
|
+
// takes two sequential rewrites, one per boundary; the second call
|
|
1260
|
+
// composes the already-corrected story). A call that was never made
|
|
1261
|
+
// (budget/infra/deadline) defers the group — untouched, never
|
|
1262
|
+
// partially applied — and holds the cursor below this candidate.
|
|
1263
|
+
const contracts = new Map(); // null = contract failed
|
|
1264
|
+
let rewritesDeferred = false;
|
|
1265
|
+
for (const group of iterGroups.values()) {
|
|
1266
|
+
if (deadlineHit()) {
|
|
1267
|
+
deadlineStopped = true;
|
|
1268
|
+
rewritesDeferred = true;
|
|
1269
|
+
break;
|
|
1270
|
+
}
|
|
1271
|
+
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
1272
|
+
rewritesDeferred = true;
|
|
1273
|
+
stopScan = true;
|
|
1274
|
+
break;
|
|
1275
|
+
}
|
|
1276
|
+
const triggersArg = group.triggers.map((t) => ({ id: t.id, content: t.memory.content }));
|
|
1277
|
+
let contract = null;
|
|
1278
|
+
let infraError = false;
|
|
1279
|
+
try {
|
|
1280
|
+
const r = await llm.complete(buildRewritePrompt(group.target.content, triggersArg));
|
|
1281
|
+
contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
|
|
1282
|
+
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
1283
|
+
}
|
|
1284
|
+
catch {
|
|
1285
|
+
infraError = true;
|
|
1286
|
+
}
|
|
1287
|
+
if (infraError) {
|
|
1288
|
+
skippedInfra++;
|
|
1289
|
+
rewritesDeferred = true; // group NOT marked, NOT rewritten — retried next run
|
|
1290
|
+
stopScan = true; // verdicts past this hold could not advance the cursor anyway
|
|
1291
|
+
break;
|
|
1292
|
+
}
|
|
1293
|
+
contracts.set(group.targetId, contract);
|
|
1294
|
+
if (!contract)
|
|
1295
|
+
contractFailed++;
|
|
1296
|
+
}
|
|
1297
|
+
if (rewritesDeferred) {
|
|
1298
|
+
boundaryHold = true;
|
|
1299
|
+
}
|
|
1300
|
+
else {
|
|
1301
|
+
// Multi-target keep rule WITHIN the boundary: the shared trigger is
|
|
1302
|
+
// the current candidate, so every group it touches resolves here —
|
|
1303
|
+
// absorbed only if EVERY contract says absorb (any keep keeps).
|
|
1304
|
+
const finalOutcome = new Map();
|
|
1305
|
+
for (const contract of contracts.values()) {
|
|
1306
|
+
if (!contract)
|
|
1307
|
+
continue;
|
|
1308
|
+
for (const t of contract.triggers) {
|
|
1309
|
+
if (t.disposition === "keep" || finalOutcome.get(t.id) === "keep")
|
|
1310
|
+
finalOutcome.set(t.id, "keep");
|
|
1311
|
+
else
|
|
1312
|
+
finalOutcome.set(t.id, "absorb");
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
// Counted from APPLIED groups only (a deferred group's dispositions
|
|
1316
|
+
// never took effect); a trigger in several applied groups counts once.
|
|
1317
|
+
const appliedOutcome = new Map();
|
|
1318
|
+
for (const group of iterGroups.values()) {
|
|
1319
|
+
const contract = contracts.get(group.targetId);
|
|
1320
|
+
if (contract === undefined)
|
|
1321
|
+
continue; // pending group — untouched this run
|
|
1322
|
+
if (contract === null) {
|
|
1323
|
+
// Failed rewrite contract → whole group mark-only, never a partial
|
|
1324
|
+
// apply. Content untouched, NO trigger absorbed.
|
|
1325
|
+
try {
|
|
1326
|
+
applyMarkOnlyGroup(db, group);
|
|
1327
|
+
}
|
|
1328
|
+
catch (err) {
|
|
1329
|
+
console.warn(`[hicortex] reconsolidation: mark-only fallback failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1330
|
+
skippedInfra++;
|
|
1331
|
+
boundaryHold = true; // retried next run
|
|
1332
|
+
continue;
|
|
1333
|
+
}
|
|
1334
|
+
markedRetracted++;
|
|
1335
|
+
console.log(`[hicortex] Reconsolidation: rewrite contract failed for ${group.targetId.slice(0, 8)} — group degraded to mark-only`);
|
|
1336
|
+
continue;
|
|
1337
|
+
}
|
|
1338
|
+
let applied = false;
|
|
1339
|
+
try {
|
|
1340
|
+
applied = await applyRewriteGroup(db, group, contract, finalOutcome, embedFn);
|
|
1341
|
+
}
|
|
1342
|
+
catch (err) {
|
|
1343
|
+
console.warn(`[hicortex] reconsolidation: rewrite apply failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1344
|
+
}
|
|
1345
|
+
if (!applied) {
|
|
1346
|
+
skippedInfra++; // defensive absorbed-target guard, or an apply error — retry next run
|
|
1347
|
+
boundaryHold = true;
|
|
1348
|
+
continue;
|
|
1349
|
+
}
|
|
1350
|
+
rewritten++;
|
|
1351
|
+
for (const t of contract.triggers) {
|
|
1352
|
+
const outcome = finalOutcome.get(t.id) ?? "keep";
|
|
1353
|
+
if (outcome === "keep" || appliedOutcome.get(t.id) === "keep")
|
|
1354
|
+
appliedOutcome.set(t.id, "keep");
|
|
1355
|
+
else
|
|
1356
|
+
appliedOutcome.set(t.id, "absorb");
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
for (const outcome of appliedOutcome.values()) {
|
|
1360
|
+
if (outcome === "absorb")
|
|
1361
|
+
absorbed++;
|
|
1362
|
+
else
|
|
1363
|
+
keptLinked++;
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
// #439 cursor advance: past this candidate ONLY when its confirmed work
|
|
1369
|
+
// landed (or was refused-with-verdict-rendered). Once anything deferred,
|
|
1370
|
+
// the latch holds the cursor below that candidate for the rest of the
|
|
1371
|
+
// run — a later iteration must never advance past an earlier hold.
|
|
1372
|
+
if (boundaryHold)
|
|
1373
|
+
cursorHold = true;
|
|
1374
|
+
if (!cursorHold)
|
|
1375
|
+
cursor = candidate.__rowid;
|
|
1376
|
+
// #402/#439: persist after EVERY candidate — AFTER the boundary apply,
|
|
1377
|
+
// so the checkpoint only ever crosses candidates whose work landed. A
|
|
1378
|
+
// SIGKILL between persists re-detects at most the in-flight candidate.
|
|
1030
1379
|
persistCursor();
|
|
1380
|
+
if (stopScan || deadlineStopped)
|
|
1381
|
+
break;
|
|
1031
1382
|
}
|
|
1032
1383
|
if (deadlineStopped) {
|
|
1033
1384
|
console.log(`[hicortex] Reconsolidation: run deadline reached (nightlyTimeBudgetMinutes) — ` +
|
|
1034
1385
|
`scan stopped at cursor ${cursor}; the next run resumes from there`);
|
|
1035
1386
|
}
|
|
1036
|
-
// ---- #
|
|
1037
|
-
//
|
|
1038
|
-
//
|
|
1039
|
-
//
|
|
1040
|
-
//
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
if (!dryRun && queuedMerges.length > 0) {
|
|
1049
|
-
const holdQueued = (from) => {
|
|
1050
|
-
for (let i = from; i < queuedMerges.length; i++) {
|
|
1051
|
-
pendingMinRowid =
|
|
1052
|
-
pendingMinRowid === null
|
|
1053
|
-
? queuedMerges[i].candidateRowid
|
|
1054
|
-
: Math.min(pendingMinRowid, queuedMerges[i].candidateRowid);
|
|
1055
|
-
}
|
|
1387
|
+
// ---- #439 final drain: lock-busy merge survivors get ONE more attempt
|
|
1388
|
+
// right after the scan (the retry list is same-run only — everything else
|
|
1389
|
+
// applied at its boundary). Still busy (or the deadline/backup refuses) →
|
|
1390
|
+
// the pairs stay un-applied, counted, and the cursor holds below the
|
|
1391
|
+
// earliest contributing candidate (dup-over-loss; logged).
|
|
1392
|
+
if (!dryRun && retryMerges.length > 0) {
|
|
1393
|
+
const batch = retryMerges.splice(0, retryMerges.length);
|
|
1394
|
+
// Hold below the earliest contributor of the UN-APPLIED tail only (fix
|
|
1395
|
+
// round, minor review note: the old whole-batch min over-held past pairs
|
|
1396
|
+
// that had just applied in the same loop).
|
|
1397
|
+
const holdBelow = (fromIndex) => {
|
|
1398
|
+
cursor = Math.min(cursor, Math.min(...batch.slice(fromIndex).map((p) => p.candidateRowid)) - 1);
|
|
1056
1399
|
};
|
|
1057
|
-
{
|
|
1400
|
+
if (deadlineHit()) {
|
|
1401
|
+
deadlineStopped = true;
|
|
1402
|
+
mergePairsDeferred += batch.length;
|
|
1403
|
+
holdBelow(0);
|
|
1404
|
+
console.log(`[hicortex] Reconsolidation: ${batch.length} confirmed merge(s) deferred — run deadline reached`);
|
|
1405
|
+
}
|
|
1406
|
+
else {
|
|
1058
1407
|
const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
|
|
1059
1408
|
const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
|
|
1060
1409
|
if (!release) {
|
|
1061
|
-
mergePairsDeferred
|
|
1062
|
-
|
|
1063
|
-
console.warn(`[hicortex] Reconsolidation: capture lock busy
|
|
1410
|
+
mergePairsDeferred += batch.length;
|
|
1411
|
+
holdBelow(0);
|
|
1412
|
+
console.warn(`[hicortex] Reconsolidation: capture lock busy at the final drain — ` +
|
|
1413
|
+
`${batch.length} confirmed merge(s) deferred to next run`);
|
|
1064
1414
|
}
|
|
1065
1415
|
else {
|
|
1066
1416
|
try {
|
|
1067
1417
|
let backupOk = true;
|
|
1068
|
-
|
|
1069
|
-
|
|
1418
|
+
if (!mergeWindowBackedUp) {
|
|
1419
|
+
try {
|
|
1420
|
+
await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
|
|
1421
|
+
mergeWindowBackedUp = true;
|
|
1422
|
+
}
|
|
1423
|
+
catch (err) {
|
|
1424
|
+
backupOk = false;
|
|
1425
|
+
console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
|
|
1426
|
+
`(${err instanceof Error ? err.message : String(err)}) — ${batch.length} merge(s) deferred`);
|
|
1427
|
+
}
|
|
1070
1428
|
}
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
`(${err instanceof Error ? err.message : String(err)}) — ${queuedMerges.length} merge(s) deferred`);
|
|
1429
|
+
if (!backupOk) {
|
|
1430
|
+
mergePairsDeferred += batch.length;
|
|
1431
|
+
holdBelow(0);
|
|
1075
1432
|
}
|
|
1076
|
-
|
|
1077
|
-
for (let i = 0; i <
|
|
1078
|
-
const pair =
|
|
1079
|
-
// #405: the deadline stop-check between local merge
|
|
1080
|
-
// transactions — a safe boundary; deferred pairs hold the
|
|
1081
|
-
// cursor below their candidates and retry next run.
|
|
1433
|
+
else {
|
|
1434
|
+
for (let i = 0; i < batch.length; i++) {
|
|
1435
|
+
const pair = batch[i];
|
|
1082
1436
|
if (deadlineHit()) {
|
|
1083
1437
|
deadlineStopped = true;
|
|
1084
|
-
mergePairsDeferred
|
|
1085
|
-
|
|
1086
|
-
console.log(`[hicortex] Reconsolidation: ${
|
|
1438
|
+
mergePairsDeferred += batch.length - i;
|
|
1439
|
+
holdBelow(i);
|
|
1440
|
+
console.log(`[hicortex] Reconsolidation: ${batch.length - i} confirmed merge(s) deferred — run deadline reached`);
|
|
1087
1441
|
break;
|
|
1088
1442
|
}
|
|
1089
1443
|
const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
|
|
@@ -1098,22 +1452,14 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1098
1452
|
`skipped (metadata mismatch) — both kept`);
|
|
1099
1453
|
}
|
|
1100
1454
|
else if (result.reason === "conflict_linked") {
|
|
1101
|
-
// #393 guard-C: the pair is conflicts-linked (operator-planted
|
|
1102
|
-
// or a prior verdict) — never blended; the cursor advances,
|
|
1103
|
-
// this verdict was rendered.
|
|
1104
1455
|
conflictSkippedJudged++;
|
|
1105
1456
|
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1106
1457
|
`skipped (conflict-flagged) — both kept`);
|
|
1107
1458
|
}
|
|
1108
1459
|
// "no_members": a member vanished/was absorbed since the
|
|
1109
|
-
// verdict — nothing to merge, nothing to hold
|
|
1110
|
-
// advances past it.
|
|
1460
|
+
// verdict — nothing to merge, nothing to hold.
|
|
1111
1461
|
}
|
|
1112
1462
|
}
|
|
1113
|
-
else {
|
|
1114
|
-
mergePairsDeferred = queuedMerges.length;
|
|
1115
|
-
holdQueued(0);
|
|
1116
|
-
}
|
|
1117
1463
|
}
|
|
1118
1464
|
finally {
|
|
1119
1465
|
release();
|
|
@@ -1121,141 +1467,19 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1121
1467
|
}
|
|
1122
1468
|
}
|
|
1123
1469
|
}
|
|
1124
|
-
// ---- Rewrite phase (AC3/AC4/AC5). Three sub-phases so the multi-target
|
|
1125
|
-
// keep rule can be honored: (R1) collect contracts, (R2) resolve every
|
|
1126
|
-
// trigger's FINAL disposition across all groups, (R3) apply one transaction
|
|
1127
|
-
// per group. A group whose rewrite call was never made (budget/infra) is
|
|
1128
|
-
// left untouched and holds the cursor — never partially applied.
|
|
1129
|
-
const contracts = new Map(); // null = contract failed
|
|
1130
|
-
// pendingMinRowid (min candidate rowid among un-applied work) is declared
|
|
1131
|
-
// above — shared with the merge phase's holdQueued.
|
|
1132
|
-
const deferFrom = (fromTargetId) => {
|
|
1133
|
-
let seen = false;
|
|
1134
|
-
for (const group of groups.values()) {
|
|
1135
|
-
if (!seen && group.targetId !== fromTargetId)
|
|
1136
|
-
continue;
|
|
1137
|
-
seen = true;
|
|
1138
|
-
for (const t of group.triggers) {
|
|
1139
|
-
pendingMinRowid = pendingMinRowid === null ? t.candidateRowid : Math.min(pendingMinRowid, t.candidateRowid);
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1142
|
-
};
|
|
1143
|
-
if (!dryRun && groups.size > 0) {
|
|
1144
|
-
for (const group of groups.values()) {
|
|
1145
|
-
// #401: the bounds stop the rewrite phase too — a group whose rewrite
|
|
1146
|
-
// call was never made is left untouched and holds the cursor (never
|
|
1147
|
-
// partially applied), exactly like the budget-exhausted path below.
|
|
1148
|
-
if (deadlineHit()) {
|
|
1149
|
-
deadlineStopped = true;
|
|
1150
|
-
deferFrom(group.targetId);
|
|
1151
|
-
break;
|
|
1152
|
-
}
|
|
1153
|
-
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
1154
|
-
deferFrom(group.targetId);
|
|
1155
|
-
break;
|
|
1156
|
-
}
|
|
1157
|
-
const triggersArg = group.triggers.map((t) => ({ id: t.id, content: t.memory.content }));
|
|
1158
|
-
let contract = null;
|
|
1159
|
-
let infraError = false;
|
|
1160
|
-
try {
|
|
1161
|
-
const r = await llm.complete(buildRewritePrompt(group.target.content, triggersArg));
|
|
1162
|
-
contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
|
|
1163
|
-
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
1164
|
-
}
|
|
1165
|
-
catch {
|
|
1166
|
-
infraError = true;
|
|
1167
|
-
}
|
|
1168
|
-
if (infraError) {
|
|
1169
|
-
skippedInfra++;
|
|
1170
|
-
deferFrom(group.targetId); // group NOT marked, NOT rewritten — retried next run
|
|
1171
|
-
break;
|
|
1172
|
-
}
|
|
1173
|
-
contracts.set(group.targetId, contract);
|
|
1174
|
-
if (!contract)
|
|
1175
|
-
contractFailed++;
|
|
1176
|
-
}
|
|
1177
|
-
// R2: final per-trigger disposition — a trigger in multiple groups is
|
|
1178
|
-
// absorbed only if EVERY disposition says absorb (any keep keeps it).
|
|
1179
|
-
const finalOutcome = new Map();
|
|
1180
|
-
for (const contract of contracts.values()) {
|
|
1181
|
-
if (!contract)
|
|
1182
|
-
continue;
|
|
1183
|
-
for (const t of contract.triggers) {
|
|
1184
|
-
if (t.disposition === "keep" || finalOutcome.get(t.id) === "keep")
|
|
1185
|
-
finalOutcome.set(t.id, "keep");
|
|
1186
|
-
else
|
|
1187
|
-
finalOutcome.set(t.id, "absorb");
|
|
1188
|
-
}
|
|
1189
|
-
}
|
|
1190
|
-
// R3: apply (one transaction per group). An apply that fails mid-flight
|
|
1191
|
-
// (embed error, DB error) writes NOTHING (the transaction never ran) —
|
|
1192
|
-
// the group is deferred like a pending one so it retries next run.
|
|
1193
|
-
const appliedOutcome = new Map();
|
|
1194
|
-
const deferGroup = (group) => {
|
|
1195
|
-
for (const t of group.triggers) {
|
|
1196
|
-
pendingMinRowid = pendingMinRowid === null ? t.candidateRowid : Math.min(pendingMinRowid, t.candidateRowid);
|
|
1197
|
-
}
|
|
1198
|
-
};
|
|
1199
|
-
for (const group of groups.values()) {
|
|
1200
|
-
const contract = contracts.get(group.targetId);
|
|
1201
|
-
if (contract === undefined)
|
|
1202
|
-
continue; // pending group — untouched this run
|
|
1203
|
-
if (contract === null) {
|
|
1204
|
-
// Failed rewrite contract → whole group mark-only, never a partial
|
|
1205
|
-
// apply. Content untouched, NO trigger absorbed.
|
|
1206
|
-
try {
|
|
1207
|
-
applyMarkOnlyGroup(db, group);
|
|
1208
|
-
}
|
|
1209
|
-
catch (err) {
|
|
1210
|
-
console.warn(`[hicortex] reconsolidation: mark-only fallback failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1211
|
-
skippedInfra++;
|
|
1212
|
-
deferGroup(group);
|
|
1213
|
-
continue;
|
|
1214
|
-
}
|
|
1215
|
-
markedRetracted++;
|
|
1216
|
-
console.log(`[hicortex] Reconsolidation: rewrite contract failed for ${group.targetId.slice(0, 8)} — group degraded to mark-only`);
|
|
1217
|
-
continue;
|
|
1218
|
-
}
|
|
1219
|
-
let applied = false;
|
|
1220
|
-
try {
|
|
1221
|
-
applied = await applyRewriteGroup(db, group, contract, finalOutcome, embedFn);
|
|
1222
|
-
}
|
|
1223
|
-
catch (err) {
|
|
1224
|
-
console.warn(`[hicortex] reconsolidation: rewrite apply failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1225
|
-
}
|
|
1226
|
-
if (!applied) {
|
|
1227
|
-
skippedInfra++; // defensive absorbed-target guard, or an apply error — retry next run
|
|
1228
|
-
deferGroup(group);
|
|
1229
|
-
continue;
|
|
1230
|
-
}
|
|
1231
|
-
rewritten++;
|
|
1232
|
-
// Counted from APPLIED groups only (a deferred group's dispositions
|
|
1233
|
-
// never took effect); a trigger in several applied groups counts once.
|
|
1234
|
-
for (const t of contract.triggers) {
|
|
1235
|
-
const outcome = finalOutcome.get(t.id) ?? "keep";
|
|
1236
|
-
if (outcome === "keep" || appliedOutcome.get(t.id) === "keep")
|
|
1237
|
-
appliedOutcome.set(t.id, "keep");
|
|
1238
|
-
else
|
|
1239
|
-
appliedOutcome.set(t.id, "absorb");
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
for (const outcome of appliedOutcome.values()) {
|
|
1243
|
-
if (outcome === "absorb")
|
|
1244
|
-
absorbed++;
|
|
1245
|
-
else
|
|
1246
|
-
keptLinked++;
|
|
1247
|
-
}
|
|
1248
|
-
}
|
|
1249
1470
|
// ---- #393 guard-C zone reorder: the deterministic merge zone (pairs >=
|
|
1250
|
-
// the ceiling) runs LAST — after the scan
|
|
1251
|
-
//
|
|
1252
|
-
// marks, and binds land first, and the zone
|
|
1253
|
-
// claimed. With the zone first, a >=0.92
|
|
1254
|
-
// before the judge ever saw it
|
|
1255
|
-
// — the planted-eval harm);
|
|
1256
|
-
//
|
|
1257
|
-
// LLM-
|
|
1258
|
-
//
|
|
1471
|
+
// the ceiling) runs LAST — after the scan (which includes every #439
|
|
1472
|
+
// boundary apply: judged merges + rewrites). Judgment outranks the
|
|
1473
|
+
// deterministic sweep: verdicts, marks, and binds land first, and the zone
|
|
1474
|
+
// merges only what no verdict claimed. With the zone first, a >=0.92
|
|
1475
|
+
// genuine-conflict pair was blended before the judge ever saw it
|
|
1476
|
+
// (canonical = oldest, the newer truth erased — the planted-eval harm);
|
|
1477
|
+
// running it last means a `conflicts` bind set by THIS run's scan guards
|
|
1478
|
+
// the SAME run's zone. LLM-free and budget-free — an LLM-less night still
|
|
1479
|
+
// drains duplicates (a deadline-deferred cluster re-detects next run at
|
|
1480
|
+
// zero token cost — content-based discovery, no cursor involvement). Its
|
|
1481
|
+
// own short lock window, pre-merge backup, and #405 deadline stop-check;
|
|
1482
|
+
// fail-soft, never a throw.
|
|
1259
1483
|
const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
|
|
1260
1484
|
stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
|
|
1261
1485
|
threshold: autoMergeThreshold,
|
|
@@ -1263,12 +1487,6 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1263
1487
|
acquireLock: options.acquireLock,
|
|
1264
1488
|
deadline,
|
|
1265
1489
|
});
|
|
1266
|
-
// Cursor hold: un-applied work (rewrite groups, confirmed merges) holds the
|
|
1267
|
-
// cursor BELOW its earliest contributing candidate so the pairs are
|
|
1268
|
-
// re-detected next run.
|
|
1269
|
-
if (pendingMinRowid !== null) {
|
|
1270
|
-
cursor = Math.min(cursor, pendingMinRowid - 1);
|
|
1271
|
-
}
|
|
1272
1490
|
// Report snapshot: the deterministic band (from the zone's own numbers —
|
|
1273
1491
|
// losers are merge verdicts at confidence 1.0; the zone persists the
|
|
1274
1492
|
// cumulative copy itself) plus this run's judged bands.
|
|
@@ -1289,10 +1507,15 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1289
1507
|
for (const [label, stat] of runBands)
|
|
1290
1508
|
bandStats[label] = stat;
|
|
1291
1509
|
if (!dryRun) {
|
|
1292
|
-
// #401: the authoritative FINAL
|
|
1293
|
-
//
|
|
1510
|
+
// #401/#439: the authoritative FINAL write — the mid-scan persists above
|
|
1511
|
+
// are checkpoints; this one applies the final-drain cursor hold (already
|
|
1512
|
+
// folded into `cursor`) and the scan high-water. The retry floor is
|
|
1513
|
+
// applied defensively too: the drain splices retryMerges empty on every
|
|
1514
|
+
// path, but a non-empty list here would mean a confirmed merge stranded
|
|
1515
|
+
// behind the cursor — clamp, never write past un-applied work.
|
|
1294
1516
|
(0, state_js_1.updateState)((s) => {
|
|
1295
|
-
s.reconsolidationCursor =
|
|
1517
|
+
s.reconsolidationCursor = clampedCursor();
|
|
1518
|
+
s.reconsolidationScannedRowid = Math.max(scannedRowidHighwater, s.reconsolidationScannedRowid ?? 0);
|
|
1296
1519
|
// Cumulative judged-band accumulation (#392) — the zone already
|
|
1297
1520
|
// persisted the deterministic band under its own label.
|
|
1298
1521
|
if (runBands.size > 0) {
|
|
@@ -1306,13 +1529,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1306
1529
|
}
|
|
1307
1530
|
}, stateDir);
|
|
1308
1531
|
}
|
|
1309
|
-
if (rows.length > 0 ||
|
|
1310
|
-
console.log(`[hicortex] Reconsolidation: ${scanned} scanned, ${pairsEvaluated} pairs evaluated
|
|
1532
|
+
if (rows.length > 0 || mergePairsApplied > 0 || mergeBelowGate > 0) {
|
|
1533
|
+
console.log(`[hicortex] Reconsolidation: ${scanned} scanned, ${pairsEvaluated} pairs evaluated ` +
|
|
1534
|
+
`(${pairsReevaluated} re-judged / ${pairsNew} new), ` +
|
|
1311
1535
|
`${rewritten} rewritten (${absorbed} triggers absorbed, ${keptLinked} kept), ` +
|
|
1312
|
-
`${mergePairsApplied} pair(s) merged
|
|
1536
|
+
`${mergePairsApplied} pair(s) merged (${mergePairsDeferred} deferred), ` +
|
|
1537
|
+
`${markedSuperseded} superseded, ` +
|
|
1313
1538
|
`${markedRetracted} retracted (${belowGate} below gate, ${mergeBelowGate} merge below gate, ` +
|
|
1314
1539
|
`${contractFailed} contract failed), ${skippedInfra} infra-skipped, ${skippedIdempotent} ` +
|
|
1315
|
-
`already-linked, ${
|
|
1540
|
+
`already-linked, ${skippedAbsorbed} absorbed-skip, ${skippedAboveCeiling} above ceiling, ` +
|
|
1541
|
+
`${explicitVerified} explicit verified, ` +
|
|
1316
1542
|
`${explicitDivergent} explicit divergent, scout ${scoutScanned} scanned / ` +
|
|
1317
1543
|
`${scoutCorrectionShaped} correction-shaped / ${scoutCandidatesFound} candidate pair(s), ` +
|
|
1318
1544
|
`${conflictFlagged} conflict-flagged, ${conflictSkippedJudged + merges.skipped_conflict} conflict-skipped ` +
|
|
@@ -1335,6 +1561,10 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1335
1561
|
explicit_verified: explicitVerified,
|
|
1336
1562
|
explicit_divergent: explicitDivergent,
|
|
1337
1563
|
cursor,
|
|
1564
|
+
pairs_reevaluated: pairsReevaluated,
|
|
1565
|
+
pairs_new: pairsNew,
|
|
1566
|
+
skipped_absorbed: skippedAbsorbed,
|
|
1567
|
+
merge_pairs_deferred: mergePairsDeferred,
|
|
1338
1568
|
merges,
|
|
1339
1569
|
merge_pairs_applied: mergePairsApplied,
|
|
1340
1570
|
merge_below_gate: mergeBelowGate,
|