@gamaze/hicortex 0.20.7 → 0.20.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -41
- package/assets/dashboard.html +3989 -836
- package/dist/calibration.d.ts +293 -0
- package/dist/calibration.js +379 -0
- package/dist/capture-health.d.ts +87 -0
- package/dist/capture-health.js +106 -0
- package/dist/capture-pause.d.ts +86 -0
- package/dist/capture-pause.js +127 -0
- package/dist/capture.d.ts +24 -3
- package/dist/capture.js +11 -1
- package/dist/classify-domains.d.ts +6 -0
- package/dist/classify-domains.js +7 -1
- package/dist/cli.js +38 -3
- package/dist/config-read.d.ts +1 -1
- package/dist/config-read.js +96 -9
- package/dist/consolidate.d.ts +114 -68
- package/dist/consolidate.js +302 -182
- package/dist/dashboard.d.ts +326 -6
- package/dist/dashboard.js +592 -7
- package/dist/db.js +105 -0
- package/dist/dedup.d.ts +34 -26
- package/dist/dedup.js +91 -57
- package/dist/distiller.js +1 -1
- package/dist/domain-classify.d.ts +7 -6
- package/dist/domain-classify.js +12 -10
- package/dist/eval/decay-eval.d.ts +3 -3
- package/dist/eval/decay-eval.js +4 -4
- package/dist/eval/importance-eval.d.ts +85 -0
- package/dist/eval/importance-eval.js +286 -0
- package/dist/eval/planted-eval.d.ts +26 -0
- package/dist/eval/planted-eval.js +97 -0
- package/dist/eval/planted-fixtures.d.ts +107 -0
- package/dist/eval/planted-fixtures.js +283 -0
- package/dist/eval/planted-harness.d.ts +176 -0
- package/dist/eval/planted-harness.js +649 -0
- package/dist/eval/ranking-battery.d.ts +78 -0
- package/dist/eval/ranking-battery.js +181 -0
- package/dist/eval/ranking-eval.d.ts +41 -0
- package/dist/eval/ranking-eval.js +391 -0
- package/dist/eval/ranking-fixtures.d.ts +77 -0
- package/dist/eval/ranking-fixtures.js +226 -0
- package/dist/identity-store.d.ts +21 -0
- package/dist/identity-store.js +49 -0
- package/dist/index.js +4 -3
- package/dist/init.d.ts +23 -3
- package/dist/init.js +84 -9
- package/dist/llm.d.ts +43 -58
- package/dist/llm.js +87 -101
- package/dist/mcp-server.d.ts +12 -0
- package/dist/mcp-server.js +213 -32
- package/dist/nightly.d.ts +9 -1
- package/dist/nightly.js +164 -110
- package/dist/nofit.d.ts +4 -11
- package/dist/nofit.js +6 -23
- package/dist/prompts.d.ts +10 -0
- package/dist/prompts.js +28 -5
- package/dist/recall-index.d.ts +30 -28
- package/dist/recall-index.js +21 -18
- package/dist/recall-registry.d.ts +2 -1
- package/dist/recall-registry.js +35 -1
- package/dist/reconsolidation.d.ts +168 -87
- package/dist/reconsolidation.js +818 -377
- package/dist/relink.js +3 -4
- package/dist/rescore-importance.d.ts +80 -0
- package/dist/rescore-importance.js +236 -0
- package/dist/retrieval.d.ts +80 -35
- package/dist/retrieval.js +322 -105
- package/dist/run-deadline.d.ts +62 -0
- package/dist/run-deadline.js +73 -0
- package/dist/schema-prototypes.d.ts +3 -3
- package/dist/schema-prototypes.js +3 -3
- package/dist/stages.d.ts +37 -0
- package/dist/stages.js +51 -0
- package/dist/state.d.ts +34 -9
- package/dist/storage.d.ts +50 -18
- package/dist/storage.js +125 -30
- package/dist/telemetry.d.ts +8 -7
- package/dist/token-budget.js +3 -4
- package/dist/type-classify.js +4 -4
- package/dist/types.d.ts +143 -155
- package/domains.example.json +4 -5
- package/hermes-plugin/hicortex/README.md +2 -2
- package/openclaw.plugin.json +1 -1
- package/package.json +4 -1
- package/pi-extension/hicortex/README.md +1 -1
- package/server.json +3 -3
package/dist/reconsolidation.js
CHANGED
|
@@ -16,11 +16,58 @@
|
|
|
16
16
|
* #392 — one zone system, ONE verdict per pair: below `correctionMinSimilarity`
|
|
17
17
|
* (floor, 0.75) pairs are not candidates; in [floor, `dedupAutoMergeThreshold`)
|
|
18
18
|
* (ceiling, 0.92) each unlinked pair gets ONE verdict call whose action is
|
|
19
|
-
* `merge` | `corrects` | `supersedes` | `none`; at/above the
|
|
20
|
-
* deterministic merge zone (dedup.ts runDeterministicMergeZone —
|
|
21
|
-
* budget-free) owns the pair. The merge disposition reuses the
|
|
22
|
-
* execution (canonical pick, link re-point, dedup_log, metadata
|
|
23
|
-
* merge verdict below `correctionRewriteMinConfidence` keeps both
|
|
19
|
+
* `merge` | `corrects` | `supersedes` | `conflicts` | `none`; at/above the
|
|
20
|
+
* ceiling the deterministic merge zone (dedup.ts runDeterministicMergeZone —
|
|
21
|
+
* LLM-free, budget-free) owns the pair. The merge disposition reuses the
|
|
22
|
+
* dedup core's execution (canonical pick, link re-point, dedup_log, metadata
|
|
23
|
+
* rails); a merge verdict below `correctionRewriteMinConfidence` keeps both
|
|
24
|
+
* memories.
|
|
25
|
+
*
|
|
26
|
+
* #393 increment B — the SCOUT, a second detection source with the SAME
|
|
27
|
+
* judge: the similarity floor is structurally blind to corrections riding
|
|
28
|
+
* inside topically unrelated memories (the field failure — cosine ~0.5-0.6 to
|
|
29
|
+
* their target, zero `corrects` verdicts in the whole corpus baseline), so
|
|
30
|
+
* per NEW memory ONE classify-tier shape call asks whether it corrects/
|
|
31
|
+
* retracts/supersedes/CONTRADICTS something previously recorded (guard-C
|
|
32
|
+
* extended the question); correction-shaped
|
|
33
|
+
* memories FTS the corpus with the referenced claim's distinctive terms (the
|
|
34
|
+
* correction CONTAINS the words of what it corrects) and the hits become
|
|
35
|
+
* candidate pairs in the SAME verdict loop — no similarity gate for this
|
|
36
|
+
* source: cosine is a ranker/link strength, never a blocker. Per-source
|
|
37
|
+
* counters (scout_scanned / scout_correction_shaped / scout_candidates_found)
|
|
38
|
+
* ride the stage report; cosine band stats stay similarity-source-only.
|
|
39
|
+
*
|
|
40
|
+
* #393 guard-C — the conflicts flag + the zone-runs-last order: judgment
|
|
41
|
+
* OUTRANKS the deterministic sweep. A `conflicts` verdict writes a symmetric
|
|
42
|
+
* `conflicts` link (the pair genuinely disagrees — cannot both be true) and
|
|
43
|
+
* NOTHING else: no status change, no rewrite, no merge queue; both records
|
|
44
|
+
* stay live so the consumer sees both truths. Both merge paths (the zone's
|
|
45
|
+
* planDedup and the judged mergeMemoryIds) refuse to blend a conflicts-linked
|
|
46
|
+
* pair, counted as conflict_skipped. The zone therefore runs AFTER the scan —
|
|
47
|
+
* with the zone first, a >=0.92 conflict pair was blended
|
|
48
|
+
* before the judge ever saw it (the planted-eval harm: canonical=older, the
|
|
49
|
+
* newer truth erased); running it last means verdicts/marks/binds land first
|
|
50
|
+
* and the zone merges only what no verdict claimed — a conflicts bind set by
|
|
51
|
+
* this run's scan guards the SAME run's zone.
|
|
52
|
+
*
|
|
53
|
+
* #439 apply-on-confirm — confirmed merges and rewrite groups apply at the
|
|
54
|
+
* candidate BOUNDARY (the end of the scan iteration that confirmed them), not
|
|
55
|
+
* in post-scan phases. The old end-of-run batch was a completion assumption
|
|
56
|
+
* written when nightlies finished in an hour; under #405 budget pressure it
|
|
57
|
+
* became a days-long queue where confirmed work never landed and every night
|
|
58
|
+
* re-paid the judgment cost (cursor held below un-applied groups, pairs
|
|
59
|
+
* re-detected, re-judged). Now each judged-merge pair applies via
|
|
60
|
+
* mergeMemoryIds in its OWN transaction at confirmation time, each rewrite
|
|
61
|
+
* group via its own rewrite call + applyRewriteGroup transaction; the cursor
|
|
62
|
+
* advances per APPLIED candidate, so a deferral holds it below exactly ONE
|
|
63
|
+
* candidate's pairs. One pre-merge backup per run (lazy, before the first
|
|
64
|
+
* application); the capture lock is taken per boundary batch with a same-run
|
|
65
|
+
* retry list + a final drain. A shared trigger IS the current candidate, so
|
|
66
|
+
* the multi-target keep rule resolves across the boundary's groups (any keep
|
|
67
|
+
* keeps). A target corrected by two different candidates takes two sequential
|
|
68
|
+
* rewrites — the second composes the already-corrected story — instead of one
|
|
69
|
+
* grouped call (the ONE-call grouping was a cost optimization, not a
|
|
70
|
+
* correctness invariant; accepted semantics change).
|
|
24
71
|
*
|
|
25
72
|
* Status vocabulary (code-defined, extensible — deliberately NOT config):
|
|
26
73
|
* NULL/'active' default | 'superseded' + 'retracted' demote in ranking |
|
|
@@ -71,8 +118,10 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
71
118
|
};
|
|
72
119
|
})();
|
|
73
120
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
74
|
-
exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.
|
|
121
|
+
exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = exports.DEFAULT_CORRECTION_MIN_SIMILARITY = exports.RECONSOLIDATION_STAGE_LABEL = void 0;
|
|
75
122
|
exports.isFactShapedTarget = isFactShapedTarget;
|
|
123
|
+
exports.buildScoutShapePrompt = buildScoutShapePrompt;
|
|
124
|
+
exports.parseScoutShape = parseScoutShape;
|
|
76
125
|
exports.buildCorrectionVerdictPrompt = buildCorrectionVerdictPrompt;
|
|
77
126
|
exports.parseCorrectionVerdict = parseCorrectionVerdict;
|
|
78
127
|
exports.buildRewritePrompt = buildRewritePrompt;
|
|
@@ -94,6 +143,7 @@ const storage = __importStar(require("./storage.js"));
|
|
|
94
143
|
const state_js_1 = require("./state.js");
|
|
95
144
|
const db_js_1 = require("./db.js");
|
|
96
145
|
const capture_js_1 = require("./capture.js");
|
|
146
|
+
const CALIBRATION = __importStar(require("./calibration.js"));
|
|
97
147
|
const paths_js_1 = require("./paths.js");
|
|
98
148
|
const dedup_js_1 = require("./dedup.js");
|
|
99
149
|
// ---------------------------------------------------------------------------
|
|
@@ -102,37 +152,21 @@ const dedup_js_1 = require("./dedup.js");
|
|
|
102
152
|
/** Stage label used for every budget.use()/recordUsage() call (#384). */
|
|
103
153
|
exports.RECONSOLIDATION_STAGE_LABEL = "reconsolidation";
|
|
104
154
|
/**
|
|
105
|
-
* Default minimum COSINE similarity for a correction candidate pair
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
155
|
+
* Default minimum COSINE similarity for a correction candidate pair —
|
|
156
|
+
* RELEASE-MANAGED since #408 (calibration.ts CORRECTION_MIN_SIMILARITY;
|
|
157
|
+
* provenance there). Lower than the supersession stage's 0.80 on purpose: a
|
|
158
|
+
* retraction often rides inside an otherwise unrelated memory (the field
|
|
159
|
+
* failure that opened this issue), so the neighborhood gate must be a touch
|
|
160
|
+
* wider while the LLM verdict + confidence gate carry the precision load.
|
|
110
161
|
*/
|
|
111
|
-
exports.DEFAULT_CORRECTION_MIN_SIMILARITY =
|
|
162
|
+
exports.DEFAULT_CORRECTION_MIN_SIMILARITY = CALIBRATION.CORRECTION_MIN_SIMILARITY;
|
|
112
163
|
/**
|
|
113
|
-
* Default minimum verdict confidence for the REWRITE fork
|
|
164
|
+
* Default minimum verdict confidence for the REWRITE fork — release-managed
|
|
165
|
+
* (calibration.ts CORRECTION_REWRITE_MIN_CONFIDENCE). Below this a
|
|
114
166
|
* `corrects` verdict degrades to mark-only — a weak mark is recoverable, a
|
|
115
167
|
* weak rewrite is corruption.
|
|
116
168
|
*/
|
|
117
|
-
exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE =
|
|
118
|
-
/**
|
|
119
|
-
* Default wall-clock bound for the stage, in minutes (#401). Checked at the
|
|
120
|
-
* top of the candidate scan loop (and before each rewrite contract call); on
|
|
121
|
-
* expiry the scan breaks cleanly at the last fully-considered candidate and
|
|
122
|
-
* the next run resumes from the persisted cursor. 120 sits safely under any
|
|
123
|
-
* sane process-level nightly timeout. 0 disables the bound. Invalid →
|
|
124
|
-
* default.
|
|
125
|
-
*/
|
|
126
|
-
exports.DEFAULT_RECONSOLIDATION_MAX_MINUTES = 120;
|
|
127
|
-
/**
|
|
128
|
-
* Default per-run classify-call ceiling for the stage (#401) — the
|
|
129
|
-
* supersessionMaxCalls pattern with a NON-ZERO default ON PURPOSE: that
|
|
130
|
-
* knob's 0=unlimited default is what let the first full-corpus pass grow
|
|
131
|
-
* unbounded. Counts EVERY classify-tier call the stage makes (mark
|
|
132
|
-
* verifications, pair verdicts, rewrite contracts). 0 disables the cap.
|
|
133
|
-
* Invalid → default.
|
|
134
|
-
*/
|
|
135
|
-
exports.DEFAULT_RECONSOLIDATION_MAX_CALLS = 600;
|
|
169
|
+
exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = CALIBRATION.CORRECTION_REWRITE_MIN_CONFIDENCE;
|
|
136
170
|
/** Neighbor pool size before older/similarity filtering narrows to top 5 (supersession mirror). */
|
|
137
171
|
const CORRECTION_NEIGHBOR_POOL = 15;
|
|
138
172
|
/** Older-neighbor pairs kept per candidate after filtering (supersession mirror). */
|
|
@@ -161,10 +195,10 @@ exports.DEMOTED_STATUSES = ["superseded", "retracted"];
|
|
|
161
195
|
function isFactShapedTarget(mem) {
|
|
162
196
|
return mem.memory_type === "knowledge" || mem.content.includes("[Facts Learned]");
|
|
163
197
|
}
|
|
164
|
-
/** True when a superseded_by
|
|
198
|
+
/** True when a superseded_by / corrected_by / conflicts link already exists between the pair, either direction. */
|
|
165
199
|
function alreadyResolutionLinked(db, oldId, newId) {
|
|
166
200
|
const row = db
|
|
167
|
-
.prepare(`SELECT 1 FROM memory_links WHERE relationship IN ('superseded_by', 'corrected_by')
|
|
201
|
+
.prepare(`SELECT 1 FROM memory_links WHERE relationship IN ('superseded_by', 'corrected_by', 'conflicts')
|
|
168
202
|
AND ((source_id = ? AND target_id = ?) OR (source_id = ? AND target_id = ?))`)
|
|
169
203
|
.get(oldId, newId, newId, oldId);
|
|
170
204
|
return !!row;
|
|
@@ -178,6 +212,58 @@ function hasLink(db, sourceId, targetId, relationship) {
|
|
|
178
212
|
function nowIso() {
|
|
179
213
|
return new Date().toISOString();
|
|
180
214
|
}
|
|
215
|
+
/**
|
|
216
|
+
* Build the constrained correction-shape prompt (classify-tier cost profile:
|
|
217
|
+
* 1500-char truncation, supersession/verdict precedent). The wording asks for
|
|
218
|
+
* the OLD claim's distinctive terms — the field-failure mechanism is that a
|
|
219
|
+
* correction CONTAINS the words of what it corrects, even when the surrounding
|
|
220
|
+
* topics (and therefore the embedding cosine) are unrelated. Guard-C extends
|
|
221
|
+
* the question to contradictions: two records that disagree on the same
|
|
222
|
+
* quantity share even MORE wording than a cross-topic correction does.
|
|
223
|
+
*/
|
|
224
|
+
function buildScoutShapePrompt(content) {
|
|
225
|
+
const trunc = (s) => (s.length > PROMPT_TRUNCATE_CHARS ? `${s.slice(0, PROMPT_TRUNCATE_CHARS)}…` : s);
|
|
226
|
+
return (`You are scanning a memory that was just added to an AI agent's long-term memory store.\n\n` +
|
|
227
|
+
`MEMORY:\n${trunc(content)}\n\n` +
|
|
228
|
+
`Does this memory correct, retract, supersede, or contradict a claim, decision, or state that was ` +
|
|
229
|
+
`previously recorded elsewhere in the store? A mere duplicate, elaboration, independent ` +
|
|
230
|
+
`fact, or new information that invalidates nothing is NOT a correction.\n` +
|
|
231
|
+
`If it is a correction/retraction/supersession/contradiction, list the most distinctive terms of the ` +
|
|
232
|
+
`OLD claim it references — words likely to appear verbatim in the older record.\n\n` +
|
|
233
|
+
`Reply with ONLY a JSON object, no prose: ` +
|
|
234
|
+
`{"correction": true | false, "references": "<distinctive terms of the referenced old claim, or empty string>", ` +
|
|
235
|
+
`"confidence": <number between 0 and 1>}`);
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Parse the scout shape reply. Null on unparseable JSON, a missing/non-boolean
|
|
239
|
+
* `correction`, or a missing/out-of-range `confidence` — the caller counts
|
|
240
|
+
* skipped_infra and moves on (parseSupersessionReply discipline: never
|
|
241
|
+
* mis-detect on ambiguity). `references` is lenient (missing/non-string → "")
|
|
242
|
+
* because an empty string simply yields no FTS hits — a harmless miss, not a
|
|
243
|
+
* mis-judgment.
|
|
244
|
+
*/
|
|
245
|
+
function parseScoutShape(reply) {
|
|
246
|
+
if (!reply)
|
|
247
|
+
return null;
|
|
248
|
+
const start = reply.indexOf("{");
|
|
249
|
+
const end = reply.lastIndexOf("}");
|
|
250
|
+
if (start === -1 || end === -1 || end <= start)
|
|
251
|
+
return null;
|
|
252
|
+
let obj;
|
|
253
|
+
try {
|
|
254
|
+
obj = JSON.parse(reply.slice(start, end + 1));
|
|
255
|
+
}
|
|
256
|
+
catch {
|
|
257
|
+
return null;
|
|
258
|
+
}
|
|
259
|
+
if (typeof obj.correction !== "boolean")
|
|
260
|
+
return null;
|
|
261
|
+
const confidence = Number(obj.confidence);
|
|
262
|
+
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1)
|
|
263
|
+
return null;
|
|
264
|
+
const references = typeof obj.references === "string" ? obj.references : "";
|
|
265
|
+
return { correction: obj.correction, references: references.trim(), confidence };
|
|
266
|
+
}
|
|
181
267
|
/** Build the constrained pair-verdict prompt (1500-char truncation, supersession precedent). */
|
|
182
268
|
function buildCorrectionVerdictPrompt(oldContent, newContent) {
|
|
183
269
|
const trunc = (s) => (s.length > PROMPT_TRUNCATE_CHARS ? `${s.slice(0, PROMPT_TRUNCATE_CHARS)}…` : s);
|
|
@@ -191,9 +277,12 @@ function buildCorrectionVerdictPrompt(oldContent, newContent) {
|
|
|
191
277
|
`wrong, no longer true, or was retracted, and the newer memory carries the corrected fact.\n` +
|
|
192
278
|
`- "supersedes": the newer memory replaces a decision, plan, or state that was valid at the time but is ` +
|
|
193
279
|
`now outdated — a replacement, not a factual correction.\n` +
|
|
280
|
+
`- "conflicts": the two memories make claims that cannot both be true — they disagree on a fact, value, ` +
|
|
281
|
+
`or state, and neither one corrects, supersedes, or restates the other (for example two sources report ` +
|
|
282
|
+
`different values for the same quantity). Keep both; flag the conflict.\n` +
|
|
194
283
|
`- "none": unrelated, merely similar, or both can still be true (an addition or elaboration).\n\n` +
|
|
195
284
|
`Reply with ONLY a JSON object, no prose: ` +
|
|
196
|
-
`{"action": "merge" | "corrects" | "supersedes" | "none", "confidence": <number between 0 and 1>}`);
|
|
285
|
+
`{"action": "merge" | "corrects" | "supersedes" | "conflicts" | "none", "confidence": <number between 0 and 1>}`);
|
|
197
286
|
}
|
|
198
287
|
/**
|
|
199
288
|
* Parse the pair verdict. Null on anything unparseable, unknown action, or an
|
|
@@ -215,8 +304,13 @@ function parseCorrectionVerdict(reply) {
|
|
|
215
304
|
return null;
|
|
216
305
|
}
|
|
217
306
|
const action = obj.action;
|
|
218
|
-
if (action !== "merge" &&
|
|
307
|
+
if (action !== "merge" &&
|
|
308
|
+
action !== "corrects" &&
|
|
309
|
+
action !== "supersedes" &&
|
|
310
|
+
action !== "conflicts" &&
|
|
311
|
+
action !== "none") {
|
|
219
312
|
return null;
|
|
313
|
+
}
|
|
220
314
|
const confidence = Number(obj.confidence);
|
|
221
315
|
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1)
|
|
222
316
|
return null;
|
|
@@ -452,7 +546,7 @@ function bandForCosine(bands, cosine) {
|
|
|
452
546
|
}
|
|
453
547
|
/** An empty band-stat record (fresh accumulation starts from zeroes). */
|
|
454
548
|
function emptyBandStat() {
|
|
455
|
-
return { pairs: 0, merge: 0, corrects: 0, supersedes: 0, none: 0, merge_below_gate: 0, conf_sum: 0 };
|
|
549
|
+
return { pairs: 0, merge: 0, corrects: 0, supersedes: 0, conflicts: 0, none: 0, merge_below_gate: 0, conf_sum: 0 };
|
|
456
550
|
}
|
|
457
551
|
/** Add a run's per-band counts into a cumulative record (in place). */
|
|
458
552
|
function accumulateBandStat(cumulative, run) {
|
|
@@ -460,6 +554,7 @@ function accumulateBandStat(cumulative, run) {
|
|
|
460
554
|
cumulative.merge += run.merge;
|
|
461
555
|
cumulative.corrects += run.corrects;
|
|
462
556
|
cumulative.supersedes += run.supersedes;
|
|
557
|
+
cumulative.conflicts += run.conflicts;
|
|
463
558
|
cumulative.none += run.none;
|
|
464
559
|
cumulative.merge_below_gate += run.merge_below_gate;
|
|
465
560
|
cumulative.conf_sum += run.conf_sum;
|
|
@@ -481,9 +576,43 @@ async function findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarit
|
|
|
481
576
|
.sort((a, b) => (0, retrieval_js_1.l2ToCosine)(b.distance) - (0, retrieval_js_1.l2ToCosine)(a.distance))
|
|
482
577
|
.slice(0, CORRECTION_NEIGHBOR_TOP_K);
|
|
483
578
|
}
|
|
579
|
+
/**
|
|
580
|
+
* The scout source (#393 increment B): for a correction-shaped NEW memory, FTS
|
|
581
|
+
* the corpus with the referenced claim's distinctive terms and return the OLDER
|
|
582
|
+
* hits as candidate pairs. This is the reference-extraction half — it finds the
|
|
583
|
+
* old claim even when the overall topics differ (and therefore the cosine sits
|
|
584
|
+
* below the similarity floor) because the correction CONTAINS the words of
|
|
585
|
+
* what it corrects. Deterministic: ONE FTS query, zero LLM. Filters: self,
|
|
586
|
+
* non-older (detection only pairs older → newer, the KNN mirror), and
|
|
587
|
+
* defensively non-absorbed hits. Pool/top-K reuse the KNN constants; FTS rank
|
|
588
|
+
* (BM25, best first) is the order. Dedup against the KNN neighbor ids is the
|
|
589
|
+
* caller's job (a pair found by both sources is judged once, as similarity).
|
|
590
|
+
*/
|
|
591
|
+
function findScoutNeighbors(db, candidate, references, candidateEmbedding) {
|
|
592
|
+
if (!references)
|
|
593
|
+
return [];
|
|
594
|
+
try {
|
|
595
|
+
const hits = storage.searchFts(db, references, CORRECTION_NEIGHBOR_POOL);
|
|
596
|
+
return hits
|
|
597
|
+
.filter((m) => m.id !== candidate.id &&
|
|
598
|
+
m.created_at < candidate.created_at &&
|
|
599
|
+
m.status !== "absorbed")
|
|
600
|
+
.slice(0, CORRECTION_NEIGHBOR_TOP_K)
|
|
601
|
+
.map((m) => {
|
|
602
|
+
const hitVec = storage.getStoredEmbedding(db, m.id);
|
|
603
|
+
const cosine = candidateEmbedding && hitVec ? (0, retrieval_js_1.cosineBetweenVectors)(candidateEmbedding, hitVec) : 0;
|
|
604
|
+
return { mem: m, cosine, source: "scout" };
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
catch {
|
|
608
|
+
// FTS is deterministic infrastructure — a throw here is a bug or a corrupt
|
|
609
|
+
// index, never a judgment question. Fail soft: no scout pairs this memory.
|
|
610
|
+
return [];
|
|
611
|
+
}
|
|
612
|
+
}
|
|
484
613
|
async function classifyPair(llm, oldContent, newContent) {
|
|
485
614
|
try {
|
|
486
|
-
const r = await llm.
|
|
615
|
+
const r = await llm.complete(buildCorrectionVerdictPrompt(oldContent, newContent));
|
|
487
616
|
return { verdict: parseCorrectionVerdict(r.text), usage: r.usage };
|
|
488
617
|
}
|
|
489
618
|
catch {
|
|
@@ -491,32 +620,55 @@ async function classifyPair(llm, oldContent, newContent) {
|
|
|
491
620
|
}
|
|
492
621
|
}
|
|
493
622
|
/**
|
|
494
|
-
* Nightly reconsolidation stage (#384, #392 — THE unified resolution stage
|
|
623
|
+
* Nightly reconsolidation stage (#384, #392 — THE unified resolution stage;
|
|
624
|
+
* #439 apply-on-confirm).
|
|
495
625
|
*
|
|
496
|
-
* Phase
|
|
497
|
-
*
|
|
626
|
+
* Phase order (#393 guard-C): the deterministic merge zone (pairs >= the
|
|
627
|
+
* ceiling) runs LAST — after the scan (which now includes every judged-merge
|
|
628
|
+
* application and rewrite, #439). Judgment outranks the deterministic sweep:
|
|
629
|
+
* verdicts, marks, and binds land first and the zone merges only what no
|
|
630
|
+
* verdict claimed. With the zone first, a >=0.92 genuine-conflict pair was
|
|
631
|
+
* blended before the judge ever saw it (the planted-eval harm); running it
|
|
632
|
+
* last means a `conflicts` bind set by this run's scan guards the SAME run's
|
|
633
|
+
* zone. Zone internals (lock, backup, deadline, persistBand, fail-soft) are
|
|
634
|
+
* unchanged.
|
|
498
635
|
*
|
|
499
636
|
* Scan: every memory with rowid > reconsolidationCursor (no shape filter;
|
|
500
637
|
* absorbed candidates are skipped — invisible memories are not re-judged).
|
|
501
|
-
* Each candidate's pairs: incoming explicit marks (verified once, AC7) then
|
|
502
|
-
*
|
|
503
|
-
*
|
|
504
|
-
*
|
|
505
|
-
*
|
|
506
|
-
*
|
|
638
|
+
* Each candidate's pairs: incoming explicit marks (verified once, AC7), then
|
|
639
|
+
* ONE scout shape call (#393 B — flags correction shape; non-corrections stop
|
|
640
|
+
* there), then up-to-5 older KNN neighbors in [floor, ceiling) (verdict call
|
|
641
|
+
* per unlinked pair, AC2 — pairs at/above the ceiling are counted, never
|
|
642
|
+
* judged) plus the scout's FTS hits for correction-shaped memories (same
|
|
643
|
+
* verdict loop, NO similarity gate; guard-C: a scout hit whose KNN twin sits
|
|
644
|
+
* at/above the ceiling is re-tagged scout so the pair IS judged instead of
|
|
645
|
+
* being left for the zone to blend). Confirmed `corrects` pairs above the
|
|
646
|
+
* confidence gate on fact-shaped targets group by target; a `conflicts`
|
|
647
|
+
* verdict writes the conflicts link and nothing else (both live); everything
|
|
648
|
+
* else is mark-only.
|
|
507
649
|
*
|
|
508
|
-
*
|
|
509
|
-
*
|
|
510
|
-
*
|
|
650
|
+
* #439 BOUNDARY apply: at the END of each candidate iteration everything it
|
|
651
|
+
* confirmed applies IMMEDIATELY — merges first (each judged-merge pair via
|
|
652
|
+
* mergeMemoryIds in its own transaction, under the boundary's short lock
|
|
653
|
+
* window; ONE lazy pre-merge backup per run), then the iteration's rewrite
|
|
654
|
+
* groups (one rewrite LLM call + one applyRewriteGroup transaction each;
|
|
655
|
+
* dispositions resolved ACROSS the boundary's groups — the multi-target keep
|
|
656
|
+
* rule: a trigger absorbed only if every group's contract says absorb). A
|
|
657
|
+
* busy capture lock pushes the boundary's merges onto a same-run retry list
|
|
658
|
+
* (retried at the next boundary and once in a final drain after the scan);
|
|
659
|
+
* a deadline, a backup failure, or a rewrite-call refusal/infra error defers
|
|
660
|
+
* the remaining work and holds the cursor.
|
|
511
661
|
*
|
|
512
|
-
* Cursor discipline
|
|
513
|
-
*
|
|
514
|
-
*
|
|
515
|
-
*
|
|
516
|
-
*
|
|
517
|
-
*
|
|
518
|
-
*
|
|
519
|
-
*
|
|
662
|
+
* Cursor discipline: the cursor advances past a candidate only when its
|
|
663
|
+
* iteration's confirmed work has LANDED (or was refused-with-verdict-rendered:
|
|
664
|
+
* metadata mismatch, conflict-linked, mark-only fallback). A deferral holds
|
|
665
|
+
* the cursor BELOW the current candidate — bounded to ONE candidate's pairs,
|
|
666
|
+
* re-detected and re-judged next run (dup-over-loss — a confirmed resolution
|
|
667
|
+
* must never be silently dropped by the cursor passing it). The separate
|
|
668
|
+
* scan high-water (state.reconsolidationScannedRowid) records the max
|
|
669
|
+
* candidate rowid ENTERED and is never held back, so the report can split
|
|
670
|
+
* verdict calls into pairs_reevaluated (at/below the prior high-water) vs
|
|
671
|
+
* pairs_new — the convergence measurement.
|
|
520
672
|
*
|
|
521
673
|
* Dry-run: the zone's discovery + the free idempotency check only — zero LLM
|
|
522
674
|
* calls, zero writes, no cursor or band-stats persistence. Gate discovery is
|
|
@@ -533,25 +685,13 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
533
685
|
const minSimilarity = validNumber(options.minSimilarity, exports.DEFAULT_CORRECTION_MIN_SIMILARITY, (n) => n > 0 && n <= 1);
|
|
534
686
|
const rewriteMinConfidence = validNumber(options.rewriteMinConfidence, exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE, (n) => n > 0 && n <= 1);
|
|
535
687
|
const autoMergeThreshold = validNumber(options.autoMergeThreshold, dedup_js_1.DEFAULT_DEDUP_MERGE_THRESHOLD, (n) => n > 0 && n <= 1);
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
//
|
|
540
|
-
//
|
|
541
|
-
|
|
542
|
-
const
|
|
543
|
-
const deadlineHit = () => Date.now() >= deadlineAt;
|
|
544
|
-
// ---- #392 phase 0: the deterministic merge zone (pairs >= the ceiling),
|
|
545
|
-
// LLM-free and budget-free — an LLM-less night still drains duplicates. Its
|
|
546
|
-
// own short lock window, pre-merge backup, and pacing cap; fail-soft, never
|
|
547
|
-
// a throw. Runs FIRST so the scan below never sees the pairs it owns.
|
|
548
|
-
const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
|
|
549
|
-
stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
|
|
550
|
-
threshold: autoMergeThreshold,
|
|
551
|
-
maxMerges,
|
|
552
|
-
dryRun,
|
|
553
|
-
acquireLock: options.acquireLock,
|
|
554
|
-
});
|
|
688
|
+
// ---- #405 runtime bound. The stage-local reconsolidationMaxMinutes clock
|
|
689
|
+
// (#401) is gone — the run-wide pipeline deadline (nightly.ts, config
|
|
690
|
+
// nightlyTimeBudgetMinutes) is the only wall-clock. The deterministic
|
|
691
|
+
// zone's runtime counts against it via the zone's own stop-check below.
|
|
692
|
+
// hit() logs event=deadline_deferred once per stage name.
|
|
693
|
+
const deadline = options.deadline;
|
|
694
|
+
const deadlineHit = (stageLabel = exports.RECONSOLIDATION_STAGE_LABEL) => deadline?.hit(stageLabel) ?? false;
|
|
555
695
|
// Per-run verdict statistics by cosine band (#392) — report snapshot here,
|
|
556
696
|
// cumulative series in state.json at stage end (never on dry-run).
|
|
557
697
|
const bands = buildResolutionBands(minSimilarity, autoMergeThreshold);
|
|
@@ -567,6 +707,14 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
567
707
|
runBands.set(band.label, stat);
|
|
568
708
|
};
|
|
569
709
|
const startCursor = (0, state_js_1.loadState)(stateDir).reconsolidationCursor ?? 0;
|
|
710
|
+
// #439 convergence measurement: the scan high-water is the max candidate
|
|
711
|
+
// rowid any run has ENTERED — never held back by un-applied work. This
|
|
712
|
+
// run's re-judged/new split keys on the PREVIOUS run's persisted value: a
|
|
713
|
+
// verdict on a candidate at/below it re-judges pairs a prior run already
|
|
714
|
+
// judged but could not apply (the cursor held below them, so they
|
|
715
|
+
// re-detect). Once the backlog drains, pairs_reevaluated reads 0.
|
|
716
|
+
const prevScannedRowid = (0, state_js_1.loadState)(stateDir).reconsolidationScannedRowid ?? startCursor;
|
|
717
|
+
let scannedRowidHighwater = startCursor;
|
|
570
718
|
// NO shape filter (AC2) — unlike stageSupersession. Absorbed rows are
|
|
571
719
|
// excluded: they are invisible to recall and must not re-enter judgment.
|
|
572
720
|
const rows = db
|
|
@@ -593,12 +741,39 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
593
741
|
let skippedAboveCeiling = 0;
|
|
594
742
|
let skippedMetadataMismatch = 0;
|
|
595
743
|
let mergePairsApplied = 0;
|
|
744
|
+
// #393 guard-C: conflicts verdicts rendered (link written, both live) and
|
|
745
|
+
// judged-path merge refusals on a conflicts-linked pair.
|
|
746
|
+
let conflictFlagged = 0;
|
|
747
|
+
let conflictSkippedJudged = 0;
|
|
748
|
+
// #393 B scout counters (per-source observability, the #394 discipline):
|
|
749
|
+
// shape calls made / correction-shaped verdicts / FTS hits that became
|
|
750
|
+
// candidate pairs. 0 on dry-run (the shape call is LLM work).
|
|
751
|
+
let scoutScanned = 0;
|
|
752
|
+
let scoutCorrectionShaped = 0;
|
|
753
|
+
let scoutCandidatesFound = 0;
|
|
754
|
+
// #439 observability: the re-judged/new verdict split (keyed on the prior
|
|
755
|
+
// run's scan high-water) + the scan-stability guard's skip count + the
|
|
756
|
+
// confirmed-merge deferral count (still un-applied at run end).
|
|
757
|
+
let pairsReevaluated = 0;
|
|
758
|
+
let pairsNew = 0;
|
|
759
|
+
let skippedAbsorbed = 0;
|
|
760
|
+
let mergePairsDeferred = 0;
|
|
596
761
|
let cursor = startCursor;
|
|
597
|
-
|
|
598
|
-
//
|
|
599
|
-
//
|
|
600
|
-
//
|
|
601
|
-
|
|
762
|
+
// Lock-busy survivors: boundary merges that could not take the capture
|
|
763
|
+
// lock, plus (fix round, #440 review finding 1) deadline/backup-dropped
|
|
764
|
+
// tails re-queued at their boundary instead of discarded. Same-run only —
|
|
765
|
+
// retried (in full) at the next boundary and once in the final drain after
|
|
766
|
+
// the scan. While the list is non-empty, every persisted checkpoint clamps
|
|
767
|
+
// below the earliest contributing candidate (pendingRetryFloor — the kill
|
|
768
|
+
// window cannot strand them); still un-applied at run end, the drain holds
|
|
769
|
+
// the cursor below that same floor (dup-over-loss).
|
|
770
|
+
const retryMerges = [];
|
|
771
|
+
// ONE pre-merge backup per run (#439): takePreDedupBackup is a full SQLite
|
|
772
|
+
// copy, so a per-boundary backup would be hundreds of full-DB copies on a
|
|
773
|
+
// backlog night. Taken LAZILY, immediately before the first judged-merge
|
|
774
|
+
// application; remembered for the rest of the run (the zone takes its own,
|
|
775
|
+
// independent backup, as before).
|
|
776
|
+
let mergeWindowBackedUp = false;
|
|
602
777
|
// Links created by THIS stage in THIS run — lets the explicit-mark pass
|
|
603
778
|
// distinguish operator marks (pre-existing) from stage output.
|
|
604
779
|
const linksCreatedThisRun = new Set();
|
|
@@ -606,42 +781,50 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
606
781
|
storage.addLink(db, oldId, newId, relationship, strength);
|
|
607
782
|
linksCreatedThisRun.add(`${oldId}|${newId}`);
|
|
608
783
|
};
|
|
609
|
-
// ---- #401
|
|
610
|
-
// path
|
|
611
|
-
//
|
|
612
|
-
//
|
|
613
|
-
//
|
|
614
|
-
//
|
|
615
|
-
//
|
|
616
|
-
//
|
|
617
|
-
|
|
784
|
+
// ---- #401/#439 mid-scan cursor persistence. Called at EVERY scan-loop
|
|
785
|
+
// exit path AND at the end of every candidate iteration — AFTER that
|
|
786
|
+
// iteration's boundary apply, so the persisted cursor only ever advances
|
|
787
|
+
// past candidates whose confirmed work has landed; a killed run loses at
|
|
788
|
+
// most the candidate in flight. The ONE exception is lock-busy retry
|
|
789
|
+
// survivors: they intentionally ride the retry list while the scan
|
|
790
|
+
// continues (the lock may clear this run), so while any are pending every
|
|
791
|
+
// persisted checkpoint CLAMPS below their earliest contributor — a SIGKILL
|
|
792
|
+
// in the window between a busy boundary and the pair landing must never
|
|
793
|
+
// strand a confirmed merge behind the cursor (the #402 orphan-floor
|
|
794
|
+
// discipline, re-scoped to the retry list; the clamp lifts automatically
|
|
795
|
+
// once a later boundary or the final drain applies them). Also persists
|
|
796
|
+
// the scan high-water (never held back). updateState is load→mutate→
|
|
797
|
+
// temp-rename atomic.
|
|
618
798
|
let deadlineStopped = false;
|
|
619
|
-
|
|
620
|
-
//
|
|
621
|
-
//
|
|
622
|
-
//
|
|
623
|
-
|
|
624
|
-
//
|
|
625
|
-
// candidate
|
|
626
|
-
//
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
scanPendingMinRowid === null ? rowid : Math.min(scanPendingMinRowid, rowid);
|
|
799
|
+
// #439: once an iteration's confirmed work deferred (deadline at the
|
|
800
|
+
// boundary, backup failure, rewrite refusal/infra error), the cursor never
|
|
801
|
+
// advances again this run — a later iteration must not push it past the
|
|
802
|
+
// held candidate's rowid.
|
|
803
|
+
let cursorHold = false;
|
|
804
|
+
// Fix round (#440 review, finding 1): the floor below the earliest
|
|
805
|
+
// candidate contributing to a still-un-applied retry merge. Applied at
|
|
806
|
+
// every persist so the kill-with-pending-retry window cannot strand them.
|
|
807
|
+
const pendingRetryFloor = () => retryMerges.length > 0
|
|
808
|
+
? Math.min(...retryMerges.map((p) => p.candidateRowid)) - 1
|
|
809
|
+
: null;
|
|
810
|
+
const clampedCursor = () => {
|
|
811
|
+
const floor = pendingRetryFloor();
|
|
812
|
+
return floor !== null ? Math.min(cursor, floor) : cursor;
|
|
634
813
|
};
|
|
635
814
|
const persistCursor = () => {
|
|
636
815
|
if (dryRun)
|
|
637
816
|
return;
|
|
638
|
-
const checkpoint = scanPendingMinRowid !== null ? Math.min(cursor, scanPendingMinRowid - 1) : cursor;
|
|
639
817
|
(0, state_js_1.updateState)((s) => {
|
|
640
|
-
s.reconsolidationCursor =
|
|
818
|
+
s.reconsolidationCursor = clampedCursor();
|
|
819
|
+
s.reconsolidationScannedRowid = Math.max(scannedRowidHighwater, s.reconsolidationScannedRowid ?? 0);
|
|
641
820
|
}, stateDir);
|
|
642
821
|
};
|
|
643
|
-
|
|
644
|
-
|
|
822
|
+
// #439: rewrite groups live ONLY inside the candidate iteration that
|
|
823
|
+
// formed them (its boundary applies or degrades them, then they are
|
|
824
|
+
// discarded). Every trigger is the current candidate — a target corrected
|
|
825
|
+
// by two different candidates takes two sequential rewrites instead of
|
|
826
|
+
// the old one grouped call.
|
|
827
|
+
const addTrigger = (groups, target, trigger, confidence, cosine, explicit) => {
|
|
645
828
|
let group = groups.get(target.id);
|
|
646
829
|
if (!group) {
|
|
647
830
|
group = { targetId: target.id, target, triggers: [] };
|
|
@@ -656,10 +839,9 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
656
839
|
candidateRowid: trigger.__rowid,
|
|
657
840
|
explicit,
|
|
658
841
|
});
|
|
659
|
-
notePendingRowid(trigger.__rowid); // orphan floor — group unapplied until the rewrite phase
|
|
660
842
|
}
|
|
661
843
|
};
|
|
662
|
-
for (const
|
|
844
|
+
for (const snapshotted of rows) {
|
|
663
845
|
// #401: runtime bounds first — exit cleanly at the last fully-considered
|
|
664
846
|
// candidate boundary (cursor = the previous candidate's rowid here).
|
|
665
847
|
if (!dryRun && deadlineHit()) {
|
|
@@ -671,15 +853,50 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
671
853
|
persistCursor();
|
|
672
854
|
break;
|
|
673
855
|
}
|
|
674
|
-
// #
|
|
675
|
-
//
|
|
676
|
-
//
|
|
677
|
-
|
|
678
|
-
|
|
856
|
+
// ---- #439 scan-stability guard: `rows` is ONE snapshot fetched at stage
|
|
857
|
+
// start with status != 'absorbed'; boundary applies (merge losers,
|
|
858
|
+
// rewrite triggers absorbed) can mark FUTURE rows of that snapshot
|
|
859
|
+
// absorbed after the filter ran. Without this re-read such a candidate
|
|
860
|
+
// would be scouted/judged on stale content and its KNN would silently
|
|
861
|
+
// re-embed it (its vector row is gone). Absorbed → skip (counted,
|
|
862
|
+
// cursor passes it); otherwise the LIVE row's content/status drives the
|
|
863
|
+
// rest of the iteration (refreshes content rewritten by an earlier
|
|
864
|
+
// boundary when created_at and rowid order diverge).
|
|
865
|
+
const live = db
|
|
866
|
+
.prepare(`SELECT rowid AS __rowid, * FROM memories WHERE rowid = ?`)
|
|
867
|
+
.get(snapshotted.__rowid);
|
|
868
|
+
if (!live) {
|
|
869
|
+
// Vanished entirely (deleted out from under the scan) — defensive;
|
|
870
|
+
// nothing to judge, the cursor passes it (never past an active hold).
|
|
871
|
+
scannedRowidHighwater = Math.max(scannedRowidHighwater, snapshotted.__rowid);
|
|
872
|
+
if (!cursorHold)
|
|
873
|
+
cursor = snapshotted.__rowid;
|
|
679
874
|
persistCursor();
|
|
680
|
-
|
|
875
|
+
continue;
|
|
876
|
+
}
|
|
877
|
+
if (live.status === "absorbed") {
|
|
878
|
+
skippedAbsorbed++;
|
|
879
|
+
scannedRowidHighwater = Math.max(scannedRowidHighwater, live.__rowid);
|
|
880
|
+
if (!cursorHold)
|
|
881
|
+
cursor = live.__rowid;
|
|
882
|
+
persistCursor();
|
|
883
|
+
continue;
|
|
681
884
|
}
|
|
885
|
+
const candidate = live;
|
|
682
886
|
scanned++;
|
|
887
|
+
// The high-water advances as candidates are ENTERED — even when the
|
|
888
|
+
// iteration's work later defers (it is the SCAN mark, never held back).
|
|
889
|
+
scannedRowidHighwater = Math.max(scannedRowidHighwater, candidate.__rowid);
|
|
890
|
+
// #439 per-iteration confirmed work, applied at the boundary below.
|
|
891
|
+
const iterMerges = [];
|
|
892
|
+
const iterGroups = new Map();
|
|
893
|
+
// This iteration's confirmed work could not land (deadline at the
|
|
894
|
+
// boundary, backup failure, rewrite refusal/infra error): the cursor
|
|
895
|
+
// holds below this candidate.
|
|
896
|
+
let boundaryHold = false;
|
|
897
|
+
// Stop the scan AFTER the boundary (budget stop / rewrite-infra
|
|
898
|
+
// deferral / backup failure): further verdicts could not land anyway.
|
|
899
|
+
let stopScan = false;
|
|
683
900
|
// ---- AC7: verify incoming explicit marks (corrected_by/superseded_by
|
|
684
901
|
// links targeting this candidate) before they can join a rewrite group.
|
|
685
902
|
// #401: only OPERATOR marks are verified — applyExplicitMark writes
|
|
@@ -709,14 +926,17 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
709
926
|
}
|
|
710
927
|
const { verdict, usage } = await classifyPair(llm, target.content, candidate.content);
|
|
711
928
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
|
|
712
|
-
callsUsed++; // #401
|
|
713
929
|
pairsEvaluated++;
|
|
930
|
+
if (candidate.__rowid <= prevScannedRowid)
|
|
931
|
+
pairsReevaluated++;
|
|
932
|
+
else
|
|
933
|
+
pairsNew++;
|
|
714
934
|
if (!verdict) {
|
|
715
935
|
skippedInfra++; // mark retained; the neighborhood is revisited via newer candidacies
|
|
716
936
|
continue;
|
|
717
937
|
}
|
|
718
938
|
if (verdict.action === "corrects" && verdict.confidence >= rewriteMinConfidence && isFactShapedTarget(target)) {
|
|
719
|
-
addTrigger(target, candidate, verdict.confidence, null, true);
|
|
939
|
+
addTrigger(iterGroups, target, candidate, verdict.confidence, null, true);
|
|
720
940
|
explicitVerified++;
|
|
721
941
|
}
|
|
722
942
|
else {
|
|
@@ -729,79 +949,179 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
729
949
|
}
|
|
730
950
|
}
|
|
731
951
|
if (markBudgetStop) {
|
|
732
|
-
|
|
733
|
-
|
|
952
|
+
// #401: this candidate's remaining marks re-verify next run — the
|
|
953
|
+
// cursor HOLDS below it (boundaryHold), and #439 still runs the
|
|
954
|
+
// boundary: marks confirmed before the stop apply, exactly as they
|
|
955
|
+
// did when the rewrite phase was post-scan.
|
|
956
|
+
stopScan = true;
|
|
957
|
+
boundaryHold = true;
|
|
734
958
|
}
|
|
735
959
|
}
|
|
736
|
-
// ----
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
960
|
+
// ---- #393 B: scout shape call — ONE classify-tier call per candidate,
|
|
961
|
+
// budget-metered under the same stage label as verdicts. Flags whether
|
|
962
|
+
// this memory corrects/retracts/supersedes something previously recorded;
|
|
963
|
+
// non-corrections stop here (zero follow-up). A parse/infra failure skips
|
|
964
|
+
// the scout source for this memory only (fail-soft — the similarity
|
|
965
|
+
// source below still runs) and counts skipped_infra, the verdict-skip
|
|
966
|
+
// discipline. Dry-run skips the call entirely: it is LLM work, and
|
|
967
|
+
// dry-runs make zero LLM calls (the scout counters read 0 there).
|
|
968
|
+
let scoutReferences = null;
|
|
969
|
+
if (!dryRun && !stopScan) {
|
|
970
|
+
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
971
|
+
// Shape call refused — this candidate re-scouts next run: the cursor
|
|
972
|
+
// HOLDS below it (the candidate was entered but not considered).
|
|
973
|
+
stopScan = true;
|
|
974
|
+
boundaryHold = true;
|
|
975
|
+
}
|
|
976
|
+
else {
|
|
977
|
+
scoutScanned++;
|
|
978
|
+
try {
|
|
979
|
+
const r = await llm.complete(buildScoutShapePrompt(candidate.content));
|
|
980
|
+
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
981
|
+
const shape = parseScoutShape(r.text);
|
|
982
|
+
if (!shape) {
|
|
983
|
+
skippedInfra++;
|
|
984
|
+
}
|
|
985
|
+
else if (shape.correction) {
|
|
986
|
+
scoutCorrectionShaped++;
|
|
987
|
+
scoutReferences = shape.references;
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
catch {
|
|
991
|
+
skippedInfra++;
|
|
992
|
+
}
|
|
993
|
+
}
|
|
740
994
|
}
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
995
|
+
// ---- AC2: detection pairs against older neighbors — TWO sources (#393 B):
|
|
996
|
+
// the KNN similarity source (floor-gated, the pre-B baseline) and, for
|
|
997
|
+
// correction-shaped candidates, the scout's FTS source (the referenced
|
|
998
|
+
// claim's terms → corpus search; NO similarity gate — cosine is a ranker,
|
|
999
|
+
// never a blocker). Merged + deduped by neighbor id: a pair found by both
|
|
1000
|
+
// sources is judged ONCE, as similarity (with band attribution).
|
|
1001
|
+
let discoveryFailed = false;
|
|
1002
|
+
let neighbors = [];
|
|
1003
|
+
if (!stopScan) {
|
|
1004
|
+
try {
|
|
1005
|
+
neighbors = await findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarity);
|
|
1006
|
+
}
|
|
1007
|
+
catch (err) {
|
|
1008
|
+
console.warn(`[hicortex] reconsolidation: discovery failed for ${candidate.id.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1009
|
+
// #439: skip this candidate's neighbor judgments but still run the
|
|
1010
|
+
// boundary below — marks confirmed earlier this iteration apply
|
|
1011
|
+
// (they did when the phases were post-scan).
|
|
1012
|
+
discoveryFailed = true;
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
const neighborEntries = neighbors.map((n) => ({
|
|
1016
|
+
mem: n,
|
|
1017
|
+
cosine: (0, retrieval_js_1.l2ToCosine)(n.distance),
|
|
1018
|
+
source: "similarity",
|
|
1019
|
+
}));
|
|
1020
|
+
if (!discoveryFailed && scoutReferences !== null) {
|
|
1021
|
+
const candidateVec = storage.getStoredEmbedding(db, candidate.id);
|
|
1022
|
+
const knnById = new Map(neighborEntries.map((e) => [e.mem.id, e]));
|
|
1023
|
+
for (const entry of findScoutNeighbors(db, candidate, scoutReferences, candidateVec)) {
|
|
1024
|
+
const knn = knnById.get(entry.mem.id);
|
|
1025
|
+
if (knn) {
|
|
1026
|
+
// Both sources found the pair. Below the ceiling the KNN entry
|
|
1027
|
+
// would be judged anyway — similarity keeps it (band attribution,
|
|
1028
|
+
// judged once). At/above the ceiling the similarity entry would be
|
|
1029
|
+
// ceiling-skipped (zone territory, never judged) — yet the zone now
|
|
1030
|
+
// runs AFTER the scan, and a genuine conflict at >=0.92 is exactly
|
|
1031
|
+
// the pair it would blend with no judge in the loop (guard-C's
|
|
1032
|
+
// harm). Re-tag the entry to the scout source so the pair IS
|
|
1033
|
+
// judged: the scout's no-gate exemption applies, a `conflicts`
|
|
1034
|
+
// verdict can plant the guard link, and the zone's own guard then
|
|
1035
|
+
// refuses the cluster in this same run.
|
|
1036
|
+
if (knn.cosine >= autoMergeThreshold)
|
|
1037
|
+
knn.source = "scout";
|
|
1038
|
+
continue;
|
|
1039
|
+
}
|
|
1040
|
+
neighborEntries.push(entry);
|
|
1041
|
+
}
|
|
746
1042
|
}
|
|
747
|
-
for (const
|
|
748
|
-
pairsDiscovered++; //
|
|
749
|
-
if (
|
|
1043
|
+
for (const entry of neighborEntries) {
|
|
1044
|
+
pairsDiscovered++; // gate discovery (#394), both sources, before any skip/judgment
|
|
1045
|
+
if (entry.source === "scout")
|
|
1046
|
+
scoutCandidatesFound++;
|
|
1047
|
+
if (alreadyResolutionLinked(db, entry.mem.id, candidate.id)) {
|
|
750
1048
|
skippedIdempotent++;
|
|
751
1049
|
continue;
|
|
752
1050
|
}
|
|
753
1051
|
pairsDiscoveredUnlinked++; // still unlinked — the actionable candidate
|
|
754
|
-
|
|
755
|
-
//
|
|
756
|
-
//
|
|
757
|
-
|
|
758
|
-
|
|
1052
|
+
const pairCosine = entry.cosine;
|
|
1053
|
+
// #392: SIMILARITY-source pairs at/above the ceiling belong to the
|
|
1054
|
+
// deterministic zone — counted here, never LLM-judged (the zone merges
|
|
1055
|
+
// them at stage end or defers them to a later run; re-detection is
|
|
1056
|
+
// structural, not cursor-based). Scout pairs are exempt (#393 B):
|
|
1057
|
+
// cosine never blocks this source — guard-C's re-tag above relies on
|
|
1058
|
+
// it, and a judged merge re-passes the same metadata/conflict rails
|
|
1059
|
+
// the zone enforces.
|
|
1060
|
+
if (entry.source === "similarity" && pairCosine >= autoMergeThreshold) {
|
|
759
1061
|
skippedAboveCeiling++;
|
|
760
1062
|
continue;
|
|
761
1063
|
}
|
|
762
1064
|
if (dryRun)
|
|
763
1065
|
continue; // preview only — no LLM call, no write
|
|
764
|
-
// #
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
callCapStopped = maxCalls > 0 && callsUsed >= maxCalls;
|
|
768
|
-
persistCursor(); // cursor still points at the last fully-considered candidate
|
|
1066
|
+
// #405: the ONE run budget's refusal is the only call cap.
|
|
1067
|
+
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
1068
|
+
stopScan = true; // the boundary below still applies what this candidate already confirmed
|
|
769
1069
|
break;
|
|
770
1070
|
}
|
|
771
|
-
const { verdict, usage } = await classifyPair(llm,
|
|
1071
|
+
const { verdict, usage } = await classifyPair(llm, entry.mem.content, candidate.content);
|
|
772
1072
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
|
|
773
|
-
callsUsed++; // #401
|
|
774
1073
|
pairsEvaluated++;
|
|
1074
|
+
if (candidate.__rowid <= prevScannedRowid)
|
|
1075
|
+
pairsReevaluated++;
|
|
1076
|
+
else
|
|
1077
|
+
pairsNew++;
|
|
775
1078
|
if (!verdict) {
|
|
776
1079
|
skippedInfra++;
|
|
777
1080
|
continue;
|
|
778
1081
|
}
|
|
779
|
-
|
|
780
|
-
//
|
|
781
|
-
//
|
|
1082
|
+
// Bands stay similarity-source-only (refine Q2 ruling): they are the
|
|
1083
|
+
// calibration evidence for the floor/ceiling boundaries, and scout
|
|
1084
|
+
// pairs reach them through a different, cosine-blind door.
|
|
1085
|
+
if (entry.source === "similarity")
|
|
1086
|
+
recordBand(pairCosine, verdict.action, verdict.confidence);
|
|
1087
|
+
// #392/#439: a merge verdict queues for THIS iteration's boundary —
|
|
1088
|
+
// no link, no write here. Below the confidence gate BOTH memories stay
|
|
782
1089
|
// live: a weak mark is recoverable, and there is nothing to mark for a
|
|
783
1090
|
// duplicate — keeping both is the recoverable outcome.
|
|
784
1091
|
if (verdict.action === "merge") {
|
|
785
1092
|
if (verdict.confidence < rewriteMinConfidence) {
|
|
786
1093
|
mergeBelowGate++;
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
1094
|
+
if (entry.source === "similarity") {
|
|
1095
|
+
const band = bandForCosine(bands, pairCosine);
|
|
1096
|
+
if (band) {
|
|
1097
|
+
const stat = runBands.get(band.label) ?? emptyBandStat();
|
|
1098
|
+
stat.merge_below_gate++;
|
|
1099
|
+
runBands.set(band.label, stat);
|
|
1100
|
+
}
|
|
792
1101
|
}
|
|
793
1102
|
}
|
|
794
1103
|
else {
|
|
795
|
-
|
|
796
|
-
notePendingRowid(candidate.__rowid); // orphan floor — merge unapplied until the merge phase
|
|
1104
|
+
iterMerges.push({ oldId: entry.mem.id, newId: candidate.id, candidateRowid: candidate.__rowid });
|
|
797
1105
|
}
|
|
798
1106
|
continue;
|
|
799
1107
|
}
|
|
800
1108
|
if (verdict.action === "supersedes") {
|
|
801
|
-
markLink(
|
|
802
|
-
storage.updateMemory(db,
|
|
1109
|
+
markLink(entry.mem.id, candidate.id, "superseded_by", pairCosine);
|
|
1110
|
+
storage.updateMemory(db, entry.mem.id, { status: "superseded" });
|
|
803
1111
|
markedSuperseded++;
|
|
804
|
-
console.log(`[hicortex] Reconsolidation: ${
|
|
1112
|
+
console.log(`[hicortex] Reconsolidation: ${entry.mem.id.slice(0, 8)} superseded_by ${candidate.id.slice(0, 8)} (mark-only)`);
|
|
1113
|
+
continue;
|
|
1114
|
+
}
|
|
1115
|
+
// #393 guard-C: a genuine conflict — link ONLY. No status change on
|
|
1116
|
+
// either memory (both stay live so the consumer sees both truths), no
|
|
1117
|
+
// rewrite, no merge queue; the link is the guard both merge paths
|
|
1118
|
+
// consult. Ungated like the other mark actions (a weak flag is
|
|
1119
|
+
// recoverable; a weak merge is not).
|
|
1120
|
+
if (verdict.action === "conflicts") {
|
|
1121
|
+
markLink(entry.mem.id, candidate.id, "conflicts", pairCosine);
|
|
1122
|
+
conflictFlagged++;
|
|
1123
|
+
console.log(`[hicortex] Reconsolidation: ${entry.mem.id.slice(0, 8)} conflicts ${candidate.id.slice(0, 8)} ` +
|
|
1124
|
+
`(flag-only) — both kept live, never merged`);
|
|
805
1125
|
continue;
|
|
806
1126
|
}
|
|
807
1127
|
if (verdict.action === "corrects") {
|
|
@@ -810,104 +1130,319 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
810
1130
|
// Below the gate: mark-only, never rewrite. The
|
|
811
1131
|
// trigger stays live — it is the only carrier of the correction.
|
|
812
1132
|
belowGate++;
|
|
813
|
-
markLink(
|
|
814
|
-
storage.updateMemory(db,
|
|
1133
|
+
markLink(entry.mem.id, candidate.id, "corrected_by", cosine);
|
|
1134
|
+
storage.updateMemory(db, entry.mem.id, { status: "retracted" });
|
|
815
1135
|
markedRetracted++;
|
|
816
1136
|
continue;
|
|
817
1137
|
}
|
|
818
|
-
if (!isFactShapedTarget(
|
|
1138
|
+
if (!isFactShapedTarget(entry.mem)) {
|
|
819
1139
|
// Decisions/plans/experiences are history, not error — mark only.
|
|
820
|
-
markLink(
|
|
821
|
-
storage.updateMemory(db,
|
|
1140
|
+
markLink(entry.mem.id, candidate.id, "corrected_by", cosine);
|
|
1141
|
+
storage.updateMemory(db, entry.mem.id, { status: "retracted" });
|
|
822
1142
|
markedRetracted++;
|
|
823
1143
|
continue;
|
|
824
1144
|
}
|
|
825
|
-
addTrigger(
|
|
1145
|
+
addTrigger(iterGroups, entry.mem, candidate, verdict.confidence, cosine, false);
|
|
826
1146
|
}
|
|
827
1147
|
// verdict "none" → nothing to do
|
|
828
1148
|
}
|
|
829
|
-
|
|
830
|
-
//
|
|
831
|
-
//
|
|
832
|
-
//
|
|
833
|
-
//
|
|
834
|
-
//
|
|
1149
|
+
// ---- #439 BOUNDARY: apply everything this candidate confirmed, NOW —
|
|
1150
|
+
// merges first, then rewrite groups (the order the old post-scan phases
|
|
1151
|
+
// used; preserves the existing tolerance where a merge loser that is
|
|
1152
|
+
// also a rewrite trigger stays absorbed while the rewrite still composes
|
|
1153
|
+
// its content). Each application is its own transaction
|
|
1154
|
+
// (mergeMemoryIds / applyRewriteGroup — group-internal atomicity
|
|
1155
|
+
// preserved); a busy capture lock defers merges to the same-run retry
|
|
1156
|
+
// list, everything else defers by holding the cursor below this
|
|
1157
|
+
// candidate (bounded to ONE candidate's pairs).
|
|
1158
|
+
if (!dryRun) {
|
|
1159
|
+
// Earlier lock-busy survivors retry FIRST (oldest verdicts land
|
|
1160
|
+
// first), ahead of this candidate's fresh confirmations.
|
|
1161
|
+
const mergeBatch = [...retryMerges.splice(0, retryMerges.length), ...iterMerges];
|
|
1162
|
+
if (mergeBatch.length > 0) {
|
|
1163
|
+
if (deadlineHit()) {
|
|
1164
|
+
deadlineStopped = true;
|
|
1165
|
+
// Fix round (#440 review, finding 1): never DROP the batch — it can
|
|
1166
|
+
// begin with lock-busy survivors contributed by EARLIER candidates
|
|
1167
|
+
// that the cursor has already passed. Re-queue for the final drain;
|
|
1168
|
+
// its hold-below-earliest-contributor (and the persist clamp) keeps
|
|
1169
|
+
// every un-applied pair re-detectable next run.
|
|
1170
|
+
retryMerges.push(...mergeBatch);
|
|
1171
|
+
boundaryHold = true;
|
|
1172
|
+
console.log(`[hicortex] Reconsolidation: ${mergeBatch.length} confirmed merge(s) re-queued for the final drain — run deadline reached`);
|
|
1173
|
+
}
|
|
1174
|
+
else {
|
|
1175
|
+
const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
|
|
1176
|
+
const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
|
|
1177
|
+
if (!release) {
|
|
1178
|
+
// Busy capture run — the batch rides the same-run retry list:
|
|
1179
|
+
// retried at the next boundary and once in the final drain. Not
|
|
1180
|
+
// a cursor hold yet (the lock may clear this run).
|
|
1181
|
+
retryMerges.push(...mergeBatch);
|
|
1182
|
+
console.warn(`[hicortex] Reconsolidation: capture lock busy — ${mergeBatch.length} confirmed merge(s) deferred to a retry this run`);
|
|
1183
|
+
}
|
|
1184
|
+
else {
|
|
1185
|
+
try {
|
|
1186
|
+
let backupOk = true;
|
|
1187
|
+
if (!mergeWindowBackedUp) {
|
|
1188
|
+
try {
|
|
1189
|
+
await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
|
|
1190
|
+
mergeWindowBackedUp = true;
|
|
1191
|
+
}
|
|
1192
|
+
catch (err) {
|
|
1193
|
+
backupOk = false;
|
|
1194
|
+
console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
|
|
1195
|
+
`(${err instanceof Error ? err.message : String(err)}) — ${mergeBatch.length} merge(s) deferred`);
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
if (!backupOk) {
|
|
1199
|
+
// Verdicts that cannot land must not keep being paid: stop
|
|
1200
|
+
// the scan. The batch re-queues for the final drain — a
|
|
1201
|
+
// TRANSIENT backup failure can still recover there; a
|
|
1202
|
+
// persistent one ends with the drain holding the cursor
|
|
1203
|
+
// below the earliest contributor (never just this candidate,
|
|
1204
|
+
// when retry survivors ride the batch).
|
|
1205
|
+
retryMerges.push(...mergeBatch);
|
|
1206
|
+
boundaryHold = true;
|
|
1207
|
+
stopScan = true;
|
|
1208
|
+
}
|
|
1209
|
+
else {
|
|
1210
|
+
for (let i = 0; i < mergeBatch.length; i++) {
|
|
1211
|
+
const pair = mergeBatch[i];
|
|
1212
|
+
// #405: the deadline stop-check between local merge
|
|
1213
|
+
// transactions — a safe boundary; deferred pairs hold the
|
|
1214
|
+
// cursor below this candidate and retry next run.
|
|
1215
|
+
if (deadlineHit()) {
|
|
1216
|
+
deadlineStopped = true;
|
|
1217
|
+
// Fix round (#440 review, finding 1): the un-applied tail
|
|
1218
|
+
// re-queues (it can contain earlier candidates' retry
|
|
1219
|
+
// survivors) — the final drain applies or holds it.
|
|
1220
|
+
retryMerges.push(...mergeBatch.slice(i));
|
|
1221
|
+
boundaryHold = true;
|
|
1222
|
+
console.log(`[hicortex] Reconsolidation: ${mergeBatch.length - i} confirmed merge(s) re-queued for the final drain — run deadline reached`);
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
|
|
1226
|
+
if (result.ok) {
|
|
1227
|
+
mergePairsApplied++;
|
|
1228
|
+
console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1229
|
+
`into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
|
|
1230
|
+
}
|
|
1231
|
+
else if (result.reason === "metadata_mismatch") {
|
|
1232
|
+
skippedMetadataMismatch++;
|
|
1233
|
+
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1234
|
+
`skipped (metadata mismatch) — both kept`);
|
|
1235
|
+
}
|
|
1236
|
+
else if (result.reason === "conflict_linked") {
|
|
1237
|
+
// #393 guard-C: the pair is conflicts-linked (operator-planted
|
|
1238
|
+
// or a prior verdict) — never blended; the cursor advances,
|
|
1239
|
+
// this verdict was rendered.
|
|
1240
|
+
conflictSkippedJudged++;
|
|
1241
|
+
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1242
|
+
`skipped (conflict-flagged) — both kept`);
|
|
1243
|
+
}
|
|
1244
|
+
// "no_members": a member vanished/was absorbed since the
|
|
1245
|
+
// verdict — nothing to merge, nothing to hold; the cursor
|
|
1246
|
+
// advances past it.
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
finally {
|
|
1251
|
+
release();
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
if (iterGroups.size > 0) {
|
|
1257
|
+
// One rewrite call per group — the group's triggers are all THIS
|
|
1258
|
+
// candidate (#439: a target corrected by two different candidates
|
|
1259
|
+
// takes two sequential rewrites, one per boundary; the second call
|
|
1260
|
+
// composes the already-corrected story). A call that was never made
|
|
1261
|
+
// (budget/infra/deadline) defers the group — untouched, never
|
|
1262
|
+
// partially applied — and holds the cursor below this candidate.
|
|
1263
|
+
const contracts = new Map(); // null = contract failed
|
|
1264
|
+
let rewritesDeferred = false;
|
|
1265
|
+
for (const group of iterGroups.values()) {
|
|
1266
|
+
if (deadlineHit()) {
|
|
1267
|
+
deadlineStopped = true;
|
|
1268
|
+
rewritesDeferred = true;
|
|
1269
|
+
break;
|
|
1270
|
+
}
|
|
1271
|
+
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
1272
|
+
rewritesDeferred = true;
|
|
1273
|
+
stopScan = true;
|
|
1274
|
+
break;
|
|
1275
|
+
}
|
|
1276
|
+
const triggersArg = group.triggers.map((t) => ({ id: t.id, content: t.memory.content }));
|
|
1277
|
+
let contract = null;
|
|
1278
|
+
let infraError = false;
|
|
1279
|
+
try {
|
|
1280
|
+
const r = await llm.complete(buildRewritePrompt(group.target.content, triggersArg));
|
|
1281
|
+
contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
|
|
1282
|
+
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
1283
|
+
}
|
|
1284
|
+
catch {
|
|
1285
|
+
infraError = true;
|
|
1286
|
+
}
|
|
1287
|
+
if (infraError) {
|
|
1288
|
+
skippedInfra++;
|
|
1289
|
+
rewritesDeferred = true; // group NOT marked, NOT rewritten — retried next run
|
|
1290
|
+
stopScan = true; // verdicts past this hold could not advance the cursor anyway
|
|
1291
|
+
break;
|
|
1292
|
+
}
|
|
1293
|
+
contracts.set(group.targetId, contract);
|
|
1294
|
+
if (!contract)
|
|
1295
|
+
contractFailed++;
|
|
1296
|
+
}
|
|
1297
|
+
if (rewritesDeferred) {
|
|
1298
|
+
boundaryHold = true;
|
|
1299
|
+
}
|
|
1300
|
+
else {
|
|
1301
|
+
// Multi-target keep rule WITHIN the boundary: the shared trigger is
|
|
1302
|
+
// the current candidate, so every group it touches resolves here —
|
|
1303
|
+
// absorbed only if EVERY contract says absorb (any keep keeps).
|
|
1304
|
+
const finalOutcome = new Map();
|
|
1305
|
+
for (const contract of contracts.values()) {
|
|
1306
|
+
if (!contract)
|
|
1307
|
+
continue;
|
|
1308
|
+
for (const t of contract.triggers) {
|
|
1309
|
+
if (t.disposition === "keep" || finalOutcome.get(t.id) === "keep")
|
|
1310
|
+
finalOutcome.set(t.id, "keep");
|
|
1311
|
+
else
|
|
1312
|
+
finalOutcome.set(t.id, "absorb");
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
// Counted from APPLIED groups only (a deferred group's dispositions
|
|
1316
|
+
// never took effect); a trigger in several applied groups counts once.
|
|
1317
|
+
const appliedOutcome = new Map();
|
|
1318
|
+
for (const group of iterGroups.values()) {
|
|
1319
|
+
const contract = contracts.get(group.targetId);
|
|
1320
|
+
if (contract === undefined)
|
|
1321
|
+
continue; // pending group — untouched this run
|
|
1322
|
+
if (contract === null) {
|
|
1323
|
+
// Failed rewrite contract → whole group mark-only, never a partial
|
|
1324
|
+
// apply. Content untouched, NO trigger absorbed.
|
|
1325
|
+
try {
|
|
1326
|
+
applyMarkOnlyGroup(db, group);
|
|
1327
|
+
}
|
|
1328
|
+
catch (err) {
|
|
1329
|
+
console.warn(`[hicortex] reconsolidation: mark-only fallback failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1330
|
+
skippedInfra++;
|
|
1331
|
+
boundaryHold = true; // retried next run
|
|
1332
|
+
continue;
|
|
1333
|
+
}
|
|
1334
|
+
markedRetracted++;
|
|
1335
|
+
console.log(`[hicortex] Reconsolidation: rewrite contract failed for ${group.targetId.slice(0, 8)} — group degraded to mark-only`);
|
|
1336
|
+
continue;
|
|
1337
|
+
}
|
|
1338
|
+
let applied = false;
|
|
1339
|
+
try {
|
|
1340
|
+
applied = await applyRewriteGroup(db, group, contract, finalOutcome, embedFn);
|
|
1341
|
+
}
|
|
1342
|
+
catch (err) {
|
|
1343
|
+
console.warn(`[hicortex] reconsolidation: rewrite apply failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1344
|
+
}
|
|
1345
|
+
if (!applied) {
|
|
1346
|
+
skippedInfra++; // defensive absorbed-target guard, or an apply error — retry next run
|
|
1347
|
+
boundaryHold = true;
|
|
1348
|
+
continue;
|
|
1349
|
+
}
|
|
1350
|
+
rewritten++;
|
|
1351
|
+
for (const t of contract.triggers) {
|
|
1352
|
+
const outcome = finalOutcome.get(t.id) ?? "keep";
|
|
1353
|
+
if (outcome === "keep" || appliedOutcome.get(t.id) === "keep")
|
|
1354
|
+
appliedOutcome.set(t.id, "keep");
|
|
1355
|
+
else
|
|
1356
|
+
appliedOutcome.set(t.id, "absorb");
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
for (const outcome of appliedOutcome.values()) {
|
|
1360
|
+
if (outcome === "absorb")
|
|
1361
|
+
absorbed++;
|
|
1362
|
+
else
|
|
1363
|
+
keptLinked++;
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
// #439 cursor advance: past this candidate ONLY when its confirmed work
|
|
1369
|
+
// landed (or was refused-with-verdict-rendered). Once anything deferred,
|
|
1370
|
+
// the latch holds the cursor below that candidate for the rest of the
|
|
1371
|
+
// run — a later iteration must never advance past an earlier hold.
|
|
1372
|
+
if (boundaryHold)
|
|
1373
|
+
cursorHold = true;
|
|
1374
|
+
if (!cursorHold)
|
|
1375
|
+
cursor = candidate.__rowid;
|
|
1376
|
+
// #402/#439: persist after EVERY candidate — AFTER the boundary apply,
|
|
1377
|
+
// so the checkpoint only ever crosses candidates whose work landed. A
|
|
1378
|
+
// SIGKILL between persists re-detects at most the in-flight candidate.
|
|
835
1379
|
persistCursor();
|
|
1380
|
+
if (stopScan || deadlineStopped)
|
|
1381
|
+
break;
|
|
836
1382
|
}
|
|
837
1383
|
if (deadlineStopped) {
|
|
838
|
-
console.log(`[hicortex] Reconsolidation:
|
|
839
|
-
`scan stopped at cursor ${cursor}; the next run resumes from there`);
|
|
840
|
-
}
|
|
841
|
-
else if (callCapStopped) {
|
|
842
|
-
console.log(`[hicortex] Reconsolidation: per-run call cap reached (reconsolidationMaxCalls) — ` +
|
|
1384
|
+
console.log(`[hicortex] Reconsolidation: run deadline reached (nightlyTimeBudgetMinutes) — ` +
|
|
843
1385
|
`scan stopped at cursor ${cursor}; the next run resumes from there`);
|
|
844
1386
|
}
|
|
845
|
-
// ---- #
|
|
846
|
-
//
|
|
847
|
-
//
|
|
848
|
-
//
|
|
849
|
-
//
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
if (!dryRun && queuedMerges.length > 0) {
|
|
858
|
-
const holdQueued = (from) => {
|
|
859
|
-
for (let i = from; i < queuedMerges.length; i++) {
|
|
860
|
-
pendingMinRowid =
|
|
861
|
-
pendingMinRowid === null
|
|
862
|
-
? queuedMerges[i].candidateRowid
|
|
863
|
-
: Math.min(pendingMinRowid, queuedMerges[i].candidateRowid);
|
|
864
|
-
}
|
|
1387
|
+
// ---- #439 final drain: lock-busy merge survivors get ONE more attempt
|
|
1388
|
+
// right after the scan (the retry list is same-run only — everything else
|
|
1389
|
+
// applied at its boundary). Still busy (or the deadline/backup refuses) →
|
|
1390
|
+
// the pairs stay un-applied, counted, and the cursor holds below the
|
|
1391
|
+
// earliest contributing candidate (dup-over-loss; logged).
|
|
1392
|
+
if (!dryRun && retryMerges.length > 0) {
|
|
1393
|
+
const batch = retryMerges.splice(0, retryMerges.length);
|
|
1394
|
+
// Hold below the earliest contributor of the UN-APPLIED tail only (fix
|
|
1395
|
+
// round, minor review note: the old whole-batch min over-held past pairs
|
|
1396
|
+
// that had just applied in the same loop).
|
|
1397
|
+
const holdBelow = (fromIndex) => {
|
|
1398
|
+
cursor = Math.min(cursor, Math.min(...batch.slice(fromIndex).map((p) => p.candidateRowid)) - 1);
|
|
865
1399
|
};
|
|
866
|
-
if (
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
console.log(`[hicortex] Reconsolidation: ${
|
|
871
|
-
`dedupNightlyMaxMerges is 0 (merge machinery disabled)`);
|
|
872
|
-
}
|
|
873
|
-
else if (mergeOpsRemaining <= 0) {
|
|
874
|
-
mergePairsDeferred = queuedMerges.length;
|
|
875
|
-
holdQueued(0); // zone consumed the whole cap — retry next run
|
|
876
|
-
console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred — ` +
|
|
877
|
-
`dedupNightlyMaxMerges exhausted by the deterministic zone`);
|
|
1400
|
+
if (deadlineHit()) {
|
|
1401
|
+
deadlineStopped = true;
|
|
1402
|
+
mergePairsDeferred += batch.length;
|
|
1403
|
+
holdBelow(0);
|
|
1404
|
+
console.log(`[hicortex] Reconsolidation: ${batch.length} confirmed merge(s) deferred — run deadline reached`);
|
|
878
1405
|
}
|
|
879
1406
|
else {
|
|
880
1407
|
const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
|
|
881
1408
|
const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
|
|
882
1409
|
if (!release) {
|
|
883
|
-
mergePairsDeferred
|
|
884
|
-
|
|
885
|
-
console.warn(`[hicortex] Reconsolidation: capture lock busy
|
|
1410
|
+
mergePairsDeferred += batch.length;
|
|
1411
|
+
holdBelow(0);
|
|
1412
|
+
console.warn(`[hicortex] Reconsolidation: capture lock busy at the final drain — ` +
|
|
1413
|
+
`${batch.length} confirmed merge(s) deferred to next run`);
|
|
886
1414
|
}
|
|
887
1415
|
else {
|
|
888
1416
|
try {
|
|
889
1417
|
let backupOk = true;
|
|
890
|
-
|
|
891
|
-
|
|
1418
|
+
if (!mergeWindowBackedUp) {
|
|
1419
|
+
try {
|
|
1420
|
+
await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
|
|
1421
|
+
mergeWindowBackedUp = true;
|
|
1422
|
+
}
|
|
1423
|
+
catch (err) {
|
|
1424
|
+
backupOk = false;
|
|
1425
|
+
console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
|
|
1426
|
+
`(${err instanceof Error ? err.message : String(err)}) — ${batch.length} merge(s) deferred`);
|
|
1427
|
+
}
|
|
892
1428
|
}
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
`(${err instanceof Error ? err.message : String(err)}) — ${queuedMerges.length} merge(s) deferred`);
|
|
1429
|
+
if (!backupOk) {
|
|
1430
|
+
mergePairsDeferred += batch.length;
|
|
1431
|
+
holdBelow(0);
|
|
897
1432
|
}
|
|
898
|
-
|
|
899
|
-
for (let i = 0; i <
|
|
900
|
-
const pair =
|
|
901
|
-
if (
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
1433
|
+
else {
|
|
1434
|
+
for (let i = 0; i < batch.length; i++) {
|
|
1435
|
+
const pair = batch[i];
|
|
1436
|
+
if (deadlineHit()) {
|
|
1437
|
+
deadlineStopped = true;
|
|
1438
|
+
mergePairsDeferred += batch.length - i;
|
|
1439
|
+
holdBelow(i);
|
|
1440
|
+
console.log(`[hicortex] Reconsolidation: ${batch.length - i} confirmed merge(s) deferred — run deadline reached`);
|
|
905
1441
|
break;
|
|
906
1442
|
}
|
|
907
1443
|
const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
|
|
908
1444
|
if (result.ok) {
|
|
909
1445
|
mergePairsApplied++;
|
|
910
|
-
mergeOpsRemaining--;
|
|
911
1446
|
console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
912
1447
|
`into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
|
|
913
1448
|
}
|
|
@@ -916,15 +1451,15 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
916
1451
|
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
917
1452
|
`skipped (metadata mismatch) — both kept`);
|
|
918
1453
|
}
|
|
1454
|
+
else if (result.reason === "conflict_linked") {
|
|
1455
|
+
conflictSkippedJudged++;
|
|
1456
|
+
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1457
|
+
`skipped (conflict-flagged) — both kept`);
|
|
1458
|
+
}
|
|
919
1459
|
// "no_members": a member vanished/was absorbed since the
|
|
920
|
-
// verdict — nothing to merge, nothing to hold
|
|
921
|
-
// advances past it.
|
|
1460
|
+
// verdict — nothing to merge, nothing to hold.
|
|
922
1461
|
}
|
|
923
1462
|
}
|
|
924
|
-
else {
|
|
925
|
-
mergePairsDeferred = queuedMerges.length;
|
|
926
|
-
holdQueued(0);
|
|
927
|
-
}
|
|
928
1463
|
}
|
|
929
1464
|
finally {
|
|
930
1465
|
release();
|
|
@@ -932,148 +1467,34 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
932
1467
|
}
|
|
933
1468
|
}
|
|
934
1469
|
}
|
|
935
|
-
// ----
|
|
936
|
-
//
|
|
937
|
-
//
|
|
938
|
-
//
|
|
939
|
-
//
|
|
940
|
-
|
|
941
|
-
//
|
|
942
|
-
//
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
for (const group of groups.values()) {
|
|
956
|
-
// #401: the bounds stop the rewrite phase too — a group whose rewrite
|
|
957
|
-
// call was never made is left untouched and holds the cursor (never
|
|
958
|
-
// partially applied), exactly like the budget-exhausted path below.
|
|
959
|
-
if (deadlineHit()) {
|
|
960
|
-
deadlineStopped = true;
|
|
961
|
-
deferFrom(group.targetId);
|
|
962
|
-
break;
|
|
963
|
-
}
|
|
964
|
-
if (maxCalls > 0 && callsUsed >= maxCalls) {
|
|
965
|
-
callCapStopped = true;
|
|
966
|
-
deferFrom(group.targetId);
|
|
967
|
-
break;
|
|
968
|
-
}
|
|
969
|
-
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
970
|
-
deferFrom(group.targetId);
|
|
971
|
-
break;
|
|
972
|
-
}
|
|
973
|
-
const triggersArg = group.triggers.map((t) => ({ id: t.id, content: t.memory.content }));
|
|
974
|
-
let contract = null;
|
|
975
|
-
let infraError = false;
|
|
976
|
-
try {
|
|
977
|
-
const r = await llm.completeClassify(buildRewritePrompt(group.target.content, triggersArg));
|
|
978
|
-
contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
|
|
979
|
-
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
980
|
-
callsUsed++; // #401: rewrite contracts count toward the stage call cap
|
|
981
|
-
}
|
|
982
|
-
catch {
|
|
983
|
-
infraError = true;
|
|
984
|
-
}
|
|
985
|
-
if (infraError) {
|
|
986
|
-
skippedInfra++;
|
|
987
|
-
deferFrom(group.targetId); // group NOT marked, NOT rewritten — retried next run
|
|
988
|
-
break;
|
|
989
|
-
}
|
|
990
|
-
contracts.set(group.targetId, contract);
|
|
991
|
-
if (!contract)
|
|
992
|
-
contractFailed++;
|
|
993
|
-
}
|
|
994
|
-
// R2: final per-trigger disposition — a trigger in multiple groups is
|
|
995
|
-
// absorbed only if EVERY disposition says absorb (any keep keeps it).
|
|
996
|
-
const finalOutcome = new Map();
|
|
997
|
-
for (const contract of contracts.values()) {
|
|
998
|
-
if (!contract)
|
|
999
|
-
continue;
|
|
1000
|
-
for (const t of contract.triggers) {
|
|
1001
|
-
if (t.disposition === "keep" || finalOutcome.get(t.id) === "keep")
|
|
1002
|
-
finalOutcome.set(t.id, "keep");
|
|
1003
|
-
else
|
|
1004
|
-
finalOutcome.set(t.id, "absorb");
|
|
1005
|
-
}
|
|
1006
|
-
}
|
|
1007
|
-
// R3: apply (one transaction per group). An apply that fails mid-flight
|
|
1008
|
-
// (embed error, DB error) writes NOTHING (the transaction never ran) —
|
|
1009
|
-
// the group is deferred like a pending one so it retries next run.
|
|
1010
|
-
const appliedOutcome = new Map();
|
|
1011
|
-
const deferGroup = (group) => {
|
|
1012
|
-
for (const t of group.triggers) {
|
|
1013
|
-
pendingMinRowid = pendingMinRowid === null ? t.candidateRowid : Math.min(pendingMinRowid, t.candidateRowid);
|
|
1014
|
-
}
|
|
1015
|
-
};
|
|
1016
|
-
for (const group of groups.values()) {
|
|
1017
|
-
const contract = contracts.get(group.targetId);
|
|
1018
|
-
if (contract === undefined)
|
|
1019
|
-
continue; // pending group — untouched this run
|
|
1020
|
-
if (contract === null) {
|
|
1021
|
-
// Failed rewrite contract → whole group mark-only, never a partial
|
|
1022
|
-
// apply. Content untouched, NO trigger absorbed.
|
|
1023
|
-
try {
|
|
1024
|
-
applyMarkOnlyGroup(db, group);
|
|
1025
|
-
}
|
|
1026
|
-
catch (err) {
|
|
1027
|
-
console.warn(`[hicortex] reconsolidation: mark-only fallback failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1028
|
-
skippedInfra++;
|
|
1029
|
-
deferGroup(group);
|
|
1030
|
-
continue;
|
|
1031
|
-
}
|
|
1032
|
-
markedRetracted++;
|
|
1033
|
-
console.log(`[hicortex] Reconsolidation: rewrite contract failed for ${group.targetId.slice(0, 8)} — group degraded to mark-only`);
|
|
1034
|
-
continue;
|
|
1035
|
-
}
|
|
1036
|
-
let applied = false;
|
|
1037
|
-
try {
|
|
1038
|
-
applied = await applyRewriteGroup(db, group, contract, finalOutcome, embedFn);
|
|
1039
|
-
}
|
|
1040
|
-
catch (err) {
|
|
1041
|
-
console.warn(`[hicortex] reconsolidation: rewrite apply failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1042
|
-
}
|
|
1043
|
-
if (!applied) {
|
|
1044
|
-
skippedInfra++; // defensive absorbed-target guard, or an apply error — retry next run
|
|
1045
|
-
deferGroup(group);
|
|
1046
|
-
continue;
|
|
1047
|
-
}
|
|
1048
|
-
rewritten++;
|
|
1049
|
-
// Counted from APPLIED groups only (a deferred group's dispositions
|
|
1050
|
-
// never took effect); a trigger in several applied groups counts once.
|
|
1051
|
-
for (const t of contract.triggers) {
|
|
1052
|
-
const outcome = finalOutcome.get(t.id) ?? "keep";
|
|
1053
|
-
if (outcome === "keep" || appliedOutcome.get(t.id) === "keep")
|
|
1054
|
-
appliedOutcome.set(t.id, "keep");
|
|
1055
|
-
else
|
|
1056
|
-
appliedOutcome.set(t.id, "absorb");
|
|
1057
|
-
}
|
|
1058
|
-
}
|
|
1059
|
-
for (const outcome of appliedOutcome.values()) {
|
|
1060
|
-
if (outcome === "absorb")
|
|
1061
|
-
absorbed++;
|
|
1062
|
-
else
|
|
1063
|
-
keptLinked++;
|
|
1064
|
-
}
|
|
1065
|
-
}
|
|
1066
|
-
// Cursor hold: un-applied work (rewrite groups, confirmed merges) holds the
|
|
1067
|
-
// cursor BELOW its earliest contributing candidate so the pairs are
|
|
1068
|
-
// re-detected next run.
|
|
1069
|
-
if (pendingMinRowid !== null) {
|
|
1070
|
-
cursor = Math.min(cursor, pendingMinRowid - 1);
|
|
1071
|
-
}
|
|
1470
|
+
// ---- #393 guard-C zone reorder: the deterministic merge zone (pairs >=
|
|
1471
|
+
// the ceiling) runs LAST — after the scan (which includes every #439
|
|
1472
|
+
// boundary apply: judged merges + rewrites). Judgment outranks the
|
|
1473
|
+
// deterministic sweep: verdicts, marks, and binds land first, and the zone
|
|
1474
|
+
// merges only what no verdict claimed. With the zone first, a >=0.92
|
|
1475
|
+
// genuine-conflict pair was blended before the judge ever saw it
|
|
1476
|
+
// (canonical = oldest, the newer truth erased — the planted-eval harm);
|
|
1477
|
+
// running it last means a `conflicts` bind set by THIS run's scan guards
|
|
1478
|
+
// the SAME run's zone. LLM-free and budget-free — an LLM-less night still
|
|
1479
|
+
// drains duplicates (a deadline-deferred cluster re-detects next run at
|
|
1480
|
+
// zero token cost — content-based discovery, no cursor involvement). Its
|
|
1481
|
+
// own short lock window, pre-merge backup, and #405 deadline stop-check;
|
|
1482
|
+
// fail-soft, never a throw.
|
|
1483
|
+
const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
|
|
1484
|
+
stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
|
|
1485
|
+
threshold: autoMergeThreshold,
|
|
1486
|
+
dryRun,
|
|
1487
|
+
acquireLock: options.acquireLock,
|
|
1488
|
+
deadline,
|
|
1489
|
+
});
|
|
1072
1490
|
// Report snapshot: the deterministic band (from the zone's own numbers —
|
|
1073
1491
|
// losers are merge verdicts at confidence 1.0; the zone persists the
|
|
1074
1492
|
// cumulative copy itself) plus this run's judged bands.
|
|
1075
1493
|
const bandStats = {};
|
|
1076
|
-
|
|
1494
|
+
{
|
|
1495
|
+
// #405: recorded whenever the zone ran (the old max_merges>0 gate was a
|
|
1496
|
+
// 0=disabled switch — the switch is gone; a clean corpus records zeros,
|
|
1497
|
+
// same as the old default-config behavior).
|
|
1077
1498
|
const det = emptyBandStat();
|
|
1078
1499
|
det.pairs = merges.losers_merged;
|
|
1079
1500
|
det.merge = merges.losers_merged;
|
|
@@ -1086,10 +1507,15 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1086
1507
|
for (const [label, stat] of runBands)
|
|
1087
1508
|
bandStats[label] = stat;
|
|
1088
1509
|
if (!dryRun) {
|
|
1089
|
-
// #401: the authoritative FINAL
|
|
1090
|
-
//
|
|
1510
|
+
// #401/#439: the authoritative FINAL write — the mid-scan persists above
|
|
1511
|
+
// are checkpoints; this one applies the final-drain cursor hold (already
|
|
1512
|
+
// folded into `cursor`) and the scan high-water. The retry floor is
|
|
1513
|
+
// applied defensively too: the drain splices retryMerges empty on every
|
|
1514
|
+
// path, but a non-empty list here would mean a confirmed merge stranded
|
|
1515
|
+
// behind the cursor — clamp, never write past un-applied work.
|
|
1091
1516
|
(0, state_js_1.updateState)((s) => {
|
|
1092
|
-
s.reconsolidationCursor =
|
|
1517
|
+
s.reconsolidationCursor = clampedCursor();
|
|
1518
|
+
s.reconsolidationScannedRowid = Math.max(scannedRowidHighwater, s.reconsolidationScannedRowid ?? 0);
|
|
1093
1519
|
// Cumulative judged-band accumulation (#392) — the zone already
|
|
1094
1520
|
// persisted the deterministic band under its own label.
|
|
1095
1521
|
if (runBands.size > 0) {
|
|
@@ -1103,14 +1529,20 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1103
1529
|
}
|
|
1104
1530
|
}, stateDir);
|
|
1105
1531
|
}
|
|
1106
|
-
if (rows.length > 0 ||
|
|
1107
|
-
console.log(`[hicortex] Reconsolidation: ${scanned} scanned, ${pairsEvaluated} pairs evaluated
|
|
1532
|
+
if (rows.length > 0 || mergePairsApplied > 0 || mergeBelowGate > 0) {
|
|
1533
|
+
console.log(`[hicortex] Reconsolidation: ${scanned} scanned, ${pairsEvaluated} pairs evaluated ` +
|
|
1534
|
+
`(${pairsReevaluated} re-judged / ${pairsNew} new), ` +
|
|
1108
1535
|
`${rewritten} rewritten (${absorbed} triggers absorbed, ${keptLinked} kept), ` +
|
|
1109
|
-
`${mergePairsApplied} pair(s) merged
|
|
1536
|
+
`${mergePairsApplied} pair(s) merged (${mergePairsDeferred} deferred), ` +
|
|
1537
|
+
`${markedSuperseded} superseded, ` +
|
|
1110
1538
|
`${markedRetracted} retracted (${belowGate} below gate, ${mergeBelowGate} merge below gate, ` +
|
|
1111
1539
|
`${contractFailed} contract failed), ${skippedInfra} infra-skipped, ${skippedIdempotent} ` +
|
|
1112
|
-
`already-linked, ${
|
|
1113
|
-
`${
|
|
1540
|
+
`already-linked, ${skippedAbsorbed} absorbed-skip, ${skippedAboveCeiling} above ceiling, ` +
|
|
1541
|
+
`${explicitVerified} explicit verified, ` +
|
|
1542
|
+
`${explicitDivergent} explicit divergent, scout ${scoutScanned} scanned / ` +
|
|
1543
|
+
`${scoutCorrectionShaped} correction-shaped / ${scoutCandidatesFound} candidate pair(s), ` +
|
|
1544
|
+
`${conflictFlagged} conflict-flagged, ${conflictSkippedJudged + merges.skipped_conflict} conflict-skipped ` +
|
|
1545
|
+
`(cursor ${cursor})`);
|
|
1114
1546
|
}
|
|
1115
1547
|
return {
|
|
1116
1548
|
scanned,
|
|
@@ -1129,11 +1561,20 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1129
1561
|
explicit_verified: explicitVerified,
|
|
1130
1562
|
explicit_divergent: explicitDivergent,
|
|
1131
1563
|
cursor,
|
|
1564
|
+
pairs_reevaluated: pairsReevaluated,
|
|
1565
|
+
pairs_new: pairsNew,
|
|
1566
|
+
skipped_absorbed: skippedAbsorbed,
|
|
1567
|
+
merge_pairs_deferred: mergePairsDeferred,
|
|
1132
1568
|
merges,
|
|
1133
1569
|
merge_pairs_applied: mergePairsApplied,
|
|
1134
1570
|
merge_below_gate: mergeBelowGate,
|
|
1135
1571
|
skipped_above_ceiling: skippedAboveCeiling,
|
|
1136
1572
|
skipped_metadata_mismatch: skippedMetadataMismatch,
|
|
1573
|
+
conflict_flagged: conflictFlagged,
|
|
1574
|
+
conflict_skipped: conflictSkippedJudged + merges.skipped_conflict,
|
|
1575
|
+
scout_scanned: scoutScanned,
|
|
1576
|
+
scout_correction_shaped: scoutCorrectionShaped,
|
|
1577
|
+
scout_candidates_found: scoutCandidatesFound,
|
|
1137
1578
|
band_stats: bandStats,
|
|
1138
1579
|
};
|
|
1139
1580
|
}
|