@gamaze/hicortex 0.20.6 → 0.20.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -41
- package/dist/calibration.d.ts +174 -0
- package/dist/calibration.js +231 -0
- package/dist/capture.d.ts +15 -3
- package/dist/capture.js +10 -1
- package/dist/classify-domains.d.ts +6 -0
- package/dist/classify-domains.js +7 -1
- package/dist/cli.js +2 -3
- package/dist/config-read.d.ts +1 -1
- package/dist/config-read.js +96 -9
- package/dist/consolidate.d.ts +79 -68
- package/dist/consolidate.js +218 -174
- package/dist/dashboard.d.ts +4 -3
- package/dist/dedup.d.ts +34 -26
- package/dist/dedup.js +91 -57
- package/dist/distiller.js +1 -1
- package/dist/domain-classify.d.ts +7 -6
- package/dist/domain-classify.js +12 -10
- package/dist/eval/decay-eval.d.ts +3 -3
- package/dist/eval/decay-eval.js +4 -4
- package/dist/eval/planted-eval.d.ts +26 -0
- package/dist/eval/planted-eval.js +97 -0
- package/dist/eval/planted-fixtures.d.ts +107 -0
- package/dist/eval/planted-fixtures.js +283 -0
- package/dist/eval/planted-harness.d.ts +176 -0
- package/dist/eval/planted-harness.js +649 -0
- package/dist/index.js +4 -3
- package/dist/init.d.ts +9 -3
- package/dist/init.js +52 -9
- package/dist/llm.d.ts +43 -58
- package/dist/llm.js +87 -101
- package/dist/mcp-server.js +29 -29
- package/dist/nightly.js +105 -103
- package/dist/nofit.d.ts +4 -11
- package/dist/nofit.js +6 -23
- package/dist/recall-index.d.ts +30 -28
- package/dist/recall-index.js +21 -18
- package/dist/recall-registry.d.ts +2 -1
- package/dist/recall-registry.js +35 -1
- package/dist/reconsolidation.d.ts +124 -72
- package/dist/reconsolidation.js +390 -169
- package/dist/relink.js +3 -4
- package/dist/retrieval.d.ts +68 -35
- package/dist/retrieval.js +292 -104
- package/dist/run-deadline.d.ts +62 -0
- package/dist/run-deadline.js +73 -0
- package/dist/schema-prototypes.d.ts +3 -3
- package/dist/schema-prototypes.js +3 -3
- package/dist/state.d.ts +2 -3
- package/dist/storage.d.ts +16 -16
- package/dist/storage.js +62 -24
- package/dist/telemetry.d.ts +8 -7
- package/dist/token-budget.js +3 -4
- package/dist/type-classify.js +4 -4
- package/dist/types.d.ts +95 -155
- package/domains.example.json +4 -5
- package/hermes-plugin/hicortex/README.md +2 -2
- package/openclaw.plugin.json +1 -1
- package/package.json +2 -1
- package/pi-extension/hicortex/README.md +1 -1
- package/server.json +3 -3
package/dist/reconsolidation.js
CHANGED
|
@@ -16,11 +16,39 @@
|
|
|
16
16
|
* #392 — one zone system, ONE verdict per pair: below `correctionMinSimilarity`
|
|
17
17
|
* (floor, 0.75) pairs are not candidates; in [floor, `dedupAutoMergeThreshold`)
|
|
18
18
|
* (ceiling, 0.92) each unlinked pair gets ONE verdict call whose action is
|
|
19
|
-
* `merge` | `corrects` | `supersedes` | `none`; at/above the
|
|
20
|
-
* deterministic merge zone (dedup.ts runDeterministicMergeZone —
|
|
21
|
-
* budget-free) owns the pair. The merge disposition reuses the
|
|
22
|
-
* execution (canonical pick, link re-point, dedup_log, metadata
|
|
23
|
-
* merge verdict below `correctionRewriteMinConfidence` keeps both
|
|
19
|
+
* `merge` | `corrects` | `supersedes` | `conflicts` | `none`; at/above the
|
|
20
|
+
* ceiling the deterministic merge zone (dedup.ts runDeterministicMergeZone —
|
|
21
|
+
* LLM-free, budget-free) owns the pair. The merge disposition reuses the
|
|
22
|
+
* dedup core's execution (canonical pick, link re-point, dedup_log, metadata
|
|
23
|
+
* rails); a merge verdict below `correctionRewriteMinConfidence` keeps both
|
|
24
|
+
* memories.
|
|
25
|
+
*
|
|
26
|
+
* #393 increment B — the SCOUT, a second detection source with the SAME
|
|
27
|
+
* judge: the similarity floor is structurally blind to corrections riding
|
|
28
|
+
* inside topically unrelated memories (the field failure — cosine ~0.5-0.6 to
|
|
29
|
+
* their target, zero `corrects` verdicts in the whole corpus baseline), so
|
|
30
|
+
* per NEW memory ONE classify-tier shape call asks whether it corrects/
|
|
31
|
+
* retracts/supersedes/CONTRADICTS something previously recorded (guard-C
|
|
32
|
+
* extended the question); correction-shaped
|
|
33
|
+
* memories FTS the corpus with the referenced claim's distinctive terms (the
|
|
34
|
+
* correction CONTAINS the words of what it corrects) and the hits become
|
|
35
|
+
* candidate pairs in the SAME verdict loop — no similarity gate for this
|
|
36
|
+
* source: cosine is a ranker/link strength, never a blocker. Per-source
|
|
37
|
+
* counters (scout_scanned / scout_correction_shaped / scout_candidates_found)
|
|
38
|
+
* ride the stage report; cosine band stats stay similarity-source-only.
|
|
39
|
+
*
|
|
40
|
+
* #393 guard-C — the conflicts flag + the zone-runs-last order: judgment
|
|
41
|
+
* OUTRANKS the deterministic sweep. A `conflicts` verdict writes a symmetric
|
|
42
|
+
* `conflicts` link (the pair genuinely disagrees — cannot both be true) and
|
|
43
|
+
* NOTHING else: no status change, no rewrite, no merge queue; both records
|
|
44
|
+
* stay live so the consumer sees both truths. Both merge paths (the zone's
|
|
45
|
+
* planDedup and the judged mergeMemoryIds) refuse to blend a conflicts-linked
|
|
46
|
+
* pair, counted as conflict_skipped. The zone therefore runs AFTER the
|
|
47
|
+
* rewrite phase — with the zone first, a >=0.92 conflict pair was blended
|
|
48
|
+
* before the judge ever saw it (the planted-eval harm: canonical=older, the
|
|
49
|
+
* newer truth erased); running it last means verdicts/marks/binds land first
|
|
50
|
+
* and the zone merges only what no verdict claimed — a conflicts bind set by
|
|
51
|
+
* this run's scan guards the SAME run's zone.
|
|
24
52
|
*
|
|
25
53
|
* Status vocabulary (code-defined, extensible — deliberately NOT config):
|
|
26
54
|
* NULL/'active' default | 'superseded' + 'retracted' demote in ranking |
|
|
@@ -71,8 +99,10 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
71
99
|
};
|
|
72
100
|
})();
|
|
73
101
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
74
|
-
exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.
|
|
102
|
+
exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = exports.DEFAULT_CORRECTION_MIN_SIMILARITY = exports.RECONSOLIDATION_STAGE_LABEL = void 0;
|
|
75
103
|
exports.isFactShapedTarget = isFactShapedTarget;
|
|
104
|
+
exports.buildScoutShapePrompt = buildScoutShapePrompt;
|
|
105
|
+
exports.parseScoutShape = parseScoutShape;
|
|
76
106
|
exports.buildCorrectionVerdictPrompt = buildCorrectionVerdictPrompt;
|
|
77
107
|
exports.parseCorrectionVerdict = parseCorrectionVerdict;
|
|
78
108
|
exports.buildRewritePrompt = buildRewritePrompt;
|
|
@@ -94,6 +124,7 @@ const storage = __importStar(require("./storage.js"));
|
|
|
94
124
|
const state_js_1 = require("./state.js");
|
|
95
125
|
const db_js_1 = require("./db.js");
|
|
96
126
|
const capture_js_1 = require("./capture.js");
|
|
127
|
+
const CALIBRATION = __importStar(require("./calibration.js"));
|
|
97
128
|
const paths_js_1 = require("./paths.js");
|
|
98
129
|
const dedup_js_1 = require("./dedup.js");
|
|
99
130
|
// ---------------------------------------------------------------------------
|
|
@@ -102,44 +133,21 @@ const dedup_js_1 = require("./dedup.js");
|
|
|
102
133
|
/** Stage label used for every budget.use()/recordUsage() call (#384). */
|
|
103
134
|
exports.RECONSOLIDATION_STAGE_LABEL = "reconsolidation";
|
|
104
135
|
/**
|
|
105
|
-
* Default minimum COSINE similarity for a correction candidate pair
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
136
|
+
* Default minimum COSINE similarity for a correction candidate pair —
|
|
137
|
+
* RELEASE-MANAGED since #408 (calibration.ts CORRECTION_MIN_SIMILARITY;
|
|
138
|
+
* provenance there). Lower than the supersession stage's 0.80 on purpose: a
|
|
139
|
+
* retraction often rides inside an otherwise unrelated memory (the field
|
|
140
|
+
* failure that opened this issue), so the neighborhood gate must be a touch
|
|
141
|
+
* wider while the LLM verdict + confidence gate carry the precision load.
|
|
110
142
|
*/
|
|
111
|
-
exports.DEFAULT_CORRECTION_MIN_SIMILARITY =
|
|
143
|
+
exports.DEFAULT_CORRECTION_MIN_SIMILARITY = CALIBRATION.CORRECTION_MIN_SIMILARITY;
|
|
112
144
|
/**
|
|
113
|
-
* Default minimum verdict confidence for the REWRITE fork
|
|
145
|
+
* Default minimum verdict confidence for the REWRITE fork — release-managed
|
|
146
|
+
* (calibration.ts CORRECTION_REWRITE_MIN_CONFIDENCE). Below this a
|
|
114
147
|
* `corrects` verdict degrades to mark-only — a weak mark is recoverable, a
|
|
115
148
|
* weak rewrite is corruption.
|
|
116
149
|
*/
|
|
117
|
-
exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE =
|
|
118
|
-
/**
|
|
119
|
-
* Default wall-clock bound for the stage, in minutes (#401). Checked at the
|
|
120
|
-
* top of the candidate scan loop (and before each rewrite contract call); on
|
|
121
|
-
* expiry the scan breaks cleanly at the last fully-considered candidate and
|
|
122
|
-
* the next run resumes from the persisted cursor. 120 sits safely under any
|
|
123
|
-
* sane process-level nightly timeout. 0 disables the bound. Invalid →
|
|
124
|
-
* default.
|
|
125
|
-
*/
|
|
126
|
-
exports.DEFAULT_RECONSOLIDATION_MAX_MINUTES = 120;
|
|
127
|
-
/**
|
|
128
|
-
* Default per-run classify-call ceiling for the stage (#401) — the
|
|
129
|
-
* supersessionMaxCalls pattern with a NON-ZERO default ON PURPOSE: that
|
|
130
|
-
* knob's 0=unlimited default is what let the first full-corpus pass grow
|
|
131
|
-
* unbounded. Counts EVERY classify-tier call the stage makes (mark
|
|
132
|
-
* verifications, pair verdicts, rewrite contracts). 0 disables the cap.
|
|
133
|
-
* Invalid → default.
|
|
134
|
-
*/
|
|
135
|
-
exports.DEFAULT_RECONSOLIDATION_MAX_CALLS = 600;
|
|
136
|
-
/**
|
|
137
|
-
* Candidates between mid-scan cursor persists (#401). The cursor also
|
|
138
|
-
* persists at EVERY scan-loop exit path (deadline, call/budget cap,
|
|
139
|
-
* discovery failure), so a killed run loses at most K-1 candidates of scan
|
|
140
|
-
* progress instead of the whole night.
|
|
141
|
-
*/
|
|
142
|
-
const RECONSOLIDATION_CURSOR_PERSIST_EVERY = 50;
|
|
150
|
+
exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = CALIBRATION.CORRECTION_REWRITE_MIN_CONFIDENCE;
|
|
143
151
|
/** Neighbor pool size before older/similarity filtering narrows to top 5 (supersession mirror). */
|
|
144
152
|
const CORRECTION_NEIGHBOR_POOL = 15;
|
|
145
153
|
/** Older-neighbor pairs kept per candidate after filtering (supersession mirror). */
|
|
@@ -168,10 +176,10 @@ exports.DEMOTED_STATUSES = ["superseded", "retracted"];
|
|
|
168
176
|
function isFactShapedTarget(mem) {
|
|
169
177
|
return mem.memory_type === "knowledge" || mem.content.includes("[Facts Learned]");
|
|
170
178
|
}
|
|
171
|
-
/** True when a superseded_by
|
|
179
|
+
/** True when a superseded_by / corrected_by / conflicts link already exists between the pair, either direction. */
|
|
172
180
|
function alreadyResolutionLinked(db, oldId, newId) {
|
|
173
181
|
const row = db
|
|
174
|
-
.prepare(`SELECT 1 FROM memory_links WHERE relationship IN ('superseded_by', 'corrected_by')
|
|
182
|
+
.prepare(`SELECT 1 FROM memory_links WHERE relationship IN ('superseded_by', 'corrected_by', 'conflicts')
|
|
175
183
|
AND ((source_id = ? AND target_id = ?) OR (source_id = ? AND target_id = ?))`)
|
|
176
184
|
.get(oldId, newId, newId, oldId);
|
|
177
185
|
return !!row;
|
|
@@ -185,6 +193,58 @@ function hasLink(db, sourceId, targetId, relationship) {
|
|
|
185
193
|
function nowIso() {
|
|
186
194
|
return new Date().toISOString();
|
|
187
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* Build the constrained correction-shape prompt (classify-tier cost profile:
|
|
198
|
+
* 1500-char truncation, supersession/verdict precedent). The wording asks for
|
|
199
|
+
* the OLD claim's distinctive terms — the field-failure mechanism is that a
|
|
200
|
+
* correction CONTAINS the words of what it corrects, even when the surrounding
|
|
201
|
+
* topics (and therefore the embedding cosine) are unrelated. Guard-C extends
|
|
202
|
+
* the question to contradictions: two records that disagree on the same
|
|
203
|
+
* quantity share even MORE wording than a cross-topic correction does.
|
|
204
|
+
*/
|
|
205
|
+
function buildScoutShapePrompt(content) {
|
|
206
|
+
const trunc = (s) => (s.length > PROMPT_TRUNCATE_CHARS ? `${s.slice(0, PROMPT_TRUNCATE_CHARS)}…` : s);
|
|
207
|
+
return (`You are scanning a memory that was just added to an AI agent's long-term memory store.\n\n` +
|
|
208
|
+
`MEMORY:\n${trunc(content)}\n\n` +
|
|
209
|
+
`Does this memory correct, retract, supersede, or contradict a claim, decision, or state that was ` +
|
|
210
|
+
`previously recorded elsewhere in the store? A mere duplicate, elaboration, independent ` +
|
|
211
|
+
`fact, or new information that invalidates nothing is NOT a correction.\n` +
|
|
212
|
+
`If it is a correction/retraction/supersession/contradiction, list the most distinctive terms of the ` +
|
|
213
|
+
`OLD claim it references — words likely to appear verbatim in the older record.\n\n` +
|
|
214
|
+
`Reply with ONLY a JSON object, no prose: ` +
|
|
215
|
+
`{"correction": true | false, "references": "<distinctive terms of the referenced old claim, or empty string>", ` +
|
|
216
|
+
`"confidence": <number between 0 and 1>}`);
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Parse the scout shape reply. Null on unparseable JSON, a missing/non-boolean
|
|
220
|
+
* `correction`, or a missing/out-of-range `confidence` — the caller counts
|
|
221
|
+
* skipped_infra and moves on (parseSupersessionReply discipline: never
|
|
222
|
+
* mis-detect on ambiguity). `references` is lenient (missing/non-string → "")
|
|
223
|
+
* because an empty string simply yields no FTS hits — a harmless miss, not a
|
|
224
|
+
* mis-judgment.
|
|
225
|
+
*/
|
|
226
|
+
function parseScoutShape(reply) {
|
|
227
|
+
if (!reply)
|
|
228
|
+
return null;
|
|
229
|
+
const start = reply.indexOf("{");
|
|
230
|
+
const end = reply.lastIndexOf("}");
|
|
231
|
+
if (start === -1 || end === -1 || end <= start)
|
|
232
|
+
return null;
|
|
233
|
+
let obj;
|
|
234
|
+
try {
|
|
235
|
+
obj = JSON.parse(reply.slice(start, end + 1));
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
if (typeof obj.correction !== "boolean")
|
|
241
|
+
return null;
|
|
242
|
+
const confidence = Number(obj.confidence);
|
|
243
|
+
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1)
|
|
244
|
+
return null;
|
|
245
|
+
const references = typeof obj.references === "string" ? obj.references : "";
|
|
246
|
+
return { correction: obj.correction, references: references.trim(), confidence };
|
|
247
|
+
}
|
|
188
248
|
/** Build the constrained pair-verdict prompt (1500-char truncation, supersession precedent). */
|
|
189
249
|
function buildCorrectionVerdictPrompt(oldContent, newContent) {
|
|
190
250
|
const trunc = (s) => (s.length > PROMPT_TRUNCATE_CHARS ? `${s.slice(0, PROMPT_TRUNCATE_CHARS)}…` : s);
|
|
@@ -198,9 +258,12 @@ function buildCorrectionVerdictPrompt(oldContent, newContent) {
|
|
|
198
258
|
`wrong, no longer true, or was retracted, and the newer memory carries the corrected fact.\n` +
|
|
199
259
|
`- "supersedes": the newer memory replaces a decision, plan, or state that was valid at the time but is ` +
|
|
200
260
|
`now outdated — a replacement, not a factual correction.\n` +
|
|
261
|
+
`- "conflicts": the two memories make claims that cannot both be true — they disagree on a fact, value, ` +
|
|
262
|
+
`or state, and neither one corrects, supersedes, or restates the other (for example two sources report ` +
|
|
263
|
+
`different values for the same quantity). Keep both; flag the conflict.\n` +
|
|
201
264
|
`- "none": unrelated, merely similar, or both can still be true (an addition or elaboration).\n\n` +
|
|
202
265
|
`Reply with ONLY a JSON object, no prose: ` +
|
|
203
|
-
`{"action": "merge" | "corrects" | "supersedes" | "none", "confidence": <number between 0 and 1>}`);
|
|
266
|
+
`{"action": "merge" | "corrects" | "supersedes" | "conflicts" | "none", "confidence": <number between 0 and 1>}`);
|
|
204
267
|
}
|
|
205
268
|
/**
|
|
206
269
|
* Parse the pair verdict. Null on anything unparseable, unknown action, or an
|
|
@@ -222,8 +285,13 @@ function parseCorrectionVerdict(reply) {
|
|
|
222
285
|
return null;
|
|
223
286
|
}
|
|
224
287
|
const action = obj.action;
|
|
225
|
-
if (action !== "merge" &&
|
|
288
|
+
if (action !== "merge" &&
|
|
289
|
+
action !== "corrects" &&
|
|
290
|
+
action !== "supersedes" &&
|
|
291
|
+
action !== "conflicts" &&
|
|
292
|
+
action !== "none") {
|
|
226
293
|
return null;
|
|
294
|
+
}
|
|
227
295
|
const confidence = Number(obj.confidence);
|
|
228
296
|
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1)
|
|
229
297
|
return null;
|
|
@@ -459,7 +527,7 @@ function bandForCosine(bands, cosine) {
|
|
|
459
527
|
}
|
|
460
528
|
/** An empty band-stat record (fresh accumulation starts from zeroes). */
|
|
461
529
|
function emptyBandStat() {
|
|
462
|
-
return { pairs: 0, merge: 0, corrects: 0, supersedes: 0, none: 0, merge_below_gate: 0, conf_sum: 0 };
|
|
530
|
+
return { pairs: 0, merge: 0, corrects: 0, supersedes: 0, conflicts: 0, none: 0, merge_below_gate: 0, conf_sum: 0 };
|
|
463
531
|
}
|
|
464
532
|
/** Add a run's per-band counts into a cumulative record (in place). */
|
|
465
533
|
function accumulateBandStat(cumulative, run) {
|
|
@@ -467,6 +535,7 @@ function accumulateBandStat(cumulative, run) {
|
|
|
467
535
|
cumulative.merge += run.merge;
|
|
468
536
|
cumulative.corrects += run.corrects;
|
|
469
537
|
cumulative.supersedes += run.supersedes;
|
|
538
|
+
cumulative.conflicts += run.conflicts;
|
|
470
539
|
cumulative.none += run.none;
|
|
471
540
|
cumulative.merge_below_gate += run.merge_below_gate;
|
|
472
541
|
cumulative.conf_sum += run.conf_sum;
|
|
@@ -488,9 +557,43 @@ async function findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarit
|
|
|
488
557
|
.sort((a, b) => (0, retrieval_js_1.l2ToCosine)(b.distance) - (0, retrieval_js_1.l2ToCosine)(a.distance))
|
|
489
558
|
.slice(0, CORRECTION_NEIGHBOR_TOP_K);
|
|
490
559
|
}
|
|
560
|
+
/**
|
|
561
|
+
* The scout source (#393 increment B): for a correction-shaped NEW memory, FTS
|
|
562
|
+
* the corpus with the referenced claim's distinctive terms and return the OLDER
|
|
563
|
+
* hits as candidate pairs. This is the reference-extraction half — it finds the
|
|
564
|
+
* old claim even when the overall topics differ (and therefore the cosine sits
|
|
565
|
+
* below the similarity floor) because the correction CONTAINS the words of
|
|
566
|
+
* what it corrects. Deterministic: ONE FTS query, zero LLM. Filters: self,
|
|
567
|
+
* non-older (detection only pairs older → newer, the KNN mirror), and
|
|
568
|
+
* defensively non-absorbed hits. Pool/top-K reuse the KNN constants; FTS rank
|
|
569
|
+
* (BM25, best first) is the order. Dedup against the KNN neighbor ids is the
|
|
570
|
+
* caller's job (a pair found by both sources is judged once, as similarity).
|
|
571
|
+
*/
|
|
572
|
+
function findScoutNeighbors(db, candidate, references, candidateEmbedding) {
|
|
573
|
+
if (!references)
|
|
574
|
+
return [];
|
|
575
|
+
try {
|
|
576
|
+
const hits = storage.searchFts(db, references, CORRECTION_NEIGHBOR_POOL);
|
|
577
|
+
return hits
|
|
578
|
+
.filter((m) => m.id !== candidate.id &&
|
|
579
|
+
m.created_at < candidate.created_at &&
|
|
580
|
+
m.status !== "absorbed")
|
|
581
|
+
.slice(0, CORRECTION_NEIGHBOR_TOP_K)
|
|
582
|
+
.map((m) => {
|
|
583
|
+
const hitVec = storage.getStoredEmbedding(db, m.id);
|
|
584
|
+
const cosine = candidateEmbedding && hitVec ? (0, retrieval_js_1.cosineBetweenVectors)(candidateEmbedding, hitVec) : 0;
|
|
585
|
+
return { mem: m, cosine, source: "scout" };
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
catch {
|
|
589
|
+
// FTS is deterministic infrastructure — a throw here is a bug or a corrupt
|
|
590
|
+
// index, never a judgment question. Fail soft: no scout pairs this memory.
|
|
591
|
+
return [];
|
|
592
|
+
}
|
|
593
|
+
}
|
|
491
594
|
async function classifyPair(llm, oldContent, newContent) {
|
|
492
595
|
try {
|
|
493
|
-
const r = await llm.
|
|
596
|
+
const r = await llm.complete(buildCorrectionVerdictPrompt(oldContent, newContent));
|
|
494
597
|
return { verdict: parseCorrectionVerdict(r.text), usage: r.usage };
|
|
495
598
|
}
|
|
496
599
|
catch {
|
|
@@ -500,21 +603,34 @@ async function classifyPair(llm, oldContent, newContent) {
|
|
|
500
603
|
/**
|
|
501
604
|
* Nightly reconsolidation stage (#384, #392 — THE unified resolution stage).
|
|
502
605
|
*
|
|
503
|
-
* Phase
|
|
504
|
-
*
|
|
606
|
+
* Phase order (#393 guard-C): the deterministic merge zone (pairs >= the
|
|
607
|
+
* ceiling) runs LAST — after the scan, the judged-merge phase, and the
|
|
608
|
+
* rewrite phase. Judgment outranks the deterministic sweep: verdicts, marks,
|
|
609
|
+
* and binds land first and the zone merges only what no verdict claimed. With
|
|
610
|
+
* the zone first, a >=0.92 genuine-conflict pair was blended before the judge
|
|
611
|
+
* ever saw it (the planted-eval harm); running it last means a `conflicts`
|
|
612
|
+
* bind set by this run's scan guards the SAME run's zone. Zone internals
|
|
613
|
+
* (lock, backup, deadline, persistBand, fail-soft) are unchanged.
|
|
505
614
|
*
|
|
506
615
|
* Scan: every memory with rowid > reconsolidationCursor (no shape filter;
|
|
507
616
|
* absorbed candidates are skipped — invisible memories are not re-judged).
|
|
508
|
-
* Each candidate's pairs: incoming explicit marks (verified once, AC7) then
|
|
509
|
-
*
|
|
510
|
-
*
|
|
617
|
+
* Each candidate's pairs: incoming explicit marks (verified once, AC7), then
|
|
618
|
+
* ONE scout shape call (#393 B — flags correction shape; non-corrections stop
|
|
619
|
+
* there), then up-to-5 older KNN neighbors in [floor, ceiling) (verdict call
|
|
620
|
+
* per unlinked pair, AC2 — pairs at/above the ceiling are counted, never
|
|
621
|
+
* judged) plus the scout's FTS hits for correction-shaped memories (same
|
|
622
|
+
* verdict loop, NO similarity gate; guard-C: a scout hit whose KNN twin sits
|
|
623
|
+
* at/above the ceiling is re-tagged scout so the pair IS judged instead of
|
|
624
|
+
* being left for the zone to blend). Confirmed
|
|
511
625
|
* `corrects` pairs above the confidence gate on fact-shaped targets group by
|
|
512
626
|
* target into ONE rewrite call each (AC3); confirmed `merge` pairs queue for
|
|
513
|
-
* the merge phase;
|
|
627
|
+
* the merge phase; a `conflicts` verdict writes the conflicts link and
|
|
628
|
+
* nothing else (both live); everything else is mark-only.
|
|
514
629
|
*
|
|
515
630
|
* Merge phase (#392): queued pairs merge through the dedup core under one
|
|
516
|
-
* lock/backup window
|
|
517
|
-
*
|
|
631
|
+
* lock/backup window. A pair that cannot apply keeps both memories and holds
|
|
632
|
+
* the cursor; a conflicts-linked or metadata-mismatched refusal keeps both
|
|
633
|
+
* and advances (the verdict was rendered).
|
|
518
634
|
*
|
|
519
635
|
* Cursor discipline mirrors stageSupersession: the cursor advances past a
|
|
520
636
|
* candidate once its neighbor set has been considered, regardless of infra
|
|
@@ -540,25 +656,13 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
540
656
|
const minSimilarity = validNumber(options.minSimilarity, exports.DEFAULT_CORRECTION_MIN_SIMILARITY, (n) => n > 0 && n <= 1);
|
|
541
657
|
const rewriteMinConfidence = validNumber(options.rewriteMinConfidence, exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE, (n) => n > 0 && n <= 1);
|
|
542
658
|
const autoMergeThreshold = validNumber(options.autoMergeThreshold, dedup_js_1.DEFAULT_DEDUP_MERGE_THRESHOLD, (n) => n > 0 && n <= 1);
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
//
|
|
547
|
-
//
|
|
548
|
-
|
|
549
|
-
const
|
|
550
|
-
const deadlineHit = () => Date.now() >= deadlineAt;
|
|
551
|
-
// ---- #392 phase 0: the deterministic merge zone (pairs >= the ceiling),
|
|
552
|
-
// LLM-free and budget-free — an LLM-less night still drains duplicates. Its
|
|
553
|
-
// own short lock window, pre-merge backup, and pacing cap; fail-soft, never
|
|
554
|
-
// a throw. Runs FIRST so the scan below never sees the pairs it owns.
|
|
555
|
-
const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
|
|
556
|
-
stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
|
|
557
|
-
threshold: autoMergeThreshold,
|
|
558
|
-
maxMerges,
|
|
559
|
-
dryRun,
|
|
560
|
-
acquireLock: options.acquireLock,
|
|
561
|
-
});
|
|
659
|
+
// ---- #405 runtime bound. The stage-local reconsolidationMaxMinutes clock
|
|
660
|
+
// (#401) is gone — the run-wide pipeline deadline (nightly.ts, config
|
|
661
|
+
// nightlyTimeBudgetMinutes) is the only wall-clock. The deterministic
|
|
662
|
+
// zone's runtime counts against it via the zone's own stop-check below.
|
|
663
|
+
// hit() logs event=deadline_deferred once per stage name.
|
|
664
|
+
const deadline = options.deadline;
|
|
665
|
+
const deadlineHit = (stageLabel = exports.RECONSOLIDATION_STAGE_LABEL) => deadline?.hit(stageLabel) ?? false;
|
|
562
666
|
// Per-run verdict statistics by cosine band (#392) — report snapshot here,
|
|
563
667
|
// cumulative series in state.json at stage end (never on dry-run).
|
|
564
668
|
const bands = buildResolutionBands(minSimilarity, autoMergeThreshold);
|
|
@@ -600,6 +704,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
600
704
|
let skippedAboveCeiling = 0;
|
|
601
705
|
let skippedMetadataMismatch = 0;
|
|
602
706
|
let mergePairsApplied = 0;
|
|
707
|
+
// #393 guard-C: conflicts verdicts rendered (link written, both live) and
|
|
708
|
+
// judged-path merge refusals on a conflicts-linked pair.
|
|
709
|
+
let conflictFlagged = 0;
|
|
710
|
+
let conflictSkippedJudged = 0;
|
|
711
|
+
// #393 B scout counters (per-source observability, the #394 discipline):
|
|
712
|
+
// shape calls made / correction-shaped verdicts / FTS hits that became
|
|
713
|
+
// candidate pairs. 0 on dry-run (the shape call is LLM work).
|
|
714
|
+
let scoutScanned = 0;
|
|
715
|
+
let scoutCorrectionShaped = 0;
|
|
716
|
+
let scoutCandidatesFound = 0;
|
|
603
717
|
let cursor = startCursor;
|
|
604
718
|
const queuedMerges = [];
|
|
605
719
|
// #392 cursor-hold anchor, shared by the merge phase and the rewrite phase:
|
|
@@ -614,22 +728,35 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
614
728
|
linksCreatedThisRun.add(`${oldId}|${newId}`);
|
|
615
729
|
};
|
|
616
730
|
// ---- #401: mid-scan cursor persistence. Called at EVERY scan-loop exit
|
|
617
|
-
// path (deadline,
|
|
618
|
-
// failure)
|
|
619
|
-
//
|
|
620
|
-
//
|
|
621
|
-
//
|
|
622
|
-
//
|
|
623
|
-
//
|
|
624
|
-
let scanBatch = 0;
|
|
625
|
-
let callsUsed = 0;
|
|
731
|
+
// path (deadline, budget cap, mark-verify budget stop, discovery
|
|
732
|
+
// failure) AND after every fully-considered candidate, so a killed run
|
|
733
|
+
// loses at most the candidate in flight. The end-of-stage updateState
|
|
734
|
+
// below stays the authoritative final write (it also applies the
|
|
735
|
+
// pendingMinRowid hold — that variable is only ever set AFTER the scan
|
|
736
|
+
// loop, so it is null at every call site here). updateState is
|
|
737
|
+
// load→mutate→temp-rename atomic.
|
|
626
738
|
let deadlineStopped = false;
|
|
627
|
-
|
|
739
|
+
// #402 follow-up (reviewer note 1): the hard-kill orphan floor. Queued
|
|
740
|
+
// merges and open rewrite groups are applied only in the POST-scan
|
|
741
|
+
// phases — until then their verdicts exist only in memory, and a
|
|
742
|
+
// SIGKILL/OOM between two persists would strand them BEHIND the persisted
|
|
743
|
+
// cursor (the next run would skip them forever). This tracks the smallest
|
|
744
|
+
// candidate rowid contributing to queued-but-unapplied work;
|
|
745
|
+
// persistCursor clamps every checkpoint below it so a resumed run
|
|
746
|
+
// re-detects the pairs (dup-over-loss). Deliberately SEPARATE from the
|
|
747
|
+
// post-loop pendingMinRowid hold above — different lifetime, different
|
|
748
|
+
// writers.
|
|
749
|
+
let scanPendingMinRowid = null;
|
|
750
|
+
const notePendingRowid = (rowid) => {
|
|
751
|
+
scanPendingMinRowid =
|
|
752
|
+
scanPendingMinRowid === null ? rowid : Math.min(scanPendingMinRowid, rowid);
|
|
753
|
+
};
|
|
628
754
|
const persistCursor = () => {
|
|
629
755
|
if (dryRun)
|
|
630
756
|
return;
|
|
757
|
+
const checkpoint = scanPendingMinRowid !== null ? Math.min(cursor, scanPendingMinRowid - 1) : cursor;
|
|
631
758
|
(0, state_js_1.updateState)((s) => {
|
|
632
|
-
s.reconsolidationCursor =
|
|
759
|
+
s.reconsolidationCursor = checkpoint;
|
|
633
760
|
}, stateDir);
|
|
634
761
|
};
|
|
635
762
|
const groups = new Map();
|
|
@@ -648,6 +775,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
648
775
|
candidateRowid: trigger.__rowid,
|
|
649
776
|
explicit,
|
|
650
777
|
});
|
|
778
|
+
notePendingRowid(trigger.__rowid); // orphan floor — group unapplied until the rewrite phase
|
|
651
779
|
}
|
|
652
780
|
};
|
|
653
781
|
for (const candidate of rows) {
|
|
@@ -662,14 +790,6 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
662
790
|
persistCursor();
|
|
663
791
|
break;
|
|
664
792
|
}
|
|
665
|
-
// #401: the per-stage call cap stops the scan at the candidate boundary
|
|
666
|
-
// (the in-loop check below is the mid-candidate backstop — supersession
|
|
667
|
-
// mirrors both).
|
|
668
|
-
if (!dryRun && maxCalls > 0 && callsUsed >= maxCalls) {
|
|
669
|
-
callCapStopped = true;
|
|
670
|
-
persistCursor();
|
|
671
|
-
break;
|
|
672
|
-
}
|
|
673
793
|
scanned++;
|
|
674
794
|
// ---- AC7: verify incoming explicit marks (corrected_by/superseded_by
|
|
675
795
|
// links targeting this candidate) before they can join a rewrite group.
|
|
@@ -700,7 +820,6 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
700
820
|
}
|
|
701
821
|
const { verdict, usage } = await classifyPair(llm, target.content, candidate.content);
|
|
702
822
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
|
|
703
|
-
callsUsed++; // #401
|
|
704
823
|
pairsEvaluated++;
|
|
705
824
|
if (!verdict) {
|
|
706
825
|
skippedInfra++; // mark retained; the neighborhood is revisited via newer candidacies
|
|
@@ -724,7 +843,43 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
724
843
|
break;
|
|
725
844
|
}
|
|
726
845
|
}
|
|
727
|
-
// ----
|
|
846
|
+
// ---- #393 B: scout shape call — ONE classify-tier call per candidate,
|
|
847
|
+
// budget-metered under the same stage label as verdicts. Flags whether
|
|
848
|
+
// this memory corrects/retracts/supersedes something previously recorded;
|
|
849
|
+
// non-corrections stop here (zero follow-up). A parse/infra failure skips
|
|
850
|
+
// the scout source for this memory only (fail-soft — the similarity
|
|
851
|
+
// source below still runs) and counts skipped_infra, the verdict-skip
|
|
852
|
+
// discipline. Dry-run skips the call entirely: it is LLM work, and
|
|
853
|
+
// dry-runs make zero LLM calls (the scout counters read 0 there).
|
|
854
|
+
let scoutReferences = null;
|
|
855
|
+
if (!dryRun) {
|
|
856
|
+
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
857
|
+
persistCursor(); // shape call refused — this candidate re-scouts next run
|
|
858
|
+
break;
|
|
859
|
+
}
|
|
860
|
+
scoutScanned++;
|
|
861
|
+
try {
|
|
862
|
+
const r = await llm.complete(buildScoutShapePrompt(candidate.content));
|
|
863
|
+
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
864
|
+
const shape = parseScoutShape(r.text);
|
|
865
|
+
if (!shape) {
|
|
866
|
+
skippedInfra++;
|
|
867
|
+
}
|
|
868
|
+
else if (shape.correction) {
|
|
869
|
+
scoutCorrectionShaped++;
|
|
870
|
+
scoutReferences = shape.references;
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
catch {
|
|
874
|
+
skippedInfra++;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
// ---- AC2: detection pairs against older neighbors — TWO sources (#393 B):
|
|
878
|
+
// the KNN similarity source (floor-gated, the pre-B baseline) and, for
|
|
879
|
+
// correction-shaped candidates, the scout's FTS source (the referenced
|
|
880
|
+
// claim's terms → corpus search; NO similarity gate — cosine is a ranker,
|
|
881
|
+
// never a blocker). Merged + deduped by neighbor id: a pair found by both
|
|
882
|
+
// sources is judged ONCE, as similarity (with band attribution).
|
|
728
883
|
let neighbors;
|
|
729
884
|
try {
|
|
730
885
|
neighbors = await findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarity);
|
|
@@ -735,39 +890,74 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
735
890
|
persistCursor(); // #401: every exit path persists
|
|
736
891
|
continue;
|
|
737
892
|
}
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
893
|
+
const neighborEntries = neighbors.map((n) => ({
|
|
894
|
+
mem: n,
|
|
895
|
+
cosine: (0, retrieval_js_1.l2ToCosine)(n.distance),
|
|
896
|
+
source: "similarity",
|
|
897
|
+
}));
|
|
898
|
+
if (scoutReferences !== null) {
|
|
899
|
+
const candidateVec = storage.getStoredEmbedding(db, candidate.id);
|
|
900
|
+
const knnById = new Map(neighborEntries.map((e) => [e.mem.id, e]));
|
|
901
|
+
for (const entry of findScoutNeighbors(db, candidate, scoutReferences, candidateVec)) {
|
|
902
|
+
const knn = knnById.get(entry.mem.id);
|
|
903
|
+
if (knn) {
|
|
904
|
+
// Both sources found the pair. Below the ceiling the KNN entry
|
|
905
|
+
// would be judged anyway — similarity keeps it (band attribution,
|
|
906
|
+
// judged once). At/above the ceiling the similarity entry would be
|
|
907
|
+
// ceiling-skipped (zone territory, never judged) — yet the zone now
|
|
908
|
+
// runs AFTER the scan, and a genuine conflict at >=0.92 is exactly
|
|
909
|
+
// the pair it would blend with no judge in the loop (guard-C's
|
|
910
|
+
// harm). Re-tag the entry to the scout source so the pair IS
|
|
911
|
+
// judged: the scout's no-gate exemption applies, a `conflicts`
|
|
912
|
+
// verdict can plant the guard link, and the zone's own guard then
|
|
913
|
+
// refuses the cluster in this same run.
|
|
914
|
+
if (knn.cosine >= autoMergeThreshold)
|
|
915
|
+
knn.source = "scout";
|
|
916
|
+
continue;
|
|
917
|
+
}
|
|
918
|
+
neighborEntries.push(entry);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
for (const entry of neighborEntries) {
|
|
922
|
+
pairsDiscovered++; // gate discovery (#394), both sources, before any skip/judgment
|
|
923
|
+
if (entry.source === "scout")
|
|
924
|
+
scoutCandidatesFound++;
|
|
925
|
+
if (alreadyResolutionLinked(db, entry.mem.id, candidate.id)) {
|
|
741
926
|
skippedIdempotent++;
|
|
742
927
|
continue;
|
|
743
928
|
}
|
|
744
929
|
pairsDiscoveredUnlinked++; // still unlinked — the actionable candidate
|
|
745
|
-
|
|
746
|
-
//
|
|
747
|
-
//
|
|
748
|
-
|
|
749
|
-
|
|
930
|
+
const pairCosine = entry.cosine;
|
|
931
|
+
// #392: SIMILARITY-source pairs at/above the ceiling belong to the
|
|
932
|
+
// deterministic zone — counted here, never LLM-judged (the zone merges
|
|
933
|
+
// them at stage end or defers them to a later run; re-detection is
|
|
934
|
+
// structural, not cursor-based). Scout pairs are exempt (#393 B):
|
|
935
|
+
// cosine never blocks this source — guard-C's re-tag above relies on
|
|
936
|
+
// it, and a judged merge re-passes the same metadata/conflict rails
|
|
937
|
+
// the zone enforces.
|
|
938
|
+
if (entry.source === "similarity" && pairCosine >= autoMergeThreshold) {
|
|
750
939
|
skippedAboveCeiling++;
|
|
751
940
|
continue;
|
|
752
941
|
}
|
|
753
942
|
if (dryRun)
|
|
754
943
|
continue; // preview only — no LLM call, no write
|
|
755
|
-
// #
|
|
756
|
-
|
|
757
|
-
if ((maxCalls > 0 && callsUsed >= maxCalls) || !budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
758
|
-
callCapStopped = maxCalls > 0 && callsUsed >= maxCalls;
|
|
944
|
+
// #405: the ONE run budget's refusal is the only call cap.
|
|
945
|
+
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
759
946
|
persistCursor(); // cursor still points at the last fully-considered candidate
|
|
760
947
|
break;
|
|
761
948
|
}
|
|
762
|
-
const { verdict, usage } = await classifyPair(llm,
|
|
949
|
+
const { verdict, usage } = await classifyPair(llm, entry.mem.content, candidate.content);
|
|
763
950
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
|
|
764
|
-
callsUsed++; // #401
|
|
765
951
|
pairsEvaluated++;
|
|
766
952
|
if (!verdict) {
|
|
767
953
|
skippedInfra++;
|
|
768
954
|
continue;
|
|
769
955
|
}
|
|
770
|
-
|
|
956
|
+
// Bands stay similarity-source-only (refine Q2 ruling): they are the
|
|
957
|
+
// calibration evidence for the floor/ceiling boundaries, and scout
|
|
958
|
+
// pairs reach them through a different, cosine-blind door.
|
|
959
|
+
if (entry.source === "similarity")
|
|
960
|
+
recordBand(pairCosine, verdict.action, verdict.confidence);
|
|
771
961
|
// #392: a merge verdict is queued for the merge phase (below) — no
|
|
772
962
|
// link, no write here. Below the confidence gate BOTH memories stay
|
|
773
963
|
// live: a weak mark is recoverable, and there is nothing to mark for a
|
|
@@ -775,23 +965,38 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
775
965
|
if (verdict.action === "merge") {
|
|
776
966
|
if (verdict.confidence < rewriteMinConfidence) {
|
|
777
967
|
mergeBelowGate++;
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
968
|
+
if (entry.source === "similarity") {
|
|
969
|
+
const band = bandForCosine(bands, pairCosine);
|
|
970
|
+
if (band) {
|
|
971
|
+
const stat = runBands.get(band.label) ?? emptyBandStat();
|
|
972
|
+
stat.merge_below_gate++;
|
|
973
|
+
runBands.set(band.label, stat);
|
|
974
|
+
}
|
|
783
975
|
}
|
|
784
976
|
}
|
|
785
977
|
else {
|
|
786
|
-
queuedMerges.push({ oldId:
|
|
978
|
+
queuedMerges.push({ oldId: entry.mem.id, newId: candidate.id, candidateRowid: candidate.__rowid });
|
|
979
|
+
notePendingRowid(candidate.__rowid); // orphan floor — merge unapplied until the merge phase
|
|
787
980
|
}
|
|
788
981
|
continue;
|
|
789
982
|
}
|
|
790
983
|
if (verdict.action === "supersedes") {
|
|
791
|
-
markLink(
|
|
792
|
-
storage.updateMemory(db,
|
|
984
|
+
markLink(entry.mem.id, candidate.id, "superseded_by", pairCosine);
|
|
985
|
+
storage.updateMemory(db, entry.mem.id, { status: "superseded" });
|
|
793
986
|
markedSuperseded++;
|
|
794
|
-
console.log(`[hicortex] Reconsolidation: ${
|
|
987
|
+
console.log(`[hicortex] Reconsolidation: ${entry.mem.id.slice(0, 8)} superseded_by ${candidate.id.slice(0, 8)} (mark-only)`);
|
|
988
|
+
continue;
|
|
989
|
+
}
|
|
990
|
+
// #393 guard-C: a genuine conflict — link ONLY. No status change on
|
|
991
|
+
// either memory (both stay live so the consumer sees both truths), no
|
|
992
|
+
// rewrite, no merge queue; the link is the guard both merge paths
|
|
993
|
+
// consult. Ungated like the other mark actions (a weak flag is
|
|
994
|
+
// recoverable; a weak merge is not).
|
|
995
|
+
if (verdict.action === "conflicts") {
|
|
996
|
+
markLink(entry.mem.id, candidate.id, "conflicts", pairCosine);
|
|
997
|
+
conflictFlagged++;
|
|
998
|
+
console.log(`[hicortex] Reconsolidation: ${entry.mem.id.slice(0, 8)} conflicts ${candidate.id.slice(0, 8)} ` +
|
|
999
|
+
`(flag-only) — both kept live, never merged`);
|
|
795
1000
|
continue;
|
|
796
1001
|
}
|
|
797
1002
|
if (verdict.action === "corrects") {
|
|
@@ -800,49 +1005,45 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
800
1005
|
// Below the gate: mark-only, never rewrite. The
|
|
801
1006
|
// trigger stays live — it is the only carrier of the correction.
|
|
802
1007
|
belowGate++;
|
|
803
|
-
markLink(
|
|
804
|
-
storage.updateMemory(db,
|
|
1008
|
+
markLink(entry.mem.id, candidate.id, "corrected_by", cosine);
|
|
1009
|
+
storage.updateMemory(db, entry.mem.id, { status: "retracted" });
|
|
805
1010
|
markedRetracted++;
|
|
806
1011
|
continue;
|
|
807
1012
|
}
|
|
808
|
-
if (!isFactShapedTarget(
|
|
1013
|
+
if (!isFactShapedTarget(entry.mem)) {
|
|
809
1014
|
// Decisions/plans/experiences are history, not error — mark only.
|
|
810
|
-
markLink(
|
|
811
|
-
storage.updateMemory(db,
|
|
1015
|
+
markLink(entry.mem.id, candidate.id, "corrected_by", cosine);
|
|
1016
|
+
storage.updateMemory(db, entry.mem.id, { status: "retracted" });
|
|
812
1017
|
markedRetracted++;
|
|
813
1018
|
continue;
|
|
814
1019
|
}
|
|
815
|
-
addTrigger(
|
|
1020
|
+
addTrigger(entry.mem, candidate, verdict.confidence, cosine, false);
|
|
816
1021
|
}
|
|
817
1022
|
// verdict "none" → nothing to do
|
|
818
1023
|
}
|
|
819
1024
|
cursor = candidate.__rowid;
|
|
820
|
-
// #
|
|
821
|
-
//
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
1025
|
+
// #402 follow-up (reviewer note 1): persist after EVERY fully-considered
|
|
1026
|
+
// candidate — the 50-candidate batch left a kill window that could
|
|
1027
|
+
// strand several candidates of scan progress. updateState is an atomic
|
|
1028
|
+
// temp-rename of a small file and the loop cadence is seconds per
|
|
1029
|
+
// candidate; the cost is negligible.
|
|
1030
|
+
persistCursor();
|
|
826
1031
|
}
|
|
827
1032
|
if (deadlineStopped) {
|
|
828
|
-
console.log(`[hicortex] Reconsolidation:
|
|
829
|
-
`scan stopped at cursor ${cursor}; the next run resumes from there`);
|
|
830
|
-
}
|
|
831
|
-
else if (callCapStopped) {
|
|
832
|
-
console.log(`[hicortex] Reconsolidation: per-run call cap reached (reconsolidationMaxCalls) — ` +
|
|
1033
|
+
console.log(`[hicortex] Reconsolidation: run deadline reached (nightlyTimeBudgetMinutes) — ` +
|
|
833
1034
|
`scan stopped at cursor ${cursor}; the next run resumes from there`);
|
|
834
1035
|
}
|
|
835
1036
|
// ---- #392 judged-merge phase: apply the queued pair merges through the
|
|
836
1037
|
// dedup core (mergeMemoryIds — same canonical pick, link re-points,
|
|
837
1038
|
// dedup_log, absorb). One short lock/backup window for the whole batch, one
|
|
838
|
-
// transaction per pair.
|
|
839
|
-
//
|
|
840
|
-
//
|
|
841
|
-
//
|
|
842
|
-
//
|
|
843
|
-
//
|
|
844
|
-
|
|
845
|
-
|
|
1039
|
+
// transaction per pair. #405: the dedupNightlyMaxMerges cap is gone — the
|
|
1040
|
+
// run deadline bounds the merge loop (a stop-check between local
|
|
1041
|
+
// transactions; the deferred pairs hold the cursor below their candidates).
|
|
1042
|
+
// A pair that cannot apply (deadline, busy lock, failed backup) keeps BOTH
|
|
1043
|
+
// memories live and holds the cursor below its candidate — a confirmed
|
|
1044
|
+
// merge is never silently dropped by the cursor passing it (dup-over-loss).
|
|
1045
|
+
// A metadata-rail refusal is different: the verdict WAS rendered, both
|
|
1046
|
+
// memories stay live, the cursor advances.
|
|
846
1047
|
let mergePairsDeferred = 0;
|
|
847
1048
|
if (!dryRun && queuedMerges.length > 0) {
|
|
848
1049
|
const holdQueued = (from) => {
|
|
@@ -853,20 +1054,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
853
1054
|
: Math.min(pendingMinRowid, queuedMerges[i].candidateRowid);
|
|
854
1055
|
}
|
|
855
1056
|
};
|
|
856
|
-
|
|
857
|
-
// Machinery disabled by config: keep both (counted in band_stats as
|
|
858
|
-
// merge verdicts) and ADVANCE — holding the cursor would re-judge the
|
|
859
|
-
// same pairs into the same disabled state forever.
|
|
860
|
-
console.log(`[hicortex] Reconsolidation: ${queuedMerges.length} confirmed merge(s) kept — ` +
|
|
861
|
-
`dedupNightlyMaxMerges is 0 (merge machinery disabled)`);
|
|
862
|
-
}
|
|
863
|
-
else if (mergeOpsRemaining <= 0) {
|
|
864
|
-
mergePairsDeferred = queuedMerges.length;
|
|
865
|
-
holdQueued(0); // zone consumed the whole cap — retry next run
|
|
866
|
-
console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred — ` +
|
|
867
|
-
`dedupNightlyMaxMerges exhausted by the deterministic zone`);
|
|
868
|
-
}
|
|
869
|
-
else {
|
|
1057
|
+
{
|
|
870
1058
|
const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
|
|
871
1059
|
const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
|
|
872
1060
|
if (!release) {
|
|
@@ -888,16 +1076,19 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
888
1076
|
if (backupOk) {
|
|
889
1077
|
for (let i = 0; i < queuedMerges.length; i++) {
|
|
890
1078
|
const pair = queuedMerges[i];
|
|
891
|
-
|
|
1079
|
+
// #405: the deadline stop-check between local merge
|
|
1080
|
+
// transactions — a safe boundary; deferred pairs hold the
|
|
1081
|
+
// cursor below their candidates and retry next run.
|
|
1082
|
+
if (deadlineHit()) {
|
|
1083
|
+
deadlineStopped = true;
|
|
892
1084
|
mergePairsDeferred = queuedMerges.length - i;
|
|
893
|
-
holdQueued(i);
|
|
894
|
-
console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred —
|
|
1085
|
+
holdQueued(i);
|
|
1086
|
+
console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred — run deadline reached`);
|
|
895
1087
|
break;
|
|
896
1088
|
}
|
|
897
1089
|
const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
|
|
898
1090
|
if (result.ok) {
|
|
899
1091
|
mergePairsApplied++;
|
|
900
|
-
mergeOpsRemaining--;
|
|
901
1092
|
console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
902
1093
|
`into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
|
|
903
1094
|
}
|
|
@@ -906,6 +1097,14 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
906
1097
|
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
907
1098
|
`skipped (metadata mismatch) — both kept`);
|
|
908
1099
|
}
|
|
1100
|
+
else if (result.reason === "conflict_linked") {
|
|
1101
|
+
// #393 guard-C: the pair is conflicts-linked (operator-planted
|
|
1102
|
+
// or a prior verdict) — never blended; the cursor advances,
|
|
1103
|
+
// this verdict was rendered.
|
|
1104
|
+
conflictSkippedJudged++;
|
|
1105
|
+
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1106
|
+
`skipped (conflict-flagged) — both kept`);
|
|
1107
|
+
}
|
|
909
1108
|
// "no_members": a member vanished/was absorbed since the
|
|
910
1109
|
// verdict — nothing to merge, nothing to hold; the cursor
|
|
911
1110
|
// advances past it.
|
|
@@ -951,11 +1150,6 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
951
1150
|
deferFrom(group.targetId);
|
|
952
1151
|
break;
|
|
953
1152
|
}
|
|
954
|
-
if (maxCalls > 0 && callsUsed >= maxCalls) {
|
|
955
|
-
callCapStopped = true;
|
|
956
|
-
deferFrom(group.targetId);
|
|
957
|
-
break;
|
|
958
|
-
}
|
|
959
1153
|
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
960
1154
|
deferFrom(group.targetId);
|
|
961
1155
|
break;
|
|
@@ -964,10 +1158,9 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
964
1158
|
let contract = null;
|
|
965
1159
|
let infraError = false;
|
|
966
1160
|
try {
|
|
967
|
-
const r = await llm.
|
|
1161
|
+
const r = await llm.complete(buildRewritePrompt(group.target.content, triggersArg));
|
|
968
1162
|
contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
|
|
969
1163
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
970
|
-
callsUsed++; // #401: rewrite contracts count toward the stage call cap
|
|
971
1164
|
}
|
|
972
1165
|
catch {
|
|
973
1166
|
infraError = true;
|
|
@@ -1053,6 +1246,23 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1053
1246
|
keptLinked++;
|
|
1054
1247
|
}
|
|
1055
1248
|
}
|
|
1249
|
+
// ---- #393 guard-C zone reorder: the deterministic merge zone (pairs >=
|
|
1250
|
+
// the ceiling) runs LAST — after the scan, the judged-merge phase, and the
|
|
1251
|
+
// rewrite phase. Judgment outranks the deterministic sweep: verdicts,
|
|
1252
|
+
// marks, and binds land first, and the zone merges only what no verdict
|
|
1253
|
+
// claimed. With the zone first, a >=0.92 genuine-conflict pair was blended
|
|
1254
|
+
// before the judge ever saw it (canonical = oldest, the newer truth erased
|
|
1255
|
+
// — the planted-eval harm); running it last means a `conflicts` bind set by
|
|
1256
|
+
// THIS run's scan guards the SAME run's zone. LLM-free and budget-free — an
|
|
1257
|
+
// LLM-less night still drains duplicates. Its own short lock window,
|
|
1258
|
+
// pre-merge backup, and #405 deadline stop-check; fail-soft, never a throw.
|
|
1259
|
+
const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
|
|
1260
|
+
stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
|
|
1261
|
+
threshold: autoMergeThreshold,
|
|
1262
|
+
dryRun,
|
|
1263
|
+
acquireLock: options.acquireLock,
|
|
1264
|
+
deadline,
|
|
1265
|
+
});
|
|
1056
1266
|
// Cursor hold: un-applied work (rewrite groups, confirmed merges) holds the
|
|
1057
1267
|
// cursor BELOW its earliest contributing candidate so the pairs are
|
|
1058
1268
|
// re-detected next run.
|
|
@@ -1063,7 +1273,10 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1063
1273
|
// losers are merge verdicts at confidence 1.0; the zone persists the
|
|
1064
1274
|
// cumulative copy itself) plus this run's judged bands.
|
|
1065
1275
|
const bandStats = {};
|
|
1066
|
-
|
|
1276
|
+
{
|
|
1277
|
+
// #405: recorded whenever the zone ran (the old max_merges>0 gate was a
|
|
1278
|
+
// 0=disabled switch — the switch is gone; a clean corpus records zeros,
|
|
1279
|
+
// same as the old default-config behavior).
|
|
1067
1280
|
const det = emptyBandStat();
|
|
1068
1281
|
det.pairs = merges.losers_merged;
|
|
1069
1282
|
det.merge = merges.losers_merged;
|
|
@@ -1100,7 +1313,10 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1100
1313
|
`${markedRetracted} retracted (${belowGate} below gate, ${mergeBelowGate} merge below gate, ` +
|
|
1101
1314
|
`${contractFailed} contract failed), ${skippedInfra} infra-skipped, ${skippedIdempotent} ` +
|
|
1102
1315
|
`already-linked, ${skippedAboveCeiling} above ceiling, ${explicitVerified} explicit verified, ` +
|
|
1103
|
-
`${explicitDivergent} explicit divergent
|
|
1316
|
+
`${explicitDivergent} explicit divergent, scout ${scoutScanned} scanned / ` +
|
|
1317
|
+
`${scoutCorrectionShaped} correction-shaped / ${scoutCandidatesFound} candidate pair(s), ` +
|
|
1318
|
+
`${conflictFlagged} conflict-flagged, ${conflictSkippedJudged + merges.skipped_conflict} conflict-skipped ` +
|
|
1319
|
+
`(cursor ${cursor})`);
|
|
1104
1320
|
}
|
|
1105
1321
|
return {
|
|
1106
1322
|
scanned,
|
|
@@ -1124,6 +1340,11 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1124
1340
|
merge_below_gate: mergeBelowGate,
|
|
1125
1341
|
skipped_above_ceiling: skippedAboveCeiling,
|
|
1126
1342
|
skipped_metadata_mismatch: skippedMetadataMismatch,
|
|
1343
|
+
conflict_flagged: conflictFlagged,
|
|
1344
|
+
conflict_skipped: conflictSkippedJudged + merges.skipped_conflict,
|
|
1345
|
+
scout_scanned: scoutScanned,
|
|
1346
|
+
scout_correction_shaped: scoutCorrectionShaped,
|
|
1347
|
+
scout_candidates_found: scoutCandidatesFound,
|
|
1127
1348
|
band_stats: bandStats,
|
|
1128
1349
|
};
|
|
1129
1350
|
}
|