@gamaze/hicortex 0.20.7 → 0.20.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -41
- package/dist/calibration.d.ts +174 -0
- package/dist/calibration.js +231 -0
- package/dist/capture.d.ts +15 -3
- package/dist/capture.js +10 -1
- package/dist/classify-domains.d.ts +6 -0
- package/dist/classify-domains.js +7 -1
- package/dist/cli.js +2 -3
- package/dist/config-read.d.ts +1 -1
- package/dist/config-read.js +96 -9
- package/dist/consolidate.d.ts +79 -68
- package/dist/consolidate.js +218 -174
- package/dist/dashboard.d.ts +4 -3
- package/dist/dedup.d.ts +34 -26
- package/dist/dedup.js +91 -57
- package/dist/distiller.js +1 -1
- package/dist/domain-classify.d.ts +7 -6
- package/dist/domain-classify.js +12 -10
- package/dist/eval/decay-eval.d.ts +3 -3
- package/dist/eval/decay-eval.js +4 -4
- package/dist/eval/planted-eval.d.ts +26 -0
- package/dist/eval/planted-eval.js +97 -0
- package/dist/eval/planted-fixtures.d.ts +107 -0
- package/dist/eval/planted-fixtures.js +283 -0
- package/dist/eval/planted-harness.d.ts +176 -0
- package/dist/eval/planted-harness.js +649 -0
- package/dist/index.js +4 -3
- package/dist/init.d.ts +9 -3
- package/dist/init.js +52 -9
- package/dist/llm.d.ts +43 -58
- package/dist/llm.js +87 -101
- package/dist/mcp-server.js +29 -29
- package/dist/nightly.js +105 -103
- package/dist/nofit.d.ts +4 -11
- package/dist/nofit.js +6 -23
- package/dist/recall-index.d.ts +30 -28
- package/dist/recall-index.js +21 -18
- package/dist/recall-registry.d.ts +2 -1
- package/dist/recall-registry.js +35 -1
- package/dist/reconsolidation.d.ts +124 -72
- package/dist/reconsolidation.js +359 -148
- package/dist/relink.js +3 -4
- package/dist/retrieval.d.ts +68 -35
- package/dist/retrieval.js +292 -104
- package/dist/run-deadline.d.ts +62 -0
- package/dist/run-deadline.js +73 -0
- package/dist/schema-prototypes.d.ts +3 -3
- package/dist/schema-prototypes.js +3 -3
- package/dist/state.d.ts +2 -3
- package/dist/storage.d.ts +16 -16
- package/dist/storage.js +62 -24
- package/dist/telemetry.d.ts +8 -7
- package/dist/token-budget.js +3 -4
- package/dist/type-classify.js +4 -4
- package/dist/types.d.ts +95 -155
- package/domains.example.json +4 -5
- package/hermes-plugin/hicortex/README.md +2 -2
- package/openclaw.plugin.json +1 -1
- package/package.json +2 -1
- package/pi-extension/hicortex/README.md +1 -1
- package/server.json +3 -3
package/dist/reconsolidation.js
CHANGED
|
@@ -16,11 +16,39 @@
|
|
|
16
16
|
* #392 — one zone system, ONE verdict per pair: below `correctionMinSimilarity`
|
|
17
17
|
* (floor, 0.75) pairs are not candidates; in [floor, `dedupAutoMergeThreshold`)
|
|
18
18
|
* (ceiling, 0.92) each unlinked pair gets ONE verdict call whose action is
|
|
19
|
-
* `merge` | `corrects` | `supersedes` | `none`; at/above the
|
|
20
|
-
* deterministic merge zone (dedup.ts runDeterministicMergeZone —
|
|
21
|
-
* budget-free) owns the pair. The merge disposition reuses the
|
|
22
|
-
* execution (canonical pick, link re-point, dedup_log, metadata
|
|
23
|
-
* merge verdict below `correctionRewriteMinConfidence` keeps both
|
|
19
|
+
* `merge` | `corrects` | `supersedes` | `conflicts` | `none`; at/above the
|
|
20
|
+
* ceiling the deterministic merge zone (dedup.ts runDeterministicMergeZone —
|
|
21
|
+
* LLM-free, budget-free) owns the pair. The merge disposition reuses the
|
|
22
|
+
* dedup core's execution (canonical pick, link re-point, dedup_log, metadata
|
|
23
|
+
* rails); a merge verdict below `correctionRewriteMinConfidence` keeps both
|
|
24
|
+
* memories.
|
|
25
|
+
*
|
|
26
|
+
* #393 increment B — the SCOUT, a second detection source with the SAME
|
|
27
|
+
* judge: the similarity floor is structurally blind to corrections riding
|
|
28
|
+
* inside topically unrelated memories (the field failure — cosine ~0.5-0.6 to
|
|
29
|
+
* their target, zero `corrects` verdicts in the whole corpus baseline), so
|
|
30
|
+
* per NEW memory ONE classify-tier shape call asks whether it corrects/
|
|
31
|
+
* retracts/supersedes/CONTRADICTS something previously recorded (guard-C
|
|
32
|
+
* extended the question); correction-shaped
|
|
33
|
+
* memories FTS the corpus with the referenced claim's distinctive terms (the
|
|
34
|
+
* correction CONTAINS the words of what it corrects) and the hits become
|
|
35
|
+
* candidate pairs in the SAME verdict loop — no similarity gate for this
|
|
36
|
+
* source: cosine is a ranker/link strength, never a blocker. Per-source
|
|
37
|
+
* counters (scout_scanned / scout_correction_shaped / scout_candidates_found)
|
|
38
|
+
* ride the stage report; cosine band stats stay similarity-source-only.
|
|
39
|
+
*
|
|
40
|
+
* #393 guard-C — the conflicts flag + the zone-runs-last order: judgment
|
|
41
|
+
* OUTRANKS the deterministic sweep. A `conflicts` verdict writes a symmetric
|
|
42
|
+
* `conflicts` link (the pair genuinely disagrees — cannot both be true) and
|
|
43
|
+
* NOTHING else: no status change, no rewrite, no merge queue; both records
|
|
44
|
+
* stay live so the consumer sees both truths. Both merge paths (the zone's
|
|
45
|
+
* planDedup and the judged mergeMemoryIds) refuse to blend a conflicts-linked
|
|
46
|
+
* pair, counted as conflict_skipped. The zone therefore runs AFTER the
|
|
47
|
+
* rewrite phase — with the zone first, a >=0.92 conflict pair was blended
|
|
48
|
+
* before the judge ever saw it (the planted-eval harm: canonical=older, the
|
|
49
|
+
* newer truth erased); running it last means verdicts/marks/binds land first
|
|
50
|
+
* and the zone merges only what no verdict claimed — a conflicts bind set by
|
|
51
|
+
* this run's scan guards the SAME run's zone.
|
|
24
52
|
*
|
|
25
53
|
* Status vocabulary (code-defined, extensible — deliberately NOT config):
|
|
26
54
|
* NULL/'active' default | 'superseded' + 'retracted' demote in ranking |
|
|
@@ -71,8 +99,10 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
71
99
|
};
|
|
72
100
|
})();
|
|
73
101
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
74
|
-
exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.
|
|
102
|
+
exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = exports.DEFAULT_CORRECTION_MIN_SIMILARITY = exports.RECONSOLIDATION_STAGE_LABEL = void 0;
|
|
75
103
|
exports.isFactShapedTarget = isFactShapedTarget;
|
|
104
|
+
exports.buildScoutShapePrompt = buildScoutShapePrompt;
|
|
105
|
+
exports.parseScoutShape = parseScoutShape;
|
|
76
106
|
exports.buildCorrectionVerdictPrompt = buildCorrectionVerdictPrompt;
|
|
77
107
|
exports.parseCorrectionVerdict = parseCorrectionVerdict;
|
|
78
108
|
exports.buildRewritePrompt = buildRewritePrompt;
|
|
@@ -94,6 +124,7 @@ const storage = __importStar(require("./storage.js"));
|
|
|
94
124
|
const state_js_1 = require("./state.js");
|
|
95
125
|
const db_js_1 = require("./db.js");
|
|
96
126
|
const capture_js_1 = require("./capture.js");
|
|
127
|
+
const CALIBRATION = __importStar(require("./calibration.js"));
|
|
97
128
|
const paths_js_1 = require("./paths.js");
|
|
98
129
|
const dedup_js_1 = require("./dedup.js");
|
|
99
130
|
// ---------------------------------------------------------------------------
|
|
@@ -102,37 +133,21 @@ const dedup_js_1 = require("./dedup.js");
|
|
|
102
133
|
/** Stage label used for every budget.use()/recordUsage() call (#384). */
|
|
103
134
|
exports.RECONSOLIDATION_STAGE_LABEL = "reconsolidation";
|
|
104
135
|
/**
|
|
105
|
-
* Default minimum COSINE similarity for a correction candidate pair
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
136
|
+
* Default minimum COSINE similarity for a correction candidate pair —
|
|
137
|
+
* RELEASE-MANAGED since #408 (calibration.ts CORRECTION_MIN_SIMILARITY;
|
|
138
|
+
* provenance there). Lower than the supersession stage's 0.80 on purpose: a
|
|
139
|
+
* retraction often rides inside an otherwise unrelated memory (the field
|
|
140
|
+
* failure that opened this issue), so the neighborhood gate must be a touch
|
|
141
|
+
* wider while the LLM verdict + confidence gate carry the precision load.
|
|
110
142
|
*/
|
|
111
|
-
exports.DEFAULT_CORRECTION_MIN_SIMILARITY =
|
|
143
|
+
exports.DEFAULT_CORRECTION_MIN_SIMILARITY = CALIBRATION.CORRECTION_MIN_SIMILARITY;
|
|
112
144
|
/**
|
|
113
|
-
* Default minimum verdict confidence for the REWRITE fork
|
|
145
|
+
* Default minimum verdict confidence for the REWRITE fork — release-managed
|
|
146
|
+
* (calibration.ts CORRECTION_REWRITE_MIN_CONFIDENCE). Below this a
|
|
114
147
|
* `corrects` verdict degrades to mark-only — a weak mark is recoverable, a
|
|
115
148
|
* weak rewrite is corruption.
|
|
116
149
|
*/
|
|
117
|
-
exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE =
|
|
118
|
-
/**
|
|
119
|
-
* Default wall-clock bound for the stage, in minutes (#401). Checked at the
|
|
120
|
-
* top of the candidate scan loop (and before each rewrite contract call); on
|
|
121
|
-
* expiry the scan breaks cleanly at the last fully-considered candidate and
|
|
122
|
-
* the next run resumes from the persisted cursor. 120 sits safely under any
|
|
123
|
-
* sane process-level nightly timeout. 0 disables the bound. Invalid →
|
|
124
|
-
* default.
|
|
125
|
-
*/
|
|
126
|
-
exports.DEFAULT_RECONSOLIDATION_MAX_MINUTES = 120;
|
|
127
|
-
/**
|
|
128
|
-
* Default per-run classify-call ceiling for the stage (#401) — the
|
|
129
|
-
* supersessionMaxCalls pattern with a NON-ZERO default ON PURPOSE: that
|
|
130
|
-
* knob's 0=unlimited default is what let the first full-corpus pass grow
|
|
131
|
-
* unbounded. Counts EVERY classify-tier call the stage makes (mark
|
|
132
|
-
* verifications, pair verdicts, rewrite contracts). 0 disables the cap.
|
|
133
|
-
* Invalid → default.
|
|
134
|
-
*/
|
|
135
|
-
exports.DEFAULT_RECONSOLIDATION_MAX_CALLS = 600;
|
|
150
|
+
exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = CALIBRATION.CORRECTION_REWRITE_MIN_CONFIDENCE;
|
|
136
151
|
/** Neighbor pool size before older/similarity filtering narrows to top 5 (supersession mirror). */
|
|
137
152
|
const CORRECTION_NEIGHBOR_POOL = 15;
|
|
138
153
|
/** Older-neighbor pairs kept per candidate after filtering (supersession mirror). */
|
|
@@ -161,10 +176,10 @@ exports.DEMOTED_STATUSES = ["superseded", "retracted"];
|
|
|
161
176
|
function isFactShapedTarget(mem) {
|
|
162
177
|
return mem.memory_type === "knowledge" || mem.content.includes("[Facts Learned]");
|
|
163
178
|
}
|
|
164
|
-
/** True when a superseded_by
|
|
179
|
+
/** True when a superseded_by / corrected_by / conflicts link already exists between the pair, either direction. */
|
|
165
180
|
function alreadyResolutionLinked(db, oldId, newId) {
|
|
166
181
|
const row = db
|
|
167
|
-
.prepare(`SELECT 1 FROM memory_links WHERE relationship IN ('superseded_by', 'corrected_by')
|
|
182
|
+
.prepare(`SELECT 1 FROM memory_links WHERE relationship IN ('superseded_by', 'corrected_by', 'conflicts')
|
|
168
183
|
AND ((source_id = ? AND target_id = ?) OR (source_id = ? AND target_id = ?))`)
|
|
169
184
|
.get(oldId, newId, newId, oldId);
|
|
170
185
|
return !!row;
|
|
@@ -178,6 +193,58 @@ function hasLink(db, sourceId, targetId, relationship) {
|
|
|
178
193
|
function nowIso() {
|
|
179
194
|
return new Date().toISOString();
|
|
180
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* Build the constrained correction-shape prompt (classify-tier cost profile:
|
|
198
|
+
* 1500-char truncation, supersession/verdict precedent). The wording asks for
|
|
199
|
+
* the OLD claim's distinctive terms — the field-failure mechanism is that a
|
|
200
|
+
* correction CONTAINS the words of what it corrects, even when the surrounding
|
|
201
|
+
* topics (and therefore the embedding cosine) are unrelated. Guard-C extends
|
|
202
|
+
* the question to contradictions: two records that disagree on the same
|
|
203
|
+
* quantity share even MORE wording than a cross-topic correction does.
|
|
204
|
+
*/
|
|
205
|
+
function buildScoutShapePrompt(content) {
|
|
206
|
+
const trunc = (s) => (s.length > PROMPT_TRUNCATE_CHARS ? `${s.slice(0, PROMPT_TRUNCATE_CHARS)}…` : s);
|
|
207
|
+
return (`You are scanning a memory that was just added to an AI agent's long-term memory store.\n\n` +
|
|
208
|
+
`MEMORY:\n${trunc(content)}\n\n` +
|
|
209
|
+
`Does this memory correct, retract, supersede, or contradict a claim, decision, or state that was ` +
|
|
210
|
+
`previously recorded elsewhere in the store? A mere duplicate, elaboration, independent ` +
|
|
211
|
+
`fact, or new information that invalidates nothing is NOT a correction.\n` +
|
|
212
|
+
`If it is a correction/retraction/supersession/contradiction, list the most distinctive terms of the ` +
|
|
213
|
+
`OLD claim it references — words likely to appear verbatim in the older record.\n\n` +
|
|
214
|
+
`Reply with ONLY a JSON object, no prose: ` +
|
|
215
|
+
`{"correction": true | false, "references": "<distinctive terms of the referenced old claim, or empty string>", ` +
|
|
216
|
+
`"confidence": <number between 0 and 1>}`);
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Parse the scout shape reply. Null on unparseable JSON, a missing/non-boolean
|
|
220
|
+
* `correction`, or a missing/out-of-range `confidence` — the caller counts
|
|
221
|
+
* skipped_infra and moves on (parseSupersessionReply discipline: never
|
|
222
|
+
* mis-detect on ambiguity). `references` is lenient (missing/non-string → "")
|
|
223
|
+
* because an empty string simply yields no FTS hits — a harmless miss, not a
|
|
224
|
+
* mis-judgment.
|
|
225
|
+
*/
|
|
226
|
+
function parseScoutShape(reply) {
|
|
227
|
+
if (!reply)
|
|
228
|
+
return null;
|
|
229
|
+
const start = reply.indexOf("{");
|
|
230
|
+
const end = reply.lastIndexOf("}");
|
|
231
|
+
if (start === -1 || end === -1 || end <= start)
|
|
232
|
+
return null;
|
|
233
|
+
let obj;
|
|
234
|
+
try {
|
|
235
|
+
obj = JSON.parse(reply.slice(start, end + 1));
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
if (typeof obj.correction !== "boolean")
|
|
241
|
+
return null;
|
|
242
|
+
const confidence = Number(obj.confidence);
|
|
243
|
+
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1)
|
|
244
|
+
return null;
|
|
245
|
+
const references = typeof obj.references === "string" ? obj.references : "";
|
|
246
|
+
return { correction: obj.correction, references: references.trim(), confidence };
|
|
247
|
+
}
|
|
181
248
|
/** Build the constrained pair-verdict prompt (1500-char truncation, supersession precedent). */
|
|
182
249
|
function buildCorrectionVerdictPrompt(oldContent, newContent) {
|
|
183
250
|
const trunc = (s) => (s.length > PROMPT_TRUNCATE_CHARS ? `${s.slice(0, PROMPT_TRUNCATE_CHARS)}…` : s);
|
|
@@ -191,9 +258,12 @@ function buildCorrectionVerdictPrompt(oldContent, newContent) {
|
|
|
191
258
|
`wrong, no longer true, or was retracted, and the newer memory carries the corrected fact.\n` +
|
|
192
259
|
`- "supersedes": the newer memory replaces a decision, plan, or state that was valid at the time but is ` +
|
|
193
260
|
`now outdated — a replacement, not a factual correction.\n` +
|
|
261
|
+
`- "conflicts": the two memories make claims that cannot both be true — they disagree on a fact, value, ` +
|
|
262
|
+
`or state, and neither one corrects, supersedes, or restates the other (for example two sources report ` +
|
|
263
|
+
`different values for the same quantity). Keep both; flag the conflict.\n` +
|
|
194
264
|
`- "none": unrelated, merely similar, or both can still be true (an addition or elaboration).\n\n` +
|
|
195
265
|
`Reply with ONLY a JSON object, no prose: ` +
|
|
196
|
-
`{"action": "merge" | "corrects" | "supersedes" | "none", "confidence": <number between 0 and 1>}`);
|
|
266
|
+
`{"action": "merge" | "corrects" | "supersedes" | "conflicts" | "none", "confidence": <number between 0 and 1>}`);
|
|
197
267
|
}
|
|
198
268
|
/**
|
|
199
269
|
* Parse the pair verdict. Null on anything unparseable, unknown action, or an
|
|
@@ -215,8 +285,13 @@ function parseCorrectionVerdict(reply) {
|
|
|
215
285
|
return null;
|
|
216
286
|
}
|
|
217
287
|
const action = obj.action;
|
|
218
|
-
if (action !== "merge" &&
|
|
288
|
+
if (action !== "merge" &&
|
|
289
|
+
action !== "corrects" &&
|
|
290
|
+
action !== "supersedes" &&
|
|
291
|
+
action !== "conflicts" &&
|
|
292
|
+
action !== "none") {
|
|
219
293
|
return null;
|
|
294
|
+
}
|
|
220
295
|
const confidence = Number(obj.confidence);
|
|
221
296
|
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1)
|
|
222
297
|
return null;
|
|
@@ -452,7 +527,7 @@ function bandForCosine(bands, cosine) {
|
|
|
452
527
|
}
|
|
453
528
|
/** An empty band-stat record (fresh accumulation starts from zeroes). */
|
|
454
529
|
function emptyBandStat() {
|
|
455
|
-
return { pairs: 0, merge: 0, corrects: 0, supersedes: 0, none: 0, merge_below_gate: 0, conf_sum: 0 };
|
|
530
|
+
return { pairs: 0, merge: 0, corrects: 0, supersedes: 0, conflicts: 0, none: 0, merge_below_gate: 0, conf_sum: 0 };
|
|
456
531
|
}
|
|
457
532
|
/** Add a run's per-band counts into a cumulative record (in place). */
|
|
458
533
|
function accumulateBandStat(cumulative, run) {
|
|
@@ -460,6 +535,7 @@ function accumulateBandStat(cumulative, run) {
|
|
|
460
535
|
cumulative.merge += run.merge;
|
|
461
536
|
cumulative.corrects += run.corrects;
|
|
462
537
|
cumulative.supersedes += run.supersedes;
|
|
538
|
+
cumulative.conflicts += run.conflicts;
|
|
463
539
|
cumulative.none += run.none;
|
|
464
540
|
cumulative.merge_below_gate += run.merge_below_gate;
|
|
465
541
|
cumulative.conf_sum += run.conf_sum;
|
|
@@ -481,9 +557,43 @@ async function findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarit
|
|
|
481
557
|
.sort((a, b) => (0, retrieval_js_1.l2ToCosine)(b.distance) - (0, retrieval_js_1.l2ToCosine)(a.distance))
|
|
482
558
|
.slice(0, CORRECTION_NEIGHBOR_TOP_K);
|
|
483
559
|
}
|
|
560
|
+
/**
|
|
561
|
+
* The scout source (#393 increment B): for a correction-shaped NEW memory, FTS
|
|
562
|
+
* the corpus with the referenced claim's distinctive terms and return the OLDER
|
|
563
|
+
* hits as candidate pairs. This is the reference-extraction half — it finds the
|
|
564
|
+
* old claim even when the overall topics differ (and therefore the cosine sits
|
|
565
|
+
* below the similarity floor) because the correction CONTAINS the words of
|
|
566
|
+
* what it corrects. Deterministic: ONE FTS query, zero LLM. Filters: self,
|
|
567
|
+
* non-older (detection only pairs older → newer, the KNN mirror), and
|
|
568
|
+
* defensively non-absorbed hits. Pool/top-K reuse the KNN constants; FTS rank
|
|
569
|
+
* (BM25, best first) is the order. Dedup against the KNN neighbor ids is the
|
|
570
|
+
* caller's job (a pair found by both sources is judged once, as similarity).
|
|
571
|
+
*/
|
|
572
|
+
function findScoutNeighbors(db, candidate, references, candidateEmbedding) {
|
|
573
|
+
if (!references)
|
|
574
|
+
return [];
|
|
575
|
+
try {
|
|
576
|
+
const hits = storage.searchFts(db, references, CORRECTION_NEIGHBOR_POOL);
|
|
577
|
+
return hits
|
|
578
|
+
.filter((m) => m.id !== candidate.id &&
|
|
579
|
+
m.created_at < candidate.created_at &&
|
|
580
|
+
m.status !== "absorbed")
|
|
581
|
+
.slice(0, CORRECTION_NEIGHBOR_TOP_K)
|
|
582
|
+
.map((m) => {
|
|
583
|
+
const hitVec = storage.getStoredEmbedding(db, m.id);
|
|
584
|
+
const cosine = candidateEmbedding && hitVec ? (0, retrieval_js_1.cosineBetweenVectors)(candidateEmbedding, hitVec) : 0;
|
|
585
|
+
return { mem: m, cosine, source: "scout" };
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
catch {
|
|
589
|
+
// FTS is deterministic infrastructure — a throw here is a bug or a corrupt
|
|
590
|
+
// index, never a judgment question. Fail soft: no scout pairs this memory.
|
|
591
|
+
return [];
|
|
592
|
+
}
|
|
593
|
+
}
|
|
484
594
|
async function classifyPair(llm, oldContent, newContent) {
|
|
485
595
|
try {
|
|
486
|
-
const r = await llm.
|
|
596
|
+
const r = await llm.complete(buildCorrectionVerdictPrompt(oldContent, newContent));
|
|
487
597
|
return { verdict: parseCorrectionVerdict(r.text), usage: r.usage };
|
|
488
598
|
}
|
|
489
599
|
catch {
|
|
@@ -493,21 +603,34 @@ async function classifyPair(llm, oldContent, newContent) {
|
|
|
493
603
|
/**
|
|
494
604
|
* Nightly reconsolidation stage (#384, #392 — THE unified resolution stage).
|
|
495
605
|
*
|
|
496
|
-
* Phase
|
|
497
|
-
*
|
|
606
|
+
* Phase order (#393 guard-C): the deterministic merge zone (pairs >= the
|
|
607
|
+
* ceiling) runs LAST — after the scan, the judged-merge phase, and the
|
|
608
|
+
* rewrite phase. Judgment outranks the deterministic sweep: verdicts, marks,
|
|
609
|
+
* and binds land first and the zone merges only what no verdict claimed. With
|
|
610
|
+
* the zone first, a >=0.92 genuine-conflict pair was blended before the judge
|
|
611
|
+
* ever saw it (the planted-eval harm); running it last means a `conflicts`
|
|
612
|
+
* bind set by this run's scan guards the SAME run's zone. Zone internals
|
|
613
|
+
* (lock, backup, deadline, persistBand, fail-soft) are unchanged.
|
|
498
614
|
*
|
|
499
615
|
* Scan: every memory with rowid > reconsolidationCursor (no shape filter;
|
|
500
616
|
* absorbed candidates are skipped — invisible memories are not re-judged).
|
|
501
|
-
* Each candidate's pairs: incoming explicit marks (verified once, AC7) then
|
|
502
|
-
*
|
|
503
|
-
*
|
|
617
|
+
* Each candidate's pairs: incoming explicit marks (verified once, AC7), then
|
|
618
|
+
* ONE scout shape call (#393 B — flags correction shape; non-corrections stop
|
|
619
|
+
* there), then up-to-5 older KNN neighbors in [floor, ceiling) (verdict call
|
|
620
|
+
* per unlinked pair, AC2 — pairs at/above the ceiling are counted, never
|
|
621
|
+
* judged) plus the scout's FTS hits for correction-shaped memories (same
|
|
622
|
+
* verdict loop, NO similarity gate; guard-C: a scout hit whose KNN twin sits
|
|
623
|
+
* at/above the ceiling is re-tagged scout so the pair IS judged instead of
|
|
624
|
+
* being left for the zone to blend). Confirmed
|
|
504
625
|
* `corrects` pairs above the confidence gate on fact-shaped targets group by
|
|
505
626
|
* target into ONE rewrite call each (AC3); confirmed `merge` pairs queue for
|
|
506
|
-
* the merge phase;
|
|
627
|
+
* the merge phase; a `conflicts` verdict writes the conflicts link and
|
|
628
|
+
* nothing else (both live); everything else is mark-only.
|
|
507
629
|
*
|
|
508
630
|
* Merge phase (#392): queued pairs merge through the dedup core under one
|
|
509
|
-
* lock/backup window
|
|
510
|
-
*
|
|
631
|
+
* lock/backup window. A pair that cannot apply keeps both memories and holds
|
|
632
|
+
* the cursor; a conflicts-linked or metadata-mismatched refusal keeps both
|
|
633
|
+
* and advances (the verdict was rendered).
|
|
511
634
|
*
|
|
512
635
|
* Cursor discipline mirrors stageSupersession: the cursor advances past a
|
|
513
636
|
* candidate once its neighbor set has been considered, regardless of infra
|
|
@@ -533,25 +656,13 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
533
656
|
const minSimilarity = validNumber(options.minSimilarity, exports.DEFAULT_CORRECTION_MIN_SIMILARITY, (n) => n > 0 && n <= 1);
|
|
534
657
|
const rewriteMinConfidence = validNumber(options.rewriteMinConfidence, exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE, (n) => n > 0 && n <= 1);
|
|
535
658
|
const autoMergeThreshold = validNumber(options.autoMergeThreshold, dedup_js_1.DEFAULT_DEDUP_MERGE_THRESHOLD, (n) => n > 0 && n <= 1);
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
//
|
|
540
|
-
//
|
|
541
|
-
|
|
542
|
-
const
|
|
543
|
-
const deadlineHit = () => Date.now() >= deadlineAt;
|
|
544
|
-
// ---- #392 phase 0: the deterministic merge zone (pairs >= the ceiling),
|
|
545
|
-
// LLM-free and budget-free — an LLM-less night still drains duplicates. Its
|
|
546
|
-
// own short lock window, pre-merge backup, and pacing cap; fail-soft, never
|
|
547
|
-
// a throw. Runs FIRST so the scan below never sees the pairs it owns.
|
|
548
|
-
const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
|
|
549
|
-
stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
|
|
550
|
-
threshold: autoMergeThreshold,
|
|
551
|
-
maxMerges,
|
|
552
|
-
dryRun,
|
|
553
|
-
acquireLock: options.acquireLock,
|
|
554
|
-
});
|
|
659
|
+
// ---- #405 runtime bound. The stage-local reconsolidationMaxMinutes clock
|
|
660
|
+
// (#401) is gone — the run-wide pipeline deadline (nightly.ts, config
|
|
661
|
+
// nightlyTimeBudgetMinutes) is the only wall-clock. The deterministic
|
|
662
|
+
// zone's runtime counts against it via the zone's own stop-check below.
|
|
663
|
+
// hit() logs event=deadline_deferred once per stage name.
|
|
664
|
+
const deadline = options.deadline;
|
|
665
|
+
const deadlineHit = (stageLabel = exports.RECONSOLIDATION_STAGE_LABEL) => deadline?.hit(stageLabel) ?? false;
|
|
555
666
|
// Per-run verdict statistics by cosine band (#392) — report snapshot here,
|
|
556
667
|
// cumulative series in state.json at stage end (never on dry-run).
|
|
557
668
|
const bands = buildResolutionBands(minSimilarity, autoMergeThreshold);
|
|
@@ -593,6 +704,16 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
593
704
|
let skippedAboveCeiling = 0;
|
|
594
705
|
let skippedMetadataMismatch = 0;
|
|
595
706
|
let mergePairsApplied = 0;
|
|
707
|
+
// #393 guard-C: conflicts verdicts rendered (link written, both live) and
|
|
708
|
+
// judged-path merge refusals on a conflicts-linked pair.
|
|
709
|
+
let conflictFlagged = 0;
|
|
710
|
+
let conflictSkippedJudged = 0;
|
|
711
|
+
// #393 B scout counters (per-source observability, the #394 discipline):
|
|
712
|
+
// shape calls made / correction-shaped verdicts / FTS hits that became
|
|
713
|
+
// candidate pairs. 0 on dry-run (the shape call is LLM work).
|
|
714
|
+
let scoutScanned = 0;
|
|
715
|
+
let scoutCorrectionShaped = 0;
|
|
716
|
+
let scoutCandidatesFound = 0;
|
|
596
717
|
let cursor = startCursor;
|
|
597
718
|
const queuedMerges = [];
|
|
598
719
|
// #392 cursor-hold anchor, shared by the merge phase and the rewrite phase:
|
|
@@ -607,16 +728,14 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
607
728
|
linksCreatedThisRun.add(`${oldId}|${newId}`);
|
|
608
729
|
};
|
|
609
730
|
// ---- #401: mid-scan cursor persistence. Called at EVERY scan-loop exit
|
|
610
|
-
// path (deadline,
|
|
731
|
+
// path (deadline, budget cap, mark-verify budget stop, discovery
|
|
611
732
|
// failure) AND after every fully-considered candidate, so a killed run
|
|
612
733
|
// loses at most the candidate in flight. The end-of-stage updateState
|
|
613
734
|
// below stays the authoritative final write (it also applies the
|
|
614
735
|
// pendingMinRowid hold — that variable is only ever set AFTER the scan
|
|
615
736
|
// loop, so it is null at every call site here). updateState is
|
|
616
737
|
// load→mutate→temp-rename atomic.
|
|
617
|
-
let callsUsed = 0;
|
|
618
738
|
let deadlineStopped = false;
|
|
619
|
-
let callCapStopped = false;
|
|
620
739
|
// #402 follow-up (reviewer note 1): the hard-kill orphan floor. Queued
|
|
621
740
|
// merges and open rewrite groups are applied only in the POST-scan
|
|
622
741
|
// phases — until then their verdicts exist only in memory, and a
|
|
@@ -671,14 +790,6 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
671
790
|
persistCursor();
|
|
672
791
|
break;
|
|
673
792
|
}
|
|
674
|
-
// #401: the per-stage call cap stops the scan at the candidate boundary
|
|
675
|
-
// (the in-loop check below is the mid-candidate backstop — supersession
|
|
676
|
-
// mirrors both).
|
|
677
|
-
if (!dryRun && maxCalls > 0 && callsUsed >= maxCalls) {
|
|
678
|
-
callCapStopped = true;
|
|
679
|
-
persistCursor();
|
|
680
|
-
break;
|
|
681
|
-
}
|
|
682
793
|
scanned++;
|
|
683
794
|
// ---- AC7: verify incoming explicit marks (corrected_by/superseded_by
|
|
684
795
|
// links targeting this candidate) before they can join a rewrite group.
|
|
@@ -709,7 +820,6 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
709
820
|
}
|
|
710
821
|
const { verdict, usage } = await classifyPair(llm, target.content, candidate.content);
|
|
711
822
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
|
|
712
|
-
callsUsed++; // #401
|
|
713
823
|
pairsEvaluated++;
|
|
714
824
|
if (!verdict) {
|
|
715
825
|
skippedInfra++; // mark retained; the neighborhood is revisited via newer candidacies
|
|
@@ -733,7 +843,43 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
733
843
|
break;
|
|
734
844
|
}
|
|
735
845
|
}
|
|
736
|
-
// ----
|
|
846
|
+
// ---- #393 B: scout shape call — ONE classify-tier call per candidate,
|
|
847
|
+
// budget-metered under the same stage label as verdicts. Flags whether
|
|
848
|
+
// this memory corrects/retracts/supersedes something previously recorded;
|
|
849
|
+
// non-corrections stop here (zero follow-up). A parse/infra failure skips
|
|
850
|
+
// the scout source for this memory only (fail-soft — the similarity
|
|
851
|
+
// source below still runs) and counts skipped_infra, the verdict-skip
|
|
852
|
+
// discipline. Dry-run skips the call entirely: it is LLM work, and
|
|
853
|
+
// dry-runs make zero LLM calls (the scout counters read 0 there).
|
|
854
|
+
let scoutReferences = null;
|
|
855
|
+
if (!dryRun) {
|
|
856
|
+
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
857
|
+
persistCursor(); // shape call refused — this candidate re-scouts next run
|
|
858
|
+
break;
|
|
859
|
+
}
|
|
860
|
+
scoutScanned++;
|
|
861
|
+
try {
|
|
862
|
+
const r = await llm.complete(buildScoutShapePrompt(candidate.content));
|
|
863
|
+
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
864
|
+
const shape = parseScoutShape(r.text);
|
|
865
|
+
if (!shape) {
|
|
866
|
+
skippedInfra++;
|
|
867
|
+
}
|
|
868
|
+
else if (shape.correction) {
|
|
869
|
+
scoutCorrectionShaped++;
|
|
870
|
+
scoutReferences = shape.references;
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
catch {
|
|
874
|
+
skippedInfra++;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
// ---- AC2: detection pairs against older neighbors — TWO sources (#393 B):
|
|
878
|
+
// the KNN similarity source (floor-gated, the pre-B baseline) and, for
|
|
879
|
+
// correction-shaped candidates, the scout's FTS source (the referenced
|
|
880
|
+
// claim's terms → corpus search; NO similarity gate — cosine is a ranker,
|
|
881
|
+
// never a blocker). Merged + deduped by neighbor id: a pair found by both
|
|
882
|
+
// sources is judged ONCE, as similarity (with band attribution).
|
|
737
883
|
let neighbors;
|
|
738
884
|
try {
|
|
739
885
|
neighbors = await findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarity);
|
|
@@ -744,39 +890,74 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
744
890
|
persistCursor(); // #401: every exit path persists
|
|
745
891
|
continue;
|
|
746
892
|
}
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
893
|
+
const neighborEntries = neighbors.map((n) => ({
|
|
894
|
+
mem: n,
|
|
895
|
+
cosine: (0, retrieval_js_1.l2ToCosine)(n.distance),
|
|
896
|
+
source: "similarity",
|
|
897
|
+
}));
|
|
898
|
+
if (scoutReferences !== null) {
|
|
899
|
+
const candidateVec = storage.getStoredEmbedding(db, candidate.id);
|
|
900
|
+
const knnById = new Map(neighborEntries.map((e) => [e.mem.id, e]));
|
|
901
|
+
for (const entry of findScoutNeighbors(db, candidate, scoutReferences, candidateVec)) {
|
|
902
|
+
const knn = knnById.get(entry.mem.id);
|
|
903
|
+
if (knn) {
|
|
904
|
+
// Both sources found the pair. Below the ceiling the KNN entry
|
|
905
|
+
// would be judged anyway — similarity keeps it (band attribution,
|
|
906
|
+
// judged once). At/above the ceiling the similarity entry would be
|
|
907
|
+
// ceiling-skipped (zone territory, never judged) — yet the zone now
|
|
908
|
+
// runs AFTER the scan, and a genuine conflict at >=0.92 is exactly
|
|
909
|
+
// the pair it would blend with no judge in the loop (guard-C's
|
|
910
|
+
// harm). Re-tag the entry to the scout source so the pair IS
|
|
911
|
+
// judged: the scout's no-gate exemption applies, a `conflicts`
|
|
912
|
+
// verdict can plant the guard link, and the zone's own guard then
|
|
913
|
+
// refuses the cluster in this same run.
|
|
914
|
+
if (knn.cosine >= autoMergeThreshold)
|
|
915
|
+
knn.source = "scout";
|
|
916
|
+
continue;
|
|
917
|
+
}
|
|
918
|
+
neighborEntries.push(entry);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
for (const entry of neighborEntries) {
|
|
922
|
+
pairsDiscovered++; // gate discovery (#394), both sources, before any skip/judgment
|
|
923
|
+
if (entry.source === "scout")
|
|
924
|
+
scoutCandidatesFound++;
|
|
925
|
+
if (alreadyResolutionLinked(db, entry.mem.id, candidate.id)) {
|
|
750
926
|
skippedIdempotent++;
|
|
751
927
|
continue;
|
|
752
928
|
}
|
|
753
929
|
pairsDiscoveredUnlinked++; // still unlinked — the actionable candidate
|
|
754
|
-
|
|
755
|
-
//
|
|
756
|
-
//
|
|
757
|
-
|
|
758
|
-
|
|
930
|
+
const pairCosine = entry.cosine;
|
|
931
|
+
// #392: SIMILARITY-source pairs at/above the ceiling belong to the
|
|
932
|
+
// deterministic zone — counted here, never LLM-judged (the zone merges
|
|
933
|
+
// them at stage end or defers them to a later run; re-detection is
|
|
934
|
+
// structural, not cursor-based). Scout pairs are exempt (#393 B):
|
|
935
|
+
// cosine never blocks this source — guard-C's re-tag above relies on
|
|
936
|
+
// it, and a judged merge re-passes the same metadata/conflict rails
|
|
937
|
+
// the zone enforces.
|
|
938
|
+
if (entry.source === "similarity" && pairCosine >= autoMergeThreshold) {
|
|
759
939
|
skippedAboveCeiling++;
|
|
760
940
|
continue;
|
|
761
941
|
}
|
|
762
942
|
if (dryRun)
|
|
763
943
|
continue; // preview only — no LLM call, no write
|
|
764
|
-
// #
|
|
765
|
-
|
|
766
|
-
if ((maxCalls > 0 && callsUsed >= maxCalls) || !budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
767
|
-
callCapStopped = maxCalls > 0 && callsUsed >= maxCalls;
|
|
944
|
+
// #405: the ONE run budget's refusal is the only call cap.
|
|
945
|
+
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
768
946
|
persistCursor(); // cursor still points at the last fully-considered candidate
|
|
769
947
|
break;
|
|
770
948
|
}
|
|
771
|
-
const { verdict, usage } = await classifyPair(llm,
|
|
949
|
+
const { verdict, usage } = await classifyPair(llm, entry.mem.content, candidate.content);
|
|
772
950
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
|
|
773
|
-
callsUsed++; // #401
|
|
774
951
|
pairsEvaluated++;
|
|
775
952
|
if (!verdict) {
|
|
776
953
|
skippedInfra++;
|
|
777
954
|
continue;
|
|
778
955
|
}
|
|
779
|
-
|
|
956
|
+
// Bands stay similarity-source-only (refine Q2 ruling): they are the
|
|
957
|
+
// calibration evidence for the floor/ceiling boundaries, and scout
|
|
958
|
+
// pairs reach them through a different, cosine-blind door.
|
|
959
|
+
if (entry.source === "similarity")
|
|
960
|
+
recordBand(pairCosine, verdict.action, verdict.confidence);
|
|
780
961
|
// #392: a merge verdict is queued for the merge phase (below) — no
|
|
781
962
|
// link, no write here. Below the confidence gate BOTH memories stay
|
|
782
963
|
// live: a weak mark is recoverable, and there is nothing to mark for a
|
|
@@ -784,24 +965,38 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
784
965
|
if (verdict.action === "merge") {
|
|
785
966
|
if (verdict.confidence < rewriteMinConfidence) {
|
|
786
967
|
mergeBelowGate++;
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
968
|
+
if (entry.source === "similarity") {
|
|
969
|
+
const band = bandForCosine(bands, pairCosine);
|
|
970
|
+
if (band) {
|
|
971
|
+
const stat = runBands.get(band.label) ?? emptyBandStat();
|
|
972
|
+
stat.merge_below_gate++;
|
|
973
|
+
runBands.set(band.label, stat);
|
|
974
|
+
}
|
|
792
975
|
}
|
|
793
976
|
}
|
|
794
977
|
else {
|
|
795
|
-
queuedMerges.push({ oldId:
|
|
978
|
+
queuedMerges.push({ oldId: entry.mem.id, newId: candidate.id, candidateRowid: candidate.__rowid });
|
|
796
979
|
notePendingRowid(candidate.__rowid); // orphan floor — merge unapplied until the merge phase
|
|
797
980
|
}
|
|
798
981
|
continue;
|
|
799
982
|
}
|
|
800
983
|
if (verdict.action === "supersedes") {
|
|
801
|
-
markLink(
|
|
802
|
-
storage.updateMemory(db,
|
|
984
|
+
markLink(entry.mem.id, candidate.id, "superseded_by", pairCosine);
|
|
985
|
+
storage.updateMemory(db, entry.mem.id, { status: "superseded" });
|
|
803
986
|
markedSuperseded++;
|
|
804
|
-
console.log(`[hicortex] Reconsolidation: ${
|
|
987
|
+
console.log(`[hicortex] Reconsolidation: ${entry.mem.id.slice(0, 8)} superseded_by ${candidate.id.slice(0, 8)} (mark-only)`);
|
|
988
|
+
continue;
|
|
989
|
+
}
|
|
990
|
+
// #393 guard-C: a genuine conflict — link ONLY. No status change on
|
|
991
|
+
// either memory (both stay live so the consumer sees both truths), no
|
|
992
|
+
// rewrite, no merge queue; the link is the guard both merge paths
|
|
993
|
+
// consult. Ungated like the other mark actions (a weak flag is
|
|
994
|
+
// recoverable; a weak merge is not).
|
|
995
|
+
if (verdict.action === "conflicts") {
|
|
996
|
+
markLink(entry.mem.id, candidate.id, "conflicts", pairCosine);
|
|
997
|
+
conflictFlagged++;
|
|
998
|
+
console.log(`[hicortex] Reconsolidation: ${entry.mem.id.slice(0, 8)} conflicts ${candidate.id.slice(0, 8)} ` +
|
|
999
|
+
`(flag-only) — both kept live, never merged`);
|
|
805
1000
|
continue;
|
|
806
1001
|
}
|
|
807
1002
|
if (verdict.action === "corrects") {
|
|
@@ -810,19 +1005,19 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
810
1005
|
// Below the gate: mark-only, never rewrite. The
|
|
811
1006
|
// trigger stays live — it is the only carrier of the correction.
|
|
812
1007
|
belowGate++;
|
|
813
|
-
markLink(
|
|
814
|
-
storage.updateMemory(db,
|
|
1008
|
+
markLink(entry.mem.id, candidate.id, "corrected_by", cosine);
|
|
1009
|
+
storage.updateMemory(db, entry.mem.id, { status: "retracted" });
|
|
815
1010
|
markedRetracted++;
|
|
816
1011
|
continue;
|
|
817
1012
|
}
|
|
818
|
-
if (!isFactShapedTarget(
|
|
1013
|
+
if (!isFactShapedTarget(entry.mem)) {
|
|
819
1014
|
// Decisions/plans/experiences are history, not error — mark only.
|
|
820
|
-
markLink(
|
|
821
|
-
storage.updateMemory(db,
|
|
1015
|
+
markLink(entry.mem.id, candidate.id, "corrected_by", cosine);
|
|
1016
|
+
storage.updateMemory(db, entry.mem.id, { status: "retracted" });
|
|
822
1017
|
markedRetracted++;
|
|
823
1018
|
continue;
|
|
824
1019
|
}
|
|
825
|
-
addTrigger(
|
|
1020
|
+
addTrigger(entry.mem, candidate, verdict.confidence, cosine, false);
|
|
826
1021
|
}
|
|
827
1022
|
// verdict "none" → nothing to do
|
|
828
1023
|
}
|
|
@@ -835,24 +1030,20 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
835
1030
|
persistCursor();
|
|
836
1031
|
}
|
|
837
1032
|
if (deadlineStopped) {
|
|
838
|
-
console.log(`[hicortex] Reconsolidation:
|
|
839
|
-
`scan stopped at cursor ${cursor}; the next run resumes from there`);
|
|
840
|
-
}
|
|
841
|
-
else if (callCapStopped) {
|
|
842
|
-
console.log(`[hicortex] Reconsolidation: per-run call cap reached (reconsolidationMaxCalls) — ` +
|
|
1033
|
+
console.log(`[hicortex] Reconsolidation: run deadline reached (nightlyTimeBudgetMinutes) — ` +
|
|
843
1034
|
`scan stopped at cursor ${cursor}; the next run resumes from there`);
|
|
844
1035
|
}
|
|
845
1036
|
// ---- #392 judged-merge phase: apply the queued pair merges through the
|
|
846
1037
|
// dedup core (mergeMemoryIds — same canonical pick, link re-points,
|
|
847
1038
|
// dedup_log, absorb). One short lock/backup window for the whole batch, one
|
|
848
|
-
// transaction per pair.
|
|
849
|
-
//
|
|
850
|
-
//
|
|
851
|
-
//
|
|
852
|
-
//
|
|
853
|
-
//
|
|
854
|
-
|
|
855
|
-
|
|
1039
|
+
// transaction per pair. #405: the dedupNightlyMaxMerges cap is gone — the
|
|
1040
|
+
// run deadline bounds the merge loop (a stop-check between local
|
|
1041
|
+
// transactions; the deferred pairs hold the cursor below their candidates).
|
|
1042
|
+
// A pair that cannot apply (deadline, busy lock, failed backup) keeps BOTH
|
|
1043
|
+
// memories live and holds the cursor below its candidate — a confirmed
|
|
1044
|
+
// merge is never silently dropped by the cursor passing it (dup-over-loss).
|
|
1045
|
+
// A metadata-rail refusal is different: the verdict WAS rendered, both
|
|
1046
|
+
// memories stay live, the cursor advances.
|
|
856
1047
|
let mergePairsDeferred = 0;
|
|
857
1048
|
if (!dryRun && queuedMerges.length > 0) {
|
|
858
1049
|
const holdQueued = (from) => {
|
|
@@ -863,20 +1054,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
863
1054
|
: Math.min(pendingMinRowid, queuedMerges[i].candidateRowid);
|
|
864
1055
|
}
|
|
865
1056
|
};
|
|
866
|
-
|
|
867
|
-
// Machinery disabled by config: keep both (counted in band_stats as
|
|
868
|
-
// merge verdicts) and ADVANCE — holding the cursor would re-judge the
|
|
869
|
-
// same pairs into the same disabled state forever.
|
|
870
|
-
console.log(`[hicortex] Reconsolidation: ${queuedMerges.length} confirmed merge(s) kept — ` +
|
|
871
|
-
`dedupNightlyMaxMerges is 0 (merge machinery disabled)`);
|
|
872
|
-
}
|
|
873
|
-
else if (mergeOpsRemaining <= 0) {
|
|
874
|
-
mergePairsDeferred = queuedMerges.length;
|
|
875
|
-
holdQueued(0); // zone consumed the whole cap — retry next run
|
|
876
|
-
console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred — ` +
|
|
877
|
-
`dedupNightlyMaxMerges exhausted by the deterministic zone`);
|
|
878
|
-
}
|
|
879
|
-
else {
|
|
1057
|
+
{
|
|
880
1058
|
const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
|
|
881
1059
|
const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
|
|
882
1060
|
if (!release) {
|
|
@@ -898,16 +1076,19 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
898
1076
|
if (backupOk) {
|
|
899
1077
|
for (let i = 0; i < queuedMerges.length; i++) {
|
|
900
1078
|
const pair = queuedMerges[i];
|
|
901
|
-
|
|
1079
|
+
// #405: the deadline stop-check between local merge
|
|
1080
|
+
// transactions — a safe boundary; deferred pairs hold the
|
|
1081
|
+
// cursor below their candidates and retry next run.
|
|
1082
|
+
if (deadlineHit()) {
|
|
1083
|
+
deadlineStopped = true;
|
|
902
1084
|
mergePairsDeferred = queuedMerges.length - i;
|
|
903
|
-
holdQueued(i);
|
|
904
|
-
console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred —
|
|
1085
|
+
holdQueued(i);
|
|
1086
|
+
console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred — run deadline reached`);
|
|
905
1087
|
break;
|
|
906
1088
|
}
|
|
907
1089
|
const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
|
|
908
1090
|
if (result.ok) {
|
|
909
1091
|
mergePairsApplied++;
|
|
910
|
-
mergeOpsRemaining--;
|
|
911
1092
|
console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
912
1093
|
`into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
|
|
913
1094
|
}
|
|
@@ -916,6 +1097,14 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
916
1097
|
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
917
1098
|
`skipped (metadata mismatch) — both kept`);
|
|
918
1099
|
}
|
|
1100
|
+
else if (result.reason === "conflict_linked") {
|
|
1101
|
+
// #393 guard-C: the pair is conflicts-linked (operator-planted
|
|
1102
|
+
// or a prior verdict) — never blended; the cursor advances,
|
|
1103
|
+
// this verdict was rendered.
|
|
1104
|
+
conflictSkippedJudged++;
|
|
1105
|
+
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1106
|
+
`skipped (conflict-flagged) — both kept`);
|
|
1107
|
+
}
|
|
919
1108
|
// "no_members": a member vanished/was absorbed since the
|
|
920
1109
|
// verdict — nothing to merge, nothing to hold; the cursor
|
|
921
1110
|
// advances past it.
|
|
@@ -961,11 +1150,6 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
961
1150
|
deferFrom(group.targetId);
|
|
962
1151
|
break;
|
|
963
1152
|
}
|
|
964
|
-
if (maxCalls > 0 && callsUsed >= maxCalls) {
|
|
965
|
-
callCapStopped = true;
|
|
966
|
-
deferFrom(group.targetId);
|
|
967
|
-
break;
|
|
968
|
-
}
|
|
969
1153
|
if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
|
|
970
1154
|
deferFrom(group.targetId);
|
|
971
1155
|
break;
|
|
@@ -974,10 +1158,9 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
974
1158
|
let contract = null;
|
|
975
1159
|
let infraError = false;
|
|
976
1160
|
try {
|
|
977
|
-
const r = await llm.
|
|
1161
|
+
const r = await llm.complete(buildRewritePrompt(group.target.content, triggersArg));
|
|
978
1162
|
contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
|
|
979
1163
|
budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
|
|
980
|
-
callsUsed++; // #401: rewrite contracts count toward the stage call cap
|
|
981
1164
|
}
|
|
982
1165
|
catch {
|
|
983
1166
|
infraError = true;
|
|
@@ -1063,6 +1246,23 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1063
1246
|
keptLinked++;
|
|
1064
1247
|
}
|
|
1065
1248
|
}
|
|
1249
|
+
// ---- #393 guard-C zone reorder: the deterministic merge zone (pairs >=
|
|
1250
|
+
// the ceiling) runs LAST — after the scan, the judged-merge phase, and the
|
|
1251
|
+
// rewrite phase. Judgment outranks the deterministic sweep: verdicts,
|
|
1252
|
+
// marks, and binds land first, and the zone merges only what no verdict
|
|
1253
|
+
// claimed. With the zone first, a >=0.92 genuine-conflict pair was blended
|
|
1254
|
+
// before the judge ever saw it (canonical = oldest, the newer truth erased
|
|
1255
|
+
// — the planted-eval harm); running it last means a `conflicts` bind set by
|
|
1256
|
+
// THIS run's scan guards the SAME run's zone. LLM-free and budget-free — an
|
|
1257
|
+
// LLM-less night still drains duplicates. Its own short lock window,
|
|
1258
|
+
// pre-merge backup, and #405 deadline stop-check; fail-soft, never a throw.
|
|
1259
|
+
const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
|
|
1260
|
+
stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
|
|
1261
|
+
threshold: autoMergeThreshold,
|
|
1262
|
+
dryRun,
|
|
1263
|
+
acquireLock: options.acquireLock,
|
|
1264
|
+
deadline,
|
|
1265
|
+
});
|
|
1066
1266
|
// Cursor hold: un-applied work (rewrite groups, confirmed merges) holds the
|
|
1067
1267
|
// cursor BELOW its earliest contributing candidate so the pairs are
|
|
1068
1268
|
// re-detected next run.
|
|
@@ -1073,7 +1273,10 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1073
1273
|
// losers are merge verdicts at confidence 1.0; the zone persists the
|
|
1074
1274
|
// cumulative copy itself) plus this run's judged bands.
|
|
1075
1275
|
const bandStats = {};
|
|
1076
|
-
|
|
1276
|
+
{
|
|
1277
|
+
// #405: recorded whenever the zone ran (the old max_merges>0 gate was a
|
|
1278
|
+
// 0=disabled switch — the switch is gone; a clean corpus records zeros,
|
|
1279
|
+
// same as the old default-config behavior).
|
|
1077
1280
|
const det = emptyBandStat();
|
|
1078
1281
|
det.pairs = merges.losers_merged;
|
|
1079
1282
|
det.merge = merges.losers_merged;
|
|
@@ -1110,7 +1313,10 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1110
1313
|
`${markedRetracted} retracted (${belowGate} below gate, ${mergeBelowGate} merge below gate, ` +
|
|
1111
1314
|
`${contractFailed} contract failed), ${skippedInfra} infra-skipped, ${skippedIdempotent} ` +
|
|
1112
1315
|
`already-linked, ${skippedAboveCeiling} above ceiling, ${explicitVerified} explicit verified, ` +
|
|
1113
|
-
`${explicitDivergent} explicit divergent
|
|
1316
|
+
`${explicitDivergent} explicit divergent, scout ${scoutScanned} scanned / ` +
|
|
1317
|
+
`${scoutCorrectionShaped} correction-shaped / ${scoutCandidatesFound} candidate pair(s), ` +
|
|
1318
|
+
`${conflictFlagged} conflict-flagged, ${conflictSkippedJudged + merges.skipped_conflict} conflict-skipped ` +
|
|
1319
|
+
`(cursor ${cursor})`);
|
|
1114
1320
|
}
|
|
1115
1321
|
return {
|
|
1116
1322
|
scanned,
|
|
@@ -1134,6 +1340,11 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1134
1340
|
merge_below_gate: mergeBelowGate,
|
|
1135
1341
|
skipped_above_ceiling: skippedAboveCeiling,
|
|
1136
1342
|
skipped_metadata_mismatch: skippedMetadataMismatch,
|
|
1343
|
+
conflict_flagged: conflictFlagged,
|
|
1344
|
+
conflict_skipped: conflictSkippedJudged + merges.skipped_conflict,
|
|
1345
|
+
scout_scanned: scoutScanned,
|
|
1346
|
+
scout_correction_shaped: scoutCorrectionShaped,
|
|
1347
|
+
scout_candidates_found: scoutCandidatesFound,
|
|
1137
1348
|
band_stats: bandStats,
|
|
1138
1349
|
};
|
|
1139
1350
|
}
|