@gamaze/hicortex 0.20.7 → 0.20.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +18 -41
  2. package/assets/dashboard.html +3989 -836
  3. package/dist/calibration.d.ts +293 -0
  4. package/dist/calibration.js +379 -0
  5. package/dist/capture-health.d.ts +87 -0
  6. package/dist/capture-health.js +106 -0
  7. package/dist/capture-pause.d.ts +86 -0
  8. package/dist/capture-pause.js +127 -0
  9. package/dist/capture.d.ts +24 -3
  10. package/dist/capture.js +11 -1
  11. package/dist/classify-domains.d.ts +6 -0
  12. package/dist/classify-domains.js +7 -1
  13. package/dist/cli.js +38 -3
  14. package/dist/config-read.d.ts +1 -1
  15. package/dist/config-read.js +96 -9
  16. package/dist/consolidate.d.ts +114 -68
  17. package/dist/consolidate.js +302 -182
  18. package/dist/dashboard.d.ts +326 -6
  19. package/dist/dashboard.js +592 -7
  20. package/dist/db.js +105 -0
  21. package/dist/dedup.d.ts +34 -26
  22. package/dist/dedup.js +91 -57
  23. package/dist/distiller.js +1 -1
  24. package/dist/domain-classify.d.ts +7 -6
  25. package/dist/domain-classify.js +12 -10
  26. package/dist/eval/decay-eval.d.ts +3 -3
  27. package/dist/eval/decay-eval.js +4 -4
  28. package/dist/eval/importance-eval.d.ts +85 -0
  29. package/dist/eval/importance-eval.js +286 -0
  30. package/dist/eval/planted-eval.d.ts +26 -0
  31. package/dist/eval/planted-eval.js +97 -0
  32. package/dist/eval/planted-fixtures.d.ts +107 -0
  33. package/dist/eval/planted-fixtures.js +283 -0
  34. package/dist/eval/planted-harness.d.ts +176 -0
  35. package/dist/eval/planted-harness.js +649 -0
  36. package/dist/eval/ranking-battery.d.ts +78 -0
  37. package/dist/eval/ranking-battery.js +181 -0
  38. package/dist/eval/ranking-eval.d.ts +41 -0
  39. package/dist/eval/ranking-eval.js +391 -0
  40. package/dist/eval/ranking-fixtures.d.ts +77 -0
  41. package/dist/eval/ranking-fixtures.js +226 -0
  42. package/dist/identity-store.d.ts +21 -0
  43. package/dist/identity-store.js +49 -0
  44. package/dist/index.js +4 -3
  45. package/dist/init.d.ts +23 -3
  46. package/dist/init.js +84 -9
  47. package/dist/llm.d.ts +43 -58
  48. package/dist/llm.js +87 -101
  49. package/dist/mcp-server.d.ts +12 -0
  50. package/dist/mcp-server.js +213 -32
  51. package/dist/nightly.d.ts +9 -1
  52. package/dist/nightly.js +164 -110
  53. package/dist/nofit.d.ts +4 -11
  54. package/dist/nofit.js +6 -23
  55. package/dist/prompts.d.ts +10 -0
  56. package/dist/prompts.js +28 -5
  57. package/dist/recall-index.d.ts +30 -28
  58. package/dist/recall-index.js +21 -18
  59. package/dist/recall-registry.d.ts +2 -1
  60. package/dist/recall-registry.js +35 -1
  61. package/dist/reconsolidation.d.ts +168 -87
  62. package/dist/reconsolidation.js +818 -377
  63. package/dist/relink.js +3 -4
  64. package/dist/rescore-importance.d.ts +80 -0
  65. package/dist/rescore-importance.js +236 -0
  66. package/dist/retrieval.d.ts +80 -35
  67. package/dist/retrieval.js +322 -105
  68. package/dist/run-deadline.d.ts +62 -0
  69. package/dist/run-deadline.js +73 -0
  70. package/dist/schema-prototypes.d.ts +3 -3
  71. package/dist/schema-prototypes.js +3 -3
  72. package/dist/stages.d.ts +37 -0
  73. package/dist/stages.js +51 -0
  74. package/dist/state.d.ts +34 -9
  75. package/dist/storage.d.ts +50 -18
  76. package/dist/storage.js +125 -30
  77. package/dist/telemetry.d.ts +8 -7
  78. package/dist/token-budget.js +3 -4
  79. package/dist/type-classify.js +4 -4
  80. package/dist/types.d.ts +143 -155
  81. package/domains.example.json +4 -5
  82. package/hermes-plugin/hicortex/README.md +2 -2
  83. package/openclaw.plugin.json +1 -1
  84. package/package.json +4 -1
  85. package/pi-extension/hicortex/README.md +1 -1
  86. package/server.json +3 -3
@@ -16,11 +16,58 @@
16
16
  * #392 — one zone system, ONE verdict per pair: below `correctionMinSimilarity`
17
17
  * (floor, 0.75) pairs are not candidates; in [floor, `dedupAutoMergeThreshold`)
18
18
  * (ceiling, 0.92) each unlinked pair gets ONE verdict call whose action is
19
- * `merge` | `corrects` | `supersedes` | `none`; at/above the ceiling the
20
- * deterministic merge zone (dedup.ts runDeterministicMergeZone — LLM-free,
21
- * budget-free) owns the pair. The merge disposition reuses the dedup core's
22
- * execution (canonical pick, link re-point, dedup_log, metadata rails); a
23
- * merge verdict below `correctionRewriteMinConfidence` keeps both memories.
19
+ * `merge` | `corrects` | `supersedes` | `conflicts` | `none`; at/above the
20
+ * ceiling the deterministic merge zone (dedup.ts runDeterministicMergeZone —
21
+ * LLM-free, budget-free) owns the pair. The merge disposition reuses the
22
+ * dedup core's execution (canonical pick, link re-point, dedup_log, metadata
23
+ * rails); a merge verdict below `correctionRewriteMinConfidence` keeps both
24
+ * memories.
25
+ *
26
+ * #393 increment B — the SCOUT, a second detection source with the SAME
27
+ * judge: the similarity floor is structurally blind to corrections riding
28
+ * inside topically unrelated memories (the field failure — cosine ~0.5-0.6 to
29
+ * their target, zero `corrects` verdicts in the whole corpus baseline), so
30
+ * per NEW memory ONE classify-tier shape call asks whether it corrects/
31
+ * retracts/supersedes/CONTRADICTS something previously recorded (guard-C
32
+ * extended the question); correction-shaped
33
+ * memories FTS the corpus with the referenced claim's distinctive terms (the
34
+ * correction CONTAINS the words of what it corrects) and the hits become
35
+ * candidate pairs in the SAME verdict loop — no similarity gate for this
36
+ * source: cosine is a ranker/link strength, never a blocker. Per-source
37
+ * counters (scout_scanned / scout_correction_shaped / scout_candidates_found)
38
+ * ride the stage report; cosine band stats stay similarity-source-only.
39
+ *
40
+ * #393 guard-C — the conflicts flag + the zone-runs-last order: judgment
41
+ * OUTRANKS the deterministic sweep. A `conflicts` verdict writes a symmetric
42
+ * `conflicts` link (the pair genuinely disagrees — cannot both be true) and
43
+ * NOTHING else: no status change, no rewrite, no merge queue; both records
44
+ * stay live so the consumer sees both truths. Both merge paths (the zone's
45
+ * planDedup and the judged mergeMemoryIds) refuse to blend a conflicts-linked
46
+ * pair, counted as conflict_skipped. The zone therefore runs AFTER the scan —
47
+ * with the zone first, a >=0.92 conflict pair was blended
48
+ * before the judge ever saw it (the planted-eval harm: canonical=older, the
49
+ * newer truth erased); running it last means verdicts/marks/binds land first
50
+ * and the zone merges only what no verdict claimed — a conflicts bind set by
51
+ * this run's scan guards the SAME run's zone.
52
+ *
53
+ * #439 apply-on-confirm — confirmed merges and rewrite groups apply at the
54
+ * candidate BOUNDARY (the end of the scan iteration that confirmed them), not
55
+ * in post-scan phases. The old end-of-run batch was a completion assumption
56
+ * written when nightlies finished in an hour; under #405 budget pressure it
57
+ * became a days-long queue where confirmed work never landed and every night
58
+ * re-paid the judgment cost (cursor held below un-applied groups, pairs
59
+ * re-detected, re-judged). Now each judged-merge pair applies via
60
+ * mergeMemoryIds in its OWN transaction at confirmation time, each rewrite
61
+ * group via its own rewrite call + applyRewriteGroup transaction; the cursor
62
+ * advances per APPLIED candidate, so a deferral holds it below exactly ONE
63
+ * candidate's pairs. One pre-merge backup per run (lazy, before the first
64
+ * application); the capture lock is taken per boundary batch with a same-run
65
+ * retry list + a final drain. A shared trigger IS the current candidate, so
66
+ * the multi-target keep rule resolves across the boundary's groups (any keep
67
+ * keeps). A target corrected by two different candidates takes two sequential
68
+ * rewrites — the second composes the already-corrected story — instead of one
69
+ * grouped call (the ONE-call grouping was a cost optimization, not a
70
+ * correctness invariant; accepted semantics change).
24
71
  *
25
72
  * Status vocabulary (code-defined, extensible — deliberately NOT config):
26
73
  * NULL/'active' default | 'superseded' + 'retracted' demote in ranking |
@@ -71,8 +118,10 @@ var __importStar = (this && this.__importStar) || (function () {
71
118
  };
72
119
  })();
73
120
  Object.defineProperty(exports, "__esModule", { value: true });
74
- exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.DEFAULT_RECONSOLIDATION_MAX_CALLS = exports.DEFAULT_RECONSOLIDATION_MAX_MINUTES = exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = exports.DEFAULT_CORRECTION_MIN_SIMILARITY = exports.RECONSOLIDATION_STAGE_LABEL = void 0;
121
+ exports.absorbTrigger = exports.DEMOTED_STATUSES = exports.FOOTER_HEAD_MAX_CHARS = exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = exports.DEFAULT_CORRECTION_MIN_SIMILARITY = exports.RECONSOLIDATION_STAGE_LABEL = void 0;
75
122
  exports.isFactShapedTarget = isFactShapedTarget;
123
+ exports.buildScoutShapePrompt = buildScoutShapePrompt;
124
+ exports.parseScoutShape = parseScoutShape;
76
125
  exports.buildCorrectionVerdictPrompt = buildCorrectionVerdictPrompt;
77
126
  exports.parseCorrectionVerdict = parseCorrectionVerdict;
78
127
  exports.buildRewritePrompt = buildRewritePrompt;
@@ -94,6 +143,7 @@ const storage = __importStar(require("./storage.js"));
94
143
  const state_js_1 = require("./state.js");
95
144
  const db_js_1 = require("./db.js");
96
145
  const capture_js_1 = require("./capture.js");
146
+ const CALIBRATION = __importStar(require("./calibration.js"));
97
147
  const paths_js_1 = require("./paths.js");
98
148
  const dedup_js_1 = require("./dedup.js");
99
149
  // ---------------------------------------------------------------------------
@@ -102,37 +152,21 @@ const dedup_js_1 = require("./dedup.js");
102
152
  /** Stage label used for every budget.use()/recordUsage() call (#384). */
103
153
  exports.RECONSOLIDATION_STAGE_LABEL = "reconsolidation";
104
154
  /**
105
- * Default minimum COSINE similarity for a correction candidate pair. Lower
106
- * than the supersession stage's 0.80 on purpose: a retraction often rides
107
- * inside an otherwise unrelated memory (the field failure that opened this
108
- * issue), so the neighborhood gate must be a touch wider while the LLM
109
- * verdict + confidence gate carry the precision load.
155
+ * Default minimum COSINE similarity for a correction candidate pair —
156
+ * RELEASE-MANAGED since #408 (calibration.ts CORRECTION_MIN_SIMILARITY;
157
+ * provenance there). Lower than the supersession stage's 0.80 on purpose: a
158
+ * retraction often rides inside an otherwise unrelated memory (the field
159
+ * failure that opened this issue), so the neighborhood gate must be a touch
160
+ * wider while the LLM verdict + confidence gate carry the precision load.
110
161
  */
111
- exports.DEFAULT_CORRECTION_MIN_SIMILARITY = 0.75;
162
+ exports.DEFAULT_CORRECTION_MIN_SIMILARITY = CALIBRATION.CORRECTION_MIN_SIMILARITY;
112
163
  /**
113
- * Default minimum verdict confidence for the REWRITE fork. Below this a
164
+ * Default minimum verdict confidence for the REWRITE fork — release-managed
165
+ * (calibration.ts CORRECTION_REWRITE_MIN_CONFIDENCE). Below this a
114
166
  * `corrects` verdict degrades to mark-only — a weak mark is recoverable, a
115
167
  * weak rewrite is corruption.
116
168
  */
117
- exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = 0.8;
118
- /**
119
- * Default wall-clock bound for the stage, in minutes (#401). Checked at the
120
- * top of the candidate scan loop (and before each rewrite contract call); on
121
- * expiry the scan breaks cleanly at the last fully-considered candidate and
122
- * the next run resumes from the persisted cursor. 120 sits safely under any
123
- * sane process-level nightly timeout. 0 disables the bound. Invalid →
124
- * default.
125
- */
126
- exports.DEFAULT_RECONSOLIDATION_MAX_MINUTES = 120;
127
- /**
128
- * Default per-run classify-call ceiling for the stage (#401) — the
129
- * supersessionMaxCalls pattern with a NON-ZERO default ON PURPOSE: that
130
- * knob's 0=unlimited default is what let the first full-corpus pass grow
131
- * unbounded. Counts EVERY classify-tier call the stage makes (mark
132
- * verifications, pair verdicts, rewrite contracts). 0 disables the cap.
133
- * Invalid → default.
134
- */
135
- exports.DEFAULT_RECONSOLIDATION_MAX_CALLS = 600;
169
+ exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = CALIBRATION.CORRECTION_REWRITE_MIN_CONFIDENCE;
136
170
  /** Neighbor pool size before older/similarity filtering narrows to top 5 (supersession mirror). */
137
171
  const CORRECTION_NEIGHBOR_POOL = 15;
138
172
  /** Older-neighbor pairs kept per candidate after filtering (supersession mirror). */
@@ -161,10 +195,10 @@ exports.DEMOTED_STATUSES = ["superseded", "retracted"];
161
195
  function isFactShapedTarget(mem) {
162
196
  return mem.memory_type === "knowledge" || mem.content.includes("[Facts Learned]");
163
197
  }
164
- /** True when a superseded_by OR corrected_by link already exists between the pair, either direction. */
198
+ /** True when a superseded_by / corrected_by / conflicts link already exists between the pair, either direction. */
165
199
  function alreadyResolutionLinked(db, oldId, newId) {
166
200
  const row = db
167
- .prepare(`SELECT 1 FROM memory_links WHERE relationship IN ('superseded_by', 'corrected_by')
201
+ .prepare(`SELECT 1 FROM memory_links WHERE relationship IN ('superseded_by', 'corrected_by', 'conflicts')
168
202
  AND ((source_id = ? AND target_id = ?) OR (source_id = ? AND target_id = ?))`)
169
203
  .get(oldId, newId, newId, oldId);
170
204
  return !!row;
@@ -178,6 +212,58 @@ function hasLink(db, sourceId, targetId, relationship) {
178
212
  function nowIso() {
179
213
  return new Date().toISOString();
180
214
  }
215
+ /**
216
+ * Build the constrained correction-shape prompt (classify-tier cost profile:
217
+ * 1500-char truncation, supersession/verdict precedent). The wording asks for
218
+ * the OLD claim's distinctive terms — the field-failure mechanism is that a
219
+ * correction CONTAINS the words of what it corrects, even when the surrounding
220
+ * topics (and therefore the embedding cosine) are unrelated. Guard-C extends
221
+ * the question to contradictions: two records that disagree on the same
222
+ * quantity share even MORE wording than a cross-topic correction does.
223
+ */
224
+ function buildScoutShapePrompt(content) {
225
+ const trunc = (s) => (s.length > PROMPT_TRUNCATE_CHARS ? `${s.slice(0, PROMPT_TRUNCATE_CHARS)}…` : s);
226
+ return (`You are scanning a memory that was just added to an AI agent's long-term memory store.\n\n` +
227
+ `MEMORY:\n${trunc(content)}\n\n` +
228
+ `Does this memory correct, retract, supersede, or contradict a claim, decision, or state that was ` +
229
+ `previously recorded elsewhere in the store? A mere duplicate, elaboration, independent ` +
230
+ `fact, or new information that invalidates nothing is NOT a correction.\n` +
231
+ `If it is a correction/retraction/supersession/contradiction, list the most distinctive terms of the ` +
232
+ `OLD claim it references — words likely to appear verbatim in the older record.\n\n` +
233
+ `Reply with ONLY a JSON object, no prose: ` +
234
+ `{"correction": true | false, "references": "<distinctive terms of the referenced old claim, or empty string>", ` +
235
+ `"confidence": <number between 0 and 1>}`);
236
+ }
237
+ /**
238
+ * Parse the scout shape reply. Null on unparseable JSON, a missing/non-boolean
239
+ * `correction`, or a missing/out-of-range `confidence` — the caller counts
240
+ * skipped_infra and moves on (parseSupersessionReply discipline: never
241
+ * mis-detect on ambiguity). `references` is lenient (missing/non-string → "")
242
+ * because an empty string simply yields no FTS hits — a harmless miss, not a
243
+ * mis-judgment.
244
+ */
245
+ function parseScoutShape(reply) {
246
+ if (!reply)
247
+ return null;
248
+ const start = reply.indexOf("{");
249
+ const end = reply.lastIndexOf("}");
250
+ if (start === -1 || end === -1 || end <= start)
251
+ return null;
252
+ let obj;
253
+ try {
254
+ obj = JSON.parse(reply.slice(start, end + 1));
255
+ }
256
+ catch {
257
+ return null;
258
+ }
259
+ if (typeof obj.correction !== "boolean")
260
+ return null;
261
+ const confidence = Number(obj.confidence);
262
+ if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1)
263
+ return null;
264
+ const references = typeof obj.references === "string" ? obj.references : "";
265
+ return { correction: obj.correction, references: references.trim(), confidence };
266
+ }
181
267
  /** Build the constrained pair-verdict prompt (1500-char truncation, supersession precedent). */
182
268
  function buildCorrectionVerdictPrompt(oldContent, newContent) {
183
269
  const trunc = (s) => (s.length > PROMPT_TRUNCATE_CHARS ? `${s.slice(0, PROMPT_TRUNCATE_CHARS)}…` : s);
@@ -191,9 +277,12 @@ function buildCorrectionVerdictPrompt(oldContent, newContent) {
191
277
  `wrong, no longer true, or was retracted, and the newer memory carries the corrected fact.\n` +
192
278
  `- "supersedes": the newer memory replaces a decision, plan, or state that was valid at the time but is ` +
193
279
  `now outdated — a replacement, not a factual correction.\n` +
280
+ `- "conflicts": the two memories make claims that cannot both be true — they disagree on a fact, value, ` +
281
+ `or state, and neither one corrects, supersedes, or restates the other (for example two sources report ` +
282
+ `different values for the same quantity). Keep both; flag the conflict.\n` +
194
283
  `- "none": unrelated, merely similar, or both can still be true (an addition or elaboration).\n\n` +
195
284
  `Reply with ONLY a JSON object, no prose: ` +
196
- `{"action": "merge" | "corrects" | "supersedes" | "none", "confidence": <number between 0 and 1>}`);
285
+ `{"action": "merge" | "corrects" | "supersedes" | "conflicts" | "none", "confidence": <number between 0 and 1>}`);
197
286
  }
198
287
  /**
199
288
  * Parse the pair verdict. Null on anything unparseable, unknown action, or an
@@ -215,8 +304,13 @@ function parseCorrectionVerdict(reply) {
215
304
  return null;
216
305
  }
217
306
  const action = obj.action;
218
- if (action !== "merge" && action !== "corrects" && action !== "supersedes" && action !== "none")
307
+ if (action !== "merge" &&
308
+ action !== "corrects" &&
309
+ action !== "supersedes" &&
310
+ action !== "conflicts" &&
311
+ action !== "none") {
219
312
  return null;
313
+ }
220
314
  const confidence = Number(obj.confidence);
221
315
  if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1)
222
316
  return null;
@@ -452,7 +546,7 @@ function bandForCosine(bands, cosine) {
452
546
  }
453
547
  /** An empty band-stat record (fresh accumulation starts from zeroes). */
454
548
  function emptyBandStat() {
455
- return { pairs: 0, merge: 0, corrects: 0, supersedes: 0, none: 0, merge_below_gate: 0, conf_sum: 0 };
549
+ return { pairs: 0, merge: 0, corrects: 0, supersedes: 0, conflicts: 0, none: 0, merge_below_gate: 0, conf_sum: 0 };
456
550
  }
457
551
  /** Add a run's per-band counts into a cumulative record (in place). */
458
552
  function accumulateBandStat(cumulative, run) {
@@ -460,6 +554,7 @@ function accumulateBandStat(cumulative, run) {
460
554
  cumulative.merge += run.merge;
461
555
  cumulative.corrects += run.corrects;
462
556
  cumulative.supersedes += run.supersedes;
557
+ cumulative.conflicts += run.conflicts;
463
558
  cumulative.none += run.none;
464
559
  cumulative.merge_below_gate += run.merge_below_gate;
465
560
  cumulative.conf_sum += run.conf_sum;
@@ -481,9 +576,43 @@ async function findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarit
481
576
  .sort((a, b) => (0, retrieval_js_1.l2ToCosine)(b.distance) - (0, retrieval_js_1.l2ToCosine)(a.distance))
482
577
  .slice(0, CORRECTION_NEIGHBOR_TOP_K);
483
578
  }
579
+ /**
580
+ * The scout source (#393 increment B): for a correction-shaped NEW memory, FTS
581
+ * the corpus with the referenced claim's distinctive terms and return the OLDER
582
+ * hits as candidate pairs. This is the reference-extraction half — it finds the
583
+ * old claim even when the overall topics differ (and therefore the cosine sits
584
+ * below the similarity floor) because the correction CONTAINS the words of
585
+ * what it corrects. Deterministic: ONE FTS query, zero LLM. Filters: self,
586
+ * non-older (detection only pairs older → newer, the KNN mirror), and
587
+ * defensively non-absorbed hits. Pool/top-K reuse the KNN constants; FTS rank
588
+ * (BM25, best first) is the order. Dedup against the KNN neighbor ids is the
589
+ * caller's job (a pair found by both sources is judged once, as similarity).
590
+ */
591
+ function findScoutNeighbors(db, candidate, references, candidateEmbedding) {
592
+ if (!references)
593
+ return [];
594
+ try {
595
+ const hits = storage.searchFts(db, references, CORRECTION_NEIGHBOR_POOL);
596
+ return hits
597
+ .filter((m) => m.id !== candidate.id &&
598
+ m.created_at < candidate.created_at &&
599
+ m.status !== "absorbed")
600
+ .slice(0, CORRECTION_NEIGHBOR_TOP_K)
601
+ .map((m) => {
602
+ const hitVec = storage.getStoredEmbedding(db, m.id);
603
+ const cosine = candidateEmbedding && hitVec ? (0, retrieval_js_1.cosineBetweenVectors)(candidateEmbedding, hitVec) : 0;
604
+ return { mem: m, cosine, source: "scout" };
605
+ });
606
+ }
607
+ catch {
608
+ // FTS is deterministic infrastructure — a throw here is a bug or a corrupt
609
+ // index, never a judgment question. Fail soft: no scout pairs this memory.
610
+ return [];
611
+ }
612
+ }
484
613
  async function classifyPair(llm, oldContent, newContent) {
485
614
  try {
486
- const r = await llm.completeClassify(buildCorrectionVerdictPrompt(oldContent, newContent));
615
+ const r = await llm.complete(buildCorrectionVerdictPrompt(oldContent, newContent));
487
616
  return { verdict: parseCorrectionVerdict(r.text), usage: r.usage };
488
617
  }
489
618
  catch {
@@ -491,32 +620,55 @@ async function classifyPair(llm, oldContent, newContent) {
491
620
  }
492
621
  }
493
622
  /**
494
- * Nightly reconsolidation stage (#384, #392 — THE unified resolution stage).
623
+ * Nightly reconsolidation stage (#384, #392 — THE unified resolution stage;
624
+ * #439 apply-on-confirm).
495
625
  *
496
- * Phase 0 (#392): the deterministic merge zone (pairs >= the ceiling) runs
497
- * first — LLM-free, budget-free, own lock/backup/cap.
626
+ * Phase order (#393 guard-C): the deterministic merge zone (pairs >= the
627
+ * ceiling) runs LAST — after the scan (which now includes every judged-merge
628
+ * application and rewrite, #439). Judgment outranks the deterministic sweep:
629
+ * verdicts, marks, and binds land first and the zone merges only what no
630
+ * verdict claimed. With the zone first, a >=0.92 genuine-conflict pair was
631
+ * blended before the judge ever saw it (the planted-eval harm); running it
632
+ * last means a `conflicts` bind set by this run's scan guards the SAME run's
633
+ * zone. Zone internals (lock, backup, deadline, persistBand, fail-soft) are
634
+ * unchanged.
498
635
  *
499
636
  * Scan: every memory with rowid > reconsolidationCursor (no shape filter;
500
637
  * absorbed candidates are skipped — invisible memories are not re-judged).
501
- * Each candidate's pairs: incoming explicit marks (verified once, AC7) then
502
- * up-to-5 older KNN neighbors in [floor, ceiling) (verdict call per unlinked
503
- * pair, AC2 — pairs at/above the ceiling are counted, never judged). Confirmed
504
- * `corrects` pairs above the confidence gate on fact-shaped targets group by
505
- * target into ONE rewrite call each (AC3); confirmed `merge` pairs queue for
506
- * the merge phase; everything else is mark-only.
638
+ * Each candidate's pairs: incoming explicit marks (verified once, AC7), then
639
+ * ONE scout shape call (#393 B — flags correction shape; non-corrections stop
640
+ * there), then up-to-5 older KNN neighbors in [floor, ceiling) (verdict call
641
+ * per unlinked pair, AC2 — pairs at/above the ceiling are counted, never
642
+ * judged) plus the scout's FTS hits for correction-shaped memories (same
643
+ * verdict loop, NO similarity gate; guard-C: a scout hit whose KNN twin sits
644
+ * at/above the ceiling is re-tagged scout so the pair IS judged instead of
645
+ * being left for the zone to blend). Confirmed `corrects` pairs above the
646
+ * confidence gate on fact-shaped targets group by target; a `conflicts`
647
+ * verdict writes the conflicts link and nothing else (both live); everything
648
+ * else is mark-only.
507
649
  *
508
- * Merge phase (#392): queued pairs merge through the dedup core under one
509
- * lock/backup window, capped with the zone by dedupNightlyMaxMerges. A pair
510
- * that cannot apply keeps both memories and holds the cursor.
650
+ * #439 BOUNDARY apply: at the END of each candidate iteration everything it
651
+ * confirmed applies IMMEDIATELY — merges first (each judged-merge pair via
652
+ * mergeMemoryIds in its own transaction, under the boundary's short lock
653
+ * window; ONE lazy pre-merge backup per run), then the iteration's rewrite
654
+ * groups (one rewrite LLM call + one applyRewriteGroup transaction each;
655
+ * dispositions resolved ACROSS the boundary's groups — the multi-target keep
656
+ * rule: a trigger absorbed only if every group's contract says absorb). A
657
+ * busy capture lock pushes the boundary's merges onto a same-run retry list
658
+ * (retried at the next boundary and once in a final drain after the scan);
659
+ * a deadline, a backup failure, or a rewrite-call refusal/infra error defers
660
+ * the remaining work and holds the cursor.
511
661
  *
512
- * Cursor discipline mirrors stageSupersession: the cursor advances past a
513
- * candidate once its neighbor set has been considered, regardless of infra
514
- * skips — EXCEPT when rewrite groups or confirmed merges could not be applied
515
- * (budget exhausted / rewrite-call infra error / merge cap or lock): the
516
- * cursor then holds BELOW the earliest candidate contributing to the
517
- * un-applied work, so those pairs are re-detected next run (dup-over-loss —
518
- * an un-marked, un-rewritten, un-merged confirmed resolution must never be
519
- * silently dropped by the cursor passing it).
662
+ * Cursor discipline: the cursor advances past a candidate only when its
663
+ * iteration's confirmed work has LANDED (or was refused-with-verdict-rendered:
664
+ * metadata mismatch, conflict-linked, mark-only fallback). A deferral holds
665
+ * the cursor BELOW the current candidate — bounded to ONE candidate's pairs,
666
+ * re-detected and re-judged next run (dup-over-loss — a confirmed resolution
667
+ * must never be silently dropped by the cursor passing it). The separate
668
+ * scan high-water (state.reconsolidationScannedRowid) records the max
669
+ * candidate rowid ENTERED and is never held back, so the report can split
670
+ * verdict calls into pairs_reevaluated (at/below the prior high-water) vs
671
+ * pairs_new — the convergence measurement.
520
672
  *
521
673
  * Dry-run: the zone's discovery + the free idempotency check only — zero LLM
522
674
  * calls, zero writes, no cursor or band-stats persistence. Gate discovery is
@@ -533,25 +685,13 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
533
685
  const minSimilarity = validNumber(options.minSimilarity, exports.DEFAULT_CORRECTION_MIN_SIMILARITY, (n) => n > 0 && n <= 1);
534
686
  const rewriteMinConfidence = validNumber(options.rewriteMinConfidence, exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE, (n) => n > 0 && n <= 1);
535
687
  const autoMergeThreshold = validNumber(options.autoMergeThreshold, dedup_js_1.DEFAULT_DEDUP_MERGE_THRESHOLD, (n) => n > 0 && n <= 1);
536
- const maxMerges = validNumber(options.maxMerges, dedup_js_1.DEFAULT_DEDUP_NIGHTLY_MAX_MERGES, (n) => n >= 0);
537
- const maxMinutes = validNumber(options.maxMinutes, exports.DEFAULT_RECONSOLIDATION_MAX_MINUTES, (n) => n >= 0);
538
- const maxCalls = validNumber(options.maxCalls, exports.DEFAULT_RECONSOLIDATION_MAX_CALLS, (n) => n >= 0);
539
- // ---- #401 runtime bounds. The wall-clock deadline is measured from stage
540
- // start (the deterministic zone's runtime counts against it — the binding
541
- // constraint must be THIS knob, never the process-level backstop).
542
- const deadlineAt = maxMinutes > 0 ? Date.now() + Math.round(maxMinutes * 60_000) : Infinity;
543
- const deadlineHit = () => Date.now() >= deadlineAt;
544
- // ---- #392 phase 0: the deterministic merge zone (pairs >= the ceiling),
545
- // LLM-free and budget-free — an LLM-less night still drains duplicates. Its
546
- // own short lock window, pre-merge backup, and pacing cap; fail-soft, never
547
- // a throw. Runs FIRST so the scan below never sees the pairs it owns.
548
- const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
549
- stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
550
- threshold: autoMergeThreshold,
551
- maxMerges,
552
- dryRun,
553
- acquireLock: options.acquireLock,
554
- });
688
+ // ---- #405 runtime bound. The stage-local reconsolidationMaxMinutes clock
689
+ // (#401) is gone — the run-wide pipeline deadline (nightly.ts, config
690
+ // nightlyTimeBudgetMinutes) is the only wall-clock. The deterministic
691
+ // zone's runtime counts against it via the zone's own stop-check below.
692
+ // hit() logs event=deadline_deferred once per stage name.
693
+ const deadline = options.deadline;
694
+ const deadlineHit = (stageLabel = exports.RECONSOLIDATION_STAGE_LABEL) => deadline?.hit(stageLabel) ?? false;
555
695
  // Per-run verdict statistics by cosine band (#392) — report snapshot here,
556
696
  // cumulative series in state.json at stage end (never on dry-run).
557
697
  const bands = buildResolutionBands(minSimilarity, autoMergeThreshold);
@@ -567,6 +707,14 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
567
707
  runBands.set(band.label, stat);
568
708
  };
569
709
  const startCursor = (0, state_js_1.loadState)(stateDir).reconsolidationCursor ?? 0;
710
+ // #439 convergence measurement: the scan high-water is the max candidate
711
+ // rowid any run has ENTERED — never held back by un-applied work. This
712
+ // run's re-judged/new split keys on the PREVIOUS run's persisted value: a
713
+ // verdict on a candidate at/below it re-judges pairs a prior run already
714
+ // judged but could not apply (the cursor held below them, so they
715
+ // re-detect). Once the backlog drains, pairs_reevaluated reads 0.
716
+ const prevScannedRowid = (0, state_js_1.loadState)(stateDir).reconsolidationScannedRowid ?? startCursor;
717
+ let scannedRowidHighwater = startCursor;
570
718
  // NO shape filter (AC2) — unlike stageSupersession. Absorbed rows are
571
719
  // excluded: they are invisible to recall and must not re-enter judgment.
572
720
  const rows = db
@@ -593,12 +741,39 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
593
741
  let skippedAboveCeiling = 0;
594
742
  let skippedMetadataMismatch = 0;
595
743
  let mergePairsApplied = 0;
744
+ // #393 guard-C: conflicts verdicts rendered (link written, both live) and
745
+ // judged-path merge refusals on a conflicts-linked pair.
746
+ let conflictFlagged = 0;
747
+ let conflictSkippedJudged = 0;
748
+ // #393 B scout counters (per-source observability, the #394 discipline):
749
+ // shape calls made / correction-shaped verdicts / FTS hits that became
750
+ // candidate pairs. 0 on dry-run (the shape call is LLM work).
751
+ let scoutScanned = 0;
752
+ let scoutCorrectionShaped = 0;
753
+ let scoutCandidatesFound = 0;
754
+ // #439 observability: the re-judged/new verdict split (keyed on the prior
755
+ // run's scan high-water) + the scan-stability guard's skip count + the
756
+ // confirmed-merge deferral count (still un-applied at run end).
757
+ let pairsReevaluated = 0;
758
+ let pairsNew = 0;
759
+ let skippedAbsorbed = 0;
760
+ let mergePairsDeferred = 0;
596
761
  let cursor = startCursor;
597
- const queuedMerges = [];
598
- // #392 cursor-hold anchor, shared by the merge phase and the rewrite phase:
599
- // un-applied work holds the cursor BELOW the earliest contributing
600
- // candidate so the pairs are re-detected next run (dup-over-loss).
601
- let pendingMinRowid = null;
762
+ // Lock-busy survivors: boundary merges that could not take the capture
763
+ // lock, plus (fix round, #440 review finding 1) deadline/backup-dropped
764
+ // tails re-queued at their boundary instead of discarded. Same-run only —
765
+ // retried (in full) at the next boundary and once in the final drain after
766
+ // the scan. While the list is non-empty, every persisted checkpoint clamps
767
+ // below the earliest contributing candidate (pendingRetryFloor — the kill
768
+ // window cannot strand them); still un-applied at run end, the drain holds
769
+ // the cursor below that same floor (dup-over-loss).
770
+ const retryMerges = [];
771
+ // ONE pre-merge backup per run (#439): takePreDedupBackup is a full SQLite
772
+ // copy, so a per-boundary backup would be hundreds of full-DB copies on a
773
+ // backlog night. Taken LAZILY, immediately before the first judged-merge
774
+ // application; remembered for the rest of the run (the zone takes its own,
775
+ // independent backup, as before).
776
+ let mergeWindowBackedUp = false;
602
777
  // Links created by THIS stage in THIS run — lets the explicit-mark pass
603
778
  // distinguish operator marks (pre-existing) from stage output.
604
779
  const linksCreatedThisRun = new Set();
@@ -606,42 +781,50 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
606
781
  storage.addLink(db, oldId, newId, relationship, strength);
607
782
  linksCreatedThisRun.add(`${oldId}|${newId}`);
608
783
  };
609
- // ---- #401: mid-scan cursor persistence. Called at EVERY scan-loop exit
610
- // path (deadline, call/budget cap, mark-verify budget stop, discovery
611
- // failure) AND after every fully-considered candidate, so a killed run
612
- // loses at most the candidate in flight. The end-of-stage updateState
613
- // below stays the authoritative final write (it also applies the
614
- // pendingMinRowid hold — that variable is only ever set AFTER the scan
615
- // loop, so it is null at every call site here). updateState is
616
- // load→mutate→temp-rename atomic.
617
- let callsUsed = 0;
784
+ // ---- #401/#439 mid-scan cursor persistence. Called at EVERY scan-loop
785
+ // exit path AND at the end of every candidate iteration — AFTER that
786
+ // iteration's boundary apply, so the persisted cursor only ever advances
787
+ // past candidates whose confirmed work has landed; a killed run loses at
788
+ // most the candidate in flight. The ONE exception is lock-busy retry
789
+ // survivors: they intentionally ride the retry list while the scan
790
+ // continues (the lock may clear this run), so while any are pending every
791
+ // persisted checkpoint CLAMPS below their earliest contributor — a SIGKILL
792
+ // in the window between a busy boundary and the pair landing must never
793
+ // strand a confirmed merge behind the cursor (the #402 orphan-floor
794
+ // discipline, re-scoped to the retry list; the clamp lifts automatically
795
+ // once a later boundary or the final drain applies them). Also persists
796
+ // the scan high-water (never held back). updateState is load→mutate→
797
+ // temp-rename atomic.
618
798
  let deadlineStopped = false;
619
- let callCapStopped = false;
620
- // #402 follow-up (reviewer note 1): the hard-kill orphan floor. Queued
621
- // merges and open rewrite groups are applied only in the POST-scan
622
- // phases — until then their verdicts exist only in memory, and a
623
- // SIGKILL/OOM between two persists would strand them BEHIND the persisted
624
- // cursor (the next run would skip them forever). This tracks the smallest
625
- // candidate rowid contributing to queued-but-unapplied work;
626
- // persistCursor clamps every checkpoint below it so a resumed run
627
- // re-detects the pairs (dup-over-loss). Deliberately SEPARATE from the
628
- // post-loop pendingMinRowid hold above — different lifetime, different
629
- // writers.
630
- let scanPendingMinRowid = null;
631
- const notePendingRowid = (rowid) => {
632
- scanPendingMinRowid =
633
- scanPendingMinRowid === null ? rowid : Math.min(scanPendingMinRowid, rowid);
799
+ // #439: once an iteration's confirmed work deferred (deadline at the
800
+ // boundary, backup failure, rewrite refusal/infra error), the cursor never
801
+ // advances again this run — a later iteration must not push it past the
802
+ // held candidate's rowid.
803
+ let cursorHold = false;
804
+ // Fix round (#440 review, finding 1): the floor below the earliest
805
+ // candidate contributing to a still-un-applied retry merge. Applied at
806
+ // every persist so the kill-with-pending-retry window cannot strand them.
807
+ const pendingRetryFloor = () => retryMerges.length > 0
808
+ ? Math.min(...retryMerges.map((p) => p.candidateRowid)) - 1
809
+ : null;
810
+ const clampedCursor = () => {
811
+ const floor = pendingRetryFloor();
812
+ return floor !== null ? Math.min(cursor, floor) : cursor;
634
813
  };
635
814
  const persistCursor = () => {
636
815
  if (dryRun)
637
816
  return;
638
- const checkpoint = scanPendingMinRowid !== null ? Math.min(cursor, scanPendingMinRowid - 1) : cursor;
639
817
  (0, state_js_1.updateState)((s) => {
640
- s.reconsolidationCursor = checkpoint;
818
+ s.reconsolidationCursor = clampedCursor();
819
+ s.reconsolidationScannedRowid = Math.max(scannedRowidHighwater, s.reconsolidationScannedRowid ?? 0);
641
820
  }, stateDir);
642
821
  };
643
- const groups = new Map();
644
- const addTrigger = (target, trigger, confidence, cosine, explicit) => {
822
+ // #439: rewrite groups live ONLY inside the candidate iteration that
823
+ // formed them (its boundary applies or degrades them, then they are
824
+ // discarded). Every trigger is the current candidate — a target corrected
825
+ // by two different candidates takes two sequential rewrites instead of
826
+ // the old one grouped call.
827
+ const addTrigger = (groups, target, trigger, confidence, cosine, explicit) => {
645
828
  let group = groups.get(target.id);
646
829
  if (!group) {
647
830
  group = { targetId: target.id, target, triggers: [] };
@@ -656,10 +839,9 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
656
839
  candidateRowid: trigger.__rowid,
657
840
  explicit,
658
841
  });
659
- notePendingRowid(trigger.__rowid); // orphan floor — group unapplied until the rewrite phase
660
842
  }
661
843
  };
662
- for (const candidate of rows) {
844
+ for (const snapshotted of rows) {
663
845
  // #401: runtime bounds first — exit cleanly at the last fully-considered
664
846
  // candidate boundary (cursor = the previous candidate's rowid here).
665
847
  if (!dryRun && deadlineHit()) {
@@ -671,15 +853,50 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
671
853
  persistCursor();
672
854
  break;
673
855
  }
674
- // #401: the per-stage call cap stops the scan at the candidate boundary
675
- // (the in-loop check below is the mid-candidate backstop — supersession
676
- // mirrors both).
677
- if (!dryRun && maxCalls > 0 && callsUsed >= maxCalls) {
678
- callCapStopped = true;
856
+ // ---- #439 scan-stability guard: `rows` is ONE snapshot fetched at stage
857
+ // start with status != 'absorbed'; boundary applies (merge losers,
858
+ // rewrite triggers absorbed) can mark FUTURE rows of that snapshot
859
+ // absorbed after the filter ran. Without this re-read such a candidate
860
+ // would be scouted/judged on stale content and its KNN would silently
861
+ // re-embed it (its vector row is gone). Absorbed → skip (counted,
862
+ // cursor passes it); otherwise the LIVE row's content/status drives the
863
+ // rest of the iteration (refreshes content rewritten by an earlier
864
+ // boundary when created_at and rowid order diverge).
865
+ const live = db
866
+ .prepare(`SELECT rowid AS __rowid, * FROM memories WHERE rowid = ?`)
867
+ .get(snapshotted.__rowid);
868
+ if (!live) {
869
+ // Vanished entirely (deleted out from under the scan) — defensive;
870
+ // nothing to judge, the cursor passes it (never past an active hold).
871
+ scannedRowidHighwater = Math.max(scannedRowidHighwater, snapshotted.__rowid);
872
+ if (!cursorHold)
873
+ cursor = snapshotted.__rowid;
679
874
  persistCursor();
680
- break;
875
+ continue;
876
+ }
877
+ if (live.status === "absorbed") {
878
+ skippedAbsorbed++;
879
+ scannedRowidHighwater = Math.max(scannedRowidHighwater, live.__rowid);
880
+ if (!cursorHold)
881
+ cursor = live.__rowid;
882
+ persistCursor();
883
+ continue;
681
884
  }
885
+ const candidate = live;
682
886
  scanned++;
887
+ // The high-water advances as candidates are ENTERED — even when the
888
+ // iteration's work later defers (it is the SCAN mark, never held back).
889
+ scannedRowidHighwater = Math.max(scannedRowidHighwater, candidate.__rowid);
890
+ // #439 per-iteration confirmed work, applied at the boundary below.
891
+ const iterMerges = [];
892
+ const iterGroups = new Map();
893
+ // This iteration's confirmed work could not land (deadline at the
894
+ // boundary, backup failure, rewrite refusal/infra error): the cursor
895
+ // holds below this candidate.
896
+ let boundaryHold = false;
897
+ // Stop the scan AFTER the boundary (budget stop / rewrite-infra
898
+ // deferral / backup failure): further verdicts could not land anyway.
899
+ let stopScan = false;
683
900
  // ---- AC7: verify incoming explicit marks (corrected_by/superseded_by
684
901
  // links targeting this candidate) before they can join a rewrite group.
685
902
  // #401: only OPERATOR marks are verified — applyExplicitMark writes
@@ -709,14 +926,17 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
709
926
  }
710
927
  const { verdict, usage } = await classifyPair(llm, target.content, candidate.content);
711
928
  budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
712
- callsUsed++; // #401
713
929
  pairsEvaluated++;
930
+ if (candidate.__rowid <= prevScannedRowid)
931
+ pairsReevaluated++;
932
+ else
933
+ pairsNew++;
714
934
  if (!verdict) {
715
935
  skippedInfra++; // mark retained; the neighborhood is revisited via newer candidacies
716
936
  continue;
717
937
  }
718
938
  if (verdict.action === "corrects" && verdict.confidence >= rewriteMinConfidence && isFactShapedTarget(target)) {
719
- addTrigger(target, candidate, verdict.confidence, null, true);
939
+ addTrigger(iterGroups, target, candidate, verdict.confidence, null, true);
720
940
  explicitVerified++;
721
941
  }
722
942
  else {
@@ -729,79 +949,179 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
729
949
  }
730
950
  }
731
951
  if (markBudgetStop) {
732
- persistCursor(); // #401: this candidate's remaining marks re-verify next run
733
- break;
952
+ // #401: this candidate's remaining marks re-verify next run — the
953
+ // cursor HOLDS below it (boundaryHold), and #439 still runs the
954
+ // boundary: marks confirmed before the stop apply, exactly as they
955
+ // did when the rewrite phase was post-scan.
956
+ stopScan = true;
957
+ boundaryHold = true;
734
958
  }
735
959
  }
736
- // ---- AC2: detection pairs against older KNN neighbors.
737
- let neighbors;
738
- try {
739
- neighbors = await findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarity);
960
+ // ---- #393 B: scout shape call — ONE classify-tier call per candidate,
961
+ // budget-metered under the same stage label as verdicts. Flags whether
962
+ // this memory corrects/retracts/supersedes something previously recorded;
963
+ // non-corrections stop here (zero follow-up). A parse/infra failure skips
964
+ // the scout source for this memory only (fail-soft — the similarity
965
+ // source below still runs) and counts skipped_infra, the verdict-skip
966
+ // discipline. Dry-run skips the call entirely: it is LLM work, and
967
+ // dry-runs make zero LLM calls (the scout counters read 0 there).
968
+ let scoutReferences = null;
969
+ if (!dryRun && !stopScan) {
970
+ if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
971
+ // Shape call refused — this candidate re-scouts next run: the cursor
972
+ // HOLDS below it (the candidate was entered but not considered).
973
+ stopScan = true;
974
+ boundaryHold = true;
975
+ }
976
+ else {
977
+ scoutScanned++;
978
+ try {
979
+ const r = await llm.complete(buildScoutShapePrompt(candidate.content));
980
+ budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
981
+ const shape = parseScoutShape(r.text);
982
+ if (!shape) {
983
+ skippedInfra++;
984
+ }
985
+ else if (shape.correction) {
986
+ scoutCorrectionShaped++;
987
+ scoutReferences = shape.references;
988
+ }
989
+ }
990
+ catch {
991
+ skippedInfra++;
992
+ }
993
+ }
740
994
  }
741
- catch (err) {
742
- console.warn(`[hicortex] reconsolidation: discovery failed for ${candidate.id.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
743
- cursor = candidate.__rowid;
744
- persistCursor(); // #401: every exit path persists
745
- continue;
995
+ // ---- AC2: detection pairs against older neighbors — TWO sources (#393 B):
996
+ // the KNN similarity source (floor-gated, the pre-B baseline) and, for
997
+ // correction-shaped candidates, the scout's FTS source (the referenced
998
+ // claim's terms → corpus search; NO similarity gate — cosine is a ranker,
999
+ // never a blocker). Merged + deduped by neighbor id: a pair found by both
1000
+ // sources is judged ONCE, as similarity (with band attribution).
1001
+ let discoveryFailed = false;
1002
+ let neighbors = [];
1003
+ if (!stopScan) {
1004
+ try {
1005
+ neighbors = await findOlderCorrectionNeighbors(db, candidate, embedFn, minSimilarity);
1006
+ }
1007
+ catch (err) {
1008
+ console.warn(`[hicortex] reconsolidation: discovery failed for ${candidate.id.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1009
+ // #439: skip this candidate's neighbor judgments but still run the
1010
+ // boundary below — marks confirmed earlier this iteration apply
1011
+ // (they did when the phases were post-scan).
1012
+ discoveryFailed = true;
1013
+ }
1014
+ }
1015
+ const neighborEntries = neighbors.map((n) => ({
1016
+ mem: n,
1017
+ cosine: (0, retrieval_js_1.l2ToCosine)(n.distance),
1018
+ source: "similarity",
1019
+ }));
1020
+ if (!discoveryFailed && scoutReferences !== null) {
1021
+ const candidateVec = storage.getStoredEmbedding(db, candidate.id);
1022
+ const knnById = new Map(neighborEntries.map((e) => [e.mem.id, e]));
1023
+ for (const entry of findScoutNeighbors(db, candidate, scoutReferences, candidateVec)) {
1024
+ const knn = knnById.get(entry.mem.id);
1025
+ if (knn) {
1026
+ // Both sources found the pair. Below the ceiling the KNN entry
1027
+ // would be judged anyway — similarity keeps it (band attribution,
1028
+ // judged once). At/above the ceiling the similarity entry would be
1029
+ // ceiling-skipped (zone territory, never judged) — yet the zone now
1030
+ // runs AFTER the scan, and a genuine conflict at >=0.92 is exactly
1031
+ // the pair it would blend with no judge in the loop (guard-C's
1032
+ // harm). Re-tag the entry to the scout source so the pair IS
1033
+ // judged: the scout's no-gate exemption applies, a `conflicts`
1034
+ // verdict can plant the guard link, and the zone's own guard then
1035
+ // refuses the cluster in this same run.
1036
+ if (knn.cosine >= autoMergeThreshold)
1037
+ knn.source = "scout";
1038
+ continue;
1039
+ }
1040
+ neighborEntries.push(entry);
1041
+ }
746
1042
  }
747
- for (const neighbor of neighbors) {
748
- pairsDiscovered++; // every neighbor passed the floor gate
749
- if (alreadyResolutionLinked(db, neighbor.id, candidate.id)) {
1043
+ for (const entry of neighborEntries) {
1044
+ pairsDiscovered++; // gate discovery (#394), both sources, before any skip/judgment
1045
+ if (entry.source === "scout")
1046
+ scoutCandidatesFound++;
1047
+ if (alreadyResolutionLinked(db, entry.mem.id, candidate.id)) {
750
1048
  skippedIdempotent++;
751
1049
  continue;
752
1050
  }
753
1051
  pairsDiscoveredUnlinked++; // still unlinked — the actionable candidate
754
- // #392: pairs at/above the ceiling belong to the deterministic zone —
755
- // counted here, never LLM-judged (the zone merges them or holds them
756
- // for its cap; re-detection is structural, not cursor-based).
757
- const pairCosine = (0, retrieval_js_1.l2ToCosine)(neighbor.distance);
758
- if (pairCosine >= autoMergeThreshold) {
1052
+ const pairCosine = entry.cosine;
1053
+ // #392: SIMILARITY-source pairs at/above the ceiling belong to the
1054
+ // deterministic zone — counted here, never LLM-judged (the zone merges
1055
+ // them at stage end or defers them to a later run; re-detection is
1056
+ // structural, not cursor-based). Scout pairs are exempt (#393 B):
1057
+ // cosine never blocks this source — guard-C's re-tag above relies on
1058
+ // it, and a judged merge re-passes the same metadata/conflict rails
1059
+ // the zone enforces.
1060
+ if (entry.source === "similarity" && pairCosine >= autoMergeThreshold) {
759
1061
  skippedAboveCeiling++;
760
1062
  continue;
761
1063
  }
762
1064
  if (dryRun)
763
1065
  continue; // preview only — no LLM call, no write
764
- // #401: the per-stage call cap rides the same boundary as the budget —
765
- // supersession-stage pattern (consolidate.ts stageSupersession).
766
- if ((maxCalls > 0 && callsUsed >= maxCalls) || !budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
767
- callCapStopped = maxCalls > 0 && callsUsed >= maxCalls;
768
- persistCursor(); // cursor still points at the last fully-considered candidate
1066
+ // #405: the ONE run budget's refusal is the only call cap.
1067
+ if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
1068
+ stopScan = true; // the boundary below still applies what this candidate already confirmed
769
1069
  break;
770
1070
  }
771
- const { verdict, usage } = await classifyPair(llm, neighbor.content, candidate.content);
1071
+ const { verdict, usage } = await classifyPair(llm, entry.mem.content, candidate.content);
772
1072
  budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, usage);
773
- callsUsed++; // #401
774
1073
  pairsEvaluated++;
1074
+ if (candidate.__rowid <= prevScannedRowid)
1075
+ pairsReevaluated++;
1076
+ else
1077
+ pairsNew++;
775
1078
  if (!verdict) {
776
1079
  skippedInfra++;
777
1080
  continue;
778
1081
  }
779
- recordBand(pairCosine, verdict.action, verdict.confidence);
780
- // #392: a merge verdict is queued for the merge phase (below) — no
781
- // link, no write here. Below the confidence gate BOTH memories stay
1082
+ // Bands stay similarity-source-only (refine Q2 ruling): they are the
1083
+ // calibration evidence for the floor/ceiling boundaries, and scout
1084
+ // pairs reach them through a different, cosine-blind door.
1085
+ if (entry.source === "similarity")
1086
+ recordBand(pairCosine, verdict.action, verdict.confidence);
1087
+ // #392/#439: a merge verdict queues for THIS iteration's boundary —
1088
+ // no link, no write here. Below the confidence gate BOTH memories stay
782
1089
  // live: a weak mark is recoverable, and there is nothing to mark for a
783
1090
  // duplicate — keeping both is the recoverable outcome.
784
1091
  if (verdict.action === "merge") {
785
1092
  if (verdict.confidence < rewriteMinConfidence) {
786
1093
  mergeBelowGate++;
787
- const band = bandForCosine(bands, pairCosine);
788
- if (band) {
789
- const stat = runBands.get(band.label) ?? emptyBandStat();
790
- stat.merge_below_gate++;
791
- runBands.set(band.label, stat);
1094
+ if (entry.source === "similarity") {
1095
+ const band = bandForCosine(bands, pairCosine);
1096
+ if (band) {
1097
+ const stat = runBands.get(band.label) ?? emptyBandStat();
1098
+ stat.merge_below_gate++;
1099
+ runBands.set(band.label, stat);
1100
+ }
792
1101
  }
793
1102
  }
794
1103
  else {
795
- queuedMerges.push({ oldId: neighbor.id, newId: candidate.id, candidateRowid: candidate.__rowid });
796
- notePendingRowid(candidate.__rowid); // orphan floor — merge unapplied until the merge phase
1104
+ iterMerges.push({ oldId: entry.mem.id, newId: candidate.id, candidateRowid: candidate.__rowid });
797
1105
  }
798
1106
  continue;
799
1107
  }
800
1108
  if (verdict.action === "supersedes") {
801
- markLink(neighbor.id, candidate.id, "superseded_by", (0, retrieval_js_1.l2ToCosine)(neighbor.distance));
802
- storage.updateMemory(db, neighbor.id, { status: "superseded" });
1109
+ markLink(entry.mem.id, candidate.id, "superseded_by", pairCosine);
1110
+ storage.updateMemory(db, entry.mem.id, { status: "superseded" });
803
1111
  markedSuperseded++;
804
- console.log(`[hicortex] Reconsolidation: ${neighbor.id.slice(0, 8)} superseded_by ${candidate.id.slice(0, 8)} (mark-only)`);
1112
+ console.log(`[hicortex] Reconsolidation: ${entry.mem.id.slice(0, 8)} superseded_by ${candidate.id.slice(0, 8)} (mark-only)`);
1113
+ continue;
1114
+ }
1115
+ // #393 guard-C: a genuine conflict — link ONLY. No status change on
1116
+ // either memory (both stay live so the consumer sees both truths), no
1117
+ // rewrite, no merge queue; the link is the guard both merge paths
1118
+ // consult. Ungated like the other mark actions (a weak flag is
1119
+ // recoverable; a weak merge is not).
1120
+ if (verdict.action === "conflicts") {
1121
+ markLink(entry.mem.id, candidate.id, "conflicts", pairCosine);
1122
+ conflictFlagged++;
1123
+ console.log(`[hicortex] Reconsolidation: ${entry.mem.id.slice(0, 8)} conflicts ${candidate.id.slice(0, 8)} ` +
1124
+ `(flag-only) — both kept live, never merged`);
805
1125
  continue;
806
1126
  }
807
1127
  if (verdict.action === "corrects") {
@@ -810,104 +1130,319 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
810
1130
  // Below the gate: mark-only, never rewrite. The
811
1131
  // trigger stays live — it is the only carrier of the correction.
812
1132
  belowGate++;
813
- markLink(neighbor.id, candidate.id, "corrected_by", cosine);
814
- storage.updateMemory(db, neighbor.id, { status: "retracted" });
1133
+ markLink(entry.mem.id, candidate.id, "corrected_by", cosine);
1134
+ storage.updateMemory(db, entry.mem.id, { status: "retracted" });
815
1135
  markedRetracted++;
816
1136
  continue;
817
1137
  }
818
- if (!isFactShapedTarget(neighbor)) {
1138
+ if (!isFactShapedTarget(entry.mem)) {
819
1139
  // Decisions/plans/experiences are history, not error — mark only.
820
- markLink(neighbor.id, candidate.id, "corrected_by", cosine);
821
- storage.updateMemory(db, neighbor.id, { status: "retracted" });
1140
+ markLink(entry.mem.id, candidate.id, "corrected_by", cosine);
1141
+ storage.updateMemory(db, entry.mem.id, { status: "retracted" });
822
1142
  markedRetracted++;
823
1143
  continue;
824
1144
  }
825
- addTrigger(neighbor, candidate, verdict.confidence, cosine, false);
1145
+ addTrigger(iterGroups, entry.mem, candidate, verdict.confidence, cosine, false);
826
1146
  }
827
1147
  // verdict "none" → nothing to do
828
1148
  }
829
- cursor = candidate.__rowid;
830
- // #402 follow-up (reviewer note 1): persist after EVERY fully-considered
831
- // candidate — the 50-candidate batch left a kill window that could
832
- // strand several candidates of scan progress. updateState is an atomic
833
- // temp-rename of a small file and the loop cadence is seconds per
834
- // candidate; the cost is negligible.
1149
+ // ---- #439 BOUNDARY: apply everything this candidate confirmed, NOW —
1150
+ // merges first, then rewrite groups (the order the old post-scan phases
1151
+ // used; preserves the existing tolerance where a merge loser that is
1152
+ // also a rewrite trigger stays absorbed while the rewrite still composes
1153
+ // its content). Each application is its own transaction
1154
+ // (mergeMemoryIds / applyRewriteGroup — group-internal atomicity
1155
+ // preserved); a busy capture lock defers merges to the same-run retry
1156
+ // list, everything else defers by holding the cursor below this
1157
+ // candidate (bounded to ONE candidate's pairs).
1158
+ if (!dryRun) {
1159
+ // Earlier lock-busy survivors retry FIRST (oldest verdicts land
1160
+ // first), ahead of this candidate's fresh confirmations.
1161
+ const mergeBatch = [...retryMerges.splice(0, retryMerges.length), ...iterMerges];
1162
+ if (mergeBatch.length > 0) {
1163
+ if (deadlineHit()) {
1164
+ deadlineStopped = true;
1165
+ // Fix round (#440 review, finding 1): never DROP the batch — it can
1166
+ // begin with lock-busy survivors contributed by EARLIER candidates
1167
+ // that the cursor has already passed. Re-queue for the final drain;
1168
+ // its hold-below-earliest-contributor (and the persist clamp) keeps
1169
+ // every un-applied pair re-detectable next run.
1170
+ retryMerges.push(...mergeBatch);
1171
+ boundaryHold = true;
1172
+ console.log(`[hicortex] Reconsolidation: ${mergeBatch.length} confirmed merge(s) re-queued for the final drain — run deadline reached`);
1173
+ }
1174
+ else {
1175
+ const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
1176
+ const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
1177
+ if (!release) {
1178
+ // Busy capture run — the batch rides the same-run retry list:
1179
+ // retried at the next boundary and once in the final drain. Not
1180
+ // a cursor hold yet (the lock may clear this run).
1181
+ retryMerges.push(...mergeBatch);
1182
+ console.warn(`[hicortex] Reconsolidation: capture lock busy — ${mergeBatch.length} confirmed merge(s) deferred to a retry this run`);
1183
+ }
1184
+ else {
1185
+ try {
1186
+ let backupOk = true;
1187
+ if (!mergeWindowBackedUp) {
1188
+ try {
1189
+ await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
1190
+ mergeWindowBackedUp = true;
1191
+ }
1192
+ catch (err) {
1193
+ backupOk = false;
1194
+ console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
1195
+ `(${err instanceof Error ? err.message : String(err)}) — ${mergeBatch.length} merge(s) deferred`);
1196
+ }
1197
+ }
1198
+ if (!backupOk) {
1199
+ // Verdicts that cannot land must not keep being paid: stop
1200
+ // the scan. The batch re-queues for the final drain — a
1201
+ // TRANSIENT backup failure can still recover there; a
1202
+ // persistent one ends with the drain holding the cursor
1203
+ // below the earliest contributor (never just this candidate,
1204
+ // when retry survivors ride the batch).
1205
+ retryMerges.push(...mergeBatch);
1206
+ boundaryHold = true;
1207
+ stopScan = true;
1208
+ }
1209
+ else {
1210
+ for (let i = 0; i < mergeBatch.length; i++) {
1211
+ const pair = mergeBatch[i];
1212
+ // #405: the deadline stop-check between local merge
1213
+ // transactions — a safe boundary; deferred pairs hold the
1214
+ // cursor below this candidate and retry next run.
1215
+ if (deadlineHit()) {
1216
+ deadlineStopped = true;
1217
+ // Fix round (#440 review, finding 1): the un-applied tail
1218
+ // re-queues (it can contain earlier candidates' retry
1219
+ // survivors) — the final drain applies or holds it.
1220
+ retryMerges.push(...mergeBatch.slice(i));
1221
+ boundaryHold = true;
1222
+ console.log(`[hicortex] Reconsolidation: ${mergeBatch.length - i} confirmed merge(s) re-queued for the final drain — run deadline reached`);
1223
+ break;
1224
+ }
1225
+ const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
1226
+ if (result.ok) {
1227
+ mergePairsApplied++;
1228
+ console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
1229
+ `into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
1230
+ }
1231
+ else if (result.reason === "metadata_mismatch") {
1232
+ skippedMetadataMismatch++;
1233
+ console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
1234
+ `skipped (metadata mismatch) — both kept`);
1235
+ }
1236
+ else if (result.reason === "conflict_linked") {
1237
+ // #393 guard-C: the pair is conflicts-linked (operator-planted
1238
+ // or a prior verdict) — never blended; the cursor advances,
1239
+ // this verdict was rendered.
1240
+ conflictSkippedJudged++;
1241
+ console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
1242
+ `skipped (conflict-flagged) — both kept`);
1243
+ }
1244
+ // "no_members": a member vanished/was absorbed since the
1245
+ // verdict — nothing to merge, nothing to hold; the cursor
1246
+ // advances past it.
1247
+ }
1248
+ }
1249
+ }
1250
+ finally {
1251
+ release();
1252
+ }
1253
+ }
1254
+ }
1255
+ }
1256
+ if (iterGroups.size > 0) {
1257
+ // One rewrite call per group — the group's triggers are all THIS
1258
+ // candidate (#439: a target corrected by two different candidates
1259
+ // takes two sequential rewrites, one per boundary; the second call
1260
+ // composes the already-corrected story). A call that was never made
1261
+ // (budget/infra/deadline) defers the group — untouched, never
1262
+ // partially applied — and holds the cursor below this candidate.
1263
+ const contracts = new Map(); // null = contract failed
1264
+ let rewritesDeferred = false;
1265
+ for (const group of iterGroups.values()) {
1266
+ if (deadlineHit()) {
1267
+ deadlineStopped = true;
1268
+ rewritesDeferred = true;
1269
+ break;
1270
+ }
1271
+ if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
1272
+ rewritesDeferred = true;
1273
+ stopScan = true;
1274
+ break;
1275
+ }
1276
+ const triggersArg = group.triggers.map((t) => ({ id: t.id, content: t.memory.content }));
1277
+ let contract = null;
1278
+ let infraError = false;
1279
+ try {
1280
+ const r = await llm.complete(buildRewritePrompt(group.target.content, triggersArg));
1281
+ contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
1282
+ budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
1283
+ }
1284
+ catch {
1285
+ infraError = true;
1286
+ }
1287
+ if (infraError) {
1288
+ skippedInfra++;
1289
+ rewritesDeferred = true; // group NOT marked, NOT rewritten — retried next run
1290
+ stopScan = true; // verdicts past this hold could not advance the cursor anyway
1291
+ break;
1292
+ }
1293
+ contracts.set(group.targetId, contract);
1294
+ if (!contract)
1295
+ contractFailed++;
1296
+ }
1297
+ if (rewritesDeferred) {
1298
+ boundaryHold = true;
1299
+ }
1300
+ else {
1301
+ // Multi-target keep rule WITHIN the boundary: the shared trigger is
1302
+ // the current candidate, so every group it touches resolves here —
1303
+ // absorbed only if EVERY contract says absorb (any keep keeps).
1304
+ const finalOutcome = new Map();
1305
+ for (const contract of contracts.values()) {
1306
+ if (!contract)
1307
+ continue;
1308
+ for (const t of contract.triggers) {
1309
+ if (t.disposition === "keep" || finalOutcome.get(t.id) === "keep")
1310
+ finalOutcome.set(t.id, "keep");
1311
+ else
1312
+ finalOutcome.set(t.id, "absorb");
1313
+ }
1314
+ }
1315
+ // Counted from APPLIED groups only (a deferred group's dispositions
1316
+ // never took effect); a trigger in several applied groups counts once.
1317
+ const appliedOutcome = new Map();
1318
+ for (const group of iterGroups.values()) {
1319
+ const contract = contracts.get(group.targetId);
1320
+ if (contract === undefined)
1321
+ continue; // pending group — untouched this run
1322
+ if (contract === null) {
1323
+ // Failed rewrite contract → whole group mark-only, never a partial
1324
+ // apply. Content untouched, NO trigger absorbed.
1325
+ try {
1326
+ applyMarkOnlyGroup(db, group);
1327
+ }
1328
+ catch (err) {
1329
+ console.warn(`[hicortex] reconsolidation: mark-only fallback failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1330
+ skippedInfra++;
1331
+ boundaryHold = true; // retried next run
1332
+ continue;
1333
+ }
1334
+ markedRetracted++;
1335
+ console.log(`[hicortex] Reconsolidation: rewrite contract failed for ${group.targetId.slice(0, 8)} — group degraded to mark-only`);
1336
+ continue;
1337
+ }
1338
+ let applied = false;
1339
+ try {
1340
+ applied = await applyRewriteGroup(db, group, contract, finalOutcome, embedFn);
1341
+ }
1342
+ catch (err) {
1343
+ console.warn(`[hicortex] reconsolidation: rewrite apply failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1344
+ }
1345
+ if (!applied) {
1346
+ skippedInfra++; // defensive absorbed-target guard, or an apply error — retry next run
1347
+ boundaryHold = true;
1348
+ continue;
1349
+ }
1350
+ rewritten++;
1351
+ for (const t of contract.triggers) {
1352
+ const outcome = finalOutcome.get(t.id) ?? "keep";
1353
+ if (outcome === "keep" || appliedOutcome.get(t.id) === "keep")
1354
+ appliedOutcome.set(t.id, "keep");
1355
+ else
1356
+ appliedOutcome.set(t.id, "absorb");
1357
+ }
1358
+ }
1359
+ for (const outcome of appliedOutcome.values()) {
1360
+ if (outcome === "absorb")
1361
+ absorbed++;
1362
+ else
1363
+ keptLinked++;
1364
+ }
1365
+ }
1366
+ }
1367
+ }
1368
+ // #439 cursor advance: past this candidate ONLY when its confirmed work
1369
+ // landed (or was refused-with-verdict-rendered). Once anything deferred,
1370
+ // the latch holds the cursor below that candidate for the rest of the
1371
+ // run — a later iteration must never advance past an earlier hold.
1372
+ if (boundaryHold)
1373
+ cursorHold = true;
1374
+ if (!cursorHold)
1375
+ cursor = candidate.__rowid;
1376
+ // #402/#439: persist after EVERY candidate — AFTER the boundary apply,
1377
+ // so the checkpoint only ever crosses candidates whose work landed. A
1378
+ // SIGKILL between persists re-detects at most the in-flight candidate.
835
1379
  persistCursor();
1380
+ if (stopScan || deadlineStopped)
1381
+ break;
836
1382
  }
837
1383
  if (deadlineStopped) {
838
- console.log(`[hicortex] Reconsolidation: wall-clock deadline reached (reconsolidationMaxMinutes) — ` +
839
- `scan stopped at cursor ${cursor}; the next run resumes from there`);
840
- }
841
- else if (callCapStopped) {
842
- console.log(`[hicortex] Reconsolidation: per-run call cap reached (reconsolidationMaxCalls) — ` +
1384
+ console.log(`[hicortex] Reconsolidation: run deadline reached (nightlyTimeBudgetMinutes) — ` +
843
1385
  `scan stopped at cursor ${cursor}; the next run resumes from there`);
844
1386
  }
845
- // ---- #392 judged-merge phase: apply the queued pair merges through the
846
- // dedup core (mergeMemoryIds — same canonical pick, link re-points,
847
- // dedup_log, absorb). One short lock/backup window for the whole batch, one
848
- // transaction per pair. Zone merge operations count against the SAME
849
- // dedupNightlyMaxMerges cap. A pair that cannot apply (cap exhausted, busy
850
- // lock, failed backup) keeps BOTH memories live and holds the cursor below
851
- // its candidate — a confirmed merge is never silently dropped by the cursor
852
- // passing it (dup-over-loss). A metadata-rail refusal is different: the
853
- // verdict WAS rendered, both memories stay live, the cursor advances.
854
- const zoneOpsUsed = merges.merged_clusters + merges.failed;
855
- let mergeOpsRemaining = maxMerges > 0 ? Math.max(0, maxMerges - zoneOpsUsed) : 0;
856
- let mergePairsDeferred = 0;
857
- if (!dryRun && queuedMerges.length > 0) {
858
- const holdQueued = (from) => {
859
- for (let i = from; i < queuedMerges.length; i++) {
860
- pendingMinRowid =
861
- pendingMinRowid === null
862
- ? queuedMerges[i].candidateRowid
863
- : Math.min(pendingMinRowid, queuedMerges[i].candidateRowid);
864
- }
1387
+ // ---- #439 final drain: lock-busy merge survivors get ONE more attempt
1388
+ // right after the scan (the retry list is same-run only — everything else
1389
+ // applied at its boundary). Still busy (or the deadline/backup refuses) →
1390
+ // the pairs stay un-applied, counted, and the cursor holds below the
1391
+ // earliest contributing candidate (dup-over-loss; logged).
1392
+ if (!dryRun && retryMerges.length > 0) {
1393
+ const batch = retryMerges.splice(0, retryMerges.length);
1394
+ // Hold below the earliest contributor of the UN-APPLIED tail only (fix
1395
+ // round, minor review note: the old whole-batch min over-held past pairs
1396
+ // that had just applied in the same loop).
1397
+ const holdBelow = (fromIndex) => {
1398
+ cursor = Math.min(cursor, Math.min(...batch.slice(fromIndex).map((p) => p.candidateRowid)) - 1);
865
1399
  };
866
- if (maxMerges === 0) {
867
- // Machinery disabled by config: keep both (counted in band_stats as
868
- // merge verdicts) and ADVANCE — holding the cursor would re-judge the
869
- // same pairs into the same disabled state forever.
870
- console.log(`[hicortex] Reconsolidation: ${queuedMerges.length} confirmed merge(s) kept — ` +
871
- `dedupNightlyMaxMerges is 0 (merge machinery disabled)`);
872
- }
873
- else if (mergeOpsRemaining <= 0) {
874
- mergePairsDeferred = queuedMerges.length;
875
- holdQueued(0); // zone consumed the whole cap — retry next run
876
- console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred — ` +
877
- `dedupNightlyMaxMerges exhausted by the deterministic zone`);
1400
+ if (deadlineHit()) {
1401
+ deadlineStopped = true;
1402
+ mergePairsDeferred += batch.length;
1403
+ holdBelow(0);
1404
+ console.log(`[hicortex] Reconsolidation: ${batch.length} confirmed merge(s) deferred — run deadline reached`);
878
1405
  }
879
1406
  else {
880
1407
  const acquire = options.acquireLock ?? capture_js_1.acquireCaptureLock;
881
1408
  const release = await acquire(stateDir ?? (0, paths_js_1.hicortexHome)(), 0);
882
1409
  if (!release) {
883
- mergePairsDeferred = queuedMerges.length;
884
- holdQueued(0); // a busy capture run defers the batch — fail-soft
885
- console.warn(`[hicortex] Reconsolidation: capture lock busy — ${mergePairsDeferred} confirmed merge(s) deferred to next run`);
1410
+ mergePairsDeferred += batch.length;
1411
+ holdBelow(0);
1412
+ console.warn(`[hicortex] Reconsolidation: capture lock busy at the final drain — ` +
1413
+ `${batch.length} confirmed merge(s) deferred to next run`);
886
1414
  }
887
1415
  else {
888
1416
  try {
889
1417
  let backupOk = true;
890
- try {
891
- await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
1418
+ if (!mergeWindowBackedUp) {
1419
+ try {
1420
+ await (0, dedup_js_1.takePreDedupBackup)(db, stateDir ?? (0, paths_js_1.hicortexHome)());
1421
+ mergeWindowBackedUp = true;
1422
+ }
1423
+ catch (err) {
1424
+ backupOk = false;
1425
+ console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
1426
+ `(${err instanceof Error ? err.message : String(err)}) — ${batch.length} merge(s) deferred`);
1427
+ }
892
1428
  }
893
- catch (err) {
894
- backupOk = false;
895
- console.error(`[hicortex] Reconsolidation: pre-merge backup failed ` +
896
- `(${err instanceof Error ? err.message : String(err)}) — ${queuedMerges.length} merge(s) deferred`);
1429
+ if (!backupOk) {
1430
+ mergePairsDeferred += batch.length;
1431
+ holdBelow(0);
897
1432
  }
898
- if (backupOk) {
899
- for (let i = 0; i < queuedMerges.length; i++) {
900
- const pair = queuedMerges[i];
901
- if (mergeOpsRemaining <= 0) {
902
- mergePairsDeferred = queuedMerges.length - i;
903
- holdQueued(i); // cap exhausted mid-batch — the rest retry next run
904
- console.log(`[hicortex] Reconsolidation: ${mergePairsDeferred} confirmed merge(s) deferred — dedupNightlyMaxMerges exhausted`);
1433
+ else {
1434
+ for (let i = 0; i < batch.length; i++) {
1435
+ const pair = batch[i];
1436
+ if (deadlineHit()) {
1437
+ deadlineStopped = true;
1438
+ mergePairsDeferred += batch.length - i;
1439
+ holdBelow(i);
1440
+ console.log(`[hicortex] Reconsolidation: ${batch.length - i} confirmed merge(s) deferred — run deadline reached`);
905
1441
  break;
906
1442
  }
907
1443
  const result = (0, dedup_js_1.mergeMemoryIds)(db, [pair.oldId, pair.newId]);
908
1444
  if (result.ok) {
909
1445
  mergePairsApplied++;
910
- mergeOpsRemaining--;
911
1446
  console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
912
1447
  `into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
913
1448
  }
@@ -916,15 +1451,15 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
916
1451
  console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
917
1452
  `skipped (metadata mismatch) — both kept`);
918
1453
  }
1454
+ else if (result.reason === "conflict_linked") {
1455
+ conflictSkippedJudged++;
1456
+ console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
1457
+ `skipped (conflict-flagged) — both kept`);
1458
+ }
919
1459
  // "no_members": a member vanished/was absorbed since the
920
- // verdict — nothing to merge, nothing to hold; the cursor
921
- // advances past it.
1460
+ // verdict — nothing to merge, nothing to hold.
922
1461
  }
923
1462
  }
924
- else {
925
- mergePairsDeferred = queuedMerges.length;
926
- holdQueued(0);
927
- }
928
1463
  }
929
1464
  finally {
930
1465
  release();
@@ -932,148 +1467,34 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
932
1467
  }
933
1468
  }
934
1469
  }
935
- // ---- Rewrite phase (AC3/AC4/AC5). Three sub-phases so the multi-target
936
- // keep rule can be honored: (R1) collect contracts, (R2) resolve every
937
- // trigger's FINAL disposition across all groups, (R3) apply one transaction
938
- // per group. A group whose rewrite call was never made (budget/infra) is
939
- // left untouched and holds the cursor — never partially applied.
940
- const contracts = new Map(); // null = contract failed
941
- // pendingMinRowid (min candidate rowid among un-applied work) is declared
942
- // above — shared with the merge phase's holdQueued.
943
- const deferFrom = (fromTargetId) => {
944
- let seen = false;
945
- for (const group of groups.values()) {
946
- if (!seen && group.targetId !== fromTargetId)
947
- continue;
948
- seen = true;
949
- for (const t of group.triggers) {
950
- pendingMinRowid = pendingMinRowid === null ? t.candidateRowid : Math.min(pendingMinRowid, t.candidateRowid);
951
- }
952
- }
953
- };
954
- if (!dryRun && groups.size > 0) {
955
- for (const group of groups.values()) {
956
- // #401: the bounds stop the rewrite phase too — a group whose rewrite
957
- // call was never made is left untouched and holds the cursor (never
958
- // partially applied), exactly like the budget-exhausted path below.
959
- if (deadlineHit()) {
960
- deadlineStopped = true;
961
- deferFrom(group.targetId);
962
- break;
963
- }
964
- if (maxCalls > 0 && callsUsed >= maxCalls) {
965
- callCapStopped = true;
966
- deferFrom(group.targetId);
967
- break;
968
- }
969
- if (!budget.use(exports.RECONSOLIDATION_STAGE_LABEL)) {
970
- deferFrom(group.targetId);
971
- break;
972
- }
973
- const triggersArg = group.triggers.map((t) => ({ id: t.id, content: t.memory.content }));
974
- let contract = null;
975
- let infraError = false;
976
- try {
977
- const r = await llm.completeClassify(buildRewritePrompt(group.target.content, triggersArg));
978
- contract = parseRewriteReply(r.text, group.triggers.map((t) => t.id), group.target.content);
979
- budget.recordUsage(exports.RECONSOLIDATION_STAGE_LABEL, r.usage);
980
- callsUsed++; // #401: rewrite contracts count toward the stage call cap
981
- }
982
- catch {
983
- infraError = true;
984
- }
985
- if (infraError) {
986
- skippedInfra++;
987
- deferFrom(group.targetId); // group NOT marked, NOT rewritten — retried next run
988
- break;
989
- }
990
- contracts.set(group.targetId, contract);
991
- if (!contract)
992
- contractFailed++;
993
- }
994
- // R2: final per-trigger disposition — a trigger in multiple groups is
995
- // absorbed only if EVERY disposition says absorb (any keep keeps it).
996
- const finalOutcome = new Map();
997
- for (const contract of contracts.values()) {
998
- if (!contract)
999
- continue;
1000
- for (const t of contract.triggers) {
1001
- if (t.disposition === "keep" || finalOutcome.get(t.id) === "keep")
1002
- finalOutcome.set(t.id, "keep");
1003
- else
1004
- finalOutcome.set(t.id, "absorb");
1005
- }
1006
- }
1007
- // R3: apply (one transaction per group). An apply that fails mid-flight
1008
- // (embed error, DB error) writes NOTHING (the transaction never ran) —
1009
- // the group is deferred like a pending one so it retries next run.
1010
- const appliedOutcome = new Map();
1011
- const deferGroup = (group) => {
1012
- for (const t of group.triggers) {
1013
- pendingMinRowid = pendingMinRowid === null ? t.candidateRowid : Math.min(pendingMinRowid, t.candidateRowid);
1014
- }
1015
- };
1016
- for (const group of groups.values()) {
1017
- const contract = contracts.get(group.targetId);
1018
- if (contract === undefined)
1019
- continue; // pending group — untouched this run
1020
- if (contract === null) {
1021
- // Failed rewrite contract → whole group mark-only, never a partial
1022
- // apply. Content untouched, NO trigger absorbed.
1023
- try {
1024
- applyMarkOnlyGroup(db, group);
1025
- }
1026
- catch (err) {
1027
- console.warn(`[hicortex] reconsolidation: mark-only fallback failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1028
- skippedInfra++;
1029
- deferGroup(group);
1030
- continue;
1031
- }
1032
- markedRetracted++;
1033
- console.log(`[hicortex] Reconsolidation: rewrite contract failed for ${group.targetId.slice(0, 8)} — group degraded to mark-only`);
1034
- continue;
1035
- }
1036
- let applied = false;
1037
- try {
1038
- applied = await applyRewriteGroup(db, group, contract, finalOutcome, embedFn);
1039
- }
1040
- catch (err) {
1041
- console.warn(`[hicortex] reconsolidation: rewrite apply failed for ${group.targetId.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
1042
- }
1043
- if (!applied) {
1044
- skippedInfra++; // defensive absorbed-target guard, or an apply error — retry next run
1045
- deferGroup(group);
1046
- continue;
1047
- }
1048
- rewritten++;
1049
- // Counted from APPLIED groups only (a deferred group's dispositions
1050
- // never took effect); a trigger in several applied groups counts once.
1051
- for (const t of contract.triggers) {
1052
- const outcome = finalOutcome.get(t.id) ?? "keep";
1053
- if (outcome === "keep" || appliedOutcome.get(t.id) === "keep")
1054
- appliedOutcome.set(t.id, "keep");
1055
- else
1056
- appliedOutcome.set(t.id, "absorb");
1057
- }
1058
- }
1059
- for (const outcome of appliedOutcome.values()) {
1060
- if (outcome === "absorb")
1061
- absorbed++;
1062
- else
1063
- keptLinked++;
1064
- }
1065
- }
1066
- // Cursor hold: un-applied work (rewrite groups, confirmed merges) holds the
1067
- // cursor BELOW its earliest contributing candidate so the pairs are
1068
- // re-detected next run.
1069
- if (pendingMinRowid !== null) {
1070
- cursor = Math.min(cursor, pendingMinRowid - 1);
1071
- }
1470
+ // ---- #393 guard-C zone reorder: the deterministic merge zone (pairs >=
1471
+ // the ceiling) runs LAST — after the scan (which includes every #439
1472
+ // boundary apply: judged merges + rewrites). Judgment outranks the
1473
+ // deterministic sweep: verdicts, marks, and binds land first, and the zone
1474
+ // merges only what no verdict claimed. With the zone first, a >=0.92
1475
+ // genuine-conflict pair was blended before the judge ever saw it
1476
+ // (canonical = oldest, the newer truth erased — the planted-eval harm);
1477
+ // running it last means a `conflicts` bind set by THIS run's scan guards
1478
+ // the SAME run's zone. LLM-free and budget-free — an LLM-less night still
1479
+ // drains duplicates (a deadline-deferred cluster re-detects next run at
1480
+ // zero token cost — content-based discovery, no cursor involvement). Its
1481
+ // own short lock window, pre-merge backup, and #405 deadline stop-check;
1482
+ // fail-soft, never a throw.
1483
+ const merges = await (0, dedup_js_1.runDeterministicMergeZone)(db, {
1484
+ stateDir: stateDir ?? (0, paths_js_1.hicortexHome)(),
1485
+ threshold: autoMergeThreshold,
1486
+ dryRun,
1487
+ acquireLock: options.acquireLock,
1488
+ deadline,
1489
+ });
1072
1490
  // Report snapshot: the deterministic band (from the zone's own numbers —
1073
1491
  // losers are merge verdicts at confidence 1.0; the zone persists the
1074
1492
  // cumulative copy itself) plus this run's judged bands.
1075
1493
  const bandStats = {};
1076
- if (merges.max_merges > 0) {
1494
+ {
1495
+ // #405: recorded whenever the zone ran (the old max_merges>0 gate was a
1496
+ // 0=disabled switch — the switch is gone; a clean corpus records zeros,
1497
+ // same as the old default-config behavior).
1077
1498
  const det = emptyBandStat();
1078
1499
  det.pairs = merges.losers_merged;
1079
1500
  det.merge = merges.losers_merged;
@@ -1086,10 +1507,15 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1086
1507
  for (const [label, stat] of runBands)
1087
1508
  bandStats[label] = stat;
1088
1509
  if (!dryRun) {
1089
- // #401: the authoritative FINAL cursor write — the mid-scan persists
1090
- // above are checkpoints; this one also applies the pendingMinRowid hold.
1510
+ // #401/#439: the authoritative FINAL write — the mid-scan persists above
1511
+ // are checkpoints; this one applies the final-drain cursor hold (already
1512
+ // folded into `cursor`) and the scan high-water. The retry floor is
1513
+ // applied defensively too: the drain splices retryMerges empty on every
1514
+ // path, but a non-empty list here would mean a confirmed merge stranded
1515
+ // behind the cursor — clamp, never write past un-applied work.
1091
1516
  (0, state_js_1.updateState)((s) => {
1092
- s.reconsolidationCursor = cursor;
1517
+ s.reconsolidationCursor = clampedCursor();
1518
+ s.reconsolidationScannedRowid = Math.max(scannedRowidHighwater, s.reconsolidationScannedRowid ?? 0);
1093
1519
  // Cumulative judged-band accumulation (#392) — the zone already
1094
1520
  // persisted the deterministic band under its own label.
1095
1521
  if (runBands.size > 0) {
@@ -1103,14 +1529,20 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1103
1529
  }
1104
1530
  }, stateDir);
1105
1531
  }
1106
- if (rows.length > 0 || groups.size > 0 || mergePairsApplied > 0 || mergeBelowGate > 0) {
1107
- console.log(`[hicortex] Reconsolidation: ${scanned} scanned, ${pairsEvaluated} pairs evaluated, ` +
1532
+ if (rows.length > 0 || mergePairsApplied > 0 || mergeBelowGate > 0) {
1533
+ console.log(`[hicortex] Reconsolidation: ${scanned} scanned, ${pairsEvaluated} pairs evaluated ` +
1534
+ `(${pairsReevaluated} re-judged / ${pairsNew} new), ` +
1108
1535
  `${rewritten} rewritten (${absorbed} triggers absorbed, ${keptLinked} kept), ` +
1109
- `${mergePairsApplied} pair(s) merged, ${markedSuperseded} superseded, ` +
1536
+ `${mergePairsApplied} pair(s) merged (${mergePairsDeferred} deferred), ` +
1537
+ `${markedSuperseded} superseded, ` +
1110
1538
  `${markedRetracted} retracted (${belowGate} below gate, ${mergeBelowGate} merge below gate, ` +
1111
1539
  `${contractFailed} contract failed), ${skippedInfra} infra-skipped, ${skippedIdempotent} ` +
1112
- `already-linked, ${skippedAboveCeiling} above ceiling, ${explicitVerified} explicit verified, ` +
1113
- `${explicitDivergent} explicit divergent (cursor ${cursor})`);
1540
+ `already-linked, ${skippedAbsorbed} absorbed-skip, ${skippedAboveCeiling} above ceiling, ` +
1541
+ `${explicitVerified} explicit verified, ` +
1542
+ `${explicitDivergent} explicit divergent, scout ${scoutScanned} scanned / ` +
1543
+ `${scoutCorrectionShaped} correction-shaped / ${scoutCandidatesFound} candidate pair(s), ` +
1544
+ `${conflictFlagged} conflict-flagged, ${conflictSkippedJudged + merges.skipped_conflict} conflict-skipped ` +
1545
+ `(cursor ${cursor})`);
1114
1546
  }
1115
1547
  return {
1116
1548
  scanned,
@@ -1129,11 +1561,20 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
1129
1561
  explicit_verified: explicitVerified,
1130
1562
  explicit_divergent: explicitDivergent,
1131
1563
  cursor,
1564
+ pairs_reevaluated: pairsReevaluated,
1565
+ pairs_new: pairsNew,
1566
+ skipped_absorbed: skippedAbsorbed,
1567
+ merge_pairs_deferred: mergePairsDeferred,
1132
1568
  merges,
1133
1569
  merge_pairs_applied: mergePairsApplied,
1134
1570
  merge_below_gate: mergeBelowGate,
1135
1571
  skipped_above_ceiling: skippedAboveCeiling,
1136
1572
  skipped_metadata_mismatch: skippedMetadataMismatch,
1573
+ conflict_flagged: conflictFlagged,
1574
+ conflict_skipped: conflictSkippedJudged + merges.skipped_conflict,
1575
+ scout_scanned: scoutScanned,
1576
+ scout_correction_shaped: scoutCorrectionShaped,
1577
+ scout_candidates_found: scoutCandidatesFound,
1137
1578
  band_stats: bandStats,
1138
1579
  };
1139
1580
  }