@gamaze/hicortex 0.22.3 → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/assets/dashboard.html +430 -140
- package/dist/calibration.d.ts +26 -11
- package/dist/calibration.js +33 -13
- package/dist/capture.js +4 -1
- package/dist/cli.d.ts +3 -0
- package/dist/cli.js +26 -0
- package/dist/cluster.d.ts +8 -5
- package/dist/cluster.js +3 -7
- package/dist/consolidate.d.ts +4 -64
- package/dist/consolidate.js +15 -285
- package/dist/db.js +21 -0
- package/dist/dedup.d.ts +14 -9
- package/dist/dedup.js +33 -25
- package/dist/distiller.d.ts +18 -0
- package/dist/distiller.js +94 -9
- package/dist/eval/planted-harness.js +1 -1
- package/dist/eval/run-eval.js +2 -4
- package/dist/llm.d.ts +12 -1
- package/dist/llm.js +14 -3
- package/dist/mcp-stdio.d.ts +61 -3
- package/dist/mcp-stdio.js +272 -51
- package/dist/nightly.js +2 -4
- package/dist/recall-hook-cli.js +10 -0
- package/dist/recall-index.js +12 -0
- package/dist/reconsolidation.d.ts +17 -11
- package/dist/reconsolidation.js +38 -28
- package/dist/relink.d.ts +2 -2
- package/dist/relink.js +2 -2
- package/dist/retrieval.d.ts +5 -4
- package/dist/retrieval.js +5 -4
- package/dist/state.d.ts +8 -8
- package/dist/storage.d.ts +2 -2
- package/dist/storage.js +2 -2
- package/dist/sweep-volatile.d.ts +55 -0
- package/dist/sweep-volatile.js +184 -0
- package/dist/types.d.ts +26 -28
- package/dist/wrapper-prompt.d.ts +43 -0
- package/dist/wrapper-prompt.js +122 -0
- package/package.json +1 -1
- package/server.json +2 -2
package/dist/reconsolidation.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Reconsolidation (#384, #392) — the store resolves its own corrections, and
|
|
4
4
|
* THE unified resolution stage.
|
|
5
5
|
*
|
|
6
|
-
* Nightly consolidation stage (runs as Stage 3.8, after
|
|
6
|
+
* Nightly consolidation stage (runs as Stage 3.8, after links, before
|
|
7
7
|
* decay/prune) that detects memories which correct, retract, supersede, or
|
|
8
8
|
* DUPLICATE older ones; REWRITES corrected facts in place (absorbing
|
|
9
9
|
* transition-only trigger memories), MERGES confirmed duplicates via the
|
|
@@ -45,8 +45,9 @@
|
|
|
45
45
|
* planDedup and the judged mergeMemoryIds) refuse to blend a conflicts-linked
|
|
46
46
|
* pair, counted as conflict_skipped. The zone therefore runs AFTER the scan —
|
|
47
47
|
* with the zone first, a >=0.92 conflict pair was blended
|
|
48
|
-
* before the judge ever saw it (the planted-eval harm:
|
|
49
|
-
*
|
|
48
|
+
* before the judge ever saw it (the planted-eval harm: an unjudged blend —
|
|
49
|
+
* whichever row lost the canonical pick, its wording was simply erased);
|
|
50
|
+
* running it last means verdicts/marks/binds land first
|
|
50
51
|
* and the zone merges only what no verdict claimed — a conflicts bind set by
|
|
51
52
|
* this run's scan guards the SAME run's zone.
|
|
52
53
|
*
|
|
@@ -154,10 +155,14 @@ exports.RECONSOLIDATION_STAGE_LABEL = "reconsolidation";
|
|
|
154
155
|
/**
|
|
155
156
|
* Default minimum COSINE similarity for a correction candidate pair —
|
|
156
157
|
* RELEASE-MANAGED since #408 (calibration.ts CORRECTION_MIN_SIMILARITY;
|
|
157
|
-
* provenance there).
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
158
|
+
* provenance there). Deliberately wide: a retraction often rides inside an
|
|
159
|
+
* otherwise unrelated memory (the field failure that opened this issue), so
|
|
160
|
+
* the neighborhood gate must be a touch wider while the LLM verdict +
|
|
161
|
+
* confidence gate carry the precision load. Since #206-B (owner decision 6)
|
|
162
|
+
* this floor also subsumes the retired Stage 3.7 supersession scan's 0.80:
|
|
163
|
+
* 3.8 is the ONLY true-update detector, scanning every new memory — no
|
|
164
|
+
* shape gate, wider floor — with `corrects`/`supersedes` partitioning what
|
|
165
|
+
* 3.7's binary prompt called a supersession.
|
|
161
166
|
*/
|
|
162
167
|
exports.DEFAULT_CORRECTION_MIN_SIMILARITY = CALIBRATION.CORRECTION_MIN_SIMILARITY;
|
|
163
168
|
/**
|
|
@@ -167,11 +172,12 @@ exports.DEFAULT_CORRECTION_MIN_SIMILARITY = CALIBRATION.CORRECTION_MIN_SIMILARIT
|
|
|
167
172
|
* weak rewrite is corruption.
|
|
168
173
|
*/
|
|
169
174
|
exports.DEFAULT_CORRECTION_REWRITE_MIN_CONFIDENCE = CALIBRATION.CORRECTION_REWRITE_MIN_CONFIDENCE;
|
|
170
|
-
/** Neighbor pool size before older/similarity filtering narrows to top 5
|
|
175
|
+
/** Neighbor pool size before older/similarity filtering narrows to top 5
|
|
176
|
+
* (formerly the supersession stage's constants — 3.8 inherited the shape). */
|
|
171
177
|
const CORRECTION_NEIGHBOR_POOL = 15;
|
|
172
|
-
/** Older-neighbor pairs kept per candidate after filtering
|
|
178
|
+
/** Older-neighbor pairs kept per candidate after filtering. */
|
|
173
179
|
const CORRECTION_NEIGHBOR_TOP_K = 5;
|
|
174
|
-
/** Content truncation for prompts (classify-tier cost profile
|
|
180
|
+
/** Content truncation for prompts (classify-tier cost profile). */
|
|
175
181
|
const PROMPT_TRUNCATE_CHARS = 1500;
|
|
176
182
|
/** Head of the old content quoted in the provenance footer. */
|
|
177
183
|
exports.FOOTER_HEAD_MAX_CHARS = 160;
|
|
@@ -214,7 +220,7 @@ function nowIso() {
|
|
|
214
220
|
}
|
|
215
221
|
/**
|
|
216
222
|
* Build the constrained correction-shape prompt (classify-tier cost profile:
|
|
217
|
-
*
|
|
223
|
+
* 1500-char truncation, verdict-prompt precedent). The wording asks for
|
|
218
224
|
* the OLD claim's distinctive terms — the field-failure mechanism is that a
|
|
219
225
|
* correction CONTAINS the words of what it corrects, even when the surrounding
|
|
220
226
|
* topics (and therefore the embedding cosine) are unrelated. Guard-C extends
|
|
@@ -237,7 +243,8 @@ function buildScoutShapePrompt(content) {
|
|
|
237
243
|
/**
|
|
238
244
|
* Parse the scout shape reply. Null on unparseable JSON, a missing/non-boolean
|
|
239
245
|
* `correction`, or a missing/out-of-range `confidence` — the caller counts
|
|
240
|
-
* skipped_infra and moves on (
|
|
246
|
+
* skipped_infra and moves on (the retired supersession stage's parse
|
|
247
|
+
* discipline, kept: never
|
|
241
248
|
* mis-detect on ambiguity). `references` is lenient (missing/non-string → "")
|
|
242
249
|
* because an empty string simply yields no FTS hits — a harmless miss, not a
|
|
243
250
|
* mis-judgment.
|
|
@@ -264,7 +271,7 @@ function parseScoutShape(reply) {
|
|
|
264
271
|
const references = typeof obj.references === "string" ? obj.references : "";
|
|
265
272
|
return { correction: obj.correction, references: references.trim(), confidence };
|
|
266
273
|
}
|
|
267
|
-
/** Build the constrained pair-verdict prompt (1500-char truncation
|
|
274
|
+
/** Build the constrained pair-verdict prompt (1500-char truncation). */
|
|
268
275
|
function buildCorrectionVerdictPrompt(oldContent, newContent) {
|
|
269
276
|
const trunc = (s) => (s.length > PROMPT_TRUNCATE_CHARS ? `${s.slice(0, PROMPT_TRUNCATE_CHARS)}…` : s);
|
|
270
277
|
return (`You are checking how a NEWER memory relates to an OLDER one in an AI agent's long-term memory.\n\n` +
|
|
@@ -287,7 +294,7 @@ function buildCorrectionVerdictPrompt(oldContent, newContent) {
|
|
|
287
294
|
/**
|
|
288
295
|
* Parse the pair verdict. Null on anything unparseable, unknown action, or an
|
|
289
296
|
* out-of-range/missing confidence — the caller counts skipped_infra and moves
|
|
290
|
-
* on (same
|
|
297
|
+
* on (the same never-mis-judge-on-ambiguity discipline).
|
|
291
298
|
*/
|
|
292
299
|
function parseCorrectionVerdict(reply) {
|
|
293
300
|
if (!reply)
|
|
@@ -558,8 +565,8 @@ function accumulateBandStat(cumulative, run) {
|
|
|
558
565
|
cumulative.none += run.none;
|
|
559
566
|
cumulative.merge_below_gate += run.merge_below_gate;
|
|
560
567
|
cumulative.conf_sum += run.conf_sum;
|
|
561
|
-
if (run.
|
|
562
|
-
cumulative.
|
|
568
|
+
if (run.project_skipped !== undefined) {
|
|
569
|
+
cumulative.project_skipped = (cumulative.project_skipped ?? 0) + run.project_skipped;
|
|
563
570
|
}
|
|
564
571
|
}
|
|
565
572
|
/**
|
|
@@ -715,7 +722,9 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
715
722
|
// re-detect). Once the backlog drains, pairs_reevaluated reads 0.
|
|
716
723
|
const prevScannedRowid = (0, state_js_1.loadState)(stateDir).reconsolidationScannedRowid ?? startCursor;
|
|
717
724
|
let scannedRowidHighwater = startCursor;
|
|
718
|
-
// NO shape filter (AC2) —
|
|
725
|
+
// NO shape filter (AC2) — also why 3.8 subsumes the retired Stage 3.7
|
|
726
|
+
// supersession scan (its shape gate would miss exactly these pairs).
|
|
727
|
+
// Absorbed rows are
|
|
719
728
|
// excluded: they are invisible to recall and must not re-enter judgment.
|
|
720
729
|
const rows = db
|
|
721
730
|
.prepare(`SELECT rowid AS __rowid, * FROM memories
|
|
@@ -739,7 +748,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
739
748
|
let explicitDivergent = 0;
|
|
740
749
|
let mergeBelowGate = 0;
|
|
741
750
|
let skippedAboveCeiling = 0;
|
|
742
|
-
let
|
|
751
|
+
let skippedProjectMismatch = 0;
|
|
743
752
|
let mergePairsApplied = 0;
|
|
744
753
|
// #393 guard-C: conflicts verdicts rendered (link written, both live) and
|
|
745
754
|
// judged-path merge refusals on a conflicts-linked pair.
|
|
@@ -1228,10 +1237,10 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1228
1237
|
console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1229
1238
|
`into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
|
|
1230
1239
|
}
|
|
1231
|
-
else if (result.reason === "
|
|
1232
|
-
|
|
1240
|
+
else if (result.reason === "project_mismatch") {
|
|
1241
|
+
skippedProjectMismatch++;
|
|
1233
1242
|
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1234
|
-
`skipped (
|
|
1243
|
+
`skipped (project mismatch) — both kept`);
|
|
1235
1244
|
}
|
|
1236
1245
|
else if (result.reason === "conflict_linked") {
|
|
1237
1246
|
// #393 guard-C: the pair is conflicts-linked (operator-planted
|
|
@@ -1446,10 +1455,10 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1446
1455
|
console.log(`[hicortex] Reconsolidation: merged ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1447
1456
|
`into canonical ${result.canonicalId.slice(0, 8)} (${result.linksRepointed} link(s) re-pointed)`);
|
|
1448
1457
|
}
|
|
1449
|
-
else if (result.reason === "
|
|
1450
|
-
|
|
1458
|
+
else if (result.reason === "project_mismatch") {
|
|
1459
|
+
skippedProjectMismatch++;
|
|
1451
1460
|
console.log(`[hicortex] Reconsolidation: merge of ${pair.oldId.slice(0, 8)} + ${pair.newId.slice(0, 8)} ` +
|
|
1452
|
-
`skipped (
|
|
1461
|
+
`skipped (project mismatch) — both kept`);
|
|
1453
1462
|
}
|
|
1454
1463
|
else if (result.reason === "conflict_linked") {
|
|
1455
1464
|
conflictSkippedJudged++;
|
|
@@ -1473,7 +1482,8 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1473
1482
|
// deterministic sweep: verdicts, marks, and binds land first, and the zone
|
|
1474
1483
|
// merges only what no verdict claimed. With the zone first, a >=0.92
|
|
1475
1484
|
// genuine-conflict pair was blended before the judge ever saw it
|
|
1476
|
-
// (
|
|
1485
|
+
// (an unjudged blend — the canonical-pick loser's wording erased —
|
|
1486
|
+
// the planted-eval harm);
|
|
1477
1487
|
// running it last means a `conflicts` bind set by THIS run's scan guards
|
|
1478
1488
|
// the SAME run's zone. LLM-free and budget-free — an LLM-less night still
|
|
1479
1489
|
// drains duplicates (a deadline-deferred cluster re-detects next run at
|
|
@@ -1499,8 +1509,8 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1499
1509
|
det.pairs = merges.losers_merged;
|
|
1500
1510
|
det.merge = merges.losers_merged;
|
|
1501
1511
|
det.conf_sum = merges.losers_merged;
|
|
1502
|
-
if (merges.
|
|
1503
|
-
det.
|
|
1512
|
+
if (merges.skipped_project_mismatch > 0) {
|
|
1513
|
+
det.project_skipped = merges.skipped_project_mismatch;
|
|
1504
1514
|
}
|
|
1505
1515
|
bandStats[`>=${autoMergeThreshold}`] = det;
|
|
1506
1516
|
}
|
|
@@ -1569,7 +1579,7 @@ async function stageReconsolidation(db, llm, budget, embedFn, dryRun, stateDir,
|
|
|
1569
1579
|
merge_pairs_applied: mergePairsApplied,
|
|
1570
1580
|
merge_below_gate: mergeBelowGate,
|
|
1571
1581
|
skipped_above_ceiling: skippedAboveCeiling,
|
|
1572
|
-
|
|
1582
|
+
skipped_project_mismatch: skippedProjectMismatch,
|
|
1573
1583
|
conflict_flagged: conflictFlagged,
|
|
1574
1584
|
conflict_skipped: conflictSkippedJudged + merges.skipped_conflict,
|
|
1575
1585
|
scout_scanned: scoutScanned,
|
package/dist/relink.d.ts
CHANGED
|
@@ -86,8 +86,8 @@ export interface RelinkReport {
|
|
|
86
86
|
/**
|
|
87
87
|
* Read the stored embedding for a memory from memory_vectors.
|
|
88
88
|
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
89
|
-
* @deprecated moved to storage.ts (shared with
|
|
90
|
-
*
|
|
89
|
+
* @deprecated moved to storage.ts (shared with the resolution stages);
|
|
90
|
+
* re-exported here so existing importers of relink.ts keep working.
|
|
91
91
|
*/
|
|
92
92
|
export { getStoredEmbedding } from "./storage.js";
|
|
93
93
|
/**
|
package/dist/relink.js
CHANGED
|
@@ -107,8 +107,8 @@ function loadExistingPairs(db) {
|
|
|
107
107
|
/**
|
|
108
108
|
* Read the stored embedding for a memory from memory_vectors.
|
|
109
109
|
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
110
|
-
* @deprecated moved to storage.ts (shared with
|
|
111
|
-
*
|
|
110
|
+
* @deprecated moved to storage.ts (shared with the resolution stages);
|
|
111
|
+
* re-exported here so existing importers of relink.ts keep working.
|
|
112
112
|
*/
|
|
113
113
|
var storage_js_1 = require("./storage.js");
|
|
114
114
|
Object.defineProperty(exports, "getStoredEmbedding", { enumerable: true, get: function () { return storage_js_1.getStoredEmbedding; } });
|
package/dist/retrieval.d.ts
CHANGED
|
@@ -155,14 +155,15 @@ export declare function recallQueryVector(registry: CentroidStore, sessionId: st
|
|
|
155
155
|
}): Float32Array;
|
|
156
156
|
/**
|
|
157
157
|
* Ids among `candidateIds` that have been superseded by a later memory — i.e.
|
|
158
|
-
* they are the SOURCE of a `superseded_by` link (
|
|
159
|
-
* old → new
|
|
158
|
+
* they are the SOURCE of a `superseded_by` link (the resolution pass links
|
|
159
|
+
* old → new; formerly also the retired Stage 3.7 supersession scan). One
|
|
160
|
+
* query, not per-candidate.
|
|
160
161
|
*/
|
|
161
162
|
export declare function findSupersededIds(db: Database.Database, candidateIds: string[]): Set<string>;
|
|
162
163
|
/**
|
|
163
164
|
* The full ranking-demotion set among `candidateIds` (#384): the UNION of
|
|
164
|
-
* (a) sources of a `superseded_by` link (legacy +
|
|
165
|
-
* driven, works on pre-v14 rows with NULL status) and (b) rows whose
|
|
165
|
+
* (a) sources of a `superseded_by` link (legacy + retired-3.7 + resolution
|
|
166
|
+
* verdicts — link driven, works on pre-v14 rows with NULL status) and (b) rows whose
|
|
166
167
|
* `memories.status` is 'superseded' or 'retracted' (reconsolidation marks +
|
|
167
168
|
* explicit ingest marks). `corrected` is deliberately NOT demoting — a
|
|
168
169
|
* rewritten memory carries the CORRECTION, and demoting it would bury the
|
package/dist/retrieval.js
CHANGED
|
@@ -275,8 +275,9 @@ function recallQueryVector(registry, sessionId, promptEmb, opts) {
|
|
|
275
275
|
}
|
|
276
276
|
/**
|
|
277
277
|
* Ids among `candidateIds` that have been superseded by a later memory — i.e.
|
|
278
|
-
* they are the SOURCE of a `superseded_by` link (
|
|
279
|
-
* old → new
|
|
278
|
+
* they are the SOURCE of a `superseded_by` link (the resolution pass links
|
|
279
|
+
* old → new; formerly also the retired Stage 3.7 supersession scan). One
|
|
280
|
+
* query, not per-candidate.
|
|
280
281
|
*/
|
|
281
282
|
function findSupersededIds(db, candidateIds) {
|
|
282
283
|
if (candidateIds.length === 0)
|
|
@@ -290,8 +291,8 @@ function findSupersededIds(db, candidateIds) {
|
|
|
290
291
|
}
|
|
291
292
|
/**
|
|
292
293
|
* The full ranking-demotion set among `candidateIds` (#384): the UNION of
|
|
293
|
-
* (a) sources of a `superseded_by` link (legacy +
|
|
294
|
-
* driven, works on pre-v14 rows with NULL status) and (b) rows whose
|
|
294
|
+
* (a) sources of a `superseded_by` link (legacy + retired-3.7 + resolution
|
|
295
|
+
* verdicts — link driven, works on pre-v14 rows with NULL status) and (b) rows whose
|
|
295
296
|
* `memories.status` is 'superseded' or 'retracted' (reconsolidation marks +
|
|
296
297
|
* explicit ingest marks). `corrected` is deliberately NOT demoting — a
|
|
297
298
|
* rewritten memory carries the CORRECTION, and demoting it would bury the
|
package/dist/state.d.ts
CHANGED
|
@@ -51,19 +51,19 @@ export interface HicortexState {
|
|
|
51
51
|
*/
|
|
52
52
|
domainCursor?: number;
|
|
53
53
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
54
|
+
* #206-B: the retired supersession stage's cursor key
|
|
55
|
+
* (`supersessionCursor`) is deliberately NOT modeled here anymore. The
|
|
56
|
+
* stage (3.7, #191 Phase B) is retired into the reconsolidation pass, the
|
|
57
|
+
* whole-corpus backfill was complete before retirement, and nothing reads
|
|
58
|
+
* the key — pre-#206-B installs keep their stale value on disk, left to
|
|
59
|
+
* rot unread (no migration, no deletion). Do not repurpose the name.
|
|
60
60
|
*/
|
|
61
|
-
supersessionCursor?: number;
|
|
62
61
|
/**
|
|
63
62
|
* Resume cursor for the nightly's reconsolidation stage (#384) — highest
|
|
64
63
|
* memories.rowid whose candidates have been evaluated (or infra-skipped)
|
|
65
64
|
* with all their CONFIRMED work applied. Absent/0 = never run. Same
|
|
66
|
-
* advance-past-considered-candidates discipline
|
|
65
|
+
* advance-past-considered-candidates discipline (formerly the retired
|
|
66
|
+
* supersessionCursor's),
|
|
67
67
|
* with one addition (#439): confirmed merges and rewrite groups apply at
|
|
68
68
|
* the candidate boundary — the END of the iteration that confirmed them —
|
|
69
69
|
* and the cursor advances past a candidate only when that apply landed.
|
package/dist/storage.d.ts
CHANGED
|
@@ -149,8 +149,8 @@ export declare function getMemoryTagsWeightedBatched(db: Database.Database, memo
|
|
|
149
149
|
* Read the stored embedding for a memory from memory_vectors.
|
|
150
150
|
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
151
151
|
*
|
|
152
|
-
* Shared by `hicortex relink` and the nightly's
|
|
153
|
-
* (consolidate.ts) — lives here (not in relink.ts) so
|
|
152
|
+
* Shared by `hicortex relink` and the nightly's resolution stages
|
|
153
|
+
* (reconsolidation.ts, consolidate.ts) — lives here (not in relink.ts) so they can use
|
|
154
154
|
* it without importing from relink.ts, which itself imports from
|
|
155
155
|
* consolidate.ts (BudgetTracker, discoverLinkCandidates).
|
|
156
156
|
*/
|
package/dist/storage.js
CHANGED
|
@@ -408,8 +408,8 @@ function getMemoryTagsWeightedBatched(db, memoryIds) {
|
|
|
408
408
|
* Read the stored embedding for a memory from memory_vectors.
|
|
409
409
|
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
410
410
|
*
|
|
411
|
-
* Shared by `hicortex relink` and the nightly's
|
|
412
|
-
* (consolidate.ts) — lives here (not in relink.ts) so
|
|
411
|
+
* Shared by `hicortex relink` and the nightly's resolution stages
|
|
412
|
+
* (reconsolidation.ts, consolidate.ts) — lives here (not in relink.ts) so they can use
|
|
413
413
|
* it without importing from relink.ts, which itself imports from
|
|
414
414
|
* consolidate.ts (BudgetTracker, discoverLinkCandidates).
|
|
415
415
|
*/
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `hicortex sweep-volatile` (#489, deliverable iii — owner decision 3).
|
|
3
|
+
*
|
|
4
|
+
* The ONE-SHOT store sweep: the live corpus already holds the GH-status /
|
|
5
|
+
* version-bump rows gold set A adjudicated ("not healthy to have GH in mem").
|
|
6
|
+
* This command reuses the SAME deterministic gate the distill path applies
|
|
7
|
+
* (distiller.ts isVolatileStatusEntry — one gate, one meaning) over the stored
|
|
8
|
+
* corpus. Mirrors `dedup`:
|
|
9
|
+
*
|
|
10
|
+
* - DRY RUN by default: report candidates only, zero writes.
|
|
11
|
+
* - `--apply` is explicit and ordered capture-lock (fail fast) →
|
|
12
|
+
* pre-sweep backup (abort ALL writes if it fails) → one transaction.
|
|
13
|
+
* - NO hard deletes: swept rows are DEMOTED via storage.absorbMemory —
|
|
14
|
+
* status 'absorbed' (recall-invisible everywhere: vector + FTS rows
|
|
15
|
+
* dropped, and every read path already handles the state), plain row +
|
|
16
|
+
* links retained as evidence. Recovery posture = the dedup posture: the
|
|
17
|
+
* pre-sweep backup IS the rollback. The shared absorb primitive means no
|
|
18
|
+
* new status vocabulary to thread through candidate paths.
|
|
19
|
+
* - Tags cleared + domain NULL'd before absorb (dedup's rule: an absorbed
|
|
20
|
+
* row must not count in moduleIndex/tag recomputes).
|
|
21
|
+
* - Audit trail: one `volatile_sweep_log` row per swept memory (migration
|
|
22
|
+
* v23) — inspectable forever, never silent. No runtime consumer; the
|
|
23
|
+
* CAPTURE-side volatility gate is the re-ingest safety net.
|
|
24
|
+
*
|
|
25
|
+
* Server-mode only (needs the local DB), like dedup/relink/classify-domains.
|
|
26
|
+
*/
|
|
27
|
+
import { acquireCaptureLock } from "./capture.js";
|
|
28
|
+
export interface SweepVolatileOptions {
|
|
29
|
+
/** Execute the sweep. Default false = dry run (report only, zero writes). */
|
|
30
|
+
apply?: boolean;
|
|
31
|
+
/** DB path override (tests / manual snapshot verification). */
|
|
32
|
+
dbPath?: string;
|
|
33
|
+
/** State dir override (tests). Defaults to ~/.hicortex. Backup lands under
|
|
34
|
+
* here/backups/. */
|
|
35
|
+
stateDir?: string;
|
|
36
|
+
/** Config override (tests). Defaults to reading stateDir/config.json. */
|
|
37
|
+
config?: Record<string, unknown> | null;
|
|
38
|
+
/** Capture-lock acquirer override (tests). Defaults to capture.ts's lock. */
|
|
39
|
+
acquireLock?: typeof acquireCaptureLock;
|
|
40
|
+
}
|
|
41
|
+
export interface SweepVolatileReport {
|
|
42
|
+
dryRun: boolean;
|
|
43
|
+
/** Live rows the gate flags, in store order. Empty when the kill-switch is
|
|
44
|
+
* off (the sweep is inert without the gate — same release-managed switch,
|
|
45
|
+
* one meaning). */
|
|
46
|
+
candidates: Array<{
|
|
47
|
+
id: string;
|
|
48
|
+
preview: string;
|
|
49
|
+
}>;
|
|
50
|
+
/** --apply only: rows actually swept (absorbed). */
|
|
51
|
+
swept?: number;
|
|
52
|
+
/** --apply only: path to the pre-sweep backup. */
|
|
53
|
+
backupPath?: string;
|
|
54
|
+
}
|
|
55
|
+
export declare function runSweepVolatile(options?: SweepVolatileOptions): Promise<SweepVolatileReport>;
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `hicortex sweep-volatile` (#489, deliverable iii — owner decision 3).
|
|
4
|
+
*
|
|
5
|
+
* The ONE-SHOT store sweep: the live corpus already holds the GH-status /
|
|
6
|
+
* version-bump rows gold set A adjudicated ("not healthy to have GH in mem").
|
|
7
|
+
* This command reuses the SAME deterministic gate the distill path applies
|
|
8
|
+
* (distiller.ts isVolatileStatusEntry — one gate, one meaning) over the stored
|
|
9
|
+
* corpus. Mirrors `dedup`:
|
|
10
|
+
*
|
|
11
|
+
* - DRY RUN by default: report candidates only, zero writes.
|
|
12
|
+
* - `--apply` is explicit and ordered capture-lock (fail fast) →
|
|
13
|
+
* pre-sweep backup (abort ALL writes if it fails) → one transaction.
|
|
14
|
+
* - NO hard deletes: swept rows are DEMOTED via storage.absorbMemory —
|
|
15
|
+
* status 'absorbed' (recall-invisible everywhere: vector + FTS rows
|
|
16
|
+
* dropped, and every read path already handles the state), plain row +
|
|
17
|
+
* links retained as evidence. Recovery posture = the dedup posture: the
|
|
18
|
+
* pre-sweep backup IS the rollback. The shared absorb primitive means no
|
|
19
|
+
* new status vocabulary to thread through candidate paths.
|
|
20
|
+
* - Tags cleared + domain NULL'd before absorb (dedup's rule: an absorbed
|
|
21
|
+
* row must not count in moduleIndex/tag recomputes).
|
|
22
|
+
* - Audit trail: one `volatile_sweep_log` row per swept memory (migration
|
|
23
|
+
* v23) — inspectable forever, never silent. No runtime consumer; the
|
|
24
|
+
* CAPTURE-side volatility gate is the re-ingest safety net.
|
|
25
|
+
*
|
|
26
|
+
* Server-mode only (needs the local DB), like dedup/relink/classify-domains.
|
|
27
|
+
*/
|
|
28
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
29
|
+
if (k2 === undefined) k2 = k;
|
|
30
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
31
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
32
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
33
|
+
}
|
|
34
|
+
Object.defineProperty(o, k2, desc);
|
|
35
|
+
}) : (function(o, m, k, k2) {
|
|
36
|
+
if (k2 === undefined) k2 = k;
|
|
37
|
+
o[k2] = m[k];
|
|
38
|
+
}));
|
|
39
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
40
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
41
|
+
}) : function(o, v) {
|
|
42
|
+
o["default"] = v;
|
|
43
|
+
});
|
|
44
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
45
|
+
var ownKeys = function(o) {
|
|
46
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
47
|
+
var ar = [];
|
|
48
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
49
|
+
return ar;
|
|
50
|
+
};
|
|
51
|
+
return ownKeys(o);
|
|
52
|
+
};
|
|
53
|
+
return function (mod) {
|
|
54
|
+
if (mod && mod.__esModule) return mod;
|
|
55
|
+
var result = {};
|
|
56
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
57
|
+
__setModuleDefault(result, mod);
|
|
58
|
+
return result;
|
|
59
|
+
};
|
|
60
|
+
})();
|
|
61
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
62
|
+
exports.runSweepVolatile = runSweepVolatile;
|
|
63
|
+
const paths_js_1 = require("./paths.js");
|
|
64
|
+
const node_fs_1 = require("node:fs");
|
|
65
|
+
const node_path_1 = require("node:path");
|
|
66
|
+
const db_js_1 = require("./db.js");
|
|
67
|
+
const storage = __importStar(require("./storage.js"));
|
|
68
|
+
const distiller_js_1 = require("./distiller.js");
|
|
69
|
+
const capture_js_1 = require("./capture.js");
|
|
70
|
+
const config_read_js_1 = require("./config-read.js");
|
|
71
|
+
const calibration_js_1 = require("./calibration.js");
|
|
72
|
+
const backup_js_1 = require("./backup.js");
|
|
73
|
+
const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
|
|
74
|
+
/** Pre-sweep backup filename pattern — scoped retention (like pre-dedup). */
|
|
75
|
+
const PRE_SWEEP_BACKUP_PATTERN = /^pre-sweep-volatile-.*\.db$/;
|
|
76
|
+
function readConfig(stateDir) {
|
|
77
|
+
try {
|
|
78
|
+
return JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(stateDir, "config.json"), "utf-8"));
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/** Live-row filter: active or content-rewritten rows only. Already-retired
|
|
85
|
+
* states are not candidates — 'absorbed' is invisible already; 'superseded'
|
|
86
|
+
* / 'retracted' are demoted already. */
|
|
87
|
+
const LIVE_ROW_WHERE = `COALESCE(status, '') NOT IN ('absorbed', 'superseded', 'retracted')`;
|
|
88
|
+
/**
|
|
89
|
+
* Pre-sweep DB backup to <stateDir>/backups/pre-sweep-volatile-<ISO>.db,
|
|
90
|
+
* pruned to `backupRetention` newest (pattern-scoped, like pre-dedup).
|
|
91
|
+
* THROWS on failure — runSweepVolatile aborts the whole sweep when it does.
|
|
92
|
+
*/
|
|
93
|
+
async function takePreSweepBackup(db, stateDir, config) {
|
|
94
|
+
const backupDir = (0, node_path_1.join)(stateDir, "backups");
|
|
95
|
+
(0, node_fs_1.mkdirSync)(backupDir, { recursive: true });
|
|
96
|
+
const backupPath = (0, node_path_1.join)(backupDir, `pre-sweep-volatile-${new Date().toISOString().replace(/[:.]/g, "-")}.db`);
|
|
97
|
+
await db.backup(backupPath);
|
|
98
|
+
const retention = (0, config_read_js_1.readNonNegativeConfig)(config ?? {}, "backupRetention", backup_js_1.DEFAULT_BACKUP_RETENTION);
|
|
99
|
+
(0, backup_js_1.pruneBackupArtifacts)(backupDir, retention, PRE_SWEEP_BACKUP_PATTERN);
|
|
100
|
+
return backupPath;
|
|
101
|
+
}
|
|
102
|
+
async function runSweepVolatile(options = {}) {
|
|
103
|
+
const stateDir = options.stateDir ?? HICORTEX_HOME;
|
|
104
|
+
const config = options.config !== undefined ? options.config : readConfig(stateDir);
|
|
105
|
+
// Server-mode only — client installs have no local DB.
|
|
106
|
+
if (config?.mode === "client") {
|
|
107
|
+
throw new Error("[hicortex] sweep-volatile is server-mode only (it needs the local DB). " +
|
|
108
|
+
`This machine is a client of ${config.serverUrl ?? "a remote server"} — run sweep-volatile on the server.`);
|
|
109
|
+
}
|
|
110
|
+
const apply = options.apply ?? false;
|
|
111
|
+
const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
|
|
112
|
+
const db = (0, db_js_1.initDb)(dbPath);
|
|
113
|
+
try {
|
|
114
|
+
console.log(`[hicortex] sweep-volatile starting (${apply ? "APPLY" : "dry-run"}): db ${dbPath}`);
|
|
115
|
+
const rows = db
|
|
116
|
+
.prepare(`SELECT id, content FROM memories WHERE ${LIVE_ROW_WHERE}`)
|
|
117
|
+
.all();
|
|
118
|
+
const candidates = calibration_js_1.VOLATILE_STATUS_FILTER
|
|
119
|
+
? rows
|
|
120
|
+
.filter((r) => (0, distiller_js_1.isVolatileStatusEntry)(r.content))
|
|
121
|
+
.map((r) => ({ id: r.id, preview: r.content.slice(0, 120) }))
|
|
122
|
+
: [];
|
|
123
|
+
if (!calibration_js_1.VOLATILE_STATUS_FILTER) {
|
|
124
|
+
console.warn("[hicortex] sweep-volatile: the volatility gate is switched off in this release (VOLATILE_STATUS_FILTER) — nothing to sweep.");
|
|
125
|
+
}
|
|
126
|
+
const report = { dryRun: !apply, candidates };
|
|
127
|
+
console.log(`[hicortex] sweep-volatile: ${candidates.length} volatile row(s) among ${rows.length} live (` +
|
|
128
|
+
`${apply ? "demoting via absorb" : "dry run — zero writes"})`);
|
|
129
|
+
if (!apply) {
|
|
130
|
+
for (const c of candidates) {
|
|
131
|
+
console.log(`[hicortex] ${c.id.slice(0, 8)}: "${c.preview}"`);
|
|
132
|
+
}
|
|
133
|
+
if (candidates.length === 0)
|
|
134
|
+
console.log("[hicortex] (nothing matched the gate)");
|
|
135
|
+
return report;
|
|
136
|
+
}
|
|
137
|
+
if (candidates.length === 0)
|
|
138
|
+
return report;
|
|
139
|
+
// --apply: fail fast on a busy capture lock — the audit rows and absorb
|
|
140
|
+
// writes must not race a nightly/capture run (dedup posture).
|
|
141
|
+
const acquireLock = options.acquireLock ?? capture_js_1.acquireCaptureLock;
|
|
142
|
+
const releaseLock = await acquireLock(stateDir, 0);
|
|
143
|
+
if (!releaseLock) {
|
|
144
|
+
throw new Error("[hicortex] sweep-volatile --apply aborted: another capture/nightly run holds the lock. Retry when it finishes.");
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
// Backup FIRST — abort entirely (no writes attempted) if it fails.
|
|
148
|
+
let backupPath;
|
|
149
|
+
try {
|
|
150
|
+
backupPath = await takePreSweepBackup(db, stateDir, config);
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
throw new Error(`[hicortex] sweep-volatile --apply aborted: backup failed (${err instanceof Error ? err.message : String(err)}). No rows swept.`);
|
|
154
|
+
}
|
|
155
|
+
console.log(`[hicortex] Backup written: ${backupPath}`);
|
|
156
|
+
report.backupPath = backupPath;
|
|
157
|
+
// ONE transaction: audit row + tag/domain clear + absorb per row. Any
|
|
158
|
+
// failure rolls back the whole sweep (all-or-nothing, like /distill's
|
|
159
|
+
// insert phase) — a partial sweep is never left behind.
|
|
160
|
+
const sweep = db.transaction(() => {
|
|
161
|
+
const clearTags = db.prepare("DELETE FROM memory_tags WHERE memory_id = ?");
|
|
162
|
+
const audit = db.prepare("INSERT OR REPLACE INTO volatile_sweep_log (memory_id, swept_at, preview) VALUES (?, ?, ?)");
|
|
163
|
+
const now = new Date().toISOString();
|
|
164
|
+
for (const c of candidates) {
|
|
165
|
+
audit.run(c.id, now, c.preview);
|
|
166
|
+
clearTags.run(c.id);
|
|
167
|
+
storage.updateMemory(db, c.id, { domain: null });
|
|
168
|
+
storage.absorbMemory(db, c.id);
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
sweep();
|
|
172
|
+
report.swept = candidates.length;
|
|
173
|
+
console.log(`[hicortex] sweep-volatile: swept (absorbed) ${report.swept} row(s); ` +
|
|
174
|
+
`rollback = restore ${backupPath}`);
|
|
175
|
+
return report;
|
|
176
|
+
}
|
|
177
|
+
finally {
|
|
178
|
+
releaseLock();
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
finally {
|
|
182
|
+
db.close();
|
|
183
|
+
}
|
|
184
|
+
}
|
package/dist/types.d.ts
CHANGED
|
@@ -37,9 +37,13 @@ export interface Memory {
|
|
|
37
37
|
* never config: NULL/absent = active (the default, and every pre-v14 row);
|
|
38
38
|
* 'superseded'/'retracted' = marked stale or wrong (demoted in ranking);
|
|
39
39
|
* 'corrected' = rewritten in place (does NOT demote — demoting it would
|
|
40
|
-
* bury the correction); 'absorbed' = invisible to recall (
|
|
41
|
-
*
|
|
42
|
-
*
|
|
40
|
+
* bury the correction); 'absorbed' = invisible to recall (no vector/FTS
|
|
41
|
+
* row, plain row + links kept as evidence and rollback reference). Writers
|
|
42
|
+
* of 'absorbed': the reconsolidation rewrite path (trigger folded into a
|
|
43
|
+
* corrected target), dedup merge losers (#392), and the one-shot
|
|
44
|
+
* `sweep-volatile` demotion (#489 — volatile rows retire through the SAME
|
|
45
|
+
* primitive, so no new status vocabulary exists to thread through read
|
|
46
|
+
* paths; recovery is the pre-sweep backup).
|
|
43
47
|
*/
|
|
44
48
|
status?: string | null;
|
|
45
49
|
/**
|
|
@@ -128,8 +132,13 @@ export interface ResolutionBandStat {
|
|
|
128
132
|
merge_below_gate: number;
|
|
129
133
|
/** Sum of verdict confidences (divide by `pairs` for the mean). Deterministic merges count 1.0 each. */
|
|
130
134
|
conf_sum: number;
|
|
131
|
-
/**
|
|
132
|
-
|
|
135
|
+
/**
|
|
136
|
+
* Deterministic band only: clusters refused by the project rail. #206
|
|
137
|
+
* decision 2 renamed this from `metadata_skipped` when the source_agent
|
|
138
|
+
* rail was removed — pre-rename cumulative values stay on disk unread
|
|
139
|
+
* (their semantics conflated both rails).
|
|
140
|
+
*/
|
|
141
|
+
project_skipped?: number;
|
|
133
142
|
}
|
|
134
143
|
/**
|
|
135
144
|
* Report of the deterministic merge zone (#392) — the band at/above the merge
|
|
@@ -151,8 +160,8 @@ export interface DeterministicMergeZoneReport {
|
|
|
151
160
|
losers_merged: number;
|
|
152
161
|
/** Loser links re-pointed onto canonicals this run. */
|
|
153
162
|
links_repointed: number;
|
|
154
|
-
/** Clusters skipped — members disagree on project
|
|
155
|
-
|
|
163
|
+
/** Clusters skipped — members disagree on project (#206 decision 2: the source_agent rail is removed). */
|
|
164
|
+
skipped_project_mismatch: number;
|
|
156
165
|
/**
|
|
157
166
|
* #393 guard-C: clusters skipped because a member pair holds a `conflicts`
|
|
158
167
|
* link — a judge-flagged genuine conflict is never blended, both records
|
|
@@ -251,27 +260,15 @@ export interface ConsolidationReport {
|
|
|
251
260
|
heuristic_fallback?: number;
|
|
252
261
|
failed: number;
|
|
253
262
|
};
|
|
254
|
-
/** Supersession detection (#191 Phase B) — runs after linking, before decay/prune. */
|
|
255
|
-
supersession?: {
|
|
256
|
-
/** Decision/correction-shaped candidates examined this run. */
|
|
257
|
-
scanned: number;
|
|
258
|
-
/** Older-neighbor pairs actually sent to the classify-tier LLM. */
|
|
259
|
-
evaluated: number;
|
|
260
|
-
/** Pairs the LLM judged superseded — a `superseded_by` link was created. */
|
|
261
|
-
superseded: number;
|
|
262
|
-
/** Pairs skipped on a parse/infra error (retried naturally next night). */
|
|
263
|
-
skipped_infra: number;
|
|
264
|
-
/** Pairs skipped because a superseded_by link already existed (either direction). */
|
|
265
|
-
skipped_idempotent: number;
|
|
266
|
-
/** supersessionCursor after this run (unchanged in dry-run). */
|
|
267
|
-
cursor: number;
|
|
268
|
-
};
|
|
269
263
|
/**
|
|
270
|
-
* Reconsolidation (#384) — runs after
|
|
264
|
+
* Reconsolidation (#384) — runs after linking, before decay/prune.
|
|
271
265
|
* Since #392 this is THE unified resolution stage: its verdict also carries
|
|
272
266
|
* a `merge` disposition, and the deterministic merge zone (pairs at/above
|
|
273
267
|
* the merge ceiling — release-managed since #408) runs inside it,
|
|
274
|
-
* LLM-free, before the scan.
|
|
268
|
+
* LLM-free, before the scan. Since #206-B (owner decision 6) it is also
|
|
269
|
+
* the ONLY true-update detector — the standalone supersession stage
|
|
270
|
+
* (3.7, #191 Phase B) is retired into its `supersedes` verdict action,
|
|
271
|
+
* and its former `supersession` report slot no longer exists.
|
|
275
272
|
*/
|
|
276
273
|
reconsolidation?: {
|
|
277
274
|
/** Candidates examined this run (rowid > cursor; no shape filter). */
|
|
@@ -358,11 +355,12 @@ export interface ConsolidationReport {
|
|
|
358
355
|
*/
|
|
359
356
|
skipped_above_ceiling: number;
|
|
360
357
|
/**
|
|
361
|
-
* #392: judged merge pairs refused by the
|
|
362
|
-
*
|
|
363
|
-
* the verdict was rendered, this
|
|
358
|
+
* #392: judged merge pairs refused by the project rail (project
|
|
359
|
+
* disagreement — the only metadata rail, #206 decision 2). Both
|
|
360
|
+
* memories kept; the cursor advances — the verdict was rendered, this
|
|
361
|
+
* is not an infra failure.
|
|
364
362
|
*/
|
|
365
|
-
|
|
363
|
+
skipped_project_mismatch: number;
|
|
366
364
|
/**
|
|
367
365
|
* #393 guard-C: verdicts that flagged a genuine conflict — a `conflicts`
|
|
368
366
|
* link was written, both memories stay live (no status change, no
|