@gamaze/hicortex 0.22.3 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/calibration.d.ts +26 -11
- package/dist/calibration.js +33 -13
- package/dist/capture.js +4 -1
- package/dist/cli.d.ts +3 -0
- package/dist/cli.js +26 -0
- package/dist/cluster.d.ts +8 -5
- package/dist/cluster.js +3 -7
- package/dist/consolidate.d.ts +4 -64
- package/dist/consolidate.js +15 -285
- package/dist/db.js +21 -0
- package/dist/dedup.d.ts +14 -9
- package/dist/dedup.js +33 -25
- package/dist/distiller.d.ts +18 -0
- package/dist/distiller.js +94 -9
- package/dist/eval/planted-harness.js +1 -1
- package/dist/eval/run-eval.js +2 -4
- package/dist/nightly.js +2 -4
- package/dist/recall-hook-cli.js +10 -0
- package/dist/recall-index.js +12 -0
- package/dist/reconsolidation.d.ts +17 -11
- package/dist/reconsolidation.js +38 -28
- package/dist/relink.d.ts +2 -2
- package/dist/relink.js +2 -2
- package/dist/retrieval.d.ts +5 -4
- package/dist/retrieval.js +5 -4
- package/dist/state.d.ts +8 -8
- package/dist/storage.d.ts +2 -2
- package/dist/storage.js +2 -2
- package/dist/sweep-volatile.d.ts +55 -0
- package/dist/sweep-volatile.js +184 -0
- package/dist/types.d.ts +26 -28
- package/dist/wrapper-prompt.d.ts +43 -0
- package/dist/wrapper-prompt.js +122 -0
- package/package.json +1 -1
- package/server.json +2 -2
package/dist/consolidate.js
CHANGED
|
@@ -38,7 +38,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
38
38
|
};
|
|
39
39
|
})();
|
|
40
40
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
41
|
-
exports.DEFAULT_MEMORY_SOFT_CAP = exports.
|
|
41
|
+
exports.DEFAULT_MEMORY_SOFT_CAP = exports.BudgetTracker = exports.REFLECTION_CONTRADICTION_MIN_COSINE = exports.l2ToCosine = exports.CROSS_PROJECT_LINK_THRESHOLD = exports.CONSOLIDATE_LINK_TOP_K = exports.CONSOLIDATE_LINK_THRESHOLD = exports.DEFAULT_NIGHTLY_LLM_CALL_BUDGET = void 0;
|
|
42
42
|
exports.resolveNightlyLlmCallBudget = resolveNightlyLlmCallBudget;
|
|
43
43
|
exports.isContradictionCandidate = isContradictionCandidate;
|
|
44
44
|
exports.warnUnmeteredTokensRun = warnUnmeteredTokensRun;
|
|
@@ -50,9 +50,6 @@ exports.rebuildContentModuleIndex = rebuildContentModuleIndex;
|
|
|
50
50
|
exports.discoverLinkCandidates = discoverLinkCandidates;
|
|
51
51
|
exports.classifyLinkCandidates = classifyLinkCandidates;
|
|
52
52
|
exports.classifyRelationship = classifyRelationship;
|
|
53
|
-
exports.buildSupersessionPrompt = buildSupersessionPrompt;
|
|
54
|
-
exports.parseSupersessionReply = parseSupersessionReply;
|
|
55
|
-
exports.stageSupersession = stageSupersession;
|
|
56
53
|
exports.stageDecayPrune = stageDecayPrune;
|
|
57
54
|
exports.applyStrengthPromotion = applyStrengthPromotion;
|
|
58
55
|
exports.stagePromotion = stagePromotion;
|
|
@@ -1031,271 +1028,6 @@ function classifyRelationship(source, target, similarity) {
|
|
|
1031
1028
|
return "relates_to";
|
|
1032
1029
|
}
|
|
1033
1030
|
// ---------------------------------------------------------------------------
|
|
1034
|
-
// Stage 3.7: Supersession Detection (#191 Phase B)
|
|
1035
|
-
// ---------------------------------------------------------------------------
|
|
1036
|
-
//
|
|
1037
|
-
// A later decision/correction can reverse, replace, or invalidate an earlier
|
|
1038
|
-
// one — e.g. "chose Ollama for distillation" superseded a month later by
|
|
1039
|
-
// "switched distillation to a local 35B model over a mesh VPN". Left
|
|
1040
|
-
// unlinked, retrieval and lesson selection can surface the stale one. This
|
|
1041
|
-
// stage links OLD → NEW with relationship `superseded_by` and accelerates the
|
|
1042
|
-
// old memory's decay, WITHOUT deleting it (unlike `hicortex dedup`'s merge —
|
|
1043
|
-
// this is a judgment call about content, not a duplicate).
|
|
1044
|
-
//
|
|
1045
|
-
// Scope: memories with `rowid > supersessionCursor` (state.json; starts 0 —
|
|
1046
|
-
// the corpus is back-processed gradually) whose shape suggests a
|
|
1047
|
-
// decision/correction. For each, KNN top-5 OLDER same-shape neighbors
|
|
1048
|
-
// at/above the release-managed similarity floor (calibration.ts); one constrained classify-tier LLM
|
|
1049
|
-
// call per pair decides `superseded: true| false`. A parse/infra error skips
|
|
1050
|
-
// just that PAIR (retried naturally next night since the cursor still
|
|
1051
|
-
// advances past the memory — see the cursor note below); it never mis-links.
|
|
1052
|
-
// #405: no per-stage call cap — the ONE run budget (nightlyLlmCallBudget)
|
|
1053
|
-
// and the run deadline are the only bounds, like every other stage.
|
|
1054
|
-
/** Default minimum COSINE similarity for a supersession candidate pair —
|
|
1055
|
-
* RELEASE-MANAGED since #408 (calibration.ts SUPERSESSION_MIN_SIMILARITY). */
|
|
1056
|
-
exports.DEFAULT_SUPERSESSION_MIN_SIMILARITY = CALIBRATION.SUPERSESSION_MIN_SIMILARITY;
|
|
1057
|
-
/** Default multiplier applied to a superseded memory's base_strength. */
|
|
1058
|
-
/** Floor under which a superseded memory's base_strength never drops. */
|
|
1059
|
-
/** Neighbor pool size before shape/older/similarity filtering narrows to top 5. */
|
|
1060
|
-
const SUPERSESSION_NEIGHBOR_POOL = 15;
|
|
1061
|
-
/** Older-neighbor pairs kept per candidate after filtering. */
|
|
1062
|
-
const SUPERSESSION_NEIGHBOR_TOP_K = 5;
|
|
1063
|
-
/**
|
|
1064
|
-
* A memory whose content/type marks it as a SUPERSEDABLE claim — one a newer
|
|
1065
|
-
* memory about the same subject can replace. Decisions and corrections were the
|
|
1066
|
-
* original scope; plain facts and project-state updates were added because an
|
|
1067
|
-
* updated fact ("scoring model is X" → later "is Y") otherwise never gets a
|
|
1068
|
-
* superseded_by link and both versions compete in recall forever. Ordinary
|
|
1069
|
-
* episodic chatter and problem/solution history stay excluded: they record
|
|
1070
|
-
* events, not mutable state, so there is nothing to supersede.
|
|
1071
|
-
*/
|
|
1072
|
-
function isSupersedableShape(mem) {
|
|
1073
|
-
return (mem.memory_type === "decisions" ||
|
|
1074
|
-
mem.content.includes("[Decisions Made]") ||
|
|
1075
|
-
mem.content.includes("[Corrections & Rejections]") ||
|
|
1076
|
-
mem.content.includes("[Facts Learned]") ||
|
|
1077
|
-
mem.content.includes("[Project State Changes]"));
|
|
1078
|
-
}
|
|
1079
|
-
/** True when a `superseded_by` link already exists between the pair, either direction. */
|
|
1080
|
-
function alreadySupersedeLinked(db, oldId, newId) {
|
|
1081
|
-
const row = db
|
|
1082
|
-
.prepare(`SELECT 1 FROM memory_links WHERE relationship = 'superseded_by'
|
|
1083
|
-
AND ((source_id = ? AND target_id = ?) OR (source_id = ? AND target_id = ?))`)
|
|
1084
|
-
.get(oldId, newId, newId, oldId);
|
|
1085
|
-
return !!row;
|
|
1086
|
-
}
|
|
1087
|
-
/**
|
|
1088
|
-
* Build the constrained supersession-check prompt. Content is truncated the
|
|
1089
|
-
* same width as domain-classify.ts's classifier (1500 chars) — this is a
|
|
1090
|
-
* classify-tier call with the same cost profile.
|
|
1091
|
-
*/
|
|
1092
|
-
function buildSupersessionPrompt(oldContent, newContent) {
|
|
1093
|
-
const trunc = (s) => (s.length > 1500 ? `${s.slice(0, 1500)}…` : s);
|
|
1094
|
-
return (`You are checking whether a NEWER memory supersedes an OLDER one in an AI agent's long-term memory.\n\n` +
|
|
1095
|
-
`OLDER MEMORY:\n${trunc(oldContent)}\n\n` +
|
|
1096
|
-
`NEWER MEMORY:\n${trunc(newContent)}\n\n` +
|
|
1097
|
-
`Does the NEWER memory reverse, replace, update, or invalidate the OLDER one — e.g. a later decision ` +
|
|
1098
|
-
`overturns an earlier one, a correction retracts a prior claim, or a later fact updates the SAME subject's ` +
|
|
1099
|
-
`value/status that has since changed (e.g. "model is X" → "model is Y")? Reply true ONLY for a genuine ` +
|
|
1100
|
-
`replacement of the same fact/decision. Two memories that are merely related, or that can both still be ` +
|
|
1101
|
-
`true — even about the same project or entity (different facts, an addition, an elaboration) — are NOT a ` +
|
|
1102
|
-
`supersession.\n` +
|
|
1103
|
-
`Reply with ONLY a JSON object, no prose: {"superseded": true} or {"superseded": false}.`);
|
|
1104
|
-
}
|
|
1105
|
-
/**
|
|
1106
|
-
* Parse the model's supersession verdict. Returns the boolean on a valid
|
|
1107
|
-
* reply, or null on anything unparseable (caller skips the pair — no retry,
|
|
1108
|
-
* unlike domain-classify's tag classifier; a missed pair is retried naturally
|
|
1109
|
-
* when this stage revisits the corpus).
|
|
1110
|
-
*/
|
|
1111
|
-
function parseSupersessionReply(reply) {
|
|
1112
|
-
if (!reply)
|
|
1113
|
-
return null;
|
|
1114
|
-
const start = reply.indexOf("{");
|
|
1115
|
-
const end = reply.lastIndexOf("}");
|
|
1116
|
-
if (start === -1 || end === -1 || end <= start)
|
|
1117
|
-
return null;
|
|
1118
|
-
try {
|
|
1119
|
-
const obj = JSON.parse(reply.slice(start, end + 1));
|
|
1120
|
-
return typeof obj.superseded === "boolean" ? obj.superseded : null;
|
|
1121
|
-
}
|
|
1122
|
-
catch {
|
|
1123
|
-
return null;
|
|
1124
|
-
}
|
|
1125
|
-
}
|
|
1126
|
-
/**
|
|
1127
|
-
* ONE classify-tier LLM call judging whether `newContent` supersedes
|
|
1128
|
-
* `oldContent`. Returns `{verdict, usage}` — verdict is null on any infra error
|
|
1129
|
-
* or unparseable reply (the caller treats null as "skip this pair", never
|
|
1130
|
-
* mis-links on ambiguity). `usage` is the call's token accounting (#246),
|
|
1131
|
-
* surfaced even on a null verdict so the BudgetTracker still meters a
|
|
1132
|
-
* network-round-tripped attempt (the cost is real even if the parse failed).
|
|
1133
|
-
*/
|
|
1134
|
-
async function classifySupersession(llm, oldContent, newContent) {
|
|
1135
|
-
try {
|
|
1136
|
-
const r = await llm.complete(buildSupersessionPrompt(oldContent, newContent));
|
|
1137
|
-
return { verdict: parseSupersessionReply(r.text), usage: r.usage };
|
|
1138
|
-
}
|
|
1139
|
-
catch {
|
|
1140
|
-
return { verdict: null, usage: undefined };
|
|
1141
|
-
}
|
|
1142
|
-
}
|
|
1143
|
-
/**
|
|
1144
|
-
* Find up to SUPERSESSION_NEIGHBOR_TOP_K OLDER, same-shape neighbors for a
|
|
1145
|
-
* candidate, at/above minSimilarity, highest cosine first. Reuses the
|
|
1146
|
-
* candidate's stored embedding when available (relink-style fallback to
|
|
1147
|
-
* embedFn otherwise).
|
|
1148
|
-
*/
|
|
1149
|
-
async function findOlderNeighbors(db, candidate, embedFn, minSimilarity) {
|
|
1150
|
-
const embedding = storage.getStoredEmbedding(db, candidate.id) ?? (await embedFn(candidate.content));
|
|
1151
|
-
return storage
|
|
1152
|
-
.vectorSearch(db, embedding, SUPERSESSION_NEIGHBOR_POOL, [candidate.id])
|
|
1153
|
-
.filter((n) => n.created_at < candidate.created_at &&
|
|
1154
|
-
isSupersedableShape(n) &&
|
|
1155
|
-
(0, retrieval_js_1.l2ToCosine)(n.distance) >= minSimilarity)
|
|
1156
|
-
.sort((a, b) => (0, retrieval_js_1.l2ToCosine)(b.distance) - (0, retrieval_js_1.l2ToCosine)(a.distance))
|
|
1157
|
-
.slice(0, SUPERSESSION_NEIGHBOR_TOP_K);
|
|
1158
|
-
}
|
|
1159
|
-
/**
|
|
1160
|
-
* Nightly supersession-detection stage. Scans memories/rowid > cursor whose
|
|
1161
|
-
* shape is supersedable (decision/correction/fact/state — isSupersedableShape),
|
|
1162
|
-
* checks each against its older same-shape neighbors, and links confirmed
|
|
1163
|
-
* supersessions. Dry-run performs discovery + the free idempotency check only —
|
|
1164
|
-
* no LLM calls, no writes, no
|
|
1165
|
-
* cursor persistence (mirrors stageImportance/stageContentDomains's dry-run
|
|
1166
|
-
* convention of never spending budget on a preview).
|
|
1167
|
-
*
|
|
1168
|
-
* Cursor discipline is DELIBERATELY simple (owner amendment): the cursor
|
|
1169
|
-
* advances past a candidate once its neighbor set has been considered,
|
|
1170
|
-
* REGARDLESS of whether every pair got an LLM call (call budget) or a clean
|
|
1171
|
-
* verdict (infra skip) — missing one pair is acceptable and self-heals next
|
|
1172
|
-
* time this memory's neighborhood is re-examined via a NEWER memory's own
|
|
1173
|
-
* candidacy. It only stops SHORT of a candidate when the budget is already
|
|
1174
|
-
* exhausted before that candidate starts, so the cursor never skips a
|
|
1175
|
-
* candidate that was never looked at.
|
|
1176
|
-
*
|
|
1177
|
-
* #405: the cursor persists after EVERY fully-considered candidate (the
|
|
1178
|
-
* post-#404 reconsolidation pattern), not at stage end — a run killed or
|
|
1179
|
-
* deadline-deferred mid-stage loses at most the candidate in flight. No
|
|
1180
|
-
* orphan clamp is needed (unlike reconsolidation): supersession applies each
|
|
1181
|
-
* verdict's link immediately, so `cursor = candidate.__rowid` always sits
|
|
1182
|
-
* after all of that candidate's writes.
|
|
1183
|
-
*/
|
|
1184
|
-
async function stageSupersession(db, llm, budget, embedFn, dryRun, stateDir, options = {}) {
|
|
1185
|
-
// Config values pass through `unknown`-typed JSON — validate rather than
|
|
1186
|
-
// trust (same discipline as retrieval.ts's configureRecall).
|
|
1187
|
-
const validNumber = (v, fallback, ok) => {
|
|
1188
|
-
const n = Number(v);
|
|
1189
|
-
return Number.isFinite(n) && ok(n) ? n : fallback;
|
|
1190
|
-
};
|
|
1191
|
-
const minSimilarity = validNumber(options.minSimilarity, exports.DEFAULT_SUPERSESSION_MIN_SIMILARITY, (n) => n > 0 && n <= 1);
|
|
1192
|
-
const startCursor = (0, state_js_1.loadState)(stateDir).supersessionCursor ?? 0;
|
|
1193
|
-
const rows = db
|
|
1194
|
-
.prepare(
|
|
1195
|
-
// Candidate shape must mirror isSupersedableShape() exactly — keep the two
|
|
1196
|
-
// in lockstep (an inline SQL copy, so drift here silently narrows scope).
|
|
1197
|
-
`SELECT rowid AS __rowid, * FROM memories
|
|
1198
|
-
WHERE rowid > ?
|
|
1199
|
-
AND (memory_type = 'decisions'
|
|
1200
|
-
OR content LIKE '%[Decisions Made]%'
|
|
1201
|
-
OR content LIKE '%[Corrections & Rejections]%'
|
|
1202
|
-
OR content LIKE '%[Facts Learned]%'
|
|
1203
|
-
OR content LIKE '%[Project State Changes]%')
|
|
1204
|
-
ORDER BY rowid ASC`)
|
|
1205
|
-
.all(startCursor);
|
|
1206
|
-
let scanned = 0;
|
|
1207
|
-
let evaluated = 0;
|
|
1208
|
-
let superseded = 0;
|
|
1209
|
-
let skippedInfra = 0;
|
|
1210
|
-
let skippedIdempotent = 0;
|
|
1211
|
-
let cursor = startCursor;
|
|
1212
|
-
// #405: per-candidate checkpoint — persists the cursor after every fully
|
|
1213
|
-
// considered candidate (updateState is an atomic temp-rename of a small
|
|
1214
|
-
// file; the loop cadence is seconds per candidate, so the cost is
|
|
1215
|
-
// negligible). The end-of-stage write below stays the authoritative final
|
|
1216
|
-
// write.
|
|
1217
|
-
const persistCursor = () => {
|
|
1218
|
-
if (dryRun)
|
|
1219
|
-
return;
|
|
1220
|
-
(0, state_js_1.updateState)((s) => {
|
|
1221
|
-
s.supersessionCursor = cursor;
|
|
1222
|
-
}, stateDir);
|
|
1223
|
-
};
|
|
1224
|
-
for (const candidate of rows) {
|
|
1225
|
-
// #405: the ONE run budget is the only call cap; the deadline stops the
|
|
1226
|
-
// scan at the candidate boundary — the cursor holds at the last
|
|
1227
|
-
// fully-considered candidate (persisted below).
|
|
1228
|
-
if (!dryRun && budget.exhausted)
|
|
1229
|
-
break;
|
|
1230
|
-
if (!dryRun && options.deadline?.hit("supersession"))
|
|
1231
|
-
break;
|
|
1232
|
-
scanned++;
|
|
1233
|
-
let neighbors;
|
|
1234
|
-
try {
|
|
1235
|
-
neighbors = await findOlderNeighbors(db, candidate, embedFn, minSimilarity);
|
|
1236
|
-
}
|
|
1237
|
-
catch (err) {
|
|
1238
|
-
console.warn(`[hicortex] supersession: discovery failed for ${candidate.id.slice(0, 8)} — ${err instanceof Error ? err.message : String(err)}`);
|
|
1239
|
-
cursor = candidate.__rowid;
|
|
1240
|
-
persistCursor(); // #405: every exit path persists
|
|
1241
|
-
continue;
|
|
1242
|
-
}
|
|
1243
|
-
for (const neighbor of neighbors) {
|
|
1244
|
-
if (alreadySupersedeLinked(db, neighbor.id, candidate.id)) {
|
|
1245
|
-
skippedIdempotent++;
|
|
1246
|
-
continue;
|
|
1247
|
-
}
|
|
1248
|
-
if (dryRun)
|
|
1249
|
-
continue; // preview only — no LLM call, no write
|
|
1250
|
-
if (!budget.use("supersession"))
|
|
1251
|
-
break; // #405: the ONE run budget
|
|
1252
|
-
const { verdict, usage } = await classifySupersession(llm, neighbor.content, candidate.content);
|
|
1253
|
-
// Meter every round-tripped attempt (#246) — even a null verdict spent
|
|
1254
|
-
// real tokens. The stage label matches the budget.use() above.
|
|
1255
|
-
budget.recordUsage("supersession", usage);
|
|
1256
|
-
evaluated++;
|
|
1257
|
-
if (verdict === null) {
|
|
1258
|
-
skippedInfra++;
|
|
1259
|
-
continue;
|
|
1260
|
-
}
|
|
1261
|
-
if (verdict) {
|
|
1262
|
-
const cosine = (0, retrieval_js_1.l2ToCosine)(neighbor.distance);
|
|
1263
|
-
// The link IS the signal (0.15.2): retrieval demotes superseded
|
|
1264
|
-
// memories via an explicit scoring multiplier (supersededDemotion,
|
|
1265
|
-
// retrieval.ts). The old base_strength penalty was retired because it
|
|
1266
|
-
// (a) fought the config-tunable strength weight and (b) leaked into
|
|
1267
|
-
// prune eligibility — a reversed decision must rank lower, not edge
|
|
1268
|
-
// toward deletion.
|
|
1269
|
-
storage.addLink(db, neighbor.id, candidate.id, "superseded_by", cosine);
|
|
1270
|
-
superseded++;
|
|
1271
|
-
console.log(`[hicortex] Supersession: ${neighbor.id.slice(0, 8)} superseded_by ${candidate.id.slice(0, 8)} (cosine ${cosine.toFixed(3)})`);
|
|
1272
|
-
}
|
|
1273
|
-
}
|
|
1274
|
-
cursor = candidate.__rowid;
|
|
1275
|
-
// #405: checkpoint after every fully-considered candidate (post-#404
|
|
1276
|
-
// reconsolidation pattern) — a killed or deadline-deferred run loses at
|
|
1277
|
-
// most the candidate in flight.
|
|
1278
|
-
persistCursor();
|
|
1279
|
-
}
|
|
1280
|
-
if (!dryRun) {
|
|
1281
|
-
(0, state_js_1.updateState)((s) => {
|
|
1282
|
-
s.supersessionCursor = cursor;
|
|
1283
|
-
}, stateDir);
|
|
1284
|
-
}
|
|
1285
|
-
if (rows.length > 0) {
|
|
1286
|
-
console.log(`[hicortex] Supersession detection: ${scanned} scanned, ${evaluated} evaluated, ${superseded} superseded, ` +
|
|
1287
|
-
`${skippedIdempotent} already-linked, ${skippedInfra} infra-skipped (cursor ${cursor})`);
|
|
1288
|
-
}
|
|
1289
|
-
return {
|
|
1290
|
-
scanned,
|
|
1291
|
-
evaluated,
|
|
1292
|
-
superseded,
|
|
1293
|
-
skipped_infra: skippedInfra,
|
|
1294
|
-
skipped_idempotent: skippedIdempotent,
|
|
1295
|
-
cursor,
|
|
1296
|
-
};
|
|
1297
|
-
}
|
|
1298
|
-
// ---------------------------------------------------------------------------
|
|
1299
1031
|
// Stage 4: Decay & Prune
|
|
1300
1032
|
// ---------------------------------------------------------------------------
|
|
1301
1033
|
/**
|
|
@@ -1413,9 +1145,9 @@ function stagePromotion(db, dryRun) {
|
|
|
1413
1145
|
});
|
|
1414
1146
|
}
|
|
1415
1147
|
// #459: the stage's one-line summary in the shared stage idiom (rows
|
|
1416
|
-
// examined / promoted / total gain, like the
|
|
1148
|
+
// examined / promoted / total gain, like the resolution-stage summary) — the
|
|
1417
1149
|
// stage writes its report in-memory only, so the log line is the soak-time
|
|
1418
|
-
// health signal. Zero examined rows stay silent (the
|
|
1150
|
+
// health signal. Zero examined rows stay silent (the quiet-stage gate).
|
|
1419
1151
|
if (dryRun) {
|
|
1420
1152
|
console.log(`[hicortex] Strength promotion (dry-run): ${rows.length} examined, would promote ` +
|
|
1421
1153
|
`${promoted} (+${totalGain.toFixed(3)} total strength, ` +
|
|
@@ -1598,8 +1330,8 @@ async function skippedRunResolutionReport(db, dryRun, stateDir, options = {}) {
|
|
|
1598
1330
|
none: 0,
|
|
1599
1331
|
merge_below_gate: 0,
|
|
1600
1332
|
conf_sum: merges.losers_merged,
|
|
1601
|
-
...(merges.
|
|
1602
|
-
? {
|
|
1333
|
+
...(merges.skipped_project_mismatch > 0
|
|
1334
|
+
? { project_skipped: merges.skipped_project_mismatch }
|
|
1603
1335
|
: {}),
|
|
1604
1336
|
};
|
|
1605
1337
|
}
|
|
@@ -1631,7 +1363,7 @@ async function skippedRunResolutionReport(db, dryRun, stateDir, options = {}) {
|
|
|
1631
1363
|
merge_pairs_applied: 0,
|
|
1632
1364
|
merge_below_gate: 0,
|
|
1633
1365
|
skipped_above_ceiling: 0,
|
|
1634
|
-
|
|
1366
|
+
skipped_project_mismatch: 0,
|
|
1635
1367
|
conflict_flagged: 0, // guard-C: no scan on a quiet night — nothing flagged
|
|
1636
1368
|
conflict_skipped: merges.skipped_conflict, // guard-C: the zone's guard still counts
|
|
1637
1369
|
scout_scanned: 0, // #393 B: the scan (and its shape calls) doesn't run on a quiet night
|
|
@@ -1640,7 +1372,7 @@ async function skippedRunResolutionReport(db, dryRun, stateDir, options = {}) {
|
|
|
1640
1372
|
band_stats: bandStats,
|
|
1641
1373
|
};
|
|
1642
1374
|
}
|
|
1643
|
-
async function runConsolidation(db, llm, embedFn, dryRun = false, skipReflection = false, stateDir, domainOptions,
|
|
1375
|
+
async function runConsolidation(db, llm, embedFn, dryRun = false, skipReflection = false, stateDir, domainOptions,
|
|
1644
1376
|
/** The ONE per-run LLM-call ceiling (#405/#241). The caller resolves
|
|
1645
1377
|
* `nightlyLlmCallBudget` from config (consolidateMaxLlmCalls is a
|
|
1646
1378
|
* deprecated alias — resolveNightlyLlmCallBudget) and passes it; unset →
|
|
@@ -1652,10 +1384,9 @@ budgetMaxCalls,
|
|
|
1652
1384
|
memorySoftCap,
|
|
1653
1385
|
/** Reconsolidation-stage knobs (#384) — eval/test seams since #408 (the
|
|
1654
1386
|
* values are release-managed calibration constants; nightly.ts threads
|
|
1655
|
-
* NOTHING)
|
|
1656
|
-
*
|
|
1657
|
-
*
|
|
1658
|
-
* argument meaning. */
|
|
1387
|
+
* NOTHING); unset fields → the stage's calibration defaults. The
|
|
1388
|
+
* pre-#206-B supersessionOptions param that sat here is retired with its
|
|
1389
|
+
* stage — callers updated in the same change. */
|
|
1659
1390
|
reconsolidationOptions,
|
|
1660
1391
|
/**
|
|
1661
1392
|
* The run-wide pipeline deadline (#405), created at nightly start and
|
|
@@ -1826,14 +1557,13 @@ deadline) {
|
|
|
1826
1557
|
if (!deadline?.hit("links")) {
|
|
1827
1558
|
report.stages.links = await stageLinks(db, precheck.newMemories, embedFn, dryRun, llm, budget);
|
|
1828
1559
|
}
|
|
1829
|
-
// Stage 3.7: Supersession Detection (#191 Phase B)
|
|
1830
|
-
if (!deadline?.hit("supersession")) {
|
|
1831
|
-
report.stages.supersession = await stageSupersession(db, llm, budget, embedFn, dryRun, stateDir, { ...supersessionOptions, deadline });
|
|
1832
|
-
}
|
|
1833
1560
|
// Stage 3.8: Reconsolidation (#384) — resolve corrections: rewrite
|
|
1834
1561
|
// fact-shaped targets in place (absorbing transition-only triggers),
|
|
1835
|
-
// mark everything else.
|
|
1836
|
-
//
|
|
1562
|
+
// mark everything else. THE unified resolution stage since #392, and
|
|
1563
|
+
// since #206-B (owner decision 6) also the ONLY true-update detector:
|
|
1564
|
+
// the standalone supersession stage (3.7) is retired into its
|
|
1565
|
+
// `supersedes` verdict action. Rides the same shared budget under its
|
|
1566
|
+
// own stage label + cursor.
|
|
1837
1567
|
if (!deadline?.hit("reconsolidation")) {
|
|
1838
1568
|
report.stages.reconsolidation = await (0, reconsolidation_js_1.stageReconsolidation)(db, llm, budget, embedFn, dryRun, stateDir, { ...reconsolidationOptions, deadline });
|
|
1839
1569
|
}
|
package/dist/db.js
CHANGED
|
@@ -742,6 +742,27 @@ const MIGRATIONS = [
|
|
|
742
742
|
db.exec("CREATE INDEX IF NOT EXISTS idx_recall_events_memory_kind ON recall_events(memory_id, kind)");
|
|
743
743
|
},
|
|
744
744
|
},
|
|
745
|
+
{
|
|
746
|
+
version: 23,
|
|
747
|
+
name: "volatile_sweep_log",
|
|
748
|
+
up: (db) => {
|
|
749
|
+
// #489 — `hicortex sweep-volatile` audit trail (owner decision 3:
|
|
750
|
+
// one-shot store sweep, mark-not-delete). Every swept memory gets a row
|
|
751
|
+
// here as it is absorbed: the sweep is inspectable forever, never
|
|
752
|
+
// silent. The dedup_log precedent (a sidecar audit table, no memories
|
|
753
|
+
// FK), but with no runtime consumer — the CAPTURE-side volatility gate
|
|
754
|
+
// is the re-ingest safety net, so nothing consults this table; it is
|
|
755
|
+
// purely the durable record of what --apply retired and when.
|
|
756
|
+
// Idempotent: IF NOT EXISTS.
|
|
757
|
+
db.exec(`
|
|
758
|
+
CREATE TABLE IF NOT EXISTS volatile_sweep_log (
|
|
759
|
+
memory_id TEXT PRIMARY KEY,
|
|
760
|
+
swept_at TEXT NOT NULL,
|
|
761
|
+
preview TEXT
|
|
762
|
+
)
|
|
763
|
+
`);
|
|
764
|
+
},
|
|
765
|
+
},
|
|
745
766
|
];
|
|
746
767
|
/**
|
|
747
768
|
* Run all pending migrations against the database.
|
package/dist/dedup.d.ts
CHANGED
|
@@ -20,8 +20,9 @@
|
|
|
20
20
|
* consolidate.ts) so one stage report covers all resolution work.
|
|
21
21
|
*
|
|
22
22
|
* Per cluster (shared `planDedup`/`mergeCluster` core — no forks):
|
|
23
|
-
* - Canonical = highest access_count (tie:
|
|
24
|
-
*
|
|
23
|
+
* - Canonical = highest access_count (tie: NEWEST created_at — newest-wins,
|
|
24
|
+
* #206 decision 3 — then lexicographically smallest id, fully
|
|
25
|
+
* deterministic for audit).
|
|
25
26
|
* - Losers' links are re-pointed onto the canonical (a link that would
|
|
26
27
|
* become a self-link, or one whose (canonical, target) ordered pair
|
|
27
28
|
* ALREADY holds an edge, is skipped rather than overwritten — see
|
|
@@ -44,8 +45,11 @@
|
|
|
44
45
|
* dedup_log (loser_id → canonical_id) + the retained loser row is the
|
|
45
46
|
* record.
|
|
46
47
|
*
|
|
47
|
-
* A cluster whose members disagree on project
|
|
48
|
-
*
|
|
48
|
+
* A cluster whose members disagree on project is SKIPPED entirely and listed
|
|
49
|
+
* for manual review (reason `project_mismatch`) — no --force in this release.
|
|
50
|
+
* The source_agent rail was REMOVED (#206 decision 2, 2026-09-21): cross-agent
|
|
51
|
+
* clusters merge — attribution is preserved on the retained evidence row and
|
|
52
|
+
* no recall path filters by agent.
|
|
49
53
|
*
|
|
50
54
|
* Safety rails when applying (CLI and zone alike):
|
|
51
55
|
* - A full DB backup (SQLite backup API) is taken FIRST, to
|
|
@@ -219,16 +223,17 @@ export type MergeMemoryIdsResult = {
|
|
|
219
223
|
linksRepointed: number;
|
|
220
224
|
} | {
|
|
221
225
|
ok: false;
|
|
222
|
-
reason: "
|
|
226
|
+
reason: "project_mismatch" | "conflict_linked" | "no_members";
|
|
223
227
|
};
|
|
224
228
|
/**
|
|
225
229
|
* Merge an explicit set of memories (the judged-pair phase of #392: the
|
|
226
230
|
* reconsolidation stage queues verdict-confirmed pairs and applies them
|
|
227
231
|
* through THIS function so the merge math stays single-definition). Loads the
|
|
228
232
|
* LIVE rows at apply time — members that vanished or were absorbed between
|
|
229
|
-
* verdict and apply are dropped defensively; a
|
|
230
|
-
* the merge (both memories stay live
|
|
231
|
-
*
|
|
233
|
+
* verdict and apply are dropped defensively; a project disagreement refuses
|
|
234
|
+
* the merge (both memories stay live — the only metadata rail left, #206
|
|
235
|
+
* decision 2); a conflicts-linked pair refuses it exactly the same way
|
|
236
|
+
* (#393 guard-C). One transaction for the whole set.
|
|
232
237
|
*/
|
|
233
238
|
export declare function mergeMemoryIds(db: Database.Database, ids: string[]): MergeMemoryIdsResult;
|
|
234
239
|
/**
|
|
@@ -267,7 +272,7 @@ export interface DeterministicMergeZoneOptions {
|
|
|
267
272
|
*
|
|
268
273
|
* Also persists the deterministic band's cumulative statistics to state.json
|
|
269
274
|
* `resolutionBandStats` (label `>=threshold`; losers count as merge verdicts
|
|
270
|
-
* at confidence 1.0, mismatch clusters as
|
|
275
|
+
* at confidence 1.0, mismatch clusters as project_skipped) — skipped
|
|
271
276
|
* entirely on dry-run. Called from the reconsolidation stage (main path) and
|
|
272
277
|
* from runConsolidation's quiet-night skip path — exactly one of the two per
|
|
273
278
|
* run.
|
package/dist/dedup.js
CHANGED
|
@@ -21,8 +21,9 @@
|
|
|
21
21
|
* consolidate.ts) so one stage report covers all resolution work.
|
|
22
22
|
*
|
|
23
23
|
* Per cluster (shared `planDedup`/`mergeCluster` core — no forks):
|
|
24
|
-
* - Canonical = highest access_count (tie:
|
|
25
|
-
*
|
|
24
|
+
* - Canonical = highest access_count (tie: NEWEST created_at — newest-wins,
|
|
25
|
+
* #206 decision 3 — then lexicographically smallest id, fully
|
|
26
|
+
* deterministic for audit).
|
|
26
27
|
* - Losers' links are re-pointed onto the canonical (a link that would
|
|
27
28
|
* become a self-link, or one whose (canonical, target) ordered pair
|
|
28
29
|
* ALREADY holds an edge, is skipped rather than overwritten — see
|
|
@@ -45,8 +46,11 @@
|
|
|
45
46
|
* dedup_log (loser_id → canonical_id) + the retained loser row is the
|
|
46
47
|
* record.
|
|
47
48
|
*
|
|
48
|
-
* A cluster whose members disagree on project
|
|
49
|
-
*
|
|
49
|
+
* A cluster whose members disagree on project is SKIPPED entirely and listed
|
|
50
|
+
* for manual review (reason `project_mismatch`) — no --force in this release.
|
|
51
|
+
* The source_agent rail was REMOVED (#206 decision 2, 2026-09-21): cross-agent
|
|
52
|
+
* clusters merge — attribution is preserved on the retained evidence row and
|
|
53
|
+
* no recall path filters by agent.
|
|
50
54
|
*
|
|
51
55
|
* Safety rails when applying (CLI and zone alike):
|
|
52
56
|
* - A full DB backup (SQLite backup API) is taken FIRST, to
|
|
@@ -160,13 +164,15 @@ function loadMembers(db, ids) {
|
|
|
160
164
|
FROM memories WHERE id IN (${placeholders})`)
|
|
161
165
|
.all(...ids);
|
|
162
166
|
}
|
|
163
|
-
/** Canonical = highest access_count; ties broken by
|
|
167
|
+
/** Canonical = highest access_count; ties broken by NEWEST created_at (newest-wins,
|
|
168
|
+
* #206 decision 3 — most-used still wins first, newest is the tie-break), then
|
|
169
|
+
* lexicographically smallest id. */
|
|
164
170
|
function pickCanonical(members) {
|
|
165
171
|
const sorted = [...members].sort((a, b) => {
|
|
166
172
|
if (b.access_count !== a.access_count)
|
|
167
173
|
return b.access_count - a.access_count;
|
|
168
174
|
if (a.created_at !== b.created_at)
|
|
169
|
-
return
|
|
175
|
+
return b.created_at.localeCompare(a.created_at);
|
|
170
176
|
return a.id.localeCompare(b.id);
|
|
171
177
|
});
|
|
172
178
|
const [canonical, ...losers] = sorted;
|
|
@@ -281,7 +287,9 @@ function planDedup(db, threshold) {
|
|
|
281
287
|
continue;
|
|
282
288
|
}
|
|
283
289
|
const mismatch = (0, cluster_js_1.clusterMetadataMismatch)(members);
|
|
284
|
-
|
|
290
|
+
// #206 decision 2: project is the ONLY rail — the source_agent rail was
|
|
291
|
+
// removed (cross-agent clusters merge; the skip reason is project_mismatch).
|
|
292
|
+
if (mismatch.projectMismatch) {
|
|
285
293
|
mismatchSkipped.push({ size: members.length, memberIds: members.map((m) => m.id), mismatch });
|
|
286
294
|
continue;
|
|
287
295
|
}
|
|
@@ -383,9 +391,10 @@ function mergeCluster(db, canonical, losers, injectFailure) {
|
|
|
383
391
|
* reconsolidation stage queues verdict-confirmed pairs and applies them
|
|
384
392
|
* through THIS function so the merge math stays single-definition). Loads the
|
|
385
393
|
* LIVE rows at apply time — members that vanished or were absorbed between
|
|
386
|
-
* verdict and apply are dropped defensively; a
|
|
387
|
-
* the merge (both memories stay live
|
|
388
|
-
*
|
|
394
|
+
* verdict and apply are dropped defensively; a project disagreement refuses
|
|
395
|
+
* the merge (both memories stay live — the only metadata rail left, #206
|
|
396
|
+
* decision 2); a conflicts-linked pair refuses it exactly the same way
|
|
397
|
+
* (#393 guard-C). One transaction for the whole set.
|
|
389
398
|
*/
|
|
390
399
|
function mergeMemoryIds(db, ids) {
|
|
391
400
|
const unique = [...new Set(ids)];
|
|
@@ -393,14 +402,14 @@ function mergeMemoryIds(db, ids) {
|
|
|
393
402
|
if (members.length < 2)
|
|
394
403
|
return { ok: false, reason: "no_members" };
|
|
395
404
|
// #393 guard-C: a conflicts-linked pair is never blended — the mirror of the
|
|
396
|
-
//
|
|
405
|
+
// project rail (both memories stay live; the caller's verdict was still
|
|
397
406
|
// rendered, so its cursor advances).
|
|
398
407
|
if (clusterHasConflictLink(db, members.map((m) => m.id))) {
|
|
399
408
|
return { ok: false, reason: "conflict_linked" };
|
|
400
409
|
}
|
|
401
410
|
const mismatch = (0, cluster_js_1.clusterMetadataMismatch)(members);
|
|
402
|
-
if (mismatch.projectMismatch
|
|
403
|
-
return { ok: false, reason: "
|
|
411
|
+
if (mismatch.projectMismatch) {
|
|
412
|
+
return { ok: false, reason: "project_mismatch" };
|
|
404
413
|
}
|
|
405
414
|
const { canonical, losers } = pickCanonical(members);
|
|
406
415
|
const tx = db.transaction(() => mergeCluster(db, canonical, losers));
|
|
@@ -440,7 +449,7 @@ async function takePreDedupBackup(db, stateDir, config) {
|
|
|
440
449
|
*
|
|
441
450
|
* Also persists the deterministic band's cumulative statistics to state.json
|
|
442
451
|
* `resolutionBandStats` (label `>=threshold`; losers count as merge verdicts
|
|
443
|
-
* at confidence 1.0, mismatch clusters as
|
|
452
|
+
* at confidence 1.0, mismatch clusters as project_skipped) — skipped
|
|
444
453
|
* entirely on dry-run. Called from the reconsolidation stage (main path) and
|
|
445
454
|
* from runConsolidation's quiet-night skip path — exactly one of the two per
|
|
446
455
|
* run.
|
|
@@ -462,7 +471,7 @@ async function runDeterministicMergeZone(db, opts = {}) {
|
|
|
462
471
|
merged_clusters: 0,
|
|
463
472
|
losers_merged: 0,
|
|
464
473
|
links_repointed: 0,
|
|
465
|
-
|
|
474
|
+
skipped_project_mismatch: plan.mismatchSkipped.length,
|
|
466
475
|
skipped_conflict: plan.conflictSkipped.length,
|
|
467
476
|
capped: 0,
|
|
468
477
|
failed: 0,
|
|
@@ -480,7 +489,7 @@ async function runDeterministicMergeZone(db, opts = {}) {
|
|
|
480
489
|
// Cumulative deterministic-band stats (state.json) — one write at zone
|
|
481
490
|
// end, on every apply-path exit, never when there is nothing to record.
|
|
482
491
|
const persistBand = () => {
|
|
483
|
-
if (report.losers_merged === 0 && report.
|
|
492
|
+
if (report.losers_merged === 0 && report.skipped_project_mismatch === 0)
|
|
484
493
|
return;
|
|
485
494
|
(0, state_js_1.updateState)((s) => {
|
|
486
495
|
const label = `>=${threshold}`;
|
|
@@ -496,7 +505,7 @@ async function runDeterministicMergeZone(db, opts = {}) {
|
|
|
496
505
|
// Deterministic merges carry no verdict — model confidence 1.0 each
|
|
497
506
|
// (the calibration line: measured ~100% same-memory at the ceiling).
|
|
498
507
|
conf_sum: b.conf_sum + report.losers_merged,
|
|
499
|
-
|
|
508
|
+
project_skipped: (b.project_skipped ?? 0) + report.skipped_project_mismatch,
|
|
500
509
|
};
|
|
501
510
|
s.resolutionBandStats = bands;
|
|
502
511
|
}, stateDir);
|
|
@@ -566,7 +575,7 @@ async function runDeterministicMergeZone(db, opts = {}) {
|
|
|
566
575
|
}
|
|
567
576
|
console.log(`[hicortex] deterministic-merge zone (>= ${threshold}): ${report.merged_clusters}/${plan.mergePlans.length} ` +
|
|
568
577
|
`cluster(s) merged, ${report.losers_merged} loser(s) absorbed, ` +
|
|
569
|
-
`${report.
|
|
578
|
+
`${report.skipped_project_mismatch} skipped (project mismatch), ` +
|
|
570
579
|
`${report.skipped_conflict} skipped (conflict-flagged)` +
|
|
571
580
|
(report.capped > 0 ? `, ${report.capped} deferred (run deadline)` : "") +
|
|
572
581
|
(report.failed > 0 ? `, ${report.failed} FAILED` : ""));
|
|
@@ -585,7 +594,7 @@ async function runDeterministicMergeZone(db, opts = {}) {
|
|
|
585
594
|
return {
|
|
586
595
|
threshold, clusters_found: 0, mergeable_clusters: 0,
|
|
587
596
|
merged_clusters: 0, losers_merged: 0, links_repointed: 0,
|
|
588
|
-
|
|
597
|
+
skipped_project_mismatch: 0, skipped_conflict: 0, capped: 0, failed: 0,
|
|
589
598
|
};
|
|
590
599
|
}
|
|
591
600
|
}
|
|
@@ -649,7 +658,7 @@ async function runDedup(options = {}) {
|
|
|
649
658
|
linksSkippedExisting: linksSkippedExistingPreview,
|
|
650
659
|
};
|
|
651
660
|
console.log(`[hicortex] dedup: ${plan.clusterCount} cluster(s) found, ${mergeable.length} mergeable ` +
|
|
652
|
-
`(${plannedMerges} row(s) would be absorbed), ${plan.mismatchSkipped.length} skipped (
|
|
661
|
+
`(${plannedMerges} row(s) would be absorbed), ${plan.mismatchSkipped.length} skipped (project mismatch), ` +
|
|
653
662
|
`${plan.conflictSkipped.length} skipped (conflict-flagged), ` +
|
|
654
663
|
`${linksSkippedExistingPreview} link(s) would be skipped (existing edge on the canonical)`);
|
|
655
664
|
if (!apply) {
|
|
@@ -660,11 +669,10 @@ async function runDedup(options = {}) {
|
|
|
660
669
|
`${c.linksSkippedSelfLink} skipped (self-link)`);
|
|
661
670
|
}
|
|
662
671
|
for (const c of plan.mismatchSkipped) {
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
console.log(`[hicortex] SKIPPED (${reasons}): ${c.memberIds.map((id) => id.slice(0, 8)).join(", ")}`);
|
|
672
|
+
// #206 decision 2: project_mismatch is the only skip reason left on
|
|
673
|
+
// this rail (the source_agent rail was removed) — the bedrock dry run
|
|
674
|
+
// sizes the project rail alone off this line.
|
|
675
|
+
console.log(`[hicortex] SKIPPED (project_mismatch): ${c.memberIds.map((id) => id.slice(0, 8)).join(", ")}`);
|
|
668
676
|
}
|
|
669
677
|
// #393 guard-C: listed for review like the mismatch clusters — a
|
|
670
678
|
// conflicts-linked near-duplicate pair is deliberate, not an error.
|
package/dist/distiller.d.ts
CHANGED
|
@@ -78,6 +78,24 @@ export declare function isNoExtractResponse(result: string): boolean;
|
|
|
78
78
|
* survive. Stripping affects only this gate's decision, never stored text.
|
|
79
79
|
*/
|
|
80
80
|
export declare function hasMinimalSubstance(entry: string): boolean;
|
|
81
|
+
/**
|
|
82
|
+
* True when the entry is a VOLATILE STATUS SHAPE — GH-ticket/workflow status,
|
|
83
|
+
* version-bump status, or branch/commit state — and carries no durability
|
|
84
|
+
* escape (#489, owner decisions 1+2). Runs in distillChunk's gate zone beside
|
|
85
|
+
* hasMinimalSubstance; drops ride the SAME #156 dropped[] trail. Also reused
|
|
86
|
+
* verbatim by `hicortex sweep-volatile` over the stored corpus — ONE gate,
|
|
87
|
+
* one meaning.
|
|
88
|
+
*
|
|
89
|
+
* PRECISION OVER RECALL (the substance gate's law, inherited): escapes are
|
|
90
|
+
* checked FIRST and override every trigger; entries longer than
|
|
91
|
+
* VOLATILE_GATE_MAX_CHARS are treated as mixed prose whose status clause is
|
|
92
|
+
* not the DOMINANT content (kept). Accepted false-positive mode, stated
|
|
93
|
+
* plainly: an entry pairing a status clause with a distinct durable clause
|
|
94
|
+
* and no escape word drops, losing the durable half — unless the distiller
|
|
95
|
+
* emitted that half as its own entry, which is exactly what the ephemera
|
|
96
|
+
* prompt tells it to do. Every drop is auditable via the trail.
|
|
97
|
+
*/
|
|
98
|
+
export declare function isVolatileStatusEntry(entry: string): boolean;
|
|
81
99
|
/**
|
|
82
100
|
* A parsed distillation entry: the stored content (type tag STRIPPED) plus the
|
|
83
101
|
* classified memory_type. `memoryType` is one of "experience" | "knowledge" |
|