@gamaze/hicortex 0.20.9 → 0.20.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/dashboard.html +3989 -836
- package/dist/calibration.d.ts +119 -0
- package/dist/calibration.js +149 -1
- package/dist/capture-health.d.ts +87 -0
- package/dist/capture-health.js +106 -0
- package/dist/capture-pause.d.ts +86 -0
- package/dist/capture-pause.js +127 -0
- package/dist/capture.d.ts +9 -0
- package/dist/capture.js +2 -1
- package/dist/cli.js +36 -0
- package/dist/consolidate.d.ts +35 -0
- package/dist/consolidate.js +85 -9
- package/dist/dashboard.d.ts +322 -3
- package/dist/dashboard.js +592 -7
- package/dist/db.js +105 -0
- package/dist/eval/importance-eval.d.ts +85 -0
- package/dist/eval/importance-eval.js +286 -0
- package/dist/eval/planted-fixtures.d.ts +1 -1
- package/dist/eval/ranking-battery.d.ts +78 -0
- package/dist/eval/ranking-battery.js +181 -0
- package/dist/eval/ranking-eval.d.ts +41 -0
- package/dist/eval/ranking-eval.js +391 -0
- package/dist/eval/ranking-fixtures.d.ts +77 -0
- package/dist/eval/ranking-fixtures.js +226 -0
- package/dist/identity-store.d.ts +21 -0
- package/dist/identity-store.js +49 -0
- package/dist/init.d.ts +14 -0
- package/dist/init.js +32 -0
- package/dist/mcp-server.d.ts +12 -0
- package/dist/mcp-server.js +184 -3
- package/dist/nightly.d.ts +9 -1
- package/dist/nightly.js +59 -7
- package/dist/prompts.d.ts +10 -0
- package/dist/prompts.js +28 -5
- package/dist/reconsolidation.d.ts +59 -30
- package/dist/reconsolidation.js +526 -296
- package/dist/rescore-importance.d.ts +80 -0
- package/dist/rescore-importance.js +236 -0
- package/dist/retrieval.d.ts +12 -0
- package/dist/retrieval.js +30 -1
- package/dist/stages.d.ts +37 -0
- package/dist/stages.js +51 -0
- package/dist/state.d.ts +32 -6
- package/dist/storage.d.ts +34 -2
- package/dist/storage.js +63 -6
- package/dist/types.d.ts +48 -0
- package/package.json +3 -1
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `hicortex rescore-importance` (#425) — the one-shot LLM backfill that
|
|
3
|
+
* re-judges the EXISTING corpus under the re-anchored importance rubric
|
|
4
|
+
* (owner decision D1, 2026-09-13: rescore via LLM, resumable, local
|
|
5
|
+
* gateway).
|
|
6
|
+
*
|
|
7
|
+
* Precedents, deliberately mixed per the issue's attribution:
|
|
8
|
+
* - classify-domains (src/classify-domains.ts): resumable rowid cursor in
|
|
9
|
+
* state.json, --batch rows per invocation, --reset, server-mode-only,
|
|
10
|
+
* infra-error abort that leaves the cursor at the last committed batch.
|
|
11
|
+
* - dedup (src/dedup.ts): dry-run DEFAULT with --apply, and a DB backup
|
|
12
|
+
* taken FIRST — the CLI aborts (nothing written) if the backup fails.
|
|
13
|
+
*
|
|
14
|
+
* The scoring itself is the SHARED production loop — consolidate.ts
|
|
15
|
+
* `scoreMemoriesImportance` (the extracted stageImportance core): batches of
|
|
16
|
+
* 10, serial calls, the 0.95 write cap and the importance_scored_at
|
|
17
|
+
* watermark identical to the nightly. No forked scoring code.
|
|
18
|
+
*
|
|
19
|
+
* Scope: every LIVE (non-absorbed) memory, rowid-ascending. A row's
|
|
20
|
+
* corroboration_count survives untouched; base_strength is re-judged (that
|
|
21
|
+
* is D1's explicit trade: one rubric for all rows, the 2026-08-21-validated
|
|
22
|
+
* ORDERING is re-derived rather than mapped).
|
|
23
|
+
*/
|
|
24
|
+
import { LlmClient } from "./llm.js";
|
|
25
|
+
export interface RescoreImportanceOptions {
|
|
26
|
+
/** Execute (default: dry run — report only, zero writes, no backup). */
|
|
27
|
+
apply?: boolean;
|
|
28
|
+
/** Rows per invocation chunk (default 500). */
|
|
29
|
+
batchSize?: number;
|
|
30
|
+
/** Ignore the saved cursor and restart from rowid 0. */
|
|
31
|
+
reset?: boolean;
|
|
32
|
+
/** DB path override (tests). Defaults to resolveDbPath(). */
|
|
33
|
+
dbPath?: string;
|
|
34
|
+
/** State dir override (tests). Defaults to ~/.hicortex. */
|
|
35
|
+
stateDir?: string;
|
|
36
|
+
/** LLM override (tests). Bypasses config resolution. */
|
|
37
|
+
llm?: LlmClient;
|
|
38
|
+
/** Config override (tests). Defaults to reading stateDir/config.json. */
|
|
39
|
+
config?: Record<string, unknown> | null;
|
|
40
|
+
}
|
|
41
|
+
export interface RescoreImportanceReport {
|
|
42
|
+
/** True when this invocation was a dry run (zero writes). */
|
|
43
|
+
dryRun: boolean;
|
|
44
|
+
/** Live rows remaining to process AFTER this invocation (whole corpus minus cursor). */
|
|
45
|
+
remaining: number;
|
|
46
|
+
/** Rows this invocation re-judged (0 on a dry run). */
|
|
47
|
+
rescored: number;
|
|
48
|
+
/** Rows whose scoring call failed (endpoint down / unusable) — untouched. */
|
|
49
|
+
failed: number;
|
|
50
|
+
/** LLM calls made (10 rows each). */
|
|
51
|
+
calls: number;
|
|
52
|
+
/** Cursor after this invocation. */
|
|
53
|
+
cursor: number;
|
|
54
|
+
/** True when the run stopped early on an infra error (cursor holds). */
|
|
55
|
+
aborted: boolean;
|
|
56
|
+
/** Path to the pre-run DB backup (apply mode only). */
|
|
57
|
+
backupPath?: string;
|
|
58
|
+
/** CURRENT base_strength percentiles over the remaining rows (preview). */
|
|
59
|
+
currentDistribution: DistributionPreview;
|
|
60
|
+
/** For apply runs over rows that were actually re-judged: before → after. */
|
|
61
|
+
before?: DistributionPreview;
|
|
62
|
+
after?: DistributionPreview;
|
|
63
|
+
}
|
|
64
|
+
export interface DistributionPreview {
|
|
65
|
+
n: number;
|
|
66
|
+
min: number;
|
|
67
|
+
p25: number;
|
|
68
|
+
median: number;
|
|
69
|
+
p75: number;
|
|
70
|
+
p90: number;
|
|
71
|
+
max: number;
|
|
72
|
+
atCeiling: number;
|
|
73
|
+
atSentinel: number;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Run the rescore pass. Returns a structured report. Throws on setup errors
|
|
77
|
+
* (client mode, no LLM, backup failure) — the cursor always reflects the
|
|
78
|
+
* last committed batch.
|
|
79
|
+
*/
|
|
80
|
+
export declare function runRescoreImportance(options?: RescoreImportanceOptions): Promise<RescoreImportanceReport>;
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `hicortex rescore-importance` (#425) — the one-shot LLM backfill that
|
|
4
|
+
* re-judges the EXISTING corpus under the re-anchored importance rubric
|
|
5
|
+
* (owner decision D1, 2026-09-13: rescore via LLM, resumable, local
|
|
6
|
+
* gateway).
|
|
7
|
+
*
|
|
8
|
+
* Precedents, deliberately mixed per the issue's attribution:
|
|
9
|
+
* - classify-domains (src/classify-domains.ts): resumable rowid cursor in
|
|
10
|
+
* state.json, --batch rows per invocation, --reset, server-mode-only,
|
|
11
|
+
* infra-error abort that leaves the cursor at the last committed batch.
|
|
12
|
+
* - dedup (src/dedup.ts): dry-run DEFAULT with --apply, and a DB backup
|
|
13
|
+
* taken FIRST — the CLI aborts (nothing written) if the backup fails.
|
|
14
|
+
*
|
|
15
|
+
* The scoring itself is the SHARED production loop — consolidate.ts
|
|
16
|
+
* `scoreMemoriesImportance` (the extracted stageImportance core): batches of
|
|
17
|
+
* 10, serial calls, the 0.95 write cap and the importance_scored_at
|
|
18
|
+
* watermark identical to the nightly. No forked scoring code.
|
|
19
|
+
*
|
|
20
|
+
* Scope: every LIVE (non-absorbed) memory, rowid-ascending. A row's
|
|
21
|
+
* corroboration_count survives untouched; base_strength is re-judged (that
|
|
22
|
+
* is D1's explicit trade: one rubric for all rows, the 2026-08-21-validated
|
|
23
|
+
* ORDERING is re-derived rather than mapped).
|
|
24
|
+
*/
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.runRescoreImportance = runRescoreImportance;
|
|
27
|
+
const node_fs_1 = require("node:fs");
|
|
28
|
+
const node_fs_2 = require("node:fs");
|
|
29
|
+
const node_path_1 = require("node:path");
|
|
30
|
+
const paths_js_1 = require("./paths.js");
|
|
31
|
+
const db_js_1 = require("./db.js");
|
|
32
|
+
const state_js_1 = require("./state.js");
|
|
33
|
+
const consolidate_js_1 = require("./consolidate.js");
|
|
34
|
+
const llm_js_1 = require("./llm.js");
|
|
35
|
+
const backup_js_1 = require("./backup.js");
|
|
36
|
+
const config_read_js_1 = require("./config-read.js");
|
|
37
|
+
const calibration_js_1 = require("./calibration.js");
|
|
38
|
+
const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
|
|
39
|
+
/** Rows per invocation chunk (LLM calls are 10 rows each inside a chunk). */
|
|
40
|
+
const DEFAULT_BATCH = 500;
|
|
41
|
+
/** The scoring loop's fixed 10-per-call slice (mirrors stageImportance). */
|
|
42
|
+
const LLM_SLICE = 10;
|
|
43
|
+
const PRE_RESCORE_BACKUP_PATTERN = /^pre-rescore-.*\.db$/;
|
|
44
|
+
function percentile(sorted, p) {
|
|
45
|
+
if (sorted.length === 0)
|
|
46
|
+
return Number.NaN;
|
|
47
|
+
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
|
|
48
|
+
return sorted[idx];
|
|
49
|
+
}
|
|
50
|
+
function distributionPreview(values) {
|
|
51
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
52
|
+
return {
|
|
53
|
+
n: values.length,
|
|
54
|
+
min: percentile(sorted, 0),
|
|
55
|
+
p25: percentile(sorted, 25),
|
|
56
|
+
median: percentile(sorted, 50),
|
|
57
|
+
p75: percentile(sorted, 75),
|
|
58
|
+
p90: percentile(sorted, 90),
|
|
59
|
+
max: percentile(sorted, 100),
|
|
60
|
+
atCeiling: values.filter((v) => v >= calibration_js_1.IMPORTANCE_CEILING).length,
|
|
61
|
+
atSentinel: values.filter((v) => v === 0.5).length,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function readConfig(stateDir) {
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(stateDir, "config.json"), "utf-8"));
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function fetchChunk(db, cursor, limit) {
|
|
73
|
+
return db
|
|
74
|
+
.prepare(`SELECT rowid AS __rowid, id, base_strength FROM memories
|
|
75
|
+
WHERE rowid > ? AND COALESCE(status, '') != 'absorbed'
|
|
76
|
+
ORDER BY rowid ASC LIMIT ?`)
|
|
77
|
+
.all(cursor, limit);
|
|
78
|
+
}
|
|
79
|
+
function countRemaining(db, cursor) {
|
|
80
|
+
return db
|
|
81
|
+
.prepare(`SELECT COUNT(*) AS n FROM memories
|
|
82
|
+
WHERE rowid > ? AND COALESCE(status, '') != 'absorbed'`)
|
|
83
|
+
.get(cursor).n;
|
|
84
|
+
}
|
|
85
|
+
/** Full Memory rows for the ids in `rows` (the scoring loop's input type). */
|
|
86
|
+
function memoriesForIds(db, rows) {
|
|
87
|
+
const byId = new Map(rows.map((r) => {
|
|
88
|
+
const mem = db
|
|
89
|
+
.prepare("SELECT * FROM memories WHERE id = ?")
|
|
90
|
+
.get(r.id);
|
|
91
|
+
return [r.id, mem];
|
|
92
|
+
}));
|
|
93
|
+
return rows.map((r) => byId.get(r.id)).filter((m) => m !== undefined);
|
|
94
|
+
}
|
|
95
|
+
/** Pre-run DB backup (the dedup precedent): throws on failure — abort. */
|
|
96
|
+
async function takePreRescoreBackup(db, stateDir, config) {
|
|
97
|
+
const backupDir = (0, node_path_1.join)(stateDir, "backups");
|
|
98
|
+
(0, node_fs_2.mkdirSync)(backupDir, { recursive: true });
|
|
99
|
+
const backupPath = (0, node_path_1.join)(backupDir, `pre-rescore-${new Date().toISOString().replace(/[:.]/g, "-")}.db`);
|
|
100
|
+
await db.backup(backupPath);
|
|
101
|
+
const retention = (0, config_read_js_1.readNonNegativeConfig)(config ?? {}, "backupRetention", backup_js_1.DEFAULT_BACKUP_RETENTION);
|
|
102
|
+
(0, backup_js_1.pruneBackupArtifacts)(backupDir, retention, PRE_RESCORE_BACKUP_PATTERN);
|
|
103
|
+
return backupPath;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Run the rescore pass. Returns a structured report. Throws on setup errors
|
|
107
|
+
* (client mode, no LLM, backup failure) — the cursor always reflects the
|
|
108
|
+
* last committed batch.
|
|
109
|
+
*/
|
|
110
|
+
async function runRescoreImportance(options = {}) {
|
|
111
|
+
const batchSize = options.batchSize ?? DEFAULT_BATCH;
|
|
112
|
+
const stateDir = options.stateDir ?? HICORTEX_HOME;
|
|
113
|
+
const apply = options.apply ?? false;
|
|
114
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
115
|
+
throw new Error(`[hicortex] rescore-importance: invalid --batch value: ${options.batchSize}`);
|
|
116
|
+
}
|
|
117
|
+
const config = options.config !== undefined ? options.config : readConfig(stateDir);
|
|
118
|
+
// Server-mode only — client installs have no local DB (classify-domains).
|
|
119
|
+
if (config?.mode === "client") {
|
|
120
|
+
throw new Error("[hicortex] rescore-importance is server-mode only (it needs the local DB). " +
|
|
121
|
+
`This machine is a client of ${config.serverUrl ?? "a remote server"} — run it on the server.`);
|
|
122
|
+
}
|
|
123
|
+
let llm;
|
|
124
|
+
if (options.llm) {
|
|
125
|
+
llm = options.llm;
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
const resolved = (0, llm_js_1.resolveSavedLlmConfig)(config);
|
|
129
|
+
if (!resolved.config) {
|
|
130
|
+
throw new Error("[hicortex] rescore-importance: no LLM configured — run `npx @gamaze/hicortex init`.");
|
|
131
|
+
}
|
|
132
|
+
llm = new llm_js_1.LlmClient(resolved.config);
|
|
133
|
+
}
|
|
134
|
+
const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
|
|
135
|
+
const db = (0, db_js_1.initDb)(dbPath);
|
|
136
|
+
try {
|
|
137
|
+
let cursor = options.reset ? 0 : ((0, state_js_1.loadState)(stateDir).rescoreImportanceCursor ?? 0);
|
|
138
|
+
const remaining = countRemaining(db, cursor);
|
|
139
|
+
const plannedCalls = Math.ceil(Math.min(remaining, batchSize) / LLM_SLICE);
|
|
140
|
+
// Current distribution preview over the REMAINING rows (what is queued).
|
|
141
|
+
const remainingRows = fetchChunk(db, cursor, Number.MAX_SAFE_INTEGER);
|
|
142
|
+
const currentDistribution = distributionPreview(remainingRows.map((r) => r.base_strength ?? 0.5));
|
|
143
|
+
console.log(`[hicortex] rescore-importance ${apply ? "APPLY" : "dry run"}: ${remaining} live rows queued, ` +
|
|
144
|
+
`batch ${batchSize}, cursor ${cursor}${options.reset ? " (reset)" : ""}, ` +
|
|
145
|
+
`~${plannedCalls} LLM calls this invocation (10 rows each)`);
|
|
146
|
+
console.log(`[hicortex] current base_strength of queued rows: median ${currentDistribution.median.toFixed(2)}, ` +
|
|
147
|
+
`p90 ${currentDistribution.p90.toFixed(2)}, max ${currentDistribution.max.toFixed(2)}, ` +
|
|
148
|
+
`at ceiling ${currentDistribution.atCeiling}, at 0.5 sentinel ${currentDistribution.atSentinel}`);
|
|
149
|
+
if (!apply) {
|
|
150
|
+
console.log("[hicortex] rescore-importance dry run complete — zero writes. Re-run with --apply to execute.");
|
|
151
|
+
return {
|
|
152
|
+
dryRun: true,
|
|
153
|
+
remaining,
|
|
154
|
+
rescored: 0,
|
|
155
|
+
failed: 0,
|
|
156
|
+
calls: 0,
|
|
157
|
+
cursor,
|
|
158
|
+
aborted: false,
|
|
159
|
+
currentDistribution,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
// Backup FIRST (dedup precedent) — abort with zero writes if it fails.
|
|
163
|
+
const backupPath = await takePreRescoreBackup(db, stateDir, config);
|
|
164
|
+
console.log(`[hicortex] rescore-importance backup written: ${backupPath}`);
|
|
165
|
+
const chunk = fetchChunk(db, cursor, batchSize);
|
|
166
|
+
const before = distributionPreview(chunk.map((r) => r.base_strength ?? 0.5));
|
|
167
|
+
let rescored = 0;
|
|
168
|
+
let failed = 0;
|
|
169
|
+
let calls = 0;
|
|
170
|
+
let committedRowid = cursor;
|
|
171
|
+
let aborted = false;
|
|
172
|
+
const touchedIds = [];
|
|
173
|
+
// LLM slices of 10 INSIDE the invocation chunk, cursor-ordered: an infra
|
|
174
|
+
// error holds the cursor at the last fully committed slice (the
|
|
175
|
+
// classify-domains posture — the failing rows are untouched, re-run
|
|
176
|
+
// resumes there).
|
|
177
|
+
for (let i = 0; i < chunk.length; i += LLM_SLICE) {
|
|
178
|
+
const sliceRows = chunk.slice(i, i + LLM_SLICE);
|
|
179
|
+
const memories = memoriesForIds(db, sliceRows);
|
|
180
|
+
const r = await (0, consolidate_js_1.scoreMemoriesImportance)(db, memories, llm, {
|
|
181
|
+
onBatch: (written, batchFailed) => {
|
|
182
|
+
calls++;
|
|
183
|
+
},
|
|
184
|
+
});
|
|
185
|
+
if (r.failed > 0) {
|
|
186
|
+
// The slice's call threw (endpoint down) or a write failed — nothing
|
|
187
|
+
// usable came out of it. Stop; the cursor stays at the last
|
|
188
|
+
// committed slice's end.
|
|
189
|
+
failed += r.failed;
|
|
190
|
+
aborted = true;
|
|
191
|
+
console.warn(`[hicortex] rescore-importance ABORTED on a scoring-endpoint error ` +
|
|
192
|
+
`(slice at rowid ${sliceRows[0].__rowid}). Cursor at last committed slice — ` +
|
|
193
|
+
`re-run when the endpoint is back up.`);
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
rescored += r.scored;
|
|
197
|
+
touchedIds.push(...sliceRows.map((row) => row.id));
|
|
198
|
+
committedRowid = sliceRows[sliceRows.length - 1].__rowid;
|
|
199
|
+
(0, state_js_1.updateState)((s) => { s.rescoreImportanceCursor = committedRowid; }, stateDir);
|
|
200
|
+
if ((i / LLM_SLICE) % 25 === 0) {
|
|
201
|
+
console.log(`[hicortex] ${rescored} rows re-judged (cursor ${committedRowid}, ` +
|
|
202
|
+
`${countRemaining(db, committedRowid)} remaining)`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
cursor = committedRowid;
|
|
206
|
+
// before → after over the rows this invocation actually touched.
|
|
207
|
+
const afterRows = touchedIds
|
|
208
|
+
.map((id) => db
|
|
209
|
+
.prepare("SELECT base_strength FROM memories WHERE id = ?")
|
|
210
|
+
.get(id))
|
|
211
|
+
.filter((r) => r !== undefined);
|
|
212
|
+
const after = distributionPreview(afterRows.map((r) => r.base_strength ?? 0.5));
|
|
213
|
+
const afterLog = (label, d) => `${label}: median ${d.median.toFixed(2)} p90 ${d.p90.toFixed(2)} max ${d.max.toFixed(2)} at-ceiling ${d.atCeiling}`;
|
|
214
|
+
console.log(`[hicortex] rescore-importance ${aborted ? "ABORTED" : "chunk complete"}: ` +
|
|
215
|
+
`${rescored} re-judged, ${failed} failed, ${calls} calls, cursor ${cursor}, ` +
|
|
216
|
+
`${countRemaining(db, cursor)} remaining`);
|
|
217
|
+
console.log(`[hicortex] re-judged rows — ${afterLog("before", before)}`);
|
|
218
|
+
console.log(`[hicortex] re-judged rows — ${afterLog("after ", after)}`);
|
|
219
|
+
return {
|
|
220
|
+
dryRun: false,
|
|
221
|
+
remaining: countRemaining(db, cursor),
|
|
222
|
+
rescored,
|
|
223
|
+
failed,
|
|
224
|
+
calls,
|
|
225
|
+
cursor,
|
|
226
|
+
aborted,
|
|
227
|
+
backupPath,
|
|
228
|
+
currentDistribution,
|
|
229
|
+
before,
|
|
230
|
+
after,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
finally {
|
|
234
|
+
db.close();
|
|
235
|
+
}
|
|
236
|
+
}
|
package/dist/retrieval.d.ts
CHANGED
|
@@ -71,6 +71,8 @@ export interface ScoringWeights {
|
|
|
71
71
|
rrfFtsWeight: number;
|
|
72
72
|
/** #205 per-list RRF weight for the vector list (KNN-driven candidates). */
|
|
73
73
|
rrfVectorWeight: number;
|
|
74
|
+
/** #425 additive boost for both-channel (vector AND FTS) candidates. */
|
|
75
|
+
bothChannelBoost: number;
|
|
74
76
|
}
|
|
75
77
|
/**
|
|
76
78
|
* Configure scoring weights + ranking knobs from RESOLVED overrides (the
|
|
@@ -208,6 +210,12 @@ export declare function cosineBetweenVectors(a: Float32Array, b: Float32Array):
|
|
|
208
210
|
/**
|
|
209
211
|
* Compute decayed strength with adaptive decay (B+E+D model).
|
|
210
212
|
* Exported for use by consolidation decay/prune stage.
|
|
213
|
+
*
|
|
214
|
+
* #425 read-side law: the decay-relevant importance is CLAMPED at the
|
|
215
|
+
* release-managed ceiling (calibration.ts IMPORTANCE_CEILING) — at importance
|
|
216
|
+
* exactly 1.0 the decay rate is exactly 1.0 and the row never decays, so
|
|
217
|
+
* legacy base-1.0 rows (and any write site that predates the cap) decay
|
|
218
|
+
* again. The clamp applies to explicit importance passes too.
|
|
211
219
|
*/
|
|
212
220
|
export declare function effectiveStrength(baseStrength: number, lastAccessed: string | null, now: Date, options?: {
|
|
213
221
|
importance?: number;
|
|
@@ -242,6 +250,10 @@ export declare function computeScore(memory: Memory, distance: number, connectio
|
|
|
242
250
|
tag: string;
|
|
243
251
|
weight: number | null;
|
|
244
252
|
}>;
|
|
253
|
+
/** #425: the candidate was matched by BOTH retrieval channels (vector
|
|
254
|
+
* KNN AND BM25 FTS) — the genuine-match signature. Adds the
|
|
255
|
+
* release-managed bothChannelBoost (zero-boost neutral). */
|
|
256
|
+
bothChannel?: boolean;
|
|
245
257
|
}): number;
|
|
246
258
|
export interface EmbedFn {
|
|
247
259
|
(text: string): Promise<Float32Array>;
|
package/dist/retrieval.js
CHANGED
|
@@ -146,6 +146,10 @@ const SCORING_DEFAULTS = {
|
|
|
146
146
|
rrfCompositeWeight: CALIBRATION.RRF_COMPOSITE_WEIGHT,
|
|
147
147
|
rrfFtsWeight: CALIBRATION.RRF_FTS_WEIGHT,
|
|
148
148
|
rrfVectorWeight: CALIBRATION.RRF_VECTOR_WEIGHT,
|
|
149
|
+
// #425 (calibration.ts): the both-channel genuine-match boost. 0.10 is
|
|
150
|
+
// the sweep-chosen size — D3's dominance margin (>= 0.10 x the similarity
|
|
151
|
+
// weight) with the battery stability gates intact.
|
|
152
|
+
bothChannelBoost: CALIBRATION.BOTH_CHANNEL_BOOST,
|
|
149
153
|
};
|
|
150
154
|
let scoringWeights = { ...SCORING_DEFAULTS };
|
|
151
155
|
/**
|
|
@@ -182,6 +186,7 @@ function configureScoring(overrides) {
|
|
|
182
186
|
rrfCompositeWeight: num(overrides?.rrfCompositeWeight, SCORING_DEFAULTS.rrfCompositeWeight, 0, 1),
|
|
183
187
|
rrfFtsWeight: numW(overrides?.rrfFtsWeight, SCORING_DEFAULTS.rrfFtsWeight),
|
|
184
188
|
rrfVectorWeight: numW(overrides?.rrfVectorWeight, SCORING_DEFAULTS.rrfVectorWeight),
|
|
189
|
+
bothChannelBoost: num(overrides?.bothChannelBoost, SCORING_DEFAULTS.bothChannelBoost, 0, 1),
|
|
185
190
|
};
|
|
186
191
|
return { ...scoringWeights };
|
|
187
192
|
}
|
|
@@ -483,9 +488,16 @@ function parseTimestamp(ts) {
|
|
|
483
488
|
/**
|
|
484
489
|
* Compute decayed strength with adaptive decay (B+E+D model).
|
|
485
490
|
* Exported for use by consolidation decay/prune stage.
|
|
491
|
+
*
|
|
492
|
+
* #425 read-side law: the decay-relevant importance is CLAMPED at the
|
|
493
|
+
* release-managed ceiling (calibration.ts IMPORTANCE_CEILING) — at importance
|
|
494
|
+
* exactly 1.0 the decay rate is exactly 1.0 and the row never decays, so
|
|
495
|
+
* legacy base-1.0 rows (and any write site that predates the cap) decay
|
|
496
|
+
* again. The clamp applies to explicit importance passes too.
|
|
486
497
|
*/
|
|
487
498
|
function effectiveStrength(baseStrength, lastAccessed, now, options) {
|
|
488
|
-
const
|
|
499
|
+
const rawImportance = options?.importance ?? baseStrength;
|
|
500
|
+
const importance = Math.min(rawImportance, CALIBRATION.IMPORTANCE_CEILING);
|
|
489
501
|
const accessCount = options?.accessCount ?? 0;
|
|
490
502
|
const linkCount = options?.linkCount ?? 0;
|
|
491
503
|
const hours = Math.max((now.getTime() - parseTimestamp(lastAccessed).getTime()) / 3_600_000, 0);
|
|
@@ -512,6 +524,11 @@ function computeScore(memory, distance, connectionCount, maxConnections, now, op
|
|
|
512
524
|
// is a data-driven follow-up if the eval shows it is needed.
|
|
513
525
|
const similarity = Math.max(0, l2ToCosine(distance));
|
|
514
526
|
const effStrength = effectiveStrength(memory.base_strength ?? 0.5, memory.last_accessed, now, {
|
|
527
|
+
// #425: importance passed EXPLICITLY (the same default value
|
|
528
|
+
// effectiveStrength would apply — base strength IS importance at read
|
|
529
|
+
// time — now stated at the call site so the triple-win coupling
|
|
530
|
+
// (score share, decay rate, floor) is visible and single-sourced).
|
|
531
|
+
importance: memory.base_strength ?? 0.5,
|
|
515
532
|
accessCount: memory.access_count ?? 0,
|
|
516
533
|
linkCount: connectionCount,
|
|
517
534
|
});
|
|
@@ -567,6 +584,15 @@ function computeScore(memory, distance, connectionCount, maxConnections, now, op
|
|
|
567
584
|
score += maxWeight * scoringWeights.domainAffinity;
|
|
568
585
|
}
|
|
569
586
|
}
|
|
587
|
+
// #425 both-channel boost: vector KNN and BM25 FTS AGREEING on a candidate
|
|
588
|
+
// is the genuine-match signature (a distinctive proper noun the user knows
|
|
589
|
+
// exists — the field failure this fixes: 0.90/0.95-strength domain-adjacent
|
|
590
|
+
// memories outranked the best-similarity exact-token match). ADDITIVE,
|
|
591
|
+
// zero-boost neutral, never a penalty; rides the composite side only (like
|
|
592
|
+
// projectAffinity — the RRF side is #205 territory); applied BEFORE the
|
|
593
|
+
// superseded multiplier so a superseded both-channel row still demotes.
|
|
594
|
+
if (options?.bothChannel)
|
|
595
|
+
score += scoringWeights.bothChannelBoost;
|
|
570
596
|
// Superseded demotion (#191 Phase B): a memory whose decision was reversed by
|
|
571
597
|
// a later one keeps its content and strength but must not outrank the
|
|
572
598
|
// decision that replaced it. Applied as an explicit multiplier here rather
|
|
@@ -802,6 +828,9 @@ async function retrieve(db, embedFn, query, options) {
|
|
|
802
828
|
superseded: supersededIds.has(mid),
|
|
803
829
|
scope,
|
|
804
830
|
tagWeights: tagWeightsByMemory?.get(mid),
|
|
831
|
+
// #425: the two retrieval channels agreeing is the genuine-match
|
|
832
|
+
// signature — graph-only and single-channel candidates add nothing.
|
|
833
|
+
bothChannel: source === "both",
|
|
805
834
|
});
|
|
806
835
|
const effStr = effectiveStrength(mem.base_strength ?? 0.5, mem.last_accessed, now, {
|
|
807
836
|
accessCount: mem.access_count ?? 0,
|
package/dist/stages.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory maturity stages — the console's derived presentation layer
|
|
3
|
+
* (#409/#421 Phase 1).
|
|
4
|
+
*
|
|
5
|
+
* The E-reframe (standing ruling in the #409 thread): stages are DERIVED
|
|
6
|
+
* presentation of the one numeric strength signal, never stored state. This
|
|
7
|
+
* module holds the pure derivation only — thresholds live in calibration.ts
|
|
8
|
+
* (the #408 single home, release-managed with the evolution contract); no
|
|
9
|
+
* db, no I/O, no config reads.
|
|
10
|
+
*
|
|
11
|
+
* Wire keys are forming/belief/truth/fading (lifecycle order). The console
|
|
12
|
+
* renders human labels over these keys; nothing else consumes them.
|
|
13
|
+
*/
|
|
14
|
+
/** A memory's derived maturity stage (lifecycle order). */
|
|
15
|
+
export type Stage = "forming" | "belief" | "truth" | "fading";
|
|
16
|
+
/** The four stages in lifecycle order: forming → belief → truth → fading. */
|
|
17
|
+
export declare const STAGE_KEYS: readonly Stage[];
|
|
18
|
+
/**
|
|
19
|
+
* Derive a memory's stage from its effective strength and recency.
|
|
20
|
+
*
|
|
21
|
+
* The recency gate runs FIRST: a memory untouched for
|
|
22
|
+
* STAGE_FADING_DAYS days is Fading no matter how strong it is — decay is a
|
|
23
|
+
* function of access in this system (effectiveStrength already folds
|
|
24
|
+
* last_accessed in, but the gate makes long silence legible on its own).
|
|
25
|
+
* Then the strength bands (calibrated against the measured production
|
|
26
|
+
* distribution — see calibration.ts): < FADING → fading (the weak cluster),
|
|
27
|
+
* ≥ TRUTH → truth, ≥ BELIEF → belief, else forming.
|
|
28
|
+
*
|
|
29
|
+
* `daysSinceAccess` is whole days (UTC day diff is fine — capture is
|
|
30
|
+
* night-resolution) since the memory was last touched. The DASHBOARD handler
|
|
31
|
+
* passes days since `last_accessed`, falling back to days since `created_at`
|
|
32
|
+
* when last_accessed is NULL (never accessed since ingest — creation is the
|
|
33
|
+
* last real signal). `null` means "recency unknown" and skips the gate
|
|
34
|
+
* (strength-only derivation) — the handler never sends it; it exists so the
|
|
35
|
+
* function is total over caller data quality.
|
|
36
|
+
*/
|
|
37
|
+
export declare function deriveStage(effectiveStrength: number, daysSinceAccess: number | null): Stage;
|
package/dist/stages.js
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Memory maturity stages — the console's derived presentation layer
|
|
4
|
+
* (#409/#421 Phase 1).
|
|
5
|
+
*
|
|
6
|
+
* The E-reframe (standing ruling in the #409 thread): stages are DERIVED
|
|
7
|
+
* presentation of the one numeric strength signal, never stored state. This
|
|
8
|
+
* module holds the pure derivation only — thresholds live in calibration.ts
|
|
9
|
+
* (the #408 single home, release-managed with the evolution contract); no
|
|
10
|
+
* db, no I/O, no config reads.
|
|
11
|
+
*
|
|
12
|
+
* Wire keys are forming/belief/truth/fading (lifecycle order). The console
|
|
13
|
+
* renders human labels over these keys; nothing else consumes them.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.STAGE_KEYS = void 0;
|
|
17
|
+
exports.deriveStage = deriveStage;
|
|
18
|
+
const calibration_js_1 = require("./calibration.js");
|
|
19
|
+
/** The four stages in lifecycle order: forming → belief → truth → fading. */
|
|
20
|
+
exports.STAGE_KEYS = ["forming", "belief", "truth", "fading"];
|
|
21
|
+
/**
|
|
22
|
+
* Derive a memory's stage from its effective strength and recency.
|
|
23
|
+
*
|
|
24
|
+
* The recency gate runs FIRST: a memory untouched for
|
|
25
|
+
* STAGE_FADING_DAYS days is Fading no matter how strong it is — decay is a
|
|
26
|
+
* function of access in this system (effectiveStrength already folds
|
|
27
|
+
* last_accessed in, but the gate makes long silence legible on its own).
|
|
28
|
+
* Then the strength bands (calibrated against the measured production
|
|
29
|
+
* distribution — see calibration.ts): < FADING → fading (the weak cluster),
|
|
30
|
+
* ≥ TRUTH → truth, ≥ BELIEF → belief, else forming.
|
|
31
|
+
*
|
|
32
|
+
* `daysSinceAccess` is whole days (UTC day diff is fine — capture is
|
|
33
|
+
* night-resolution) since the memory was last touched. The DASHBOARD handler
|
|
34
|
+
* passes days since `last_accessed`, falling back to days since `created_at`
|
|
35
|
+
* when last_accessed is NULL (never accessed since ingest — creation is the
|
|
36
|
+
* last real signal). `null` means "recency unknown" and skips the gate
|
|
37
|
+
* (strength-only derivation) — the handler never sends it; it exists so the
|
|
38
|
+
* function is total over caller data quality.
|
|
39
|
+
*/
|
|
40
|
+
function deriveStage(effectiveStrength, daysSinceAccess) {
|
|
41
|
+
if (daysSinceAccess !== null && daysSinceAccess >= calibration_js_1.STAGE_FADING_DAYS) {
|
|
42
|
+
return "fading";
|
|
43
|
+
}
|
|
44
|
+
if (effectiveStrength < calibration_js_1.STAGE_FADING_STRENGTH)
|
|
45
|
+
return "fading";
|
|
46
|
+
if (effectiveStrength >= calibration_js_1.STAGE_TRUTH_STRENGTH)
|
|
47
|
+
return "truth";
|
|
48
|
+
if (effectiveStrength >= calibration_js_1.STAGE_BELIEF_STRENGTH)
|
|
49
|
+
return "belief";
|
|
50
|
+
return "forming";
|
|
51
|
+
}
|
package/dist/state.d.ts
CHANGED
|
@@ -62,14 +62,32 @@ export interface HicortexState {
|
|
|
62
62
|
/**
|
|
63
63
|
* Resume cursor for the nightly's reconsolidation stage (#384) — highest
|
|
64
64
|
* memories.rowid whose candidates have been evaluated (or infra-skipped)
|
|
65
|
-
*
|
|
66
|
-
* discipline as supersessionCursor,
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
65
|
+
* with all their CONFIRMED work applied. Absent/0 = never run. Same
|
|
66
|
+
* advance-past-considered-candidates discipline as supersessionCursor,
|
|
67
|
+
* with one addition (#439): confirmed merges and rewrite groups apply at
|
|
68
|
+
* the candidate boundary — the END of the iteration that confirmed them —
|
|
69
|
+
* and the cursor advances past a candidate only when that apply landed.
|
|
70
|
+
* A deferral (budget refusal at the rewrite call, deadline at the
|
|
71
|
+
* boundary, backup failure, lock-busy merge surviving the final drain)
|
|
72
|
+
* holds the cursor BELOW the current candidate, so the hold is bounded to
|
|
73
|
+
* ONE candidate's pairs: next run re-detects and re-judges exactly those
|
|
74
|
+
* (dup-over-loss — a confirmed resolution is never silently dropped by the
|
|
75
|
+
* cursor passing it). The ONE cross-candidate window is the lock-busy
|
|
76
|
+
* retry list (plus deadline/backup-dropped tails re-queued at their
|
|
77
|
+
* boundary): while any retry merge is pending, every persisted checkpoint
|
|
78
|
+
* clamps below its earliest contributor, so a killed or stopped run can
|
|
79
|
+
* never strand a confirmed merge behind the cursor (fix round, #440).
|
|
71
80
|
*/
|
|
72
81
|
reconsolidationCursor?: number;
|
|
82
|
+
/**
|
|
83
|
+
* #439 scan high-water for the reconsolidation stage — the highest
|
|
84
|
+
* memories.rowid any run has ENTERED, never held back by un-applied work
|
|
85
|
+
* (unlike reconsolidationCursor, which holds below deferred applies).
|
|
86
|
+
* Seeds the re-judged/new verdict split: a verdict call on a candidate
|
|
87
|
+
* at/below this mark is a re-judgment of previously judged work. Absent =
|
|
88
|
+
* never run; persisted alongside the cursor at every checkpoint.
|
|
89
|
+
*/
|
|
90
|
+
reconsolidationScannedRowid?: number;
|
|
73
91
|
/**
|
|
74
92
|
* Resume cursor for `hicortex classify-types` (#216) — highest memories.rowid
|
|
75
93
|
* whose batch has been fully committed. Absent/0 = never run (or reset).
|
|
@@ -77,6 +95,14 @@ export interface HicortexState {
|
|
|
77
95
|
* interruption never loses more than the in-flight batch.
|
|
78
96
|
*/
|
|
79
97
|
typeCursor?: number;
|
|
98
|
+
/**
|
|
99
|
+
* Resume cursor for `hicortex rescore-importance` (#425) — highest live
|
|
100
|
+
* memories.rowid re-judged under the current rubric. Absent/0 = never run
|
|
101
|
+
* (or reset). Advances per committed batch (LLM calls are 10 rows each
|
|
102
|
+
* inside a `--batch` slice); an infra error holds it at the last fully
|
|
103
|
+
* committed slice so a re-run resumes cleanly.
|
|
104
|
+
*/
|
|
105
|
+
rescoreImportanceCursor?: number;
|
|
80
106
|
/**
|
|
81
107
|
* LLM token usage accrued this billing period (#246). Period reset is
|
|
82
108
|
* monthly: when `periodStart` is in a previous calendar month, the totals
|
package/dist/storage.d.ts
CHANGED
|
@@ -8,6 +8,13 @@ import type { Memory, MemoryLink, InsertMemoryOptions } from "./types.js";
|
|
|
8
8
|
* Serialize a Float32Array embedding to a Buffer for sqlite-vec.
|
|
9
9
|
*/
|
|
10
10
|
export declare function embedToBlob(embedding: Float32Array): Buffer;
|
|
11
|
+
/**
|
|
12
|
+
* #421 machine × harness identity: optional capture-machine provenance on
|
|
13
|
+
* /ingest + /distill. A string, trimmed, capped at 128 chars; anything else
|
|
14
|
+
* (absent, wrong type, blank) becomes NULL — a wrong machine name is worse
|
|
15
|
+
* than none. Never filtered; presentation grouping only.
|
|
16
|
+
*/
|
|
17
|
+
export declare function sanitizeSourceMachine(v: unknown): string | null;
|
|
11
18
|
/**
|
|
12
19
|
* Insert a memory and its vector embedding. Returns the memory's UUID.
|
|
13
20
|
*
|
|
@@ -35,6 +42,28 @@ export declare function updateMemory(db: Database.Database, memoryId: string, fi
|
|
|
35
42
|
* Atomically increment access_count and reset last_accessed.
|
|
36
43
|
*/
|
|
37
44
|
export declare function strengthenMemory(db: Database.Database, memoryId: string, nowIsoStr: string): void;
|
|
45
|
+
/**
|
|
46
|
+
* Record an owner corroboration (#423 phase 3): one UPDATE that bumps
|
|
47
|
+
* corroboration_count, nudges base_strength up by the calibration delta
|
|
48
|
+
* (MIN-capped at the importance ceiling, #425; COALESCE keeps NULL bases
|
|
49
|
+
* honest — the unscored default), stamps the importance watermark, and
|
|
50
|
+
* refreshes last_accessed (the console's "last confirmed" cell reads it: an
|
|
51
|
+
* enrich IS a confirmation, and it keeps the stage fading gate honest).
|
|
52
|
+
* EVIDENCE ABOUT IMPORTANCE ONLY — never access_count (reserved for real
|
|
53
|
+
* recall use) nor shown_count (index exposure): faking either corrupts the
|
|
54
|
+
* uses-per-showing adoption metric.
|
|
55
|
+
*
|
|
56
|
+
* Nightly interaction: the importance pool keys on importance_scored_at IS
|
|
57
|
+
* NULL (#425), so the watermark stamp here keeps the pre-#425 contract — an
|
|
58
|
+
* enriched memory leaves the pool and the owner's mark stands in for (and is
|
|
59
|
+
* never stomped by) the first LLM score. Dedup merges take
|
|
60
|
+
* max(base_strength), so a merge never loses the mark. Returns the fresh row
|
|
61
|
+
* values (re-read after the update); null when the id matches nothing.
|
|
62
|
+
*/
|
|
63
|
+
export declare function enrichMemory(db: Database.Database, memoryId: string, nowIsoStr: string): {
|
|
64
|
+
corroborationCount: number;
|
|
65
|
+
baseStrength: number;
|
|
66
|
+
} | null;
|
|
38
67
|
/**
|
|
39
68
|
* Record that memories appeared in a pushed recall index (#192): bump
|
|
40
69
|
* shown_count and refresh last_accessed (a mild strengthen — the decay clock
|
|
@@ -267,8 +296,11 @@ export declare function getPruneCandidates(db: Database.Database, cutoffIso: str
|
|
|
267
296
|
*/
|
|
268
297
|
export declare function getAllLinkCounts(db: Database.Database): Map<string, number>;
|
|
269
298
|
/**
|
|
270
|
-
* Get all memories
|
|
271
|
-
*
|
|
299
|
+
* Get all never-scored memories — the nightly importance pool, keyed on the
|
|
300
|
+
* importance_scored_at watermark (#425, migration v19). The pre-v19 sentinel
|
|
301
|
+
* (`base_strength = 0.5`) re-rolled every row the model genuinely scored
|
|
302
|
+
* 0.5, every night; NULL watermark = never scored under ANY rubric. Absorbed
|
|
303
|
+
* memories are excluded (#384): they are invisible to recall, so
|
|
272
304
|
* importance-scoring one would spend an LLM call on dead evidence.
|
|
273
305
|
*/
|
|
274
306
|
export declare function getUnscoredMemories(db: Database.Database): Memory[];
|