@gamaze/hicortex 0.20.7 → 0.20.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -41
- package/assets/dashboard.html +3989 -836
- package/dist/calibration.d.ts +293 -0
- package/dist/calibration.js +379 -0
- package/dist/capture-health.d.ts +87 -0
- package/dist/capture-health.js +106 -0
- package/dist/capture-pause.d.ts +86 -0
- package/dist/capture-pause.js +127 -0
- package/dist/capture.d.ts +24 -3
- package/dist/capture.js +11 -1
- package/dist/classify-domains.d.ts +6 -0
- package/dist/classify-domains.js +7 -1
- package/dist/cli.js +38 -3
- package/dist/config-read.d.ts +1 -1
- package/dist/config-read.js +96 -9
- package/dist/consolidate.d.ts +114 -68
- package/dist/consolidate.js +302 -182
- package/dist/dashboard.d.ts +326 -6
- package/dist/dashboard.js +592 -7
- package/dist/db.js +105 -0
- package/dist/dedup.d.ts +34 -26
- package/dist/dedup.js +91 -57
- package/dist/distiller.js +1 -1
- package/dist/domain-classify.d.ts +7 -6
- package/dist/domain-classify.js +12 -10
- package/dist/eval/decay-eval.d.ts +3 -3
- package/dist/eval/decay-eval.js +4 -4
- package/dist/eval/importance-eval.d.ts +85 -0
- package/dist/eval/importance-eval.js +286 -0
- package/dist/eval/planted-eval.d.ts +26 -0
- package/dist/eval/planted-eval.js +97 -0
- package/dist/eval/planted-fixtures.d.ts +107 -0
- package/dist/eval/planted-fixtures.js +283 -0
- package/dist/eval/planted-harness.d.ts +176 -0
- package/dist/eval/planted-harness.js +649 -0
- package/dist/eval/ranking-battery.d.ts +78 -0
- package/dist/eval/ranking-battery.js +181 -0
- package/dist/eval/ranking-eval.d.ts +41 -0
- package/dist/eval/ranking-eval.js +391 -0
- package/dist/eval/ranking-fixtures.d.ts +77 -0
- package/dist/eval/ranking-fixtures.js +226 -0
- package/dist/identity-store.d.ts +21 -0
- package/dist/identity-store.js +49 -0
- package/dist/index.js +4 -3
- package/dist/init.d.ts +23 -3
- package/dist/init.js +84 -9
- package/dist/llm.d.ts +43 -58
- package/dist/llm.js +87 -101
- package/dist/mcp-server.d.ts +12 -0
- package/dist/mcp-server.js +213 -32
- package/dist/nightly.d.ts +9 -1
- package/dist/nightly.js +164 -110
- package/dist/nofit.d.ts +4 -11
- package/dist/nofit.js +6 -23
- package/dist/prompts.d.ts +10 -0
- package/dist/prompts.js +28 -5
- package/dist/recall-index.d.ts +30 -28
- package/dist/recall-index.js +21 -18
- package/dist/recall-registry.d.ts +2 -1
- package/dist/recall-registry.js +35 -1
- package/dist/reconsolidation.d.ts +168 -87
- package/dist/reconsolidation.js +818 -377
- package/dist/relink.js +3 -4
- package/dist/rescore-importance.d.ts +80 -0
- package/dist/rescore-importance.js +236 -0
- package/dist/retrieval.d.ts +80 -35
- package/dist/retrieval.js +322 -105
- package/dist/run-deadline.d.ts +62 -0
- package/dist/run-deadline.js +73 -0
- package/dist/schema-prototypes.d.ts +3 -3
- package/dist/schema-prototypes.js +3 -3
- package/dist/stages.d.ts +37 -0
- package/dist/stages.js +51 -0
- package/dist/state.d.ts +34 -9
- package/dist/storage.d.ts +50 -18
- package/dist/storage.js +125 -30
- package/dist/telemetry.d.ts +8 -7
- package/dist/token-budget.js +3 -4
- package/dist/type-classify.js +4 -4
- package/dist/types.d.ts +143 -155
- package/domains.example.json +4 -5
- package/hermes-plugin/hicortex/README.md +2 -2
- package/openclaw.plugin.json +1 -1
- package/package.json +4 -1
- package/pi-extension/hicortex/README.md +1 -1
- package/server.json +3 -3
package/dist/relink.js
CHANGED
|
@@ -134,7 +134,6 @@ async function runRelink(options = {}) {
|
|
|
134
134
|
`This machine is a client of ${config.serverUrl ?? "a remote server"} — run relink on the server.`);
|
|
135
135
|
}
|
|
136
136
|
// Classification is heuristic-only — no LLM client, no budget cap.
|
|
137
|
-
const budget = new consolidate_js_1.BudgetTracker(Number.MAX_SAFE_INTEGER);
|
|
138
137
|
const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
|
|
139
138
|
const db = (0, db_js_1.initDb)(dbPath);
|
|
140
139
|
const report = {
|
|
@@ -215,9 +214,9 @@ async function runRelink(options = {}) {
|
|
|
215
214
|
report.skippedExisting += batchSkippedExisting;
|
|
216
215
|
report.skippedDuplicate += batchSkippedDuplicate;
|
|
217
216
|
// Phase B: classification — shared heuristic-only path (LLM retired).
|
|
218
|
-
// classifyLinkCandidates
|
|
219
|
-
//
|
|
220
|
-
const classified = await (0, consolidate_js_1.classifyLinkCandidates)(candidates
|
|
217
|
+
// classifyLinkCandidates returns each candidate's heuristic type
|
|
218
|
+
// (extends/relates_to); #405 dropped its dead llm/budget params.
|
|
219
|
+
const classified = await (0, consolidate_js_1.classifyLinkCandidates)(candidates);
|
|
221
220
|
const types = classified.types;
|
|
222
221
|
report.llmClassified += classified.llmClassified; // always 0
|
|
223
222
|
report.heuristicFallback += classified.heuristicFallback;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `hicortex rescore-importance` (#425) — the one-shot LLM backfill that
|
|
3
|
+
* re-judges the EXISTING corpus under the re-anchored importance rubric
|
|
4
|
+
* (owner decision D1, 2026-09-13: rescore via LLM, resumable, local
|
|
5
|
+
* gateway).
|
|
6
|
+
*
|
|
7
|
+
* Precedents, deliberately mixed per the issue's attribution:
|
|
8
|
+
* - classify-domains (src/classify-domains.ts): resumable rowid cursor in
|
|
9
|
+
* state.json, --batch rows per invocation, --reset, server-mode-only,
|
|
10
|
+
* infra-error abort that leaves the cursor at the last committed batch.
|
|
11
|
+
* - dedup (src/dedup.ts): dry-run DEFAULT with --apply, and a DB backup
|
|
12
|
+
* taken FIRST — the CLI aborts (nothing written) if the backup fails.
|
|
13
|
+
*
|
|
14
|
+
* The scoring itself is the SHARED production loop — consolidate.ts
|
|
15
|
+
* `scoreMemoriesImportance` (the extracted stageImportance core): batches of
|
|
16
|
+
* 10, serial calls, the 0.95 write cap and the importance_scored_at
|
|
17
|
+
* watermark identical to the nightly. No forked scoring code.
|
|
18
|
+
*
|
|
19
|
+
* Scope: every LIVE (non-absorbed) memory, rowid-ascending. A row's
|
|
20
|
+
* corroboration_count survives untouched; base_strength is re-judged (that
|
|
21
|
+
* is D1's explicit trade: one rubric for all rows, the 2026-08-21-validated
|
|
22
|
+
* ORDERING is re-derived rather than mapped).
|
|
23
|
+
*/
|
|
24
|
+
import { LlmClient } from "./llm.js";
|
|
25
|
+
export interface RescoreImportanceOptions {
|
|
26
|
+
/** Execute (default: dry run — report only, zero writes, no backup). */
|
|
27
|
+
apply?: boolean;
|
|
28
|
+
/** Rows per invocation chunk (default 500). */
|
|
29
|
+
batchSize?: number;
|
|
30
|
+
/** Ignore the saved cursor and restart from rowid 0. */
|
|
31
|
+
reset?: boolean;
|
|
32
|
+
/** DB path override (tests). Defaults to resolveDbPath(). */
|
|
33
|
+
dbPath?: string;
|
|
34
|
+
/** State dir override (tests). Defaults to ~/.hicortex. */
|
|
35
|
+
stateDir?: string;
|
|
36
|
+
/** LLM override (tests). Bypasses config resolution. */
|
|
37
|
+
llm?: LlmClient;
|
|
38
|
+
/** Config override (tests). Defaults to reading stateDir/config.json. */
|
|
39
|
+
config?: Record<string, unknown> | null;
|
|
40
|
+
}
|
|
41
|
+
export interface RescoreImportanceReport {
|
|
42
|
+
/** True when this invocation was a dry run (zero writes). */
|
|
43
|
+
dryRun: boolean;
|
|
44
|
+
/** Live rows remaining to process AFTER this invocation (whole corpus minus cursor). */
|
|
45
|
+
remaining: number;
|
|
46
|
+
/** Rows this invocation re-judged (0 on a dry run). */
|
|
47
|
+
rescored: number;
|
|
48
|
+
/** Rows whose scoring call failed (endpoint down / unusable) — untouched. */
|
|
49
|
+
failed: number;
|
|
50
|
+
/** LLM calls made (10 rows each). */
|
|
51
|
+
calls: number;
|
|
52
|
+
/** Cursor after this invocation. */
|
|
53
|
+
cursor: number;
|
|
54
|
+
/** True when the run stopped early on an infra error (cursor holds). */
|
|
55
|
+
aborted: boolean;
|
|
56
|
+
/** Path to the pre-run DB backup (apply mode only). */
|
|
57
|
+
backupPath?: string;
|
|
58
|
+
/** CURRENT base_strength percentiles over the remaining rows (preview). */
|
|
59
|
+
currentDistribution: DistributionPreview;
|
|
60
|
+
/** For apply runs over rows that were actually re-judged: before → after. */
|
|
61
|
+
before?: DistributionPreview;
|
|
62
|
+
after?: DistributionPreview;
|
|
63
|
+
}
|
|
64
|
+
export interface DistributionPreview {
|
|
65
|
+
n: number;
|
|
66
|
+
min: number;
|
|
67
|
+
p25: number;
|
|
68
|
+
median: number;
|
|
69
|
+
p75: number;
|
|
70
|
+
p90: number;
|
|
71
|
+
max: number;
|
|
72
|
+
atCeiling: number;
|
|
73
|
+
atSentinel: number;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Run the rescore pass. Returns a structured report. Throws on setup errors
|
|
77
|
+
* (client mode, no LLM, backup failure) — the cursor always reflects the
|
|
78
|
+
* last committed batch.
|
|
79
|
+
*/
|
|
80
|
+
export declare function runRescoreImportance(options?: RescoreImportanceOptions): Promise<RescoreImportanceReport>;
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `hicortex rescore-importance` (#425) — the one-shot LLM backfill that
|
|
4
|
+
* re-judges the EXISTING corpus under the re-anchored importance rubric
|
|
5
|
+
* (owner decision D1, 2026-09-13: rescore via LLM, resumable, local
|
|
6
|
+
* gateway).
|
|
7
|
+
*
|
|
8
|
+
* Precedents, deliberately mixed per the issue's attribution:
|
|
9
|
+
* - classify-domains (src/classify-domains.ts): resumable rowid cursor in
|
|
10
|
+
* state.json, --batch rows per invocation, --reset, server-mode-only,
|
|
11
|
+
* infra-error abort that leaves the cursor at the last committed batch.
|
|
12
|
+
* - dedup (src/dedup.ts): dry-run DEFAULT with --apply, and a DB backup
|
|
13
|
+
* taken FIRST — the CLI aborts (nothing written) if the backup fails.
|
|
14
|
+
*
|
|
15
|
+
* The scoring itself is the SHARED production loop — consolidate.ts
|
|
16
|
+
* `scoreMemoriesImportance` (the extracted stageImportance core): batches of
|
|
17
|
+
* 10, serial calls, the 0.95 write cap and the importance_scored_at
|
|
18
|
+
* watermark identical to the nightly. No forked scoring code.
|
|
19
|
+
*
|
|
20
|
+
* Scope: every LIVE (non-absorbed) memory, rowid-ascending. A row's
|
|
21
|
+
* corroboration_count survives untouched; base_strength is re-judged (that
|
|
22
|
+
* is D1's explicit trade: one rubric for all rows, the 2026-08-21-validated
|
|
23
|
+
* ORDERING is re-derived rather than mapped).
|
|
24
|
+
*/
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.runRescoreImportance = runRescoreImportance;
|
|
27
|
+
const node_fs_1 = require("node:fs");
|
|
28
|
+
const node_fs_2 = require("node:fs");
|
|
29
|
+
const node_path_1 = require("node:path");
|
|
30
|
+
const paths_js_1 = require("./paths.js");
|
|
31
|
+
const db_js_1 = require("./db.js");
|
|
32
|
+
const state_js_1 = require("./state.js");
|
|
33
|
+
const consolidate_js_1 = require("./consolidate.js");
|
|
34
|
+
const llm_js_1 = require("./llm.js");
|
|
35
|
+
const backup_js_1 = require("./backup.js");
|
|
36
|
+
const config_read_js_1 = require("./config-read.js");
|
|
37
|
+
const calibration_js_1 = require("./calibration.js");
|
|
38
|
+
const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
|
|
39
|
+
/** Rows per invocation chunk (LLM calls are 10 rows each inside a chunk). */
|
|
40
|
+
const DEFAULT_BATCH = 500;
|
|
41
|
+
/** The scoring loop's fixed 10-per-call slice (mirrors stageImportance). */
|
|
42
|
+
const LLM_SLICE = 10;
|
|
43
|
+
const PRE_RESCORE_BACKUP_PATTERN = /^pre-rescore-.*\.db$/;
|
|
44
|
+
function percentile(sorted, p) {
|
|
45
|
+
if (sorted.length === 0)
|
|
46
|
+
return Number.NaN;
|
|
47
|
+
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
|
|
48
|
+
return sorted[idx];
|
|
49
|
+
}
|
|
50
|
+
function distributionPreview(values) {
|
|
51
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
52
|
+
return {
|
|
53
|
+
n: values.length,
|
|
54
|
+
min: percentile(sorted, 0),
|
|
55
|
+
p25: percentile(sorted, 25),
|
|
56
|
+
median: percentile(sorted, 50),
|
|
57
|
+
p75: percentile(sorted, 75),
|
|
58
|
+
p90: percentile(sorted, 90),
|
|
59
|
+
max: percentile(sorted, 100),
|
|
60
|
+
atCeiling: values.filter((v) => v >= calibration_js_1.IMPORTANCE_CEILING).length,
|
|
61
|
+
atSentinel: values.filter((v) => v === 0.5).length,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function readConfig(stateDir) {
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(stateDir, "config.json"), "utf-8"));
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function fetchChunk(db, cursor, limit) {
|
|
73
|
+
return db
|
|
74
|
+
.prepare(`SELECT rowid AS __rowid, id, base_strength FROM memories
|
|
75
|
+
WHERE rowid > ? AND COALESCE(status, '') != 'absorbed'
|
|
76
|
+
ORDER BY rowid ASC LIMIT ?`)
|
|
77
|
+
.all(cursor, limit);
|
|
78
|
+
}
|
|
79
|
+
function countRemaining(db, cursor) {
|
|
80
|
+
return db
|
|
81
|
+
.prepare(`SELECT COUNT(*) AS n FROM memories
|
|
82
|
+
WHERE rowid > ? AND COALESCE(status, '') != 'absorbed'`)
|
|
83
|
+
.get(cursor).n;
|
|
84
|
+
}
|
|
85
|
+
/** Full Memory rows for the ids in `rows` (the scoring loop's input type). */
|
|
86
|
+
function memoriesForIds(db, rows) {
|
|
87
|
+
const byId = new Map(rows.map((r) => {
|
|
88
|
+
const mem = db
|
|
89
|
+
.prepare("SELECT * FROM memories WHERE id = ?")
|
|
90
|
+
.get(r.id);
|
|
91
|
+
return [r.id, mem];
|
|
92
|
+
}));
|
|
93
|
+
return rows.map((r) => byId.get(r.id)).filter((m) => m !== undefined);
|
|
94
|
+
}
|
|
95
|
+
/** Pre-run DB backup (the dedup precedent): throws on failure — abort. */
|
|
96
|
+
async function takePreRescoreBackup(db, stateDir, config) {
|
|
97
|
+
const backupDir = (0, node_path_1.join)(stateDir, "backups");
|
|
98
|
+
(0, node_fs_2.mkdirSync)(backupDir, { recursive: true });
|
|
99
|
+
const backupPath = (0, node_path_1.join)(backupDir, `pre-rescore-${new Date().toISOString().replace(/[:.]/g, "-")}.db`);
|
|
100
|
+
await db.backup(backupPath);
|
|
101
|
+
const retention = (0, config_read_js_1.readNonNegativeConfig)(config ?? {}, "backupRetention", backup_js_1.DEFAULT_BACKUP_RETENTION);
|
|
102
|
+
(0, backup_js_1.pruneBackupArtifacts)(backupDir, retention, PRE_RESCORE_BACKUP_PATTERN);
|
|
103
|
+
return backupPath;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Run the rescore pass. Returns a structured report. Throws on setup errors
|
|
107
|
+
* (client mode, no LLM, backup failure) — the cursor always reflects the
|
|
108
|
+
* last committed batch.
|
|
109
|
+
*/
|
|
110
|
+
async function runRescoreImportance(options = {}) {
|
|
111
|
+
const batchSize = options.batchSize ?? DEFAULT_BATCH;
|
|
112
|
+
const stateDir = options.stateDir ?? HICORTEX_HOME;
|
|
113
|
+
const apply = options.apply ?? false;
|
|
114
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
115
|
+
throw new Error(`[hicortex] rescore-importance: invalid --batch value: ${options.batchSize}`);
|
|
116
|
+
}
|
|
117
|
+
const config = options.config !== undefined ? options.config : readConfig(stateDir);
|
|
118
|
+
// Server-mode only — client installs have no local DB (classify-domains).
|
|
119
|
+
if (config?.mode === "client") {
|
|
120
|
+
throw new Error("[hicortex] rescore-importance is server-mode only (it needs the local DB). " +
|
|
121
|
+
`This machine is a client of ${config.serverUrl ?? "a remote server"} — run it on the server.`);
|
|
122
|
+
}
|
|
123
|
+
let llm;
|
|
124
|
+
if (options.llm) {
|
|
125
|
+
llm = options.llm;
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
const resolved = (0, llm_js_1.resolveSavedLlmConfig)(config);
|
|
129
|
+
if (!resolved.config) {
|
|
130
|
+
throw new Error("[hicortex] rescore-importance: no LLM configured — run `npx @gamaze/hicortex init`.");
|
|
131
|
+
}
|
|
132
|
+
llm = new llm_js_1.LlmClient(resolved.config);
|
|
133
|
+
}
|
|
134
|
+
const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
|
|
135
|
+
const db = (0, db_js_1.initDb)(dbPath);
|
|
136
|
+
try {
|
|
137
|
+
let cursor = options.reset ? 0 : ((0, state_js_1.loadState)(stateDir).rescoreImportanceCursor ?? 0);
|
|
138
|
+
const remaining = countRemaining(db, cursor);
|
|
139
|
+
const plannedCalls = Math.ceil(Math.min(remaining, batchSize) / LLM_SLICE);
|
|
140
|
+
// Current distribution preview over the REMAINING rows (what is queued).
|
|
141
|
+
const remainingRows = fetchChunk(db, cursor, Number.MAX_SAFE_INTEGER);
|
|
142
|
+
const currentDistribution = distributionPreview(remainingRows.map((r) => r.base_strength ?? 0.5));
|
|
143
|
+
console.log(`[hicortex] rescore-importance ${apply ? "APPLY" : "dry run"}: ${remaining} live rows queued, ` +
|
|
144
|
+
`batch ${batchSize}, cursor ${cursor}${options.reset ? " (reset)" : ""}, ` +
|
|
145
|
+
`~${plannedCalls} LLM calls this invocation (10 rows each)`);
|
|
146
|
+
console.log(`[hicortex] current base_strength of queued rows: median ${currentDistribution.median.toFixed(2)}, ` +
|
|
147
|
+
`p90 ${currentDistribution.p90.toFixed(2)}, max ${currentDistribution.max.toFixed(2)}, ` +
|
|
148
|
+
`at ceiling ${currentDistribution.atCeiling}, at 0.5 sentinel ${currentDistribution.atSentinel}`);
|
|
149
|
+
if (!apply) {
|
|
150
|
+
console.log("[hicortex] rescore-importance dry run complete — zero writes. Re-run with --apply to execute.");
|
|
151
|
+
return {
|
|
152
|
+
dryRun: true,
|
|
153
|
+
remaining,
|
|
154
|
+
rescored: 0,
|
|
155
|
+
failed: 0,
|
|
156
|
+
calls: 0,
|
|
157
|
+
cursor,
|
|
158
|
+
aborted: false,
|
|
159
|
+
currentDistribution,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
// Backup FIRST (dedup precedent) — abort with zero writes if it fails.
|
|
163
|
+
const backupPath = await takePreRescoreBackup(db, stateDir, config);
|
|
164
|
+
console.log(`[hicortex] rescore-importance backup written: ${backupPath}`);
|
|
165
|
+
const chunk = fetchChunk(db, cursor, batchSize);
|
|
166
|
+
const before = distributionPreview(chunk.map((r) => r.base_strength ?? 0.5));
|
|
167
|
+
let rescored = 0;
|
|
168
|
+
let failed = 0;
|
|
169
|
+
let calls = 0;
|
|
170
|
+
let committedRowid = cursor;
|
|
171
|
+
let aborted = false;
|
|
172
|
+
const touchedIds = [];
|
|
173
|
+
// LLM slices of 10 INSIDE the invocation chunk, cursor-ordered: an infra
|
|
174
|
+
// error holds the cursor at the last fully committed slice (the
|
|
175
|
+
// classify-domains posture — the failing rows are untouched, re-run
|
|
176
|
+
// resumes there).
|
|
177
|
+
for (let i = 0; i < chunk.length; i += LLM_SLICE) {
|
|
178
|
+
const sliceRows = chunk.slice(i, i + LLM_SLICE);
|
|
179
|
+
const memories = memoriesForIds(db, sliceRows);
|
|
180
|
+
const r = await (0, consolidate_js_1.scoreMemoriesImportance)(db, memories, llm, {
|
|
181
|
+
onBatch: (written, batchFailed) => {
|
|
182
|
+
calls++;
|
|
183
|
+
},
|
|
184
|
+
});
|
|
185
|
+
if (r.failed > 0) {
|
|
186
|
+
// The slice's call threw (endpoint down) or a write failed — nothing
|
|
187
|
+
// usable came out of it. Stop; the cursor stays at the last
|
|
188
|
+
// committed slice's end.
|
|
189
|
+
failed += r.failed;
|
|
190
|
+
aborted = true;
|
|
191
|
+
console.warn(`[hicortex] rescore-importance ABORTED on a scoring-endpoint error ` +
|
|
192
|
+
`(slice at rowid ${sliceRows[0].__rowid}). Cursor at last committed slice — ` +
|
|
193
|
+
`re-run when the endpoint is back up.`);
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
rescored += r.scored;
|
|
197
|
+
touchedIds.push(...sliceRows.map((row) => row.id));
|
|
198
|
+
committedRowid = sliceRows[sliceRows.length - 1].__rowid;
|
|
199
|
+
(0, state_js_1.updateState)((s) => { s.rescoreImportanceCursor = committedRowid; }, stateDir);
|
|
200
|
+
if ((i / LLM_SLICE) % 25 === 0) {
|
|
201
|
+
console.log(`[hicortex] ${rescored} rows re-judged (cursor ${committedRowid}, ` +
|
|
202
|
+
`${countRemaining(db, committedRowid)} remaining)`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
cursor = committedRowid;
|
|
206
|
+
// before → after over the rows this invocation actually touched.
|
|
207
|
+
const afterRows = touchedIds
|
|
208
|
+
.map((id) => db
|
|
209
|
+
.prepare("SELECT base_strength FROM memories WHERE id = ?")
|
|
210
|
+
.get(id))
|
|
211
|
+
.filter((r) => r !== undefined);
|
|
212
|
+
const after = distributionPreview(afterRows.map((r) => r.base_strength ?? 0.5));
|
|
213
|
+
const afterLog = (label, d) => `${label}: median ${d.median.toFixed(2)} p90 ${d.p90.toFixed(2)} max ${d.max.toFixed(2)} at-ceiling ${d.atCeiling}`;
|
|
214
|
+
console.log(`[hicortex] rescore-importance ${aborted ? "ABORTED" : "chunk complete"}: ` +
|
|
215
|
+
`${rescored} re-judged, ${failed} failed, ${calls} calls, cursor ${cursor}, ` +
|
|
216
|
+
`${countRemaining(db, cursor)} remaining`);
|
|
217
|
+
console.log(`[hicortex] re-judged rows — ${afterLog("before", before)}`);
|
|
218
|
+
console.log(`[hicortex] re-judged rows — ${afterLog("after ", after)}`);
|
|
219
|
+
return {
|
|
220
|
+
dryRun: false,
|
|
221
|
+
remaining: countRemaining(db, cursor),
|
|
222
|
+
rescored,
|
|
223
|
+
failed,
|
|
224
|
+
calls,
|
|
225
|
+
cursor,
|
|
226
|
+
aborted,
|
|
227
|
+
backupPath,
|
|
228
|
+
currentDistribution,
|
|
229
|
+
before,
|
|
230
|
+
after,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
finally {
|
|
234
|
+
db.close();
|
|
235
|
+
}
|
|
236
|
+
}
|
package/dist/retrieval.d.ts
CHANGED
|
@@ -2,14 +2,15 @@
|
|
|
2
2
|
* Retrieval layer with composite scoring, RRF fusion, and graph traversal.
|
|
3
3
|
* Ported from hicortex/retrieval.py — same scoring model and weights.
|
|
4
4
|
*
|
|
5
|
-
* Scoring model (weights are
|
|
5
|
+
* Scoring model (weights are RELEASE-MANAGED since #408 — calibration.ts;
|
|
6
|
+
* configureScoring is the eval/test seam only):
|
|
6
7
|
* score = similarity * 0.50 + effective_strength * 0.20
|
|
7
8
|
* + connection_score * 0.15 + recency * 0.15
|
|
8
9
|
* + fresh-memory bonus (≤ 0.15, linear over the first 7 days)
|
|
9
10
|
* then × 0.50 if the memory was superseded by a later decision
|
|
10
11
|
*
|
|
11
12
|
* Decay model (B+E+D):
|
|
12
|
-
* base_decay = derived from
|
|
13
|
+
* base_decay = derived from the calibration half-life (365 → ~1-year
|
|
13
14
|
* half-life at importance 0.5, importance-scaled either way)
|
|
14
15
|
* decay_rate = 1 - base_decay * (1 - importance)
|
|
15
16
|
* decay_rate = 1 - (1 - decay_rate) * 0.7^access_count
|
|
@@ -19,10 +20,8 @@
|
|
|
19
20
|
*/
|
|
20
21
|
import type Database from "better-sqlite3";
|
|
21
22
|
import type { Memory, MemorySearchResult } from "./types.js";
|
|
22
|
-
/** Default decay half-life (days) at importance 0.5
|
|
23
|
-
* (
|
|
24
|
-
* in ranking. Long-term remembering is the product; time preference stays,
|
|
25
|
-
* but mild. */
|
|
23
|
+
/** Default decay half-life (days) at importance 0.5 — release-managed
|
|
24
|
+
* (#408): the constant lives in calibration.ts with its provenance. */
|
|
26
25
|
export declare const DEFAULT_DECAY_HALF_LIFE_DAYS = 365;
|
|
27
26
|
/**
|
|
28
27
|
* Derive the per-hour base decay constant from a half-life target: for the
|
|
@@ -33,26 +32,26 @@ export declare const DEFAULT_DECAY_HALF_LIFE_DAYS = 365;
|
|
|
33
32
|
*/
|
|
34
33
|
export declare function decayConstantForHalfLife(days: number): number;
|
|
35
34
|
/**
|
|
36
|
-
* Configure the decay speed
|
|
37
|
-
*
|
|
38
|
-
*
|
|
35
|
+
* Configure the decay speed for THIS process (the eval/test seam — #408).
|
|
36
|
+
* Production NEVER passes an argument: every process scores with the
|
|
37
|
+
* calibration half-life (calibration.ts DECAY_HALF_LIFE_DAYS). An
|
|
38
|
+
* invalid/absent value keeps the default. Exported value for tests.
|
|
39
39
|
*/
|
|
40
|
-
export declare function configureDecay(
|
|
41
|
-
|
|
42
|
-
}): number;
|
|
43
|
-
interface RecallDefaults {
|
|
40
|
+
export declare function configureDecay(halfLifeDays?: number): number;
|
|
41
|
+
export interface RecallDefaults {
|
|
44
42
|
searchLimit: number;
|
|
45
43
|
recentLimit: number;
|
|
46
44
|
recentWindowDays: number;
|
|
47
45
|
coldExposureSlots: number;
|
|
48
46
|
}
|
|
49
47
|
/**
|
|
50
|
-
* Configure recall breadth from
|
|
51
|
-
*
|
|
52
|
-
*
|
|
48
|
+
* Configure recall breadth from RESOLVED overrides (the eval/test seam —
|
|
49
|
+
* #408). Production calls this with no argument: the calibration defaults
|
|
50
|
+
* (calibration.ts) apply. Invalid/absent values keep the shipped default per
|
|
51
|
+
* key. Returns the resolved values (for logging + tests).
|
|
53
52
|
*/
|
|
54
|
-
export declare function configureRecall(
|
|
55
|
-
interface ScoringWeights {
|
|
53
|
+
export declare function configureRecall(overrides?: Partial<RecallDefaults> | null): RecallDefaults;
|
|
54
|
+
export interface ScoringWeights {
|
|
56
55
|
similarity: number;
|
|
57
56
|
strength: number;
|
|
58
57
|
connections: number;
|
|
@@ -72,31 +71,32 @@ interface ScoringWeights {
|
|
|
72
71
|
rrfFtsWeight: number;
|
|
73
72
|
/** #205 per-list RRF weight for the vector list (KNN-driven candidates). */
|
|
74
73
|
rrfVectorWeight: number;
|
|
74
|
+
/** #425 additive boost for both-channel (vector AND FTS) candidates. */
|
|
75
|
+
bothChannelBoost: number;
|
|
75
76
|
}
|
|
76
77
|
/**
|
|
77
|
-
* Configure scoring weights + ranking knobs from
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
* SAME config object for one-place tuning.
|
|
78
|
+
* Configure scoring weights + ranking knobs from RESOLVED overrides (the
|
|
79
|
+
* eval/test seam — #408). Production calls this with no argument: the
|
|
80
|
+
* calibration defaults (calibration.ts) apply, identically in the daemon and
|
|
81
|
+
* the nightly. Invalid/absent values keep the shipped default per key.
|
|
82
|
+
* Returns the resolved set for logging/tests. (The #205 BM25F field weights
|
|
83
|
+
* are NOT touched here — they live in storage.ts and resolve from the same
|
|
84
|
+
* calibration module via storage.configureBm25Fts.)
|
|
85
85
|
*/
|
|
86
|
-
export declare function configureScoring(
|
|
86
|
+
export declare function configureScoring(overrides?: Partial<ScoringWeights> | null): ScoringWeights;
|
|
87
87
|
/** Current resolved weights (tests + status output). */
|
|
88
88
|
export declare function getScoringWeights(): ScoringWeights;
|
|
89
89
|
/** EMA rate for the session-intent centroid: centroid_new = (1-α)·old + α·prompt. */
|
|
90
90
|
export declare const SESSION_INTENT_ALPHA = 0.4;
|
|
91
91
|
/**
|
|
92
|
-
* Configure session-intent keying
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
* one call.
|
|
92
|
+
* Configure session-intent keying for THIS process (the eval/test seam —
|
|
93
|
+
* #408). Production calls this with no argument: the calibration weight
|
|
94
|
+
* (calibration.ts SESSION_INTENT_WEIGHT) applies. `weight` is [0,1] (0 =
|
|
95
|
+
* disabled — the eval kill-switch); invalid/out-of-range values keep the
|
|
96
|
+
* shipped default. Returns `{ weight, alpha }` — alpha is the fixed constant,
|
|
97
|
+
* surfaced so the recall closure passes it to the registry in one call.
|
|
98
98
|
*/
|
|
99
|
-
export declare function configureSessionIntent(
|
|
99
|
+
export declare function configureSessionIntent(weight?: number): {
|
|
100
100
|
weight: number;
|
|
101
101
|
alpha: number;
|
|
102
102
|
};
|
|
@@ -161,6 +161,32 @@ export declare function findSupersededIds(db: Database.Database, candidateIds: s
|
|
|
161
161
|
* (NULL status, no link) — they never match either arm.
|
|
162
162
|
*/
|
|
163
163
|
export declare function findDemotedIds(db: Database.Database, candidateIds: string[]): Set<string>;
|
|
164
|
+
/**
|
|
165
|
+
* Hop cap for the belief walk (#393 D). Supersession edges advance
|
|
166
|
+
* created_at monotonically (the stage only links old → new), so chains are
|
|
167
|
+
* acyclic by construction and 10 hops is far beyond any real revision depth;
|
|
168
|
+
* the cap is cheap insurance (see beliefWalkTerminal for why it is needed
|
|
169
|
+
* anyway).
|
|
170
|
+
*/
|
|
171
|
+
export declare const BELIEF_WALK_MAX_HOPS = 10;
|
|
172
|
+
/**
|
|
173
|
+
* Terminal of the supersession chain starting at `id` (#393 D): follow
|
|
174
|
+
* superseded_by edges transitively until a memory with no outgoing edge and
|
|
175
|
+
* return it — `id` itself when there is nothing to walk, or whichever node
|
|
176
|
+
* the walk stopped on when it aborts. Shared by retrieval (the belief-walk
|
|
177
|
+
* splice in retrieve()/searchRecent()) and the eval harness (the
|
|
178
|
+
* planted-pairs version_chain class probe), so both agree on what "the
|
|
179
|
+
* chain's current truth" is.
|
|
180
|
+
*
|
|
181
|
+
* Edges advance created_at monotonically (acyclic by construction), BUT
|
|
182
|
+
* applyExplicitMark does no age check and created_at is backdatable from
|
|
183
|
+
* session_date — so a cycle or an absurdly long chain is not impossible.
|
|
184
|
+
* The visited set (seeded with `id`) and the BELIEF_WALK_MAX_HOPS cap are
|
|
185
|
+
* cheap insurance against exactly that; the supersededDemotion multiplier
|
|
186
|
+
* in computeScore remains the safety net for rows the walk does not fully
|
|
187
|
+
* resolve (no edge, cycle, cap abort, absorbed terminal).
|
|
188
|
+
*/
|
|
189
|
+
export declare function beliefWalkTerminal(db: Database.Database, id: string, maxHops?: number): string;
|
|
164
190
|
/**
|
|
165
191
|
* Convert an L2 distance (as returned by sqlite-vec's vec0 `distance`) to
|
|
166
192
|
* cosine similarity. Valid because our embeddings are L2-normalized
|
|
@@ -171,9 +197,25 @@ export declare function findDemotedIds(db: Database.Database, candidateIds: stri
|
|
|
171
197
|
* by consolidate.ts so pre-#145 importers keep working.
|
|
172
198
|
*/
|
|
173
199
|
export declare function l2ToCosine(distance: number): number;
|
|
200
|
+
/**
|
|
201
|
+
* Cosine similarity between two stored embeddings (#393 increment B). The
|
|
202
|
+
* similarity source measures cosines transitively via vec0 L2 distances; the
|
|
203
|
+
* scout source finds its candidates through FTS (no vec0 query), so it
|
|
204
|
+
* measures the pair cosine directly from the stored vectors instead —
|
|
205
|
+
* valid because every embedding we store is L2-normalized (embedder.ts).
|
|
206
|
+
* Used as link strength / a ranker, never as a gate (the scout has no
|
|
207
|
+
* similarity floor — that is the point of the increment).
|
|
208
|
+
*/
|
|
209
|
+
export declare function cosineBetweenVectors(a: Float32Array, b: Float32Array): number;
|
|
174
210
|
/**
|
|
175
211
|
* Compute decayed strength with adaptive decay (B+E+D model).
|
|
176
212
|
* Exported for use by consolidation decay/prune stage.
|
|
213
|
+
*
|
|
214
|
+
* #425 read-side law: the decay-relevant importance is CLAMPED at the
|
|
215
|
+
* release-managed ceiling (calibration.ts IMPORTANCE_CEILING) — at importance
|
|
216
|
+
* exactly 1.0 the decay rate is exactly 1.0 and the row never decays, so
|
|
217
|
+
* legacy base-1.0 rows (and any write site that predates the cap) decay
|
|
218
|
+
* again. The clamp applies to explicit importance passes too.
|
|
177
219
|
*/
|
|
178
220
|
export declare function effectiveStrength(baseStrength: number, lastAccessed: string | null, now: Date, options?: {
|
|
179
221
|
importance?: number;
|
|
@@ -208,6 +250,10 @@ export declare function computeScore(memory: Memory, distance: number, connectio
|
|
|
208
250
|
tag: string;
|
|
209
251
|
weight: number | null;
|
|
210
252
|
}>;
|
|
253
|
+
/** #425: the candidate was matched by BOTH retrieval channels (vector
|
|
254
|
+
* KNN AND BM25 FTS) — the genuine-match signature. Adds the
|
|
255
|
+
* release-managed bothChannelBoost (zero-boost neutral). */
|
|
256
|
+
bothChannel?: boolean;
|
|
211
257
|
}): number;
|
|
212
258
|
export interface EmbedFn {
|
|
213
259
|
(text: string): Promise<Float32Array>;
|
|
@@ -264,4 +310,3 @@ export declare function searchRecent(db: Database.Database, options?: {
|
|
|
264
310
|
project?: string | null;
|
|
265
311
|
limit?: number;
|
|
266
312
|
}): MemorySearchResult[];
|
|
267
|
-
export {};
|