@gamaze/hicortex 0.20.7 → 0.20.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -41
- package/assets/dashboard.html +3989 -836
- package/dist/calibration.d.ts +293 -0
- package/dist/calibration.js +379 -0
- package/dist/capture-health.d.ts +87 -0
- package/dist/capture-health.js +106 -0
- package/dist/capture-pause.d.ts +86 -0
- package/dist/capture-pause.js +127 -0
- package/dist/capture.d.ts +24 -3
- package/dist/capture.js +11 -1
- package/dist/classify-domains.d.ts +6 -0
- package/dist/classify-domains.js +7 -1
- package/dist/cli.js +38 -3
- package/dist/config-read.d.ts +1 -1
- package/dist/config-read.js +96 -9
- package/dist/consolidate.d.ts +114 -68
- package/dist/consolidate.js +302 -182
- package/dist/dashboard.d.ts +326 -6
- package/dist/dashboard.js +592 -7
- package/dist/db.js +105 -0
- package/dist/dedup.d.ts +34 -26
- package/dist/dedup.js +91 -57
- package/dist/distiller.js +1 -1
- package/dist/domain-classify.d.ts +7 -6
- package/dist/domain-classify.js +12 -10
- package/dist/eval/decay-eval.d.ts +3 -3
- package/dist/eval/decay-eval.js +4 -4
- package/dist/eval/importance-eval.d.ts +85 -0
- package/dist/eval/importance-eval.js +286 -0
- package/dist/eval/planted-eval.d.ts +26 -0
- package/dist/eval/planted-eval.js +97 -0
- package/dist/eval/planted-fixtures.d.ts +107 -0
- package/dist/eval/planted-fixtures.js +283 -0
- package/dist/eval/planted-harness.d.ts +176 -0
- package/dist/eval/planted-harness.js +649 -0
- package/dist/eval/ranking-battery.d.ts +78 -0
- package/dist/eval/ranking-battery.js +181 -0
- package/dist/eval/ranking-eval.d.ts +41 -0
- package/dist/eval/ranking-eval.js +391 -0
- package/dist/eval/ranking-fixtures.d.ts +77 -0
- package/dist/eval/ranking-fixtures.js +226 -0
- package/dist/identity-store.d.ts +21 -0
- package/dist/identity-store.js +49 -0
- package/dist/index.js +4 -3
- package/dist/init.d.ts +23 -3
- package/dist/init.js +84 -9
- package/dist/llm.d.ts +43 -58
- package/dist/llm.js +87 -101
- package/dist/mcp-server.d.ts +12 -0
- package/dist/mcp-server.js +213 -32
- package/dist/nightly.d.ts +9 -1
- package/dist/nightly.js +164 -110
- package/dist/nofit.d.ts +4 -11
- package/dist/nofit.js +6 -23
- package/dist/prompts.d.ts +10 -0
- package/dist/prompts.js +28 -5
- package/dist/recall-index.d.ts +30 -28
- package/dist/recall-index.js +21 -18
- package/dist/recall-registry.d.ts +2 -1
- package/dist/recall-registry.js +35 -1
- package/dist/reconsolidation.d.ts +168 -87
- package/dist/reconsolidation.js +818 -377
- package/dist/relink.js +3 -4
- package/dist/rescore-importance.d.ts +80 -0
- package/dist/rescore-importance.js +236 -0
- package/dist/retrieval.d.ts +80 -35
- package/dist/retrieval.js +322 -105
- package/dist/run-deadline.d.ts +62 -0
- package/dist/run-deadline.js +73 -0
- package/dist/schema-prototypes.d.ts +3 -3
- package/dist/schema-prototypes.js +3 -3
- package/dist/stages.d.ts +37 -0
- package/dist/stages.js +51 -0
- package/dist/state.d.ts +34 -9
- package/dist/storage.d.ts +50 -18
- package/dist/storage.js +125 -30
- package/dist/telemetry.d.ts +8 -7
- package/dist/token-budget.js +3 -4
- package/dist/type-classify.js +4 -4
- package/dist/types.d.ts +143 -155
- package/domains.example.json +4 -5
- package/hermes-plugin/hicortex/README.md +2 -2
- package/openclaw.plugin.json +1 -1
- package/package.json +4 -1
- package/pi-extension/hicortex/README.md +1 -1
- package/server.json +3 -3
package/dist/nightly.js
CHANGED
|
@@ -52,6 +52,7 @@ exports.runNightly = runNightly;
|
|
|
52
52
|
const paths_js_1 = require("./paths.js");
|
|
53
53
|
const node_fs_1 = require("node:fs");
|
|
54
54
|
const node_path_1 = require("node:path");
|
|
55
|
+
const node_os_1 = require("node:os");
|
|
55
56
|
let VERSION = "0.0.0";
|
|
56
57
|
try {
|
|
57
58
|
VERSION = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(__dirname, "..", "package.json"), "utf-8")).version;
|
|
@@ -59,12 +60,12 @@ try {
|
|
|
59
60
|
catch { }
|
|
60
61
|
const db_js_1 = require("./db.js");
|
|
61
62
|
const config_read_js_1 = require("./config-read.js");
|
|
63
|
+
const calibration_js_1 = require("./calibration.js");
|
|
62
64
|
const llm_js_1 = require("./llm.js");
|
|
63
65
|
const embedder_js_1 = require("./embedder.js");
|
|
64
66
|
const storage = __importStar(require("./storage.js"));
|
|
65
67
|
const consolidate_js_1 = require("./consolidate.js");
|
|
66
68
|
const domain_classify_js_1 = require("./domain-classify.js");
|
|
67
|
-
const nofit_js_1 = require("./nofit.js");
|
|
68
69
|
const transcript_reader_js_1 = require("./transcript-reader.js");
|
|
69
70
|
const hermes_transcript_reader_js_1 = require("./hermes-transcript-reader.js");
|
|
70
71
|
const pi_transcript_reader_js_1 = require("./pi-transcript-reader.js");
|
|
@@ -76,6 +77,7 @@ const state_js_1 = require("./state.js");
|
|
|
76
77
|
const identity_store_js_1 = require("./identity-store.js");
|
|
77
78
|
const capture_cursors_js_1 = require("./capture-cursors.js");
|
|
78
79
|
const capture_js_1 = require("./capture.js");
|
|
80
|
+
const run_deadline_js_1 = require("./run-deadline.js");
|
|
79
81
|
const dashboard_js_1 = require("./dashboard.js");
|
|
80
82
|
const telemetry_js_1 = require("./telemetry.js");
|
|
81
83
|
const init_js_1 = require("./init.js");
|
|
@@ -92,6 +94,19 @@ const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
|
|
|
92
94
|
* count stays bounded.
|
|
93
95
|
*/
|
|
94
96
|
const CONSOLIDATE_ONLY_BACKUP_MIN_AGE_MS = 20 * 60 * 60 * 1000;
|
|
97
|
+
/**
|
|
98
|
+
* #421 machine × harness identity: the stamp every captured segment carries
|
|
99
|
+
* as `source_machine`. Config `machineName` wins when set (same style as
|
|
100
|
+
* `agentName` — an owner-set identity key); otherwise the hostname. Applies
|
|
101
|
+
* to BOTH capture paths (server-local and client-remote) — same package,
|
|
102
|
+
* same stamp.
|
|
103
|
+
*/
|
|
104
|
+
function resolveCaptureMachine(config) {
|
|
105
|
+
const v = config?.machineName;
|
|
106
|
+
if (typeof v === "string" && v.trim())
|
|
107
|
+
return v.trim().slice(0, 128);
|
|
108
|
+
return (0, node_os_1.hostname)();
|
|
109
|
+
}
|
|
95
110
|
function readNightlyConfig(stateDir) {
|
|
96
111
|
const configPath = (0, node_path_1.join)(stateDir, "config.json");
|
|
97
112
|
let loaded;
|
|
@@ -127,9 +142,9 @@ function readConfigLicenseKey(stateDir) {
|
|
|
127
142
|
function readLastRun(stateDir = HICORTEX_HOME) {
|
|
128
143
|
const ts = (0, state_js_1.loadState)(stateDir).lastNightly;
|
|
129
144
|
if (!ts)
|
|
130
|
-
return
|
|
145
|
+
return null; // First run — no watermark yet; #436 applies the lookback cap
|
|
131
146
|
const d = new Date(ts);
|
|
132
|
-
return isNaN(d.getTime()) ?
|
|
147
|
+
return isNaN(d.getTime()) ? null : d; // corrupt stamp → treated as first run
|
|
133
148
|
}
|
|
134
149
|
/**
|
|
135
150
|
* Discovery watermark. Normally the last-nightly timestamp; with
|
|
@@ -141,6 +156,14 @@ function readLastRun(stateDir = HICORTEX_HOME) {
|
|
|
141
156
|
* (#189 review, fix 3). Per-session cursors keep the wide re-scan cheap: an
|
|
142
157
|
* already-captured session yields an empty delta.
|
|
143
158
|
*
|
|
159
|
+
* First run (#436): with NO watermark yet, since = now − firstRunLookbackDays
|
|
160
|
+
* (default 7 — owner ruling 2026-09-14: full-history first-run ingestion is
|
|
161
|
+
* not feasible; a long-term AI user's entire session store must not be
|
|
162
|
+
* discovered night one). The widening-only invariant composes unchanged:
|
|
163
|
+
* min(now−7d, now−N) — a recapture window wider than the default widens, a
|
|
164
|
+
* narrower one leaves the floor. Installing more history is the deliberate
|
|
165
|
+
* `--recapture-window` act, not a default.
|
|
166
|
+
*
|
|
144
167
|
* Clock-jump clamp (#327): a FUTURE-dated lastNightly (client clock error —
|
|
145
168
|
* NTP not yet synced at write time) would, once the clock corrects, sit ahead
|
|
146
169
|
* of every session mtime and permanently skip quiet sessions (their mtimes
|
|
@@ -148,10 +171,10 @@ function readLastRun(stateDir = HICORTEX_HOME) {
|
|
|
148
171
|
* fires once per affected run (this function runs once per nightly).
|
|
149
172
|
* `now` is injectable for tests.
|
|
150
173
|
*/
|
|
151
|
-
function computeSince(stateDir, recaptureWindowDays, now = new Date()) {
|
|
174
|
+
function computeSince(stateDir, recaptureWindowDays, now = new Date(), firstRunLookbackDays = calibration_js_1.DEFAULT_FIRST_RUN_LOOKBACK_DAYS) {
|
|
152
175
|
const lastRun = readLastRun(stateDir);
|
|
153
|
-
let effective = lastRun;
|
|
154
|
-
if (lastRun.getTime() > now.getTime()) {
|
|
176
|
+
let effective = lastRun ?? new Date(now.getTime() - firstRunLookbackDays * 24 * 60 * 60 * 1000);
|
|
177
|
+
if (lastRun && lastRun.getTime() > now.getTime()) {
|
|
155
178
|
console.warn(`[hicortex] state lastNightly (${lastRun.toISOString()}) is ahead of the clock ` +
|
|
156
179
|
`(${now.toISOString()}) — clamping discovery to now. A future watermark permanently ` +
|
|
157
180
|
`skips quiet sessions once the clock corrects; check the machine's clock/NTP. ` +
|
|
@@ -167,7 +190,7 @@ function computeSince(stateDir, recaptureWindowDays, now = new Date()) {
|
|
|
167
190
|
}
|
|
168
191
|
/** POST /distill transport for server mode — localhost. Sends authToken so
|
|
169
192
|
* self-capture works regardless of the localhost-bypass marker (#271 root-cause fix). */
|
|
170
|
-
function makeLocalPost(port, authToken) {
|
|
193
|
+
function makeLocalPost(port, authToken, deadline) {
|
|
171
194
|
return async (body) => {
|
|
172
195
|
const resp = await fetch(`http://127.0.0.1:${port}/distill`, {
|
|
173
196
|
method: "POST",
|
|
@@ -177,13 +200,16 @@ function makeLocalPost(port, authToken) {
|
|
|
177
200
|
},
|
|
178
201
|
body: JSON.stringify(body),
|
|
179
202
|
// Synchronous 35B distillation of a large segment can take minutes.
|
|
180
|
-
|
|
203
|
+
// #405: never wait past the run deadline — the capture loop's
|
|
204
|
+
// between-segments check is the clean stop; this is the hard bound
|
|
205
|
+
// for the POST in flight when the deadline expires mid-wait.
|
|
206
|
+
signal: AbortSignal.timeout(postTimeoutMs(deadline)),
|
|
181
207
|
});
|
|
182
208
|
return normalizePostResult(resp);
|
|
183
209
|
};
|
|
184
210
|
}
|
|
185
211
|
/** POST /distill transport for client mode — remote URL + optional bearer token. */
|
|
186
|
-
function makeRemotePost(serverUrl, authToken) {
|
|
212
|
+
function makeRemotePost(serverUrl, authToken, deadline) {
|
|
187
213
|
return async (body) => {
|
|
188
214
|
const resp = await fetch(`${serverUrl}/distill`, {
|
|
189
215
|
method: "POST",
|
|
@@ -192,11 +218,20 @@ function makeRemotePost(serverUrl, authToken) {
|
|
|
192
218
|
...(authToken ? { Authorization: `Bearer ${authToken}` } : {}),
|
|
193
219
|
},
|
|
194
220
|
body: JSON.stringify(body),
|
|
195
|
-
signal: AbortSignal.timeout(
|
|
221
|
+
signal: AbortSignal.timeout(postTimeoutMs(deadline)),
|
|
196
222
|
});
|
|
197
223
|
return normalizePostResult(resp);
|
|
198
224
|
};
|
|
199
225
|
}
|
|
226
|
+
/**
|
|
227
|
+
* Per-POST timeout: 20 min (synchronous distillation of a large segment can
|
|
228
|
+
* take minutes), clamped to the run deadline's remaining time when one is in
|
|
229
|
+
* force (#405) — a POST launched near expiry never outlives the deadline.
|
|
230
|
+
*/
|
|
231
|
+
function postTimeoutMs(deadline) {
|
|
232
|
+
const CAP_MS = 20 * 60 * 1000;
|
|
233
|
+
return Math.max(1, Math.min(CAP_MS, deadline ? deadline.remainingMs() : Infinity));
|
|
234
|
+
}
|
|
200
235
|
async function normalizePostResult(resp) {
|
|
201
236
|
if (resp.status === 201) {
|
|
202
237
|
const data = (await resp.json().catch(() => ({})));
|
|
@@ -272,9 +307,6 @@ function captureLockWaitMs() {
|
|
|
272
307
|
const env = Number(process.env.HICORTEX_CAPTURE_LOCK_WAIT_MS);
|
|
273
308
|
return Number.isFinite(env) && env >= 0 ? env : CAPTURE_LOCK_WAIT_MS;
|
|
274
309
|
}
|
|
275
|
-
// readPositiveConfig moved to ./config-read.ts (shared with the distill-tier
|
|
276
|
-
// overlay in llm.ts / mcp-server.ts). Validates positive-number config knobs
|
|
277
|
-
// at the disk→runtime boundary with a warn-on-rejected-value.
|
|
278
310
|
const NIGHTLY_LOG_MAX_BYTES = 1024 * 1024; // 1 MB — years of normal runs
|
|
279
311
|
/**
|
|
280
312
|
* Keep ~/.hicortex/nightly.log bounded. The launchd plist and systemd unit
|
|
@@ -320,10 +352,11 @@ function runEvictionOnly(options = {}) {
|
|
|
320
352
|
const stateDir = options.stateDir ?? HICORTEX_HOME;
|
|
321
353
|
const savedConfig = readNightlyConfig(stateDir);
|
|
322
354
|
// Same clock as the full nightly (see runNightly's identical trio) — the
|
|
323
|
-
// eviction ranker reads these module knobs.
|
|
324
|
-
|
|
325
|
-
(0, retrieval_js_1.
|
|
326
|
-
(0, retrieval_js_1.
|
|
355
|
+
// eviction ranker reads these module knobs. #408: calibration constants,
|
|
356
|
+
// never user config — the calls just pin the process to the shipped values.
|
|
357
|
+
(0, retrieval_js_1.configureDecay)();
|
|
358
|
+
(0, retrieval_js_1.configureRecall)();
|
|
359
|
+
(0, retrieval_js_1.configureScoring)();
|
|
327
360
|
const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
|
|
328
361
|
console.log(`[hicortex] evict-only run${options.dryRun ? " (dry run)" : ""} — DB: ${dbPath}`);
|
|
329
362
|
const db = (0, db_js_1.initDb)(dbPath);
|
|
@@ -460,16 +493,31 @@ async function runNightly(options = {}) {
|
|
|
460
493
|
}
|
|
461
494
|
const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
|
|
462
495
|
// #192: consolidation's decay/prune stage must score with the same clock as
|
|
463
|
-
// the server's retrieval path
|
|
464
|
-
(
|
|
465
|
-
(0, retrieval_js_1.
|
|
466
|
-
(0, retrieval_js_1.
|
|
496
|
+
// the server's retrieval path. #408: that clock is the calibration constant
|
|
497
|
+
// set (calibration.ts) — both processes resolve identically by construction.
|
|
498
|
+
(0, retrieval_js_1.configureDecay)();
|
|
499
|
+
(0, retrieval_js_1.configureRecall)();
|
|
500
|
+
(0, retrieval_js_1.configureScoring)();
|
|
467
501
|
const modeLabel = consolidateOnly ? " (consolidate-only)" : captureOnly ? " (capture-only)" : dryRun ? " (dry run)" : "";
|
|
468
502
|
console.log(`[hicortex] Nightly pipeline starting${modeLabel}`);
|
|
469
503
|
if (captureOnly) {
|
|
470
504
|
console.log(`[hicortex] capture-only run — consolidation skipped`);
|
|
471
505
|
}
|
|
472
506
|
console.log(`[hicortex] DB: ${dbPath}`);
|
|
507
|
+
// #405: the ONE pipeline deadline. Full and consolidate-only runs create
|
|
508
|
+
// it at start and share it across capture + every consolidation stage
|
|
509
|
+
// (checked at safe boundaries; a hit defers cleanly — see run-deadline.ts).
|
|
510
|
+
// Capture-only/watchdog runs do NOT: they keep their 30-min systemd
|
|
511
|
+
// backstop (capture is no-LLM and short). Dry-run is exempt (writes
|
|
512
|
+
// nothing, spends nothing). 0/invalid config → the 240 default — a
|
|
513
|
+
// deadline always exists (the old reconsolidationMaxMinutes 0=off is gone).
|
|
514
|
+
const deadline = !dryRun && !captureOnly
|
|
515
|
+
? (0, run_deadline_js_1.createRunDeadline)((0, run_deadline_js_1.resolveNightlyTimeBudgetMinutes)(savedConfig))
|
|
516
|
+
: undefined;
|
|
517
|
+
if (deadline) {
|
|
518
|
+
console.log(`[hicortex] Nightly time budget: ${Math.round((deadline.deadlineAt - Date.now()) / 60_000)} min ` +
|
|
519
|
+
`(deadline at ${new Date(deadline.deadlineAt).toISOString()})`);
|
|
520
|
+
}
|
|
473
521
|
// Init DB — consolidation reads the DB directly; capture goes via HTTP.
|
|
474
522
|
const db = (0, db_js_1.initDb)(dbPath);
|
|
475
523
|
try {
|
|
@@ -515,8 +563,11 @@ async function runNightly(options = {}) {
|
|
|
515
563
|
else {
|
|
516
564
|
// Full nightly waits out a transient --capture-only overlap (each segment
|
|
517
565
|
// POST can block up to 20 min); capture-only fails fast. dry-run writes
|
|
518
|
-
// nothing so it needs no lock.
|
|
519
|
-
|
|
566
|
+
// nothing so it needs no lock. #405: the wait never outlives the run
|
|
567
|
+
// deadline — waiting the full 30 min with 5 min left would burn the
|
|
568
|
+
// whole pipeline on a lock queue. captureLockWaitMs() keeps its env
|
|
569
|
+
// override (tests); min() clamps to whatever remains.
|
|
570
|
+
const lockWaitMs = Math.max(0, Math.min(captureLockWaitMs(), deadline ? deadline.remainingMs() : Infinity));
|
|
520
571
|
const releaseLock = dryRun
|
|
521
572
|
? (() => { })
|
|
522
573
|
: await (0, capture_js_1.acquireCaptureLock)(stateDir, captureOnly ? 0 : lockWaitMs);
|
|
@@ -537,7 +588,7 @@ async function runNightly(options = {}) {
|
|
|
537
588
|
// Step 1: Read new transcripts (CC + Hermes + Pi + OpenClaw). Discovery
|
|
538
589
|
// is whole-session by mtime/ended_at; per-session cursors slice each
|
|
539
590
|
// discovered session down to its unseen delta (#189).
|
|
540
|
-
const since = computeSince(stateDir, recaptureWindowDays);
|
|
591
|
+
const since = computeSince(stateDir, recaptureWindowDays, undefined, (0, config_read_js_1.readPositiveConfig)(savedConfig ?? {}, "firstRunLookbackDays", calibration_js_1.DEFAULT_FIRST_RUN_LOOKBACK_DAYS));
|
|
541
592
|
if (recaptureWindowDays) {
|
|
542
593
|
console.log(`[hicortex] --recapture-window ${recaptureWindowDays}d: reading transcripts since ${since.toISOString()}`);
|
|
543
594
|
}
|
|
@@ -581,11 +632,13 @@ async function runNightly(options = {}) {
|
|
|
581
632
|
// source_agent_id / source_domain are per-client provenance from
|
|
582
633
|
// config.json (agentId / sourceDomain) — attribution only, no filtering.
|
|
583
634
|
const result = await (0, capture_js_1.captureBatches)(batches, {
|
|
584
|
-
post: makeLocalPost(port, savedConfig?.authToken),
|
|
635
|
+
post: makeLocalPost(port, savedConfig?.authToken, deadline),
|
|
585
636
|
cursorStore,
|
|
586
637
|
dryRun,
|
|
587
638
|
sourceAgentId: savedConfig?.agentId,
|
|
588
639
|
sourceDomain: savedConfig?.sourceDomain,
|
|
640
|
+
sourceMachine: resolveCaptureMachine(savedConfig),
|
|
641
|
+
deadline,
|
|
589
642
|
});
|
|
590
643
|
memoriesIngested = result.memoriesIngested;
|
|
591
644
|
distillUsage = result.distillUsage;
|
|
@@ -620,6 +673,14 @@ async function runNightly(options = {}) {
|
|
|
620
673
|
// snapshot). 0 is a real value (under cap), so this stays undefined only
|
|
621
674
|
// when consolidation didn't run at all (capture-only / no_llm / skipped).
|
|
622
675
|
let evictedCount;
|
|
676
|
+
// #427: reconsolidation scout counters (hoisted for the dashboard
|
|
677
|
+
// snapshot). Flat snake_case on the wire, mirroring the stage report.
|
|
678
|
+
// Undefined only when consolidation didn't run; quiet-night zeros are
|
|
679
|
+
// REAL values from the stage's quiet-night report shape (the scan doesn't
|
|
680
|
+
// run on a quiet night — that's a fact about the night, not a gap).
|
|
681
|
+
let scoutScanned;
|
|
682
|
+
let scoutCorrectionShaped;
|
|
683
|
+
let scoutCandidatesFound;
|
|
623
684
|
// Resolved cap (#245) for the dashboard snapshot. Hoisted so the snapshot
|
|
624
685
|
// writer (outside the consolidation block) can stamp `capacity` even when
|
|
625
686
|
// consolidation was skipped (the cap is still "in force" config-wise).
|
|
@@ -633,10 +694,12 @@ async function runNightly(options = {}) {
|
|
|
633
694
|
// nothing-to-do short-circuit (zero LLM calls), NOT a failure.
|
|
634
695
|
// "throttled" (#246) = the llmTokensPerMonth fair-use cap was projected to
|
|
635
696
|
// be exceeded, so consolidation was skipped before any LLM call.
|
|
636
|
-
// "endpoint_down" (#337) = the
|
|
637
|
-
//
|
|
638
|
-
//
|
|
639
|
-
//
|
|
697
|
+
// "endpoint_down" (#337) = the LLM circuit breaker was open after the run —
|
|
698
|
+
// transient (retried next run), and NEVER "completed": the stages fail
|
|
699
|
+
// soft, so without this override a dead-endpoint run would report clean.
|
|
700
|
+
// "deferred" (#405) = the run-wide wall-clock deadline fired — at least one
|
|
701
|
+
// stage stopped at a safe boundary and drains next run (lastConsolidated
|
|
702
|
+
// held, exactly like endpoint_down).
|
|
640
703
|
let consolidationStatus;
|
|
641
704
|
// #246: total consolidation tokens consumed this run (hoisted for telemetry
|
|
642
705
|
// + the dashboard snapshot). Undefined when consolidation didn't run at all
|
|
@@ -688,12 +751,11 @@ async function runNightly(options = {}) {
|
|
|
688
751
|
// estimate so the next month starts clean.
|
|
689
752
|
if (!dryRun) {
|
|
690
753
|
(0, state_js_1.updateState)((s) => {
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
if (
|
|
695
|
-
|
|
696
|
-
s.llmTokensThisPeriod = { prompt: 0, completion: 0, total: 0, periodStart: now.toISOString() };
|
|
754
|
+
// #405: the ONE monthly-reset staleness helper — shared by
|
|
755
|
+
// shouldThrottleTokens, this file's post-consolidation
|
|
756
|
+
// accrual write, and token-budget.ts recordDistillUsage.
|
|
757
|
+
if ((0, consolidate_js_1.isStaleTokenPeriod)(s.llmTokensThisPeriod?.periodStart)) {
|
|
758
|
+
s.llmTokensThisPeriod = { prompt: 0, completion: 0, total: 0, periodStart: new Date().toISOString() };
|
|
697
759
|
s.llmTokensLastRun = 0;
|
|
698
760
|
console.log("[hicortex] Token fair-use period reset (new month) — throttle cleared.");
|
|
699
761
|
}
|
|
@@ -702,59 +764,41 @@ async function runNightly(options = {}) {
|
|
|
702
764
|
}
|
|
703
765
|
}
|
|
704
766
|
if (consolidationStatus !== "throttled") {
|
|
705
|
-
// #
|
|
706
|
-
//
|
|
707
|
-
//
|
|
708
|
-
//
|
|
709
|
-
//
|
|
710
|
-
//
|
|
711
|
-
//
|
|
712
|
-
//
|
|
713
|
-
|
|
714
|
-
// nightly re-runs 2-4×/day, and capture is unaffected (the daemon's
|
|
715
|
-
// /distill path has its own probe). Zero LLM phases run when the
|
|
716
|
-
// probe fails — one fast failure is the whole cost.
|
|
717
|
-
const probeOk = await llm.probe();
|
|
718
|
-
if (!probeOk) {
|
|
719
|
-
console.error("[hicortex] LLM endpoint not generating — consolidation skipped " +
|
|
720
|
-
"(endpoint_down, will retry next run). See the ops runbook's " +
|
|
721
|
-
"known failure signatures; llmProbeTimeoutMs tunes the probe's patience.");
|
|
722
|
-
consolidationStatus = "endpoint_down";
|
|
723
|
-
}
|
|
724
|
-
else {
|
|
767
|
+
// #405: the pre-consolidation readiness probe gate is DELETED. The
|
|
768
|
+
// breaker's first-fast-failure semantics took over its job: a
|
|
769
|
+
// wedged endpoint costs at most 3 ladder-exhausted logical calls
|
|
770
|
+
// (~31 min at the 900 s ceiling, once per dead night, bounded by
|
|
771
|
+
// the run deadline) before the post-run breakerOpen override below
|
|
772
|
+
// reports endpoint_down. The probe SURVIVES in the daemon's
|
|
773
|
+
// /distill gate (mcp-server.ts resolveDistillProbeGate) — a
|
|
774
|
+
// different process with a different job.
|
|
775
|
+
{
|
|
725
776
|
const cfgDomains = (0, domain_classify_js_1.parseConfigDomains)(savedConfig);
|
|
726
777
|
console.log(`[hicortex] Running consolidation...`);
|
|
727
778
|
const report = await (0, consolidate_js_1.runConsolidation)(db, llm, embedder_js_1.embed, dryRun, false, undefined, {
|
|
728
779
|
domains: cfgDomains,
|
|
729
780
|
contentDomainsReady: true,
|
|
730
|
-
|
|
781
|
+
// #408: weakPrimaryFloor is a release-managed calibration
|
|
782
|
+
// constant now — no config threading; the Options field stays
|
|
783
|
+
// as the eval/test seam.
|
|
731
784
|
}, {
|
|
732
|
-
|
|
733
|
-
|
|
785
|
+
// #405: no supersessionMaxCalls — the ONE run budget is the
|
|
786
|
+
// only call cap. #408: minSimilarity defaults to the
|
|
787
|
+
// calibration constant (seam only).
|
|
734
788
|
},
|
|
735
|
-
// #
|
|
736
|
-
|
|
789
|
+
// #405: the ONE per-run LLM-call ceiling (default 5000;
|
|
790
|
+
// consolidateMaxLlmCalls honored as a deprecated alias).
|
|
791
|
+
(0, consolidate_js_1.resolveNightlyLlmCallBudget)(savedConfig),
|
|
737
792
|
// #245: soft cap on the corpus (default 10000; 0 disables eviction).
|
|
738
793
|
memorySoftCapResolved, {
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
// — the stage defaults to 0.92 / 250.
|
|
748
|
-
autoMergeThreshold: (savedConfig?.dedupAutoMergeThreshold ??
|
|
749
|
-
savedConfig?.dedupMergeThreshold),
|
|
750
|
-
maxMerges: savedConfig?.dedupNightlyMaxMerges,
|
|
751
|
-
// #401 runtime bounds: the wall-clock deadline (default 120
|
|
752
|
-
// min, 0 disables) and the per-run classify-call ceiling
|
|
753
|
-
// (default 600, 0 disables). Same posture — the stage
|
|
754
|
-
// validates and falls back on invalid/absent values.
|
|
755
|
-
maxMinutes: savedConfig?.reconsolidationMaxMinutes,
|
|
756
|
-
maxCalls: savedConfig?.reconsolidationMaxCalls,
|
|
757
|
-
});
|
|
794
|
+
// #384/#392 reconsolidation knobs — eval/test seams since
|
|
795
|
+
// #408 (release-managed calibration constants; nothing is
|
|
796
|
+
// threaded from config). The stage validates and falls back
|
|
797
|
+
// to its calibration defaults on invalid/absent values.
|
|
798
|
+
},
|
|
799
|
+
// #405: the ONE run-wide deadline — capture and every
|
|
800
|
+
// consolidation stage check this same handle.
|
|
801
|
+
deadline);
|
|
758
802
|
console.log(`[hicortex] Consolidation ${report.status} in ${report.elapsed_seconds}s` +
|
|
759
803
|
(report.stages.reflection ? ` (${report.stages.reflection.lessons_generated} lessons)` : ""));
|
|
760
804
|
consolidationStatus = report.status;
|
|
@@ -775,6 +819,13 @@ async function runNightly(options = {}) {
|
|
|
775
819
|
// stage always returns `evicted` (0 when under cap / disabled); report
|
|
776
820
|
// it as 0 (a real value), not undefined, when the stage ran.
|
|
777
821
|
evictedCount = report.stages.memory_cap?.evicted ?? 0;
|
|
822
|
+
// #427: forward the scout counters whenever the stage ran. The
|
|
823
|
+
// stage report always carries them (quiet-night shape = zeros),
|
|
824
|
+
// so `?.` only falls through when the whole stage is absent
|
|
825
|
+
// (skipped run) — same skip-keys style as lessonsGenerated.
|
|
826
|
+
scoutScanned = report.stages.reconsolidation?.scout_scanned;
|
|
827
|
+
scoutCorrectionShaped = report.stages.reconsolidation?.scout_correction_shaped;
|
|
828
|
+
scoutCandidatesFound = report.stages.reconsolidation?.scout_candidates_found;
|
|
778
829
|
// Only set when reflection actually RAN (not skipped). A skipped stage
|
|
779
830
|
// (e.g. endpoint offline, #232 fail-soft) must NOT collapse to 0 — that
|
|
780
831
|
// would make "endpoint down" indistinguishable from "prompt too tight"
|
|
@@ -791,6 +842,13 @@ async function runNightly(options = {}) {
|
|
|
791
842
|
tokensThisRun = tokensTotal.total;
|
|
792
843
|
tokensByStage = report.budget?.tokens_by_stage;
|
|
793
844
|
}
|
|
845
|
+
// #427 observability: calls happened but ZERO tokens metered —
|
|
846
|
+
// the endpoint returned no usage objects (recordUsage no-ops by
|
|
847
|
+
// design). The snapshot would carry token nulls for such a run;
|
|
848
|
+
// make the blind spot GREPPABLE in journald instead of silent,
|
|
849
|
+
// the same structured-event style as budget_exhausted.
|
|
850
|
+
if (report.budget)
|
|
851
|
+
(0, consolidate_js_1.warnUnmeteredTokensRun)(report.budget);
|
|
794
852
|
// #255: budget exhaustion — always populated when consolidation ran
|
|
795
853
|
// (report.budget.exhausted is a boolean). The dashboard + telemetry
|
|
796
854
|
// treat true as a quality-degradation health signal. The
|
|
@@ -812,27 +870,18 @@ async function runNightly(options = {}) {
|
|
|
812
870
|
(0, state_js_1.updateState)((s) => {
|
|
813
871
|
const now = new Date();
|
|
814
872
|
const cur = s.llmTokensThisPeriod;
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
//
|
|
820
|
-
|
|
821
|
-
const
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
periodStart = now.toISOString();
|
|
825
|
-
prompt = 0;
|
|
826
|
-
completion = 0;
|
|
827
|
-
total = 0;
|
|
828
|
-
}
|
|
829
|
-
if (tokensTotal) {
|
|
830
|
-
prompt += tokensTotal.prompt;
|
|
831
|
-
completion += tokensTotal.completion;
|
|
832
|
-
total += tokensTotal.total;
|
|
833
|
-
}
|
|
873
|
+
// Monthly reset — the ONE staleness helper (#405): zero the
|
|
874
|
+
// accrual before adding this run's contribution when
|
|
875
|
+
// periodStart sits in a previous UTC calendar month. Shared
|
|
876
|
+
// with shouldThrottleTokens, this file's throttle-branch
|
|
877
|
+
// reset, and token-budget.ts recordDistillUsage.
|
|
878
|
+
const stale = (0, consolidate_js_1.isStaleTokenPeriod)(cur?.periodStart, now);
|
|
879
|
+
const prompt = (stale ? 0 : cur?.prompt ?? 0) + (tokensTotal?.prompt ?? 0);
|
|
880
|
+
const completion = (stale ? 0 : cur?.completion ?? 0) + (tokensTotal?.completion ?? 0);
|
|
881
|
+
const total = (stale ? 0 : cur?.total ?? 0) + (tokensTotal?.total ?? 0);
|
|
834
882
|
s.llmTokensThisPeriod = {
|
|
835
|
-
prompt, completion, total,
|
|
883
|
+
prompt, completion, total,
|
|
884
|
+
periodStart: stale ? now.toISOString() : cur.periodStart,
|
|
836
885
|
};
|
|
837
886
|
s.llmTokensLastRun = tokensTotal?.total ?? 0;
|
|
838
887
|
}, stateDir);
|
|
@@ -981,6 +1030,11 @@ async function runNightly(options = {}) {
|
|
|
981
1030
|
dedup,
|
|
982
1031
|
supersession,
|
|
983
1032
|
evicted: evictedCount,
|
|
1033
|
+
// #427: scout counters — forwarded whenever consolidation ran
|
|
1034
|
+
// (writeSnapshot omits the keys when undefined).
|
|
1035
|
+
scoutScanned,
|
|
1036
|
+
scoutCorrectionShaped,
|
|
1037
|
+
scoutCandidatesFound,
|
|
984
1038
|
// #246: token accounting from this run's consolidation (undefined
|
|
985
1039
|
// when consolidation didn't run or made no metered calls).
|
|
986
1040
|
tokensThisRun,
|
|
@@ -1089,13 +1143,12 @@ async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recapt
|
|
|
1089
1143
|
// the whole run — the pre-flight only needs the link back, which can take
|
|
1090
1144
|
// ~1 min after wake.
|
|
1091
1145
|
//
|
|
1092
|
-
//
|
|
1093
|
-
//
|
|
1094
|
-
//
|
|
1095
|
-
//
|
|
1096
|
-
//
|
|
1097
|
-
//
|
|
1098
|
-
// timeout length helps — the capture watchdog's frequent retry handles that (#239).
|
|
1146
|
+
// #163/#405: constants (the preflightTimeoutMs/preflightAttempts/
|
|
1147
|
+
// preflightRetryGapMs config keys are removed — no incident in the #403
|
|
1148
|
+
// inventory ever required tuning them). 20s per attempt absorbs a slow link
|
|
1149
|
+
// re-establishing after the client wakes (bumped from 15s in 0.17); for a
|
|
1150
|
+
// genuinely DOWN link no timeout length helps — the capture watchdog's
|
|
1151
|
+
// frequent retry handles that (#239).
|
|
1099
1152
|
//
|
|
1100
1153
|
// WALL-CLOCK NOTE: setTimeout and AbortSignal.timeout do NOT advance while
|
|
1101
1154
|
// macOS is asleep, so the ~3m worst case (3×20s + 2×60s) is wall-clock-
|
|
@@ -1103,9 +1156,9 @@ async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recapt
|
|
|
1103
1156
|
// elapsed time can exceed it. Not a defect: the capture lock isn't held
|
|
1104
1157
|
// during the retry and the cursor design is dup-over-loss, so a late success
|
|
1105
1158
|
// is harmless. Just don't treat 3m as a hard wall-clock bound.
|
|
1106
|
-
const PREFLIGHT_TIMEOUT_MS =
|
|
1107
|
-
const PREFLIGHT_ATTEMPTS =
|
|
1108
|
-
const PREFLIGHT_RETRY_GAP_MS =
|
|
1159
|
+
const PREFLIGHT_TIMEOUT_MS = 20_000;
|
|
1160
|
+
const PREFLIGHT_ATTEMPTS = 3;
|
|
1161
|
+
const PREFLIGHT_RETRY_GAP_MS = 60_000;
|
|
1109
1162
|
let reachable = false;
|
|
1110
1163
|
for (let attempt = 1; attempt <= PREFLIGHT_ATTEMPTS; attempt++) {
|
|
1111
1164
|
try {
|
|
@@ -1194,7 +1247,7 @@ async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recapt
|
|
|
1194
1247
|
// logs, denoises, and POSTs the denoised text to the server's /distill
|
|
1195
1248
|
// endpoint. All readers no-op when their harness isn't installed. Per-session
|
|
1196
1249
|
// cursors slice each discovered session to its unseen delta (#189).
|
|
1197
|
-
const since = computeSince(stateDir, recaptureWindowDays);
|
|
1250
|
+
const since = computeSince(stateDir, recaptureWindowDays, undefined, (0, config_read_js_1.readPositiveConfig)(config ?? {}, "firstRunLookbackDays", calibration_js_1.DEFAULT_FIRST_RUN_LOOKBACK_DAYS));
|
|
1198
1251
|
if (recaptureWindowDays) {
|
|
1199
1252
|
console.log(`[hicortex] --recapture-window ${recaptureWindowDays}d: reading transcripts since ${since.toISOString()}`);
|
|
1200
1253
|
}
|
|
@@ -1232,6 +1285,7 @@ async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recapt
|
|
|
1232
1285
|
// server stores these alongside source_agent; nothing filters on them.
|
|
1233
1286
|
sourceAgentId: config.agentId,
|
|
1234
1287
|
sourceDomain: config.sourceDomain,
|
|
1288
|
+
sourceMachine: resolveCaptureMachine(config),
|
|
1235
1289
|
});
|
|
1236
1290
|
memoriesIngested = result.memoriesIngested;
|
|
1237
1291
|
sessionsSent = result.sessionsSent;
|
package/dist/nofit.d.ts
CHANGED
|
@@ -54,21 +54,14 @@ import type { DomainDef } from "./types.js";
|
|
|
54
54
|
* Default weak-primary floor: minimum cosine(memory embedding, best domain
|
|
55
55
|
* prototype) for a no-fit memory to earn a weak primary.
|
|
56
56
|
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
* ~/.hicortex/config.json.
|
|
57
|
+
* #408: RELEASE-MANAGED — the constant (and its provenance: a starting point
|
|
58
|
+
* for bge-small-en-v1.5 embeddings) lives in calibration.ts. There is no
|
|
59
|
+
* config override anymore; the DomainStageOptions.weakPrimaryFloor field
|
|
60
|
+
* stays as the eval/test seam.
|
|
62
61
|
*/
|
|
63
62
|
export declare const DEFAULT_WEAK_PRIMARY_FLOOR = 0.45;
|
|
64
63
|
/** Halving never takes base_strength below this (survivable, not zeroed). */
|
|
65
64
|
export declare const NO_ASSOCIATION_MIN_STRENGTH = 0.05;
|
|
66
|
-
/**
|
|
67
|
-
* Resolve the weak-primary floor from a raw config object. Accepts a finite
|
|
68
|
-
* number in (0, 1); anything else (absent, wrong type, out of range) falls
|
|
69
|
-
* back to DEFAULT_WEAK_PRIMARY_FLOOR with a warning for invalid values.
|
|
70
|
-
*/
|
|
71
|
-
export declare function resolveWeakPrimaryFloor(config: Record<string, unknown> | null | undefined): number;
|
|
72
65
|
/** Outcome of resolving a no-fit memory against the domain prototypes. */
|
|
73
66
|
export type NoFitResolution = {
|
|
74
67
|
kind: "weak_primary";
|
package/dist/nofit.js
CHANGED
|
@@ -84,41 +84,24 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
84
84
|
})();
|
|
85
85
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
86
86
|
exports.NO_ASSOCIATION_MIN_STRENGTH = exports.DEFAULT_WEAK_PRIMARY_FLOOR = void 0;
|
|
87
|
-
exports.resolveWeakPrimaryFloor = resolveWeakPrimaryFloor;
|
|
88
87
|
exports.resolveNoFit = resolveNoFit;
|
|
89
88
|
exports.applyWeakPrimary = applyWeakPrimary;
|
|
90
89
|
exports.applyNoAssociationDecay = applyNoAssociationDecay;
|
|
91
90
|
const storage = __importStar(require("./storage.js"));
|
|
92
91
|
const schema_prototypes_js_1 = require("./schema-prototypes.js");
|
|
92
|
+
const CALIBRATION = __importStar(require("./calibration.js"));
|
|
93
93
|
/**
|
|
94
94
|
* Default weak-primary floor: minimum cosine(memory embedding, best domain
|
|
95
95
|
* prototype) for a no-fit memory to earn a weak primary.
|
|
96
96
|
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* ~/.hicortex/config.json.
|
|
97
|
+
* #408: RELEASE-MANAGED — the constant (and its provenance: a starting point
|
|
98
|
+
* for bge-small-en-v1.5 embeddings) lives in calibration.ts. There is no
|
|
99
|
+
* config override anymore; the DomainStageOptions.weakPrimaryFloor field
|
|
100
|
+
* stays as the eval/test seam.
|
|
102
101
|
*/
|
|
103
|
-
exports.DEFAULT_WEAK_PRIMARY_FLOOR =
|
|
102
|
+
exports.DEFAULT_WEAK_PRIMARY_FLOOR = CALIBRATION.WEAK_PRIMARY_FLOOR;
|
|
104
103
|
/** Halving never takes base_strength below this (survivable, not zeroed). */
|
|
105
104
|
exports.NO_ASSOCIATION_MIN_STRENGTH = 0.05;
|
|
106
|
-
/**
|
|
107
|
-
* Resolve the weak-primary floor from a raw config object. Accepts a finite
|
|
108
|
-
* number in (0, 1); anything else (absent, wrong type, out of range) falls
|
|
109
|
-
* back to DEFAULT_WEAK_PRIMARY_FLOOR with a warning for invalid values.
|
|
110
|
-
*/
|
|
111
|
-
function resolveWeakPrimaryFloor(config) {
|
|
112
|
-
const raw = config?.weakPrimaryFloor;
|
|
113
|
-
if (raw === undefined || raw === null)
|
|
114
|
-
return exports.DEFAULT_WEAK_PRIMARY_FLOOR;
|
|
115
|
-
if (typeof raw === "number" && Number.isFinite(raw) && raw > 0 && raw < 1) {
|
|
116
|
-
return raw;
|
|
117
|
-
}
|
|
118
|
-
console.warn(`[hicortex] invalid weakPrimaryFloor in config (${JSON.stringify(raw)}) — ` +
|
|
119
|
-
`must be a number in (0, 1); using default ${exports.DEFAULT_WEAK_PRIMARY_FLOOR}`);
|
|
120
|
-
return exports.DEFAULT_WEAK_PRIMARY_FLOOR;
|
|
121
|
-
}
|
|
122
105
|
/**
|
|
123
106
|
* Decide what happens to a no-fit memory (READ-ONLY — no writes, so callers
|
|
124
107
|
* batching writes into a transaction can decide during the scan phase):
|
package/dist/prompts.d.ts
CHANGED
|
@@ -10,6 +10,16 @@
|
|
|
10
10
|
*/
|
|
11
11
|
/**
|
|
12
12
|
* Importance scoring prompt. Takes a {memories_block} with indexed memories.
|
|
13
|
+
*
|
|
14
|
+
* RE-ANCHORED (#425): the pre-fix anchors put "useful context" at 0.3-0.5 —
|
|
15
|
+
* but the distiller's ephemera gate already removes trivia before anything
|
|
16
|
+
* reaches this scorer, so the model only ever saw curated material and the
|
|
17
|
+
* distribution compressed upward (measured on the production snapshot via
|
|
18
|
+
* eval:importance: median base 0.8, ~11% at exactly 1.0, which the decay
|
|
19
|
+
* model never forgets). The anchors now place routine-but-curated content
|
|
20
|
+
* LOW and add an explicit distribution instruction (owner decision D2,
|
|
21
|
+
* 2026-09-13: target median 0.30-0.40, p90 <= 0.75; 1.0 is never a valid
|
|
22
|
+
* score). The strict JSON-array response contract is unchanged.
|
|
13
23
|
*/
|
|
14
24
|
export declare function importanceScoring(memoriesBlock: string): string;
|
|
15
25
|
/**
|