@gamaze/hicortex 0.20.9 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +8 -0
  2. package/assets/dashboard.html +4174 -835
  3. package/dist/calibration.d.ts +119 -0
  4. package/dist/calibration.js +149 -1
  5. package/dist/capture-health.d.ts +87 -0
  6. package/dist/capture-health.js +106 -0
  7. package/dist/capture-pause.d.ts +86 -0
  8. package/dist/capture-pause.js +127 -0
  9. package/dist/capture.d.ts +9 -0
  10. package/dist/capture.js +2 -1
  11. package/dist/cli.js +36 -0
  12. package/dist/consolidate.d.ts +35 -0
  13. package/dist/consolidate.js +85 -9
  14. package/dist/dashboard.d.ts +322 -3
  15. package/dist/dashboard.js +592 -7
  16. package/dist/db.js +105 -0
  17. package/dist/eval/importance-eval.d.ts +85 -0
  18. package/dist/eval/importance-eval.js +286 -0
  19. package/dist/eval/planted-fixtures.d.ts +1 -1
  20. package/dist/eval/ranking-battery.d.ts +78 -0
  21. package/dist/eval/ranking-battery.js +181 -0
  22. package/dist/eval/ranking-eval.d.ts +41 -0
  23. package/dist/eval/ranking-eval.js +391 -0
  24. package/dist/eval/ranking-fixtures.d.ts +77 -0
  25. package/dist/eval/ranking-fixtures.js +226 -0
  26. package/dist/identity-store.d.ts +21 -0
  27. package/dist/identity-store.js +49 -0
  28. package/dist/init.d.ts +14 -0
  29. package/dist/init.js +32 -0
  30. package/dist/mcp-server.d.ts +12 -0
  31. package/dist/mcp-server.js +184 -3
  32. package/dist/nightly.d.ts +9 -1
  33. package/dist/nightly.js +59 -7
  34. package/dist/prompts.d.ts +10 -0
  35. package/dist/prompts.js +28 -5
  36. package/dist/reconsolidation.d.ts +59 -30
  37. package/dist/reconsolidation.js +526 -296
  38. package/dist/rescore-importance.d.ts +80 -0
  39. package/dist/rescore-importance.js +236 -0
  40. package/dist/retrieval.d.ts +12 -0
  41. package/dist/retrieval.js +30 -1
  42. package/dist/stages.d.ts +37 -0
  43. package/dist/stages.js +51 -0
  44. package/dist/state.d.ts +32 -6
  45. package/dist/storage.d.ts +34 -2
  46. package/dist/storage.js +63 -6
  47. package/dist/types.d.ts +48 -0
  48. package/package.json +3 -1
@@ -51,6 +51,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
51
51
  exports.resolveDistillProbeGate = resolveDistillProbeGate;
52
52
  exports.createMcpServer = createMcpServer;
53
53
  exports.resolveBodyLimitMb = resolveBodyLimitMb;
54
+ exports.resolveSearchSimilarityFloor = resolveSearchSimilarityFloor;
54
55
  exports.makeBodyLimitErrorHandler = makeBodyLimitErrorHandler;
55
56
  exports.makeContentLengthGate = makeContentLengthGate;
56
57
  exports.startServer = startServer;
@@ -88,6 +89,8 @@ const distiller_js_1 = require("./distiller.js");
88
89
  const dedup_js_1 = require("./dedup.js");
89
90
  const reconsolidation_js_1 = require("./reconsolidation.js");
90
91
  const redact_js_1 = require("./redact.js");
92
+ const capture_health_js_1 = require("./capture-health.js");
93
+ const capture_pause_js_1 = require("./capture-pause.js");
91
94
  const init_js_1 = require("./init.js");
92
95
  // ---------------------------------------------------------------------------
93
96
  // Server state
@@ -498,6 +501,25 @@ function resolveBodyLimitMb(configVal, hostedMode) {
498
501
  return cfg;
499
502
  return hostedMode ? 5 : 25;
500
503
  }
504
+ /**
505
+ * Resolve the OPTIONAL /search relevance floor (`minSimilarity` query param,
506
+ * #409 console polish). Pure — exported for tests. Absent/blank/invalid →
507
+ * undefined = NO gate (byte-identical to every pre-existing caller: agents,
508
+ * plugins, MCP tools never send the param). A finite number in [0, 1] → that
509
+ * floor, clamped into range so a hostile `?minSimilarity=42` cannot widen or
510
+ * invert the gate. Applied AFTER retrieve() with the exported recall gate
511
+ * (passesRelevanceGate: FTS/`both` hits pass regardless — a token match is
512
+ * real evidence; vector-only hits must clear the floor) — the same post-hoc
513
+ * shape /recall-index uses, so the two recall surfaces gate identically.
514
+ */
515
+ function resolveSearchSimilarityFloor(raw) {
516
+ if (typeof raw !== "string" || raw.trim() === "")
517
+ return undefined;
518
+ const v = Number(raw);
519
+ if (!Number.isFinite(v))
520
+ return undefined;
521
+ return Math.min(1, Math.max(0, v));
522
+ }
501
523
  /**
502
524
  * Express error middleware (#7): translate express.json's default HTML 413
503
525
  * (entity.too.large) into a consistent JSON response. Catches body-parser
@@ -915,7 +937,7 @@ async function startServer(options = {}) {
915
937
  res.status(503).json({ error: "Server not initialized" });
916
938
  return;
917
939
  }
918
- const { content, source_agent, source_agent_id, source_domain, project, memory_type, privacy, source_session, session_date, corrects, supersedes } = req.body ?? {};
940
+ const { content, source_agent, source_agent_id, source_domain, source_machine, project, memory_type, privacy, source_session, session_date, corrects, supersedes } = req.body ?? {};
919
941
  if (!content || typeof content !== "string") {
920
942
  res.status(400).json({ error: "Missing or invalid 'content' field" });
921
943
  return;
@@ -966,6 +988,7 @@ async function startServer(options = {}) {
966
988
  // Attribution + provenance passthrough (0.16.x); null when absent.
967
989
  sourceAgentId: typeof source_agent_id === "string" ? source_agent_id : null,
968
990
  sourceDomain: typeof source_domain === "string" ? source_domain : null,
991
+ sourceMachine: storage.sanitizeSourceMachine(source_machine),
969
992
  sourceSession: source_session ?? undefined,
970
993
  project: project ?? undefined,
971
994
  memoryType: normalizedType ?? "experience",
@@ -1007,8 +1030,16 @@ async function startServer(options = {}) {
1007
1030
  // clients/plugins still send it) but no longer read — retrieval ignores
1008
1031
  // privacy entirely (the column is vestigial, never filtered).
1009
1032
  warnDeprecatedPrivacyParamIfPresent(req.query, "search");
1033
+ // #409 console polish: OPTIONAL relevance floor. Absent (every legacy
1034
+ // caller) = ungated, byte-identical behavior. The console sends the
1035
+ // recall floor (RECALL_MIN_SIMILARITY, via the /dashboard/field echo) so
1036
+ // vector nearest-neighbor junk below it never renders as "results".
1037
+ const minSimilarity = resolveSearchSimilarityFloor(req.query.minSimilarity);
1010
1038
  try {
1011
- const results = await retrieval.retrieve(db, embedder_js_1.embed, query, { limit, project });
1039
+ let results = await retrieval.retrieve(db, embedder_js_1.embed, query, { limit, project });
1040
+ if (minSimilarity !== undefined) {
1041
+ results = results.filter((r) => (0, recall_index_js_1.passesRelevanceGate)(r, minSimilarity));
1042
+ }
1012
1043
  res.json({ results });
1013
1044
  }
1014
1045
  catch (err) {
@@ -1171,6 +1202,35 @@ async function startServer(options = {}) {
1171
1202
  (0, health_js_1.logAndSendInternalError)(res, "context", err);
1172
1203
  }
1173
1204
  });
1205
+ // PUT /identity/mode — switch ONE agent's identity scope (#423 phase 3).
1206
+ // No /context/mode alias: this is a NEW endpoint with no legacy callers.
1207
+ //
1208
+ // Single-writer discipline: the daemon's boot-time identityAgents map is
1209
+ // normally read-once-at-boot — THIS route is the one live writer. The
1210
+ // adapter reads the FRESH config (not the boot snapshot) so two switches in
1211
+ // a row can never drop each other's writes, then on success (1) persists
1212
+ // the merged map as config `identityAgents` (survives restarts) and (2)
1213
+ // sets the module-level map to the SAME map — the switch is live on the
1214
+ // very next GET /identity?agent= (applies: "immediate"). Externally
1215
+ // hand-edited config still needs a restart (pre-existing posture). Note
1216
+ // the interplay: PUT /identity's black-hole guard 409s section writes while
1217
+ // config forces off/global — switching to 'override' here first unblocks
1218
+ // section editing. Bearer-only (standard auth middleware, no exemption).
1219
+ app.put("/identity/mode", (req, res) => {
1220
+ try {
1221
+ const fresh = readConfigFile(stateDir) ?? {};
1222
+ const r = (0, identity_store_js_1.handleIdentityModePut)(identityDir, req.body ?? null, req.query, (0, identity_store_js_1.resolveIdentityAgentsConfig)(fresh).agents);
1223
+ if (r.status === 200 && r.agents) {
1224
+ const merged = r.agents;
1225
+ (0, init_js_1.persistConfigUpdates)((0, node_path_1.join)(stateDir, "config.json"), { identityAgents: merged });
1226
+ identityAgents = merged;
1227
+ }
1228
+ res.status(r.status).json(r.body);
1229
+ }
1230
+ catch (err) {
1231
+ (0, health_js_1.logAndSendInternalError)(res, "identity/mode", err);
1232
+ }
1233
+ });
1174
1234
  // REST /distill — canonical capture endpoint (0.9.0+).
1175
1235
  // Every machine (including the server itself) POSTs denoised session text here.
1176
1236
  // The server distills, embeds, stores. Body limit: see `distillBodyLimitMb`
@@ -1184,11 +1244,42 @@ async function startServer(options = {}) {
1184
1244
  res.status(503).json({ error: "Server not initialized" });
1185
1245
  return;
1186
1246
  }
1247
+ // #422: destructured ABOVE the no-LLM guard (it used to sit below) so the
1248
+ // attribution fields exist at EVERY exit incl. the 503 — capture-health
1249
+ // accounting records held posts there too. Nothing else reads them before
1250
+ // the guard, so semantics are unchanged.
1251
+ const { text, messages, source_agent, source_agent_id, source_domain, source_machine, project, session_id, segment_id, session_date, privacy } = req.body ?? {};
1252
+ // #422 capture health: normalize the wire fields ONCE so every exit below
1253
+ // records with a one-liner (recordDistillActivity re-sanitizes machine/
1254
+ // agent — the builder only forwards + picks the byte count). bytes = the
1255
+ // resolved (post-redaction) conversationText length, 0 while unresolved.
1256
+ const capEntry = (bytes, outcome) => ({
1257
+ machine: source_machine,
1258
+ agent: source_agent,
1259
+ sessionId: session_id,
1260
+ segmentId: segment_id,
1261
+ bytes,
1262
+ outcome,
1263
+ });
1264
+ // #423 phase 3 (D3): operator capture pause — server-side 200-skip. The
1265
+ // pause table is read per POST, so a pause takes effect on the very next
1266
+ // /distill (no restart). Deliberately ABOVE the no-LLM guard: a paused
1267
+ // bundle must skip regardless of LLM state — a 503 there would hold the
1268
+ // client's cursor on a box that is deliberately not capturing. The 200 is
1269
+ // the point: capture.ts treats every 200 as confirmed and advances its
1270
+ // cursor, so paused sessions are deliberately NOT captured and never
1271
+ // re-sent/backfilled. Zero client changes.
1272
+ const pause = (0, capture_pause_js_1.capturePauseKey)(source_machine, source_agent);
1273
+ if ((0, capture_pause_js_1.isCapturePaused)(db, pause.machine, pause.harness)) {
1274
+ (0, capture_health_js_1.recordDistillActivity)(db, capEntry(0, "paused"));
1275
+ res.status(200).json({ skipped: true, paused: true, machine: pause.machine, harness: pause.harness });
1276
+ return;
1277
+ }
1187
1278
  if (!llm || !llmConfig) {
1279
+ (0, capture_health_js_1.recordDistillActivity)(db, capEntry(0, "held"));
1188
1280
  res.status(503).json({ error: "No LLM configured — run npx @gamaze/hicortex init. Session will be retried." });
1189
1281
  return;
1190
1282
  }
1191
- const { text, messages, source_agent, source_agent_id, source_domain, project, session_id, segment_id, session_date, privacy } = req.body ?? {};
1192
1283
  // Resolve the conversation text from either the pre-denoised string or raw messages array.
1193
1284
  // `fromTextBranch` is captured once so the redaction gate below uses the SAME
1194
1285
  // discriminator as the resolution (avoids re-redacting the messages-derived
@@ -1238,6 +1329,7 @@ async function startServer(options = {}) {
1238
1329
  if (session_id && segment_id) {
1239
1330
  const existingCount = (0, dedup_js_1.countExistingSegment)(db, session_id, segment_id);
1240
1331
  if (existingCount > 0) {
1332
+ (0, capture_health_js_1.recordDistillActivity)(db, capEntry(conversationText.length, "skipped"));
1241
1333
  res.status(200).json({ skipped: true, existing_count: existingCount });
1242
1334
  return;
1243
1335
  }
@@ -1249,6 +1341,7 @@ async function startServer(options = {}) {
1249
1341
  if (session_id && !segment_id) {
1250
1342
  const existingCount = (0, dedup_js_1.countExistingSession)(db, session_id);
1251
1343
  if (existingCount > 0) {
1344
+ (0, capture_health_js_1.recordDistillActivity)(db, capEntry(conversationText.length, "skipped"));
1252
1345
  res.status(200).json({ skipped: true, existing_count: existingCount });
1253
1346
  return;
1254
1347
  }
@@ -1260,6 +1353,7 @@ async function startServer(options = {}) {
1260
1353
  // diagnosis — the capture client treats non-201/200 as transient and holds
1261
1354
  // its cursor (capture.ts), so the segment is retried next run, never lost.
1262
1355
  if (!(await resolveDistillProbeGate(llm, llmConfig))) {
1356
+ (0, capture_health_js_1.recordDistillActivity)(db, capEntry(conversationText.length, "held"));
1263
1357
  res.status(503).json({ error: "LLM endpoint not generating — session will be retried" });
1264
1358
  return;
1265
1359
  }
@@ -1285,6 +1379,7 @@ async function startServer(options = {}) {
1285
1379
  // a skipped duplicate neither trips the gate nor consumes budget. The client
1286
1380
  // capture loop holds its cursor on 429 (dup-over-loss, capture.ts:303).
1287
1381
  if ((0, token_budget_js_1.isTokenBudgetExceeded)(stateDir)) {
1382
+ (0, capture_health_js_1.recordDistillActivity)(db, capEntry(conversationText.length, "held"));
1288
1383
  res.status(429).json({ error: "token budget exceeded", retry: "next billing period" });
1289
1384
  return;
1290
1385
  }
@@ -1333,6 +1428,7 @@ async function startServer(options = {}) {
1333
1428
  // filtered. Default null for older clients that don't send them.
1334
1429
  sourceAgentId: typeof source_agent_id === "string" ? source_agent_id : null,
1335
1430
  sourceDomain: typeof source_domain === "string" ? source_domain : null,
1431
+ sourceMachine: storage.sanitizeSourceMachine(source_machine),
1336
1432
  // Per-chunk key: "<session_id>[#<segment_id>]#<i>". The prefix
1337
1433
  // matches the dedup checks above, so a re-run is idempotent.
1338
1434
  sourceSession: sourcePrefix ? `${sourcePrefix}#${i}` : undefined,
@@ -1351,6 +1447,7 @@ async function startServer(options = {}) {
1351
1447
  return out;
1352
1448
  });
1353
1449
  const ids = insertAll();
1450
+ (0, capture_health_js_1.recordDistillActivity)(db, capEntry(conversationText.length, "ok"));
1354
1451
  res.status(201).json({
1355
1452
  ids,
1356
1453
  distilled: ids.length,
@@ -1364,6 +1461,7 @@ async function startServer(options = {}) {
1364
1461
  });
1365
1462
  }
1366
1463
  catch (err) {
1464
+ (0, capture_health_js_1.recordDistillActivity)(db, capEntry(conversationText.length, "held"));
1367
1465
  res.status(500).json({ error: "Distillation failed" });
1368
1466
  console.error(`[hicortex] /distill: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`);
1369
1467
  }
@@ -1446,6 +1544,52 @@ async function startServer(options = {}) {
1446
1544
  }
1447
1545
  });
1448
1546
  // -------------------------------------------------------------------------
1547
+ // REST /enrich — owner corroboration (#423 phase 3).
1548
+ //
1549
+ // An enrich is EVIDENCE ABOUT IMPORTANCE: it bumps corroboration_count and
1550
+ // base_strength (+the calibration delta, capped at 1.0) — the same anchor
1551
+ // the nightly's LLM scoring and hub-boost write. It must NOT touch
1552
+ // access_count (reserved for real recall use) or shown_count (index
1553
+ // exposure) — faking either corrupts the uses-per-showing adoption metric.
1554
+ // Never a stage write: stages are derived presentation (the E-reframe).
1555
+ // An absorbed memory is invisible evidence and cannot be corroborated
1556
+ // (409, same posture as /update).
1557
+ // -------------------------------------------------------------------------
1558
+ app.post("/enrich", (req, res) => {
1559
+ if (!db) {
1560
+ res.status(503).json({ error: "Server not initialized" });
1561
+ return;
1562
+ }
1563
+ const { id } = req.body ?? {};
1564
+ if (!id || typeof id !== "string") {
1565
+ res.status(400).json({ error: "Missing or invalid 'id' field" });
1566
+ return;
1567
+ }
1568
+ const fullId = resolveMemoryId(db, id);
1569
+ if (!fullId) {
1570
+ res.status(404).json({ error: "Memory not found" });
1571
+ return;
1572
+ }
1573
+ // Same absorbed guard as /update: corroborating an invisible row would
1574
+ // strengthen evidence the store deliberately folded into another memory.
1575
+ const target = storage.getMemory(db, fullId);
1576
+ if (target?.status === "absorbed") {
1577
+ res.status(409).json({ error: "Memory is absorbed — invisible evidence cannot be corroborated" });
1578
+ return;
1579
+ }
1580
+ try {
1581
+ const r = storage.enrichMemory(db, fullId, new Date().toISOString());
1582
+ if (!r) {
1583
+ res.status(404).json({ error: "Memory not found" });
1584
+ return;
1585
+ }
1586
+ res.status(200).json({ id: fullId, corroboration_count: r.corroborationCount, base_strength: r.baseStrength });
1587
+ }
1588
+ catch (err) {
1589
+ (0, health_js_1.logAndSendInternalError)(res, "enrich", err);
1590
+ }
1591
+ });
1592
+ // -------------------------------------------------------------------------
1449
1593
  // REST /delete — permanently delete a memory and its links.
1450
1594
  //
1451
1595
  // NOTE for #124: returns {deleted: true, id} — clean JSON for future /viz.
@@ -1662,6 +1806,43 @@ async function startServer(options = {}) {
1662
1806
  // express adapter that injects the live db + config. STRICTLY view-only —
1663
1807
  // no mutation endpoints on the dashboard surface.
1664
1808
  app.get("/dashboard/data", (0, dashboard_js_1.dashboardDataHandler)(() => db, () => readConfigFile(stateDir)));
1809
+ // GET /dashboard/field — the console flight-field payload (#409/#421
1810
+ // Phase 1): the whole live store as minimal fields (titles ≤100 via the
1811
+ // production memoryTitle, derived stage + effective strength, no content
1812
+ // bodies) plus every link edge {a, b, rel}. Bearer-only (auth middleware,
1813
+ // no shell exemption — it carries data); localhost bypass applies. Handler
1814
+ // + gzip adapter live in src/dashboard.ts next to its /data sibling; the
1815
+ // field is one row per memory, so Accept-Encoding: gzip clients get the
1816
+ // compressed wire form.
1817
+ app.get("/dashboard/field", (0, dashboard_js_1.dashboardFieldHandler)(() => db));
1818
+ // GET /dashboard/events?days=N — the console replay ledger (#409/#421
1819
+ // Phase 1): night-resolution synthesis over existing tables (created_at /
1820
+ // memory_history / dedup_log / memory_links; no schema change, no
1821
+ // event-sourcing store). ids only. Bearer-only like /dashboard/field;
1822
+ // localhost bypass applies.
1823
+ app.get("/dashboard/events", (0, dashboard_js_1.dashboardEventsHandler)(() => db));
1824
+ // GET/PUT /dashboard/model — the console's model-settings surface (#422
1825
+ // Phase 2): the ONE scoped writer on the dashboard (view-only everywhere
1826
+ // else). GET echoes the config's model knobs raw (null = unset) + the
1827
+ // boot-resolved provider from the daemon's in-memory llmConfig; api_key_set
1828
+ // carries ONLY the key's presence — no key material on the wire, ever. PUT
1829
+ // validates an allowlisted subset (null clears a config key) and persists
1830
+ // via init.ts persistConfigUpdates — strict load, so a malformed config.json
1831
+ // throws → 500 with the file untouched. Bearer-only (auth middleware, no
1832
+ // shell exemption — it carries install config); localhost bypass applies.
1833
+ // Applies on restart: the daemon resolves config at boot (llmConfig is the
1834
+ // boot snapshot; the card footnotes this).
1835
+ app.get("/dashboard/model", (0, dashboard_js_1.dashboardModelGetHandler)(() => readConfigFile(stateDir), () => llmConfig));
1836
+ app.put("/dashboard/model", (0, dashboard_js_1.dashboardModelPutHandler)((updates) => (0, init_js_1.persistConfigUpdates)((0, node_path_1.join)(stateDir, "config.json"), updates), () => llmConfig));
1837
+ // PUT /dashboard/capture-pause — the console's pause/resume toggle (#423
1838
+ // phase 3, D3). Body {machine?, harness, paused}: a pause makes /distill
1839
+ // 200-skip the bundle's posts — deliberate NON-capture, the sessions are
1840
+ // not backfilled (the client cursor advances on the 200, by design). The
1841
+ // effect is IMMEDIATE — no restart — because the /distill handler reads the
1842
+ // capture_pauses table on every post. Bearer-only like /dashboard/model
1843
+ // (auth middleware, no shell exemption — it mutates operator state);
1844
+ // localhost bypass applies. Handler + adapter live in src/dashboard.ts.
1845
+ app.put("/dashboard/capture-pause", (0, dashboard_js_1.dashboardCapturePausePutHandler)(() => db));
1665
1846
  // GET /account — account identity for the console nav (name/org/plan from
1666
1847
  // config). The LIGHTWEIGHT twin of the account block inside /dashboard/data:
1667
1848
  // the /viz and /identity/ui pages need only this, not the metric payload;
package/dist/nightly.d.ts CHANGED
@@ -20,6 +20,14 @@
20
20
  * (#189 review, fix 3). Per-session cursors keep the wide re-scan cheap: an
21
21
  * already-captured session yields an empty delta.
22
22
  *
23
+ * First run (#436): with NO watermark yet, since = now − firstRunLookbackDays
24
+ * (default 7 — owner ruling 2026-09-14: full-history first-run ingestion is
25
+ * not feasible; a long-term AI user's entire session store must not be
26
+ * discovered night one). The widening-only invariant composes unchanged:
27
+ * min(now−7d, now−N) — a recapture window wider than the default widens, a
28
+ * narrower one leaves the floor. Installing more history is the deliberate
29
+ * `--recapture-window` act, not a default.
30
+ *
23
31
  * Clock-jump clamp (#327): a FUTURE-dated lastNightly (client clock error —
24
32
  * NTP not yet synced at write time) would, once the clock corrects, sit ahead
25
33
  * of every session mtime and permanently skip quiet sessions (their mtimes
@@ -27,7 +35,7 @@
27
35
  * fires once per affected run (this function runs once per nightly).
28
36
  * `now` is injectable for tests.
29
37
  */
30
- export declare function computeSince(stateDir: string, recaptureWindowDays?: number, now?: Date): Date;
38
+ export declare function computeSince(stateDir: string, recaptureWindowDays?: number, now?: Date, firstRunLookbackDays?: number): Date;
31
39
  /**
32
40
  * Parse a `Retry-After` header into ms (#327). Handles both RFC forms —
33
41
  * delay-seconds (`"30"`) and HTTP-date — and returns undefined for anything
package/dist/nightly.js CHANGED
@@ -52,6 +52,7 @@ exports.runNightly = runNightly;
52
52
  const paths_js_1 = require("./paths.js");
53
53
  const node_fs_1 = require("node:fs");
54
54
  const node_path_1 = require("node:path");
55
+ const node_os_1 = require("node:os");
55
56
  let VERSION = "0.0.0";
56
57
  try {
57
58
  VERSION = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(__dirname, "..", "package.json"), "utf-8")).version;
@@ -59,6 +60,7 @@ try {
59
60
  catch { }
60
61
  const db_js_1 = require("./db.js");
61
62
  const config_read_js_1 = require("./config-read.js");
63
+ const calibration_js_1 = require("./calibration.js");
62
64
  const llm_js_1 = require("./llm.js");
63
65
  const embedder_js_1 = require("./embedder.js");
64
66
  const storage = __importStar(require("./storage.js"));
@@ -92,6 +94,19 @@ const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
92
94
  * count stays bounded.
93
95
  */
94
96
  const CONSOLIDATE_ONLY_BACKUP_MIN_AGE_MS = 20 * 60 * 60 * 1000;
97
+ /**
98
+ * #421 machine × harness identity: the stamp every captured segment carries
99
+ * as `source_machine`. Config `machineName` wins when set (same style as
100
+ * `agentName` — an owner-set identity key); otherwise the hostname. Applies
101
+ * to BOTH capture paths (server-local and client-remote) — same package,
102
+ * same stamp.
103
+ */
104
+ function resolveCaptureMachine(config) {
105
+ const v = config?.machineName;
106
+ if (typeof v === "string" && v.trim())
107
+ return v.trim().slice(0, 128);
108
+ return (0, node_os_1.hostname)();
109
+ }
95
110
  function readNightlyConfig(stateDir) {
96
111
  const configPath = (0, node_path_1.join)(stateDir, "config.json");
97
112
  let loaded;
@@ -127,9 +142,9 @@ function readConfigLicenseKey(stateDir) {
127
142
  function readLastRun(stateDir = HICORTEX_HOME) {
128
143
  const ts = (0, state_js_1.loadState)(stateDir).lastNightly;
129
144
  if (!ts)
130
- return new Date(0); // First run — process everything
145
+ return null; // First run — no watermark yet; #436 applies the lookback cap
131
146
  const d = new Date(ts);
132
- return isNaN(d.getTime()) ? new Date(0) : d;
147
+ return isNaN(d.getTime()) ? null : d; // corrupt stamp → treated as first run
133
148
  }
134
149
  /**
135
150
  * Discovery watermark. Normally the last-nightly timestamp; with
@@ -141,6 +156,14 @@ function readLastRun(stateDir = HICORTEX_HOME) {
141
156
  * (#189 review, fix 3). Per-session cursors keep the wide re-scan cheap: an
142
157
  * already-captured session yields an empty delta.
143
158
  *
159
+ * First run (#436): with NO watermark yet, since = now − firstRunLookbackDays
160
+ * (default 7 — owner ruling 2026-09-14: full-history first-run ingestion is
161
+ * not feasible; a long-term AI user's entire session store must not be
162
+ * discovered night one). The widening-only invariant composes unchanged:
163
+ * min(now−7d, now−N) — a recapture window wider than the default widens, a
164
+ * narrower one leaves the floor. Installing more history is the deliberate
165
+ * `--recapture-window` act, not a default.
166
+ *
144
167
  * Clock-jump clamp (#327): a FUTURE-dated lastNightly (client clock error —
145
168
  * NTP not yet synced at write time) would, once the clock corrects, sit ahead
146
169
  * of every session mtime and permanently skip quiet sessions (their mtimes
@@ -148,10 +171,10 @@ function readLastRun(stateDir = HICORTEX_HOME) {
148
171
  * fires once per affected run (this function runs once per nightly).
149
172
  * `now` is injectable for tests.
150
173
  */
151
- function computeSince(stateDir, recaptureWindowDays, now = new Date()) {
174
+ function computeSince(stateDir, recaptureWindowDays, now = new Date(), firstRunLookbackDays = calibration_js_1.DEFAULT_FIRST_RUN_LOOKBACK_DAYS) {
152
175
  const lastRun = readLastRun(stateDir);
153
- let effective = lastRun;
154
- if (lastRun.getTime() > now.getTime()) {
176
+ let effective = lastRun ?? new Date(now.getTime() - firstRunLookbackDays * 24 * 60 * 60 * 1000);
177
+ if (lastRun && lastRun.getTime() > now.getTime()) {
155
178
  console.warn(`[hicortex] state lastNightly (${lastRun.toISOString()}) is ahead of the clock ` +
156
179
  `(${now.toISOString()}) — clamping discovery to now. A future watermark permanently ` +
157
180
  `skips quiet sessions once the clock corrects; check the machine's clock/NTP. ` +
@@ -565,7 +588,7 @@ async function runNightly(options = {}) {
565
588
  // Step 1: Read new transcripts (CC + Hermes + Pi + OpenClaw). Discovery
566
589
  // is whole-session by mtime/ended_at; per-session cursors slice each
567
590
  // discovered session down to its unseen delta (#189).
568
- const since = computeSince(stateDir, recaptureWindowDays);
591
+ const since = computeSince(stateDir, recaptureWindowDays, undefined, (0, config_read_js_1.readPositiveConfig)(savedConfig ?? {}, "firstRunLookbackDays", calibration_js_1.DEFAULT_FIRST_RUN_LOOKBACK_DAYS));
569
592
  if (recaptureWindowDays) {
570
593
  console.log(`[hicortex] --recapture-window ${recaptureWindowDays}d: reading transcripts since ${since.toISOString()}`);
571
594
  }
@@ -614,6 +637,7 @@ async function runNightly(options = {}) {
614
637
  dryRun,
615
638
  sourceAgentId: savedConfig?.agentId,
616
639
  sourceDomain: savedConfig?.sourceDomain,
640
+ sourceMachine: resolveCaptureMachine(savedConfig),
617
641
  deadline,
618
642
  });
619
643
  memoriesIngested = result.memoriesIngested;
@@ -649,6 +673,14 @@ async function runNightly(options = {}) {
649
673
  // snapshot). 0 is a real value (under cap), so this stays undefined only
650
674
  // when consolidation didn't run at all (capture-only / no_llm / skipped).
651
675
  let evictedCount;
676
+ // #427: reconsolidation scout counters (hoisted for the dashboard
677
+ // snapshot). Flat snake_case on the wire, mirroring the stage report.
678
+ // Undefined only when consolidation didn't run; quiet-night zeros are
679
+ // REAL values from the stage's quiet-night report shape (the scan doesn't
680
+ // run on a quiet night — that's a fact about the night, not a gap).
681
+ let scoutScanned;
682
+ let scoutCorrectionShaped;
683
+ let scoutCandidatesFound;
652
684
  // Resolved cap (#245) for the dashboard snapshot. Hoisted so the snapshot
653
685
  // writer (outside the consolidation block) can stamp `capacity` even when
654
686
  // consolidation was skipped (the cap is still "in force" config-wise).
@@ -787,6 +819,13 @@ async function runNightly(options = {}) {
787
819
  // stage always returns `evicted` (0 when under cap / disabled); report
788
820
  // it as 0 (a real value), not undefined, when the stage ran.
789
821
  evictedCount = report.stages.memory_cap?.evicted ?? 0;
822
+ // #427: forward the scout counters whenever the stage ran. The
823
+ // stage report always carries them (quiet-night shape = zeros),
824
+ // so `?.` only falls through when the whole stage is absent
825
+ // (skipped run) — same skip-keys style as lessonsGenerated.
826
+ scoutScanned = report.stages.reconsolidation?.scout_scanned;
827
+ scoutCorrectionShaped = report.stages.reconsolidation?.scout_correction_shaped;
828
+ scoutCandidatesFound = report.stages.reconsolidation?.scout_candidates_found;
790
829
  // Only set when reflection actually RAN (not skipped). A skipped stage
791
830
  // (e.g. endpoint offline, #232 fail-soft) must NOT collapse to 0 — that
792
831
  // would make "endpoint down" indistinguishable from "prompt too tight"
@@ -803,6 +842,13 @@ async function runNightly(options = {}) {
803
842
  tokensThisRun = tokensTotal.total;
804
843
  tokensByStage = report.budget?.tokens_by_stage;
805
844
  }
845
+ // #427 observability: calls happened but ZERO tokens metered —
846
+ // the endpoint returned no usage objects (recordUsage no-ops by
847
+ // design). The snapshot would carry token nulls for such a run;
848
+ // make the blind spot GREPPABLE in journald instead of silent,
849
+ // the same structured-event style as budget_exhausted.
850
+ if (report.budget)
851
+ (0, consolidate_js_1.warnUnmeteredTokensRun)(report.budget);
806
852
  // #255: budget exhaustion — always populated when consolidation ran
807
853
  // (report.budget.exhausted is a boolean). The dashboard + telemetry
808
854
  // treat true as a quality-degradation health signal. The
@@ -984,6 +1030,11 @@ async function runNightly(options = {}) {
984
1030
  dedup,
985
1031
  supersession,
986
1032
  evicted: evictedCount,
1033
+ // #427: scout counters — forwarded whenever consolidation ran
1034
+ // (writeSnapshot omits the keys when undefined).
1035
+ scoutScanned,
1036
+ scoutCorrectionShaped,
1037
+ scoutCandidatesFound,
987
1038
  // #246: token accounting from this run's consolidation (undefined
988
1039
  // when consolidation didn't run or made no metered calls).
989
1040
  tokensThisRun,
@@ -1196,7 +1247,7 @@ async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recapt
1196
1247
  // logs, denoises, and POSTs the denoised text to the server's /distill
1197
1248
  // endpoint. All readers no-op when their harness isn't installed. Per-session
1198
1249
  // cursors slice each discovered session to its unseen delta (#189).
1199
- const since = computeSince(stateDir, recaptureWindowDays);
1250
+ const since = computeSince(stateDir, recaptureWindowDays, undefined, (0, config_read_js_1.readPositiveConfig)(config ?? {}, "firstRunLookbackDays", calibration_js_1.DEFAULT_FIRST_RUN_LOOKBACK_DAYS));
1200
1251
  if (recaptureWindowDays) {
1201
1252
  console.log(`[hicortex] --recapture-window ${recaptureWindowDays}d: reading transcripts since ${since.toISOString()}`);
1202
1253
  }
@@ -1234,6 +1285,7 @@ async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recapt
1234
1285
  // server stores these alongside source_agent; nothing filters on them.
1235
1286
  sourceAgentId: config.agentId,
1236
1287
  sourceDomain: config.sourceDomain,
1288
+ sourceMachine: resolveCaptureMachine(config),
1237
1289
  });
1238
1290
  memoriesIngested = result.memoriesIngested;
1239
1291
  sessionsSent = result.sessionsSent;
package/dist/prompts.d.ts CHANGED
@@ -10,6 +10,16 @@
10
10
  */
11
11
  /**
12
12
  * Importance scoring prompt. Takes a {memories_block} with indexed memories.
13
+ *
14
+ * RE-ANCHORED (#425): the pre-fix anchors put "useful context" at 0.3-0.5 —
15
+ * but the distiller's ephemera gate already removes trivia before anything
16
+ * reaches this scorer, so the model only ever saw curated material and the
17
+ * distribution compressed upward (measured on the production snapshot via
18
+ * eval:importance: median base 0.8, ~11% at exactly 1.0, which the decay
19
+ * model never forgets). The anchors now place routine-but-curated content
20
+ * LOW and add an explicit distribution instruction (owner decision D2,
21
+ * 2026-09-13: target median 0.30-0.40, p90 <= 0.75; 1.0 is never a valid
22
+ * score). The strict JSON-array response contract is unchanged.
13
23
  */
14
24
  export declare function importanceScoring(memoriesBlock: string): string;
15
25
  /**
package/dist/prompts.js CHANGED
@@ -16,23 +16,46 @@ exports.distillation = distillation;
16
16
  exports.domainCuration = domainCuration;
17
17
  /**
18
18
  * Importance scoring prompt. Takes a {memories_block} with indexed memories.
19
+ *
20
+ * RE-ANCHORED (#425): the pre-fix anchors put "useful context" at 0.3-0.5 —
21
+ * but the distiller's ephemera gate already removes trivia before anything
22
+ * reaches this scorer, so the model only ever saw curated material and the
23
+ * distribution compressed upward (measured on the production snapshot via
24
+ * eval:importance: median base 0.8, ~11% at exactly 1.0, which the decay
25
+ * model never forgets). The anchors now place routine-but-curated content
26
+ * LOW and add an explicit distribution instruction (owner decision D2,
27
+ * 2026-09-13: target median 0.30-0.40, p90 <= 0.75; 1.0 is never a valid
28
+ * score). The strict JSON-array response contract is unchanged.
19
29
  */
20
30
  function importanceScoring(memoriesBlock) {
21
31
  return `You are a memory importance scorer. Rate each memory's long-term value.
22
32
 
23
33
  Score each memory from 0.0 (trivial/ephemeral) to 1.0 (critical/foundational).
34
+ These memories are pre-filtered for durability, so ordinary competent work is
35
+ the NORMAL case — use the full scale and keep the bulk of a batch in the lower
36
+ half. When torn between two bands, choose the LOWER.
24
37
 
25
38
  Scoring guide:
26
- - 0.0-0.2: Routine actions, transient state, trivial fixes
27
- - 0.3-0.5: Useful context, minor decisions, standard patterns
28
- - 0.6-0.8: Important decisions, debugging breakthroughs, architectural choices
29
- - 0.9-1.0: Foundational principles, critical constraints, core identity facts
39
+ - 0.0-0.2: Ephemeral state, routine actions, one-off fixes
40
+ - 0.2-0.4: Useful context, ordinary decisions, standard patterns — THE DEFAULT
41
+ BAND; most memories belong here
42
+ - 0.4-0.6: Notable decisions, recurring patterns, project-shaping context —
43
+ only rows that clearly stand above ordinary work
44
+ - 0.6-0.8: Important decisions, debugging breakthroughs, architectural
45
+ choices — rare; at most one or two in a typical batch
46
+ - 0.8-0.95: ONLY genuinely foundational principles, critical constraints, core
47
+ identity facts — material that would be serious to lose. Never score 1.0.
48
+
49
+ Distribution: a typical batch should have a median around 0.35 — half the
50
+ batch sits at 0.2-0.4. A score of 0.5 or more says "among the more important
51
+ quarter of everything in long-term memory"; a score above 0.8 must be rare
52
+ and immediately defensible as costly to lose.
30
53
 
31
54
  MEMORIES:
32
55
  ${memoriesBlock}
33
56
 
34
57
  Respond with ONLY a JSON array of scores in the same order, e.g.:
35
- [0.3, 0.7, 0.5, 0.9]
58
+ [0.3, 0.4, 0.2, 0.7]
36
59
 
37
60
  No explanations. Just the JSON array.`;
38
61
  }