@gamaze/hicortex 0.18.3 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,12 +42,21 @@ export interface DashboardMetrics {
42
42
  * cap; undefined on backfill rows (a stage outcome, not reconstructable). */
43
43
  evicted?: number;
44
44
  /**
45
- * Total LLM tokens consumed by this run's consolidation (#246). Undefined
46
- * in lockstep with `tokens_by_stage` (and on backfill rows, which can't
47
- * reconstruct a per-run meter).
45
+ * Total LLM tokens consumed by this run (#246 consolidation meter; #287
46
+ * widened to the TRUE total distill + consolidation). Undefined in
47
+ * lockstep with `tokens_by_stage` (and on backfill rows, which can't
48
+ * reconstruct a per-run meter). Older snapshots are consolidation-only:
49
+ * historical rows can't be reconstructed, which is accepted (#287).
50
+ * Inherent under-count, same acceptance: attribution is response-based,
51
+ * so tokens a FAILED distill already spent (500 after spend, response
52
+ * lost after commit) reach the monthly meter but never a run's total —
53
+ * after such a night, the month's bars sum slightly below the headline.
48
54
  */
49
55
  tokens?: number;
50
- /** Per-stage breakdown of `tokens` (#246). Undefined on backfill rows. */
56
+ /**
57
+ * Per-stage breakdown of `tokens` (#246; #287 adds a `distill` entry for
58
+ * capture-time distillation). Undefined on backfill rows.
59
+ */
51
60
  tokens_by_stage?: Record<string, {
52
61
  prompt: number;
53
62
  completion: number;
@@ -79,6 +88,19 @@ export interface DashboardMetrics {
79
88
  * shape is clean on healthy runs.
80
89
  */
81
90
  budget_deferred_by_stage?: Record<string, number>;
91
+ /**
92
+ * #6 backup stage outcome (Phase 0B). Present whenever the backup stage
93
+ * ran (full nightly); absent on capture-only / dry-run / backfill rows.
94
+ * `ok` is false when the snapshot OR the operator's offsite hook failed —
95
+ * the page flags a night the offsite copy didn't land. `bytes` is the
96
+ * compressed artifact size; `path` is the on-disk artifact (for "where
97
+ * did the last backup land?" debugging — not a restore button).
98
+ */
99
+ backup?: {
100
+ ok: boolean;
101
+ bytes: number;
102
+ path?: string;
103
+ };
82
104
  };
83
105
  /** Corpus capacity (#245). `memory_soft_cap` is the configured ceiling (0 =
84
106
  * disabled); always present in real snapshots, undefined on backfilled
@@ -102,6 +124,17 @@ export interface DashboardSnapshot {
102
124
  }
103
125
  /** The /dashboard/data response — the full payload the page renders. */
104
126
  export interface DashboardData {
127
+ /**
128
+ * Account identity (hosted): who the viewer is, so a user holding two
129
+ * tenant tokens can tell whose data the page shows. Each field is null when
130
+ * its config key (displayName/orgName/planLabel) is absent — the page
131
+ * renders nothing when ALL are null (the self-hosted default).
132
+ */
133
+ account: {
134
+ name: string | null;
135
+ org: string | null;
136
+ plan: string | null;
137
+ };
105
138
  headline: {
106
139
  total_memories: number;
107
140
  uses_per_showing: number | null;
@@ -149,9 +182,10 @@ export interface DashboardData {
149
182
  supersession: number;
150
183
  added: number;
151
184
  evicted?: number;
152
- /** Total tokens consumed that run (#246). Undefined = no metered run. */
185
+ /** Total tokens consumed that run (#246; #287: distill + consolidation).
186
+ * Undefined = no metered run. */
153
187
  tokens?: number;
154
- /** Per-stage breakdown of `tokens` (#246). */
188
+ /** Per-stage breakdown of `tokens` (#246; #287 adds `distill`). */
155
189
  tokens_by_stage?: Record<string, {
156
190
  prompt: number;
157
191
  completion: number;
@@ -207,6 +241,19 @@ export interface NightlyDelta {
207
241
  completion: number;
208
242
  total: number;
209
243
  }>;
244
+ /**
245
+ * Distill tokens metered by the daemon across this run's capture POSTs
246
+ * (#287) — summed from the /distill responses by the capture loop. Merged
247
+ * into the snapshot so `new_this_run.tokens` is the run's TRUE total
248
+ * (distill + consolidation) and `distill` joins `tokens_by_stage`. Zero
249
+ * (a daemon predating the usage field, or nothing distilled) is a no-op:
250
+ * the row keeps its consolidation-only shape.
251
+ */
252
+ distillUsage?: {
253
+ prompt: number;
254
+ completion: number;
255
+ total: number;
256
+ };
210
257
  /**
211
258
  * Always-on consolidation-budget usage (#255 CR). Forwarded whenever
212
259
  * consolidation ran so the dashboard renders a continuous used/max bar.
@@ -227,6 +274,15 @@ export interface NightlyDelta {
227
274
  * with `budgetExhausted`.
228
275
  */
229
276
  budgetDeferredByStage?: Record<string, number>;
277
+ /**
278
+ * #6 backup stage (Phase 0B). Hoisted from the nightly backup block. Present
279
+ * whenever the backup stage ran (full nightly); undefined on capture-only /
280
+ * dry-run. `backupOk` flips to false on snapshot OR hook failure so the
281
+ * digest can flag a night the offsite copy didn't land.
282
+ */
283
+ backupPath?: string;
284
+ backupBytes?: number;
285
+ backupOk?: boolean;
230
286
  }
231
287
  /**
232
288
  * Write one snapshot row for `runAt` (an ISO timestamp the caller chooses —
@@ -286,3 +342,12 @@ export declare function handleDashboardData(db: Database.Database, query: {
286
342
  * Failures surface as a 500 with the usual {error} shape — no silent degrade.
287
343
  */
288
344
  export declare function dashboardDataHandler(getDb: () => Database.Database, getConfig: () => Record<string, unknown> | null | undefined): express.RequestHandler;
345
+ /**
346
+ * Express adapter for GET /account — the account identity ONLY (name/org/plan
347
+ * from config), so the /viz and /identity/ui pages can render the nav account
348
+ * element without pulling the full /dashboard/data payload. Same readAccount()
349
+ * construction as the dashboard payload — one shape, two surfaces. Also the
350
+ * natural whoami for the future OAuth session (#292). Failures surface as a
351
+ * 500 with the usual {error} shape (same as dashboardDataHandler).
352
+ */
353
+ export declare function accountHandler(getConfig: () => Record<string, unknown> | null | undefined): express.RequestHandler;
package/dist/dashboard.js CHANGED
@@ -25,6 +25,7 @@ exports.writeSnapshot = writeSnapshot;
25
25
  exports.backfillSnapshots = backfillSnapshots;
26
26
  exports.handleDashboardData = handleDashboardData;
27
27
  exports.dashboardDataHandler = dashboardDataHandler;
28
+ exports.accountHandler = accountHandler;
28
29
  const recall_index_js_1 = require("./recall-index.js");
29
30
  const config_read_js_1 = require("./config-read.js");
30
31
  const consolidate_js_1 = require("./consolidate.js");
@@ -92,17 +93,36 @@ function computeDashboardMetrics(db) {
92
93
  */
93
94
  function writeSnapshot(db, runAt, delta, memorySoftCap) {
94
95
  const metrics = computeDashboardMetrics(db);
96
+ // #287: merge the run's two meters into the customer-facing total. `tokens`
97
+ // = consolidation (tokensThisRun) + distill (distillUsage.total); the distill
98
+ // share joins the stage map under its own key. Both fields stay in lockstep —
99
+ // emitted when EITHER phase metered, omitted when neither did (the page
100
+ // treats undefined as "no data for this day"). A zero/absent distillUsage
101
+ // (old daemon, nothing distilled) changes nothing: tokens/tokens_by_stage
102
+ // come through exactly as the consolidation report produced them.
103
+ const hasDistill = (delta.distillUsage?.total ?? 0) > 0;
104
+ const metered = delta.tokensThisRun !== undefined || hasDistill;
105
+ const mergedTokens = metered
106
+ ? (delta.tokensThisRun ?? 0) + (hasDistill ? delta.distillUsage.total : 0)
107
+ : undefined;
108
+ const mergedStages = metered
109
+ ? { ...(delta.tokensByStage ?? {}), ...(hasDistill ? { distill: delta.distillUsage } : {}) }
110
+ : undefined;
111
+ // Shape fidelity: `tokens_by_stage` with zero keys never existed pre-#287
112
+ // (the key was simply absent) — keep it that way so consumers that treat
113
+ // "present" as "has a breakdown" stay right.
114
+ const emitStages = mergedStages && Object.keys(mergedStages).length > 0 ? mergedStages : undefined;
95
115
  metrics.new_this_run = {
96
116
  added: delta.added,
97
117
  lessonsGenerated: delta.lessonsGenerated,
98
118
  dedup: delta.dedup,
99
119
  supersession: delta.supersession,
100
120
  evicted: delta.evicted,
101
- // #246: forward only when consolidation actually metered tokens this run.
102
- // Absent on capture-only / throttled / no-LLM / no-metered-call runs — the
103
- // page treats undefined as "no data for this day", matching adoption.
104
- ...(delta.tokensThisRun !== undefined ? { tokens: delta.tokensThisRun } : {}),
105
- ...(delta.tokensByStage !== undefined ? { tokens_by_stage: delta.tokensByStage } : {}),
121
+ // #246: forward only when a phase actually metered tokens this run. Absent
122
+ // on capture-only / throttled / no-LLM / no-metered-call runs — the page
123
+ // treats undefined as "no data for this day", matching adoption.
124
+ ...(mergedTokens !== undefined ? { tokens: mergedTokens } : {}),
125
+ ...(emitStages !== undefined ? { tokens_by_stage: emitStages } : {}),
106
126
  // #255 CR: always-on usage metric — forward calls_used + max_calls
107
127
  // whenever consolidation ran (regardless of exhaustion) so the page can
108
128
  // render a continuous used/max bar. Presence = a run happened; absence =
@@ -121,6 +141,17 @@ function writeSnapshot(db, runAt, delta, memorySoftCap) {
121
141
  : {}),
122
142
  }
123
143
  : {}),
144
+ // #6 backup stage — forwarded as a nested object only when the stage ran
145
+ // (backupOk !== undefined). Absent on capture-only / dry-run / backfill.
146
+ ...(delta.backupOk !== undefined
147
+ ? {
148
+ backup: {
149
+ ok: delta.backupOk === true,
150
+ bytes: delta.backupBytes ?? 0,
151
+ ...(delta.backupPath ? { path: delta.backupPath } : {}),
152
+ },
153
+ }
154
+ : {}),
124
155
  };
125
156
  if (memorySoftCap !== undefined) {
126
157
  metrics.capacity = { memory_soft_cap: memorySoftCap };
@@ -418,8 +449,9 @@ function handleDashboardData(db, query, config) {
418
449
  supersession: dayMetrics?.new_this_run?.supersession ?? supersessionCount,
419
450
  added: dayMetrics?.new_this_run?.added ?? sampleRows.length,
420
451
  evicted: dayMetrics?.new_this_run?.evicted,
421
- // #246: only present when the day's nightly metered tokens. Both fields
422
- // are forwarded together — the page renders either the breakdown or nothing.
452
+ // #246/#287: only present when the day's nightly metered tokens (distill
453
+ // or consolidation). Both fields are forwarded together — the page
454
+ // renders either the breakdown or nothing.
423
455
  tokens: dayMetrics?.new_this_run?.tokens,
424
456
  tokens_by_stage: dayMetrics?.new_this_run?.tokens_by_stage,
425
457
  // #255 CR: always-on usage metric — present whenever consolidation ran
@@ -441,6 +473,10 @@ function handleDashboardData(db, query, config) {
441
473
  return {
442
474
  status: 200,
443
475
  body: {
476
+ // Account identity — read defensively like the numeric knobs above:
477
+ // null when absent/not a string (page renders nothing, never "null").
478
+ // Shared readAccount() so GET /account renders the identical shape.
479
+ account: (0, config_read_js_1.readAccount)(config),
444
480
  range: rangeParam,
445
481
  headline,
446
482
  series,
@@ -469,3 +505,21 @@ function dashboardDataHandler(getDb, getConfig) {
469
505
  }
470
506
  };
471
507
  }
508
+ /**
509
+ * Express adapter for GET /account — the account identity ONLY (name/org/plan
510
+ * from config), so the /viz and /identity/ui pages can render the nav account
511
+ * element without pulling the full /dashboard/data payload. Same readAccount()
512
+ * construction as the dashboard payload — one shape, two surfaces. Also the
513
+ * natural whoami for the future OAuth session (#292). Failures surface as a
514
+ * 500 with the usual {error} shape (same as dashboardDataHandler).
515
+ */
516
+ function accountHandler(getConfig) {
517
+ return (_req, res) => {
518
+ try {
519
+ res.status(200).json({ account: (0, config_read_js_1.readAccount)(getConfig()) });
520
+ }
521
+ catch (err) {
522
+ res.status(500).json({ error: err instanceof Error ? err.message : String(err) });
523
+ }
524
+ };
525
+ }
@@ -10,7 +10,25 @@
10
10
  * GET /sse — SSE stream for MCP clients
11
11
  * POST /messages — message endpoint for MCP clients
12
12
  */
13
+ import express from "express";
13
14
  import type { MemorySearchResult } from "./types.js";
15
+ /**
16
+ * Resolve the request body-size limit in MB (#7). Pure — exported for tests.
17
+ * Precedence: an explicit config value > hosted-mode default (5) > self-hosted
18
+ * default (25, the historical fixed value → no regression). A finite positive
19
+ * config value wins; invalid/absent falls through.
20
+ */
21
+ export declare function resolveBodyLimitMb(configVal: unknown, hostedMode: boolean): number;
22
+ /**
23
+ * Express error middleware (#7): translate express.json's default HTML 413
24
+ * (entity.too.large) into a consistent JSON response. Catches body-parser
25
+ * errors only — which express.json emits BEFORE any route runs — so by
26
+ * registration order (this sits ahead of the routes) it never intercepts an
27
+ * error thrown inside a route handler; those reach Express's default handler.
28
+ * The `status === 413 || type === "entity.too.large"` check is defense-in-depth
29
+ * on top of that ordering. Exported so tests exercise the real handler.
30
+ */
31
+ export declare function makeBodyLimitErrorHandler(limitMb: number): express.ErrorRequestHandler;
14
32
  export declare function startServer(options?: {
15
33
  port?: number;
16
34
  host?: string;
@@ -48,6 +48,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
48
48
  return (mod && mod.__esModule) ? mod : { "default": mod };
49
49
  };
50
50
  Object.defineProperty(exports, "__esModule", { value: true });
51
+ exports.resolveBodyLimitMb = resolveBodyLimitMb;
52
+ exports.makeBodyLimitErrorHandler = makeBodyLimitErrorHandler;
51
53
  exports.startServer = startServer;
52
54
  exports.formatResults = formatResults;
53
55
  const express_1 = __importDefault(require("express"));
@@ -407,6 +409,38 @@ function createMcpServer() {
407
409
  // ---------------------------------------------------------------------------
408
410
  // HTTP server with SSE transport
409
411
  // ---------------------------------------------------------------------------
412
+ /**
413
+ * Resolve the request body-size limit in MB (#7). Pure — exported for tests.
414
+ * Precedence: an explicit config value > hosted-mode default (5) > self-hosted
415
+ * default (25, the historical fixed value → no regression). A finite positive
416
+ * config value wins; invalid/absent falls through.
417
+ */
418
+ function resolveBodyLimitMb(configVal, hostedMode) {
419
+ const cfg = Number(configVal);
420
+ if (Number.isFinite(cfg) && cfg > 0)
421
+ return cfg;
422
+ return hostedMode ? 5 : 25;
423
+ }
424
+ /**
425
+ * Express error middleware (#7): translate express.json's default HTML 413
426
+ * (entity.too.large) into a consistent JSON response. Catches body-parser
427
+ * errors only — which express.json emits BEFORE any route runs — so by
428
+ * registration order (this sits ahead of the routes) it never intercepts an
429
+ * error thrown inside a route handler; those reach Express's default handler.
430
+ * The `status === 413 || type === "entity.too.large"` check is defense-in-depth
431
+ * on top of that ordering. Exported so tests exercise the real handler.
432
+ */
433
+ function makeBodyLimitErrorHandler(limitMb) {
434
+ return (err, _req, res, next) => {
435
+ const status = err.status;
436
+ const type = err.type;
437
+ if (status === 413 || type === "entity.too.large") {
438
+ res.status(413).json({ error: "request body too large", limit_mb: limitMb });
439
+ return;
440
+ }
441
+ next(err);
442
+ };
443
+ }
410
444
  async function startServer(options = {}) {
411
445
  const port = options.port ?? 8787;
412
446
  const host = options.host ?? "0.0.0.0";
@@ -493,6 +527,12 @@ async function startServer(options = {}) {
493
527
  // env (provider-set, tenant-immutable) which takes precedence. Initialised here
494
528
  // (after stateDir + savedConfig are known) so the warn-dedup can seed from state.
495
529
  (0, token_budget_js_1.initTokenBudget)(stateDir, savedConfig?.llmTokensPerMonth);
530
+ // #7: request body-size limit. Config key wins; else 5 MB hosted / 25 MB
531
+ // self-hosted (the prior fixed value → no regression). Guards the OOM vector
532
+ // (the body is fully parsed into memory before the distiller truncates to 80K
533
+ // chars). Legitimate capture segments are ≤60K chars (~200KB), so this never
534
+ // constrains real flow — it's an abuse/backstop. Oversized → 413.
535
+ const bodyLimitMb = resolveBodyLimitMb(savedConfig?.distillBodyLimitMb, hostedMode);
496
536
  if (savedConfig?.llmBackend === "claude-cli") {
497
537
  const claudePath = (0, llm_js_1.findClaudeBinary)();
498
538
  if (claudePath) {
@@ -636,7 +676,11 @@ async function startServer(options = {}) {
636
676
  // Express app
637
677
  const app = (0, express_1.default)();
638
678
  // Raise the body limit — whole-session denoised transcripts exceed the 100 kB default.
639
- app.use(express_1.default.json({ limit: "25mb" }));
679
+ app.use(express_1.default.json({ limit: `${bodyLimitMb}mb` }));
680
+ // #7: JSON 413 on body-limit exceed (see makeBodyLimitErrorHandler). Server-side
681
+ // only — the client capture loop treats 413 like any non-2xx (holds cursor);
682
+ // it never fires for legitimate capture (segments ≤200KB ≪ the limit).
683
+ app.use(makeBodyLimitErrorHandler(bodyLimitMb));
640
684
  // CORS: reflect ONLY explicitly-allowlisted origins (config.corsAllowedOrigins),
641
685
  // and never send Access-Control-Allow-Credentials. Reflecting any origin with
642
686
  // credentials — combined with the localhost auth bypass and the default 0.0.0.0
@@ -997,7 +1041,8 @@ async function startServer(options = {}) {
997
1041
  });
998
1042
  // REST /distill — canonical capture endpoint (0.9.0+).
999
1043
  // Every machine (including the server itself) POSTs denoised session text here.
1000
- // The server distills, embeds, stores. Body limit: 25 MB (raised at app init).
1044
+ // The server distills, embeds, stores. Body limit: see `distillBodyLimitMb`
1045
+ // (default 25 MB self-hosted / 5 MB hosted); oversized → 413 (#7).
1001
1046
  //
1002
1047
  // Accepts text (string, preferred nightly path) OR messages (array, legacy).
1003
1048
  // Performs session-level dedup when session_id is present without segment_id.
@@ -1161,6 +1206,12 @@ async function startServer(options = {}) {
1161
1206
  ids,
1162
1207
  distilled: ids.length,
1163
1208
  dropped: dropped.map((d) => (d.length > 120 ? `${d.slice(0, 120)}…` : d)),
1209
+ // #287: this segment's metered usage — the same breakdown
1210
+ // recordDistillUsage accrues below. Lets the capturing nightly
1211
+ // attribute distill tokens in its dashboard snapshot
1212
+ // (new_this_run.tokens_by_stage.distill). Always present (zeros when
1213
+ // no chunk reached an LLM call); pre-#287 clients ignore it.
1214
+ usage: distillUsage,
1164
1215
  });
1165
1216
  }
1166
1217
  catch (err) {
@@ -1452,6 +1503,13 @@ async function startServer(options = {}) {
1452
1503
  // express adapter that injects the live db + config. STRICTLY view-only —
1453
1504
  // no mutation endpoints on the dashboard surface.
1454
1505
  app.get("/dashboard/data", (0, dashboard_js_1.dashboardDataHandler)(() => db, () => readConfigFile(stateDir)));
1506
+ // GET /account — account identity for the console nav (name/org/plan from
1507
+ // config). The LIGHTWEIGHT twin of the account block inside /dashboard/data:
1508
+ // the /viz and /identity/ui pages need only this, not the metric payload;
1509
+ // also the natural whoami for the future OAuth session (#292). Bearer-only
1510
+ // (standard auth middleware, no shell exemption — it carries data); localhost
1511
+ // bypass applies. Handler lives in src/dashboard.ts next to its twin.
1512
+ app.get("/account", (0, dashboard_js_1.accountHandler)(() => readConfigFile(stateDir)));
1455
1513
  // SSE endpoint — each connection gets its own McpServer + transport
1456
1514
  app.get("/sse", async (req, res) => {
1457
1515
  const transport = new sse_js_1.SSEServerTransport("/messages", res);
@@ -30,7 +30,7 @@ exports.MEMORY_SECTION_NAME = "memory";
30
30
  * injected once per session into every agent on the fleet. */
31
31
  function renderMemoryInstructions() {
32
32
  return [
33
- "Your long-term memory is Hicortex — shared across all agents and sessions.",
33
+ "Hicortex is your persistent identity and long-term memory: what you learn, decide, and correct survives every session, compaction, and model switch one memory shared by all your agents.",
34
34
  "- A `## Memory recall (auto)` index may arrive with prompts: it is a MENU, not content. Fetch a full memory with `hicortex_get(id)` when the entry could change how you handle the current task.",
35
35
  "- Recall before assuming: `hicortex_search` for prior decisions/facts/preferences, `hicortex_recent` to catch up on a project.",
36
36
  "- Cite any memory you rely on by id + date, and mark it `FETCHED` (you read the full memory via `hicortex_get`) or `SNIPPET` (the one-line entry only). Don't present a SNIPPET citation as established. On conflicts, newer memories supersede older.",
package/dist/nightly.js CHANGED
@@ -75,6 +75,7 @@ const capture_js_1 = require("./capture.js");
75
75
  const dashboard_js_1 = require("./dashboard.js");
76
76
  const telemetry_js_1 = require("./telemetry.js");
77
77
  const init_js_1 = require("./init.js");
78
+ const backup_js_1 = require("./backup.js");
78
79
  const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
79
80
  function readNightlyConfig(stateDir) {
80
81
  const configPath = (0, node_path_1.join)(stateDir, "config.json");
@@ -168,7 +169,16 @@ function makeRemotePost(serverUrl, authToken) {
168
169
  async function normalizePostResult(resp) {
169
170
  if (resp.status === 201) {
170
171
  const data = (await resp.json().catch(() => ({})));
171
- return { status: 201, distilled: data.distilled ?? 0, dropped: data.dropped ?? [] };
172
+ // #287: the daemon reports the segment's metered usage. Shape-validated
173
+ // so a partial payload can't NaN the run's totals; a pre-#287 daemon
174
+ // simply omits it → capture sums zero (snapshot stays consolidation-only).
175
+ const usage = parseUsage(data.usage);
176
+ return {
177
+ status: 201,
178
+ distilled: data.distilled ?? 0,
179
+ dropped: data.dropped ?? [],
180
+ ...(usage ? { usage } : {}),
181
+ };
172
182
  }
173
183
  if (resp.status === 200) {
174
184
  const data = (await resp.json().catch(() => ({})));
@@ -177,6 +187,18 @@ async function normalizePostResult(resp) {
177
187
  const data = (await resp.json().catch(() => ({})));
178
188
  return { status: resp.status, error: data.error ?? "unknown error" };
179
189
  }
190
+ /** Strict {prompt, completion, total} parser for /distill's usage field (#287);
191
+ * undefined on anything malformed — the caller then treats it as unmetered. */
192
+ function parseUsage(u) {
193
+ if (typeof u !== "object" || u === null)
194
+ return undefined;
195
+ const { prompt, completion, total } = u;
196
+ const num = (n) => typeof n === "number" && Number.isFinite(n) && n >= 0 ? n : null;
197
+ const p = num(prompt), c = num(completion), t = num(total);
198
+ if (p === null || c === null || t === null)
199
+ return undefined;
200
+ return { prompt: p, completion: c, total: t };
201
+ }
180
202
  function writeLastRun(stateDir = HICORTEX_HOME) {
181
203
  (0, state_js_1.updateState)((s) => {
182
204
  s.lastNightly = new Date().toISOString();
@@ -347,6 +369,13 @@ async function runNightly(options = {}) {
347
369
  let batches = [];
348
370
  let memoriesIngested = 0;
349
371
  let hadTransientFailure = false;
372
+ // #287: distill tokens metered by the daemon across this run's segment
373
+ // POSTs (summed from the /distill responses by the capture loop).
374
+ // Forwarded to the dashboard snapshot so new_this_run.tokens is the run's
375
+ // TRUE total (distill + consolidation) and the distill share lands in
376
+ // tokens_by_stage. Zero when the daemon predates the usage field — the
377
+ // snapshot writer gates on total > 0, so old servers keep today's shape.
378
+ let distillUsage;
350
379
  // consolidateOnly (hosted service): skip capture entirely. The hosted
351
380
  // consolidation timer uses this so per-tenant nightly runs don't ingest the
352
381
  // operator's local sessions into the tenant's DB.
@@ -423,6 +452,7 @@ async function runNightly(options = {}) {
423
452
  sourceDomain: savedConfig?.sourceDomain,
424
453
  });
425
454
  memoriesIngested = result.memoriesIngested;
455
+ distillUsage = result.distillUsage;
426
456
  // A 429/401 stop must hold the watermark too (fix 1): the loop abandoned
427
457
  // the remaining sessions, and mtime discovery would never re-find them.
428
458
  hadTransientFailure = result.hadTransientFailure || result.stopped !== undefined;
@@ -430,7 +460,12 @@ async function runNightly(options = {}) {
430
460
  finally {
431
461
  releaseLock();
432
462
  }
433
- console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories`);
463
+ console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories` +
464
+ // #287: distill tokens the daemon metered for those segments (absent
465
+ // when the daemon predates the usage field or nothing distilled).
466
+ (distillUsage && distillUsage.total > 0
467
+ ? ` · ${distillUsage.total.toLocaleString()} distill tokens`
468
+ : ""));
434
469
  // Prune aged-out cursors (90d) — only on a clean run so a transient
435
470
  // failure doesn't drop a still-needed cursor.
436
471
  if (!dryRun && !hadTransientFailure) {
@@ -630,6 +665,53 @@ async function runNightly(options = {}) {
630
665
  }
631
666
  }
632
667
  console.log(`[hicortex] Nightly pipeline complete.`);
668
+ // Backup stage (#6, Phase 0B) — a transactionally-consistent snapshot of
669
+ // the irreplaceable data (DB + identity + state), packaged as one tar.gz
670
+ // the operator ships offsite via the optional `backupCommand` hook. Runs
671
+ // ONLY on a full nightly (capture-only is frequent + stateless; dry-run
672
+ // writes nothing). Backup failure must NOT fail the nightly — capture +
673
+ // consolidation have already succeeded; the snapshot is on disk and the
674
+ // failure surfaces as `backupOk:false` in the dashboard snapshot + telemetry
675
+ // for alerting (the operator's hook owns active alerting; no in-product
676
+ // channel yet — Phase 3).
677
+ let backupPath;
678
+ let backupBytes;
679
+ let backupOk;
680
+ if (!dryRun && !captureOnly) {
681
+ try {
682
+ const backupDir = typeof savedConfig?.backupDir === "string" && savedConfig.backupDir.trim()
683
+ ? savedConfig.backupDir
684
+ : undefined;
685
+ const bRes = await (0, backup_js_1.createBackup)({ db, home: stateDir, outDir: backupDir });
686
+ backupPath = bRes.path;
687
+ backupBytes = bRes.bytes;
688
+ backupOk = true;
689
+ console.log(`[hicortex] Backup: ${bRes.files} files, ${bRes.bytes.toLocaleString()} bytes -> ${bRes.path}`);
690
+ const cmd = typeof savedConfig?.backupCommand === "string" && savedConfig.backupCommand.trim()
691
+ ? savedConfig.backupCommand
692
+ : undefined;
693
+ if (cmd && backupPath) {
694
+ const hook = await (0, backup_js_1.runBackupHook)(backupPath, cmd);
695
+ if (!hook.ok) {
696
+ // The artifact is on disk; only the offsite copy failed. Keep
697
+ // backupPath/backupBytes (the snapshot records what was produced)
698
+ // but flip backupOk so the aggregate can alert.
699
+ backupOk = false;
700
+ console.error(`[hicortex] Backup hook failed (exit ${hook.exitCode ?? "n/a"}). ` +
701
+ `Artifact is on disk; offsite copy did NOT complete.`);
702
+ }
703
+ else {
704
+ console.log(`[hicortex] Backup hook ok (exit 0).`);
705
+ }
706
+ }
707
+ }
708
+ catch (err) {
709
+ // The whole backup stage failed (snapshot, tar, or write). Do NOT
710
+ // propagate — capture/consolidation already succeeded. Surface + continue.
711
+ backupOk = false;
712
+ console.error(`[hicortex] Backup FAILED: ${err instanceof Error ? err.message : String(err)}`);
713
+ }
714
+ }
633
715
  // Dashboard snapshot (#224) — full nightly only. The snapshot reflects
634
716
  // corpus state regardless of whether consolidation/LLM ran, so it is
635
717
  // ALWAYS written here (the use case is history; an LLM-less install still
@@ -673,6 +755,12 @@ async function runNightly(options = {}) {
673
755
  // when consolidation didn't run or made no metered calls).
674
756
  tokensThisRun,
675
757
  tokensByStage,
758
+ // #287: the capture phase's distill tokens (from the /distill
759
+ // responses). The writer merges them into `tokens` +
760
+ // `tokens_by_stage.distill` so the customer-facing total is the
761
+ // run's TRUE spend; zero (old daemon / nothing distilled) is a no-op
762
+ // and the row keeps its consolidation-only shape.
763
+ distillUsage,
676
764
  // #255 CR: always-on budget usage — undefined when consolidation
677
765
  // didn't run (capture-only / no_llm / throttled). Forwarded whenever
678
766
  // consolidation ran so the digest renders a continuous used/max bar.
@@ -682,6 +770,13 @@ async function runNightly(options = {}) {
682
770
  // (capture-only / no_llm / throttled) or didn't exhaust.
683
771
  budgetExhausted,
684
772
  budgetDeferredByStage,
773
+ // #6 backup stage — hoisted from the block above. Present whenever
774
+ // the backup stage ran (full nightly); undefined on capture-only /
775
+ // dry-run. backupOk flips to false on snapshot OR hook failure so the
776
+ // dashboard digest can flag a night the offsite copy didn't complete.
777
+ backupPath,
778
+ backupBytes,
779
+ backupOk,
685
780
  }, memorySoftCapResolved);
686
781
  }
687
782
  catch (snapErr) {
@@ -728,6 +823,13 @@ async function runNightly(options = {}) {
728
823
  // exhausted (false is omitted to keep the ping minimal; the aggregate
729
824
  // treats absent as "not exhausted / not measurable").
730
825
  ...(budgetExhausted ? { budget_exhausted: true } : {}),
826
+ // #6 backup stage outcome — forwarded only when the backup stage ran
827
+ // (full nightly). `ok` is false on snapshot OR hook failure; the fleet
828
+ // aggregate surfaces a sustained drop in backup_ok as a data-loss risk.
829
+ // Absent on capture-only / dry-run / client runs (no backup ran).
830
+ ...(backupOk !== undefined
831
+ ? { backup: { ok: backupOk === true, bytes: backupBytes ?? 0 } }
832
+ : {}),
731
833
  sessions: batches.length,
732
834
  ok: !hadTransientFailure,
733
835
  shown: adoption.shown,
package/dist/prompts.js CHANGED
@@ -120,6 +120,7 @@ EXTRACT into this markdown format:
120
120
 
121
121
  ### Decisions Made
122
122
  - [D] [SUBJECT]: [decision] — [reasoning] (${date})
123
+ (ONLY decisions the user explicitly made or confirmed — never an AI proposal)
123
124
 
124
125
  ### Knowledge Learned
125
126
  - [K] [SUBJECT]: [knowledge] — [context/source] (${date})
@@ -145,10 +146,19 @@ TYPE TAG (critical — prefix EVERY bullet with exactly one letter + space):
145
146
  - [K] KNOWLEDGE — a durable truth that will hold across sessions: "the API is at
146
147
  :8787", "uv is used for packages", "config lives in ~/.hicortex/". Not tied
147
148
  to a single moment.
148
- - [D] DECISIONS — a choice made that future work builds on, and that a later
149
- decision can SUPERSEDE: "switched from gemma4 to qwen3.5", "adopted the
150
- graded-schema tag model". Not knowledge (it can change) and not experience
151
- (it persists and constrains).
149
+ - [D] DECISIONS — a choice the USER explicitly made or confirmed in the
150
+ transcript: they said "do X / agreed / ok / go ahead", approved the plan, or
151
+ the transcript shows the change actually being carried out. It must be a
152
+ choice future work builds on and that a later decision can SUPERSEDE:
153
+ "switched from gemma4 to qwen3.5", "adopted the graded-schema tag model".
154
+ Not knowledge (it can change) and not experience (it persists and constrains).
155
+ - An AI recommendation or proposal is NEVER a decision, however detailed or
156
+ well-reasoned — even if the user seemed receptive. If the user declined,
157
+ deferred ("hold", "later", "wait for X"), or did not answer: record it as
158
+ [E] EXPERIENCE with proposal framing ("AI proposed X → user declined/held
159
+ because Y"), or under Corrections & Rejections.
160
+ - SELF-CHECK: if an item's text says "AI recommended/proposed" or the user
161
+ "has not (yet) confirmed" it, that item is NOT [D] — re-tag it [E].
152
162
  - NEVER use [L] (learnings). Learnings are extracted by a SEPARATE reflection stage,
153
163
  not here. If the model emits [L], it is wrong — re-tag as experience/knowledge/decisions.
154
164
  The type tag goes BEFORE the subject, never as a section/category bracket.
@@ -161,6 +171,8 @@ entity. The subject is what a future reader would search for.
161
171
  - NOT: "[E] User rejected AI's bundling of unknown loads"
162
172
  - Write: "[K] Nightly capture (Hermes): cron sessions are excluded — source='cron' is skipped before distillation"
163
173
  - NOT: "[K] Discovered that cron sessions are filtered out"
174
+ - Write: "[E] Qwen3.8-27B swap: AI proposed switching → user held on Qwen3.6-35B-A3B until an MoE variant ships"
175
+ - NOT: "[D] Qwen3.8-27B: switch from Qwen3.6-27B — drop-in upgrade"
164
176
  Reason: each item's first words (after the type tag) become the memory's one-line
165
177
  index entry AND dominate its search embedding. An item that opens with a category
166
178
  label, a sentiment ("Strong Negative"), or "User rejected…" is unfindable — it
@@ -169,6 +181,9 @@ the subject; put reaction, intensity and reasoning AFTER it.
169
181
 
170
182
  RULES:
171
183
  - Extract MAX 20 items total (quality over quantity)
184
+ - Use EXACT names/versions/paths/numbers as they appear in the transcript —
185
+ never substitute what seems current or more standard (writing "Qwen3.6-27B"
186
+ when the transcript says "Qwen3.6-35B-A3B" is a fabrication)
172
187
  - Each must be useful if recalled in a future session
173
188
  - Skip: routine code edits, standard tool usage, trivial fixes
174
189
  - Include: architectural decisions, debugging breakthroughs, user preferences,
@@ -116,6 +116,20 @@ export interface TelemetryPayload {
116
116
  * the dashboard snapshot, not on the wire (mirrors `tokens_this_run`).
117
117
  */
118
118
  budget_exhausted?: boolean;
119
+ /**
120
+ * Backup stage outcome (#6, Phase 0B). Present on every full nightly (absent
121
+ * on capture-only / dry-run / client runs — no backup stage runs there).
122
+ * `ok` is false when EITHER the snapshot write OR the operator's
123
+ * `backupCommand` hook failed — the aggregate data-loss-risk signal (a
124
+ * sustained drop in `backup.ok` means offsite copies are silently not
125
+ * landing). `bytes` is the compressed artifact size (0 when the snapshot
126
+ * itself failed before producing a file). The operator's hook owns active
127
+ * alerting (email/Discord); this field is the passive fleet-health signal.
128
+ */
129
+ backup?: {
130
+ ok: boolean;
131
+ bytes: number;
132
+ };
119
133
  }
120
134
  /**
121
135
  * Check if telemetry is enabled. Disabled by: