@gamaze/hicortex 0.18.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,27 @@
1
+ /** Marker filename inside the Hicortex home dir. */
2
+ export declare const LOCALHOST_BYPASS_MARKER = ".allow-localhost-bypass";
3
+ /** Marker contents — a one-line note. Its mere PRESENCE is the signal. */
4
+ export declare const LOCALHOST_BYPASS_MARKER_CONTENT: string;
5
+ /**
6
+ * Resolve the marker file path for a given home dir. Defaults to the canonical
7
+ * Hicortex home (honors HICORTEX_HOME), so callers in tests can point the env
8
+ * override at a temp dir.
9
+ */
10
+ export declare function localhostBypassMarkerPath(home?: string): string;
11
+ /**
12
+ * Does the localhost auth-bypass marker exist? Pure filesystem check — no
13
+ * logging, no side-effects. Used by both createAuthMiddleware (gates the
14
+ * bypass per-request via a boot-time capture in mcp-server.ts) and the
15
+ * hosted-mode boot assertion.
16
+ */
17
+ export declare function localhostBypassEnabled(home?: string): boolean;
18
+ /**
19
+ * Write the localhost auth-bypass marker file (self-hosted init only — never
20
+ * in hosted mode). Idempotent: overwrites an existing marker so a re-init
21
+ * refreshes the explanatory note. Ensures the parent dir exists. Does NOT
22
+ * touch auth or any other config — just the one marker file.
23
+ *
24
+ * Returns true when a NEW marker was created (for init's "✓" reporting), false
25
+ * when one already existed (refreshed in place).
26
+ */
27
+ export declare function writeLocalhostBypassMarker(home?: string): boolean;
@@ -0,0 +1,71 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.LOCALHOST_BYPASS_MARKER_CONTENT = exports.LOCALHOST_BYPASS_MARKER = void 0;
4
+ exports.localhostBypassMarkerPath = localhostBypassMarkerPath;
5
+ exports.localhostBypassEnabled = localhostBypassEnabled;
6
+ exports.writeLocalhostBypassMarker = writeLocalhostBypassMarker;
7
+ /**
8
+ * Localhost auth-bypass marker file (#110 §2, #271 — Phase 0B).
9
+ *
10
+ * The localhost auth bypass in createAuthMiddleware (viz.ts) is marker-GATED
11
+ * from 0.18: it applies ONLY when this marker file exists in the Hicortex
12
+ * home dir. Self-hosted `init` writes the marker, so existing installs keep
13
+ * working after upgrade + re-init; a hosted tenant dir provisioned by any
14
+ * means (script, hand, restored tar) is fail-closed by default — no marker,
15
+ * no bypass, every connection (localhost included) needs the bearer token.
16
+ *
17
+ * Rationale (spec 2026-07-27 §2): with the bypass unconditional, a future
18
+ * `trust proxy` enablement would make `req.ip` header-spoofable and the
19
+ * bypass remotely triggerable. Inverting the default to "off unless marked"
20
+ * makes the bypass opt-in via a filesystem side-effect of self-hosted init,
21
+ * so a tenant home built from a bare config + DB restore cannot accidentally
22
+ * ship with the bypass active. The hosted-mode boot assertion (mcp-server.ts)
23
+ * refuses to start if BOTH hostedMode=true AND the marker is present, so even
24
+ * a stray marker cannot open a hosted tenant.
25
+ *
26
+ * Marker file name: `.allow-localhost-bypass` (dot-prefixed; not a secret —
27
+ * its mere presence is the signal; no contents needed).
28
+ */
29
+ const node_fs_1 = require("node:fs");
30
+ const node_path_1 = require("node:path");
31
+ const paths_js_1 = require("./paths.js");
32
+ /** Marker filename inside the Hicortex home dir. */
33
+ exports.LOCALHOST_BYPASS_MARKER = ".allow-localhost-bypass";
34
+ /** Marker contents — a one-line note. Its mere PRESENCE is the signal. */
35
+ exports.LOCALHOST_BYPASS_MARKER_CONTENT = "# Written by `hicortex init` (self-hosted). Opt-in to the localhost auth\n" +
36
+ "# bypass. DELETE this file to require the bearer token on localhost too\n" +
37
+ "# (fail-closed). Hosted-mode (hostedMode:true) refuses to start with this\n" +
38
+ "# marker present — see specs/2026-07-27-hosted-service.md §2.\n";
39
+ /**
40
+ * Resolve the marker file path for a given home dir. Defaults to the canonical
41
+ * Hicortex home (honors HICORTEX_HOME), so callers in tests can point the env
42
+ * override at a temp dir.
43
+ */
44
+ function localhostBypassMarkerPath(home = (0, paths_js_1.hicortexHome)()) {
45
+ return (0, node_path_1.join)(home, exports.LOCALHOST_BYPASS_MARKER);
46
+ }
47
+ /**
48
+ * Does the localhost auth-bypass marker exist? Pure filesystem check — no
49
+ * logging, no side-effects. Used by both createAuthMiddleware (gates the
50
+ * bypass per-request via a boot-time capture in mcp-server.ts) and the
51
+ * hosted-mode boot assertion.
52
+ */
53
+ function localhostBypassEnabled(home = (0, paths_js_1.hicortexHome)()) {
54
+ return (0, node_fs_1.existsSync)(localhostBypassMarkerPath(home));
55
+ }
56
+ /**
57
+ * Write the localhost auth-bypass marker file (self-hosted init only — never
58
+ * in hosted mode). Idempotent: overwrites an existing marker so a re-init
59
+ * refreshes the explanatory note. Ensures the parent dir exists. Does NOT
60
+ * touch auth or any other config — just the one marker file.
61
+ *
62
+ * Returns true when a NEW marker was created (for init's "✓" reporting), false
63
+ * when one already existed (refreshed in place).
64
+ */
65
+ function writeLocalhostBypassMarker(home = (0, paths_js_1.hicortexHome)()) {
66
+ const markerPath = localhostBypassMarkerPath(home);
67
+ const existed = (0, node_fs_1.existsSync)(markerPath);
68
+ (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(markerPath), { recursive: true });
69
+ (0, node_fs_1.writeFileSync)(markerPath, exports.LOCALHOST_BYPASS_MARKER_CONTENT, { mode: 0o644 });
70
+ return !existed;
71
+ }
@@ -10,7 +10,25 @@
10
10
  * GET /sse — SSE stream for MCP clients
11
11
  * POST /messages — message endpoint for MCP clients
12
12
  */
13
+ import express from "express";
13
14
  import type { MemorySearchResult } from "./types.js";
15
+ /**
16
+ * Resolve the request body-size limit in MB (#7). Pure — exported for tests.
17
+ * Precedence: an explicit config value > hosted-mode default (5) > self-hosted
18
+ * default (25, the historical fixed value → no regression). A finite positive
19
+ * config value wins; invalid/absent falls through.
20
+ */
21
+ export declare function resolveBodyLimitMb(configVal: unknown, hostedMode: boolean): number;
22
+ /**
23
+ * Express error middleware (#7): translate express.json's default HTML 413
24
+ * (entity.too.large) into a consistent JSON response. Catches body-parser
25
+ * errors only — which express.json emits BEFORE any route runs — so by
26
+ * registration order (this sits ahead of the routes) it never intercepts an
27
+ * error thrown inside a route handler; those reach Express's default handler.
28
+ * The `status === 413 || type === "entity.too.large"` check is defense-in-depth
29
+ * on top of that ordering. Exported so tests exercise the real handler.
30
+ */
31
+ export declare function makeBodyLimitErrorHandler(limitMb: number): express.ErrorRequestHandler;
14
32
  export declare function startServer(options?: {
15
33
  port?: number;
16
34
  host?: string;
@@ -48,6 +48,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
48
48
  return (mod && mod.__esModule) ? mod : { "default": mod };
49
49
  };
50
50
  Object.defineProperty(exports, "__esModule", { value: true });
51
+ exports.resolveBodyLimitMb = resolveBodyLimitMb;
52
+ exports.makeBodyLimitErrorHandler = makeBodyLimitErrorHandler;
51
53
  exports.startServer = startServer;
52
54
  exports.formatResults = formatResults;
53
55
  const express_1 = __importDefault(require("express"));
@@ -59,6 +61,10 @@ const db_js_1 = require("./db.js");
59
61
  const llm_js_1 = require("./llm.js");
60
62
  const features_js_1 = require("./features.js");
61
63
  const config_read_js_1 = require("./config-read.js");
64
+ const localhost_bypass_js_1 = require("./localhost-bypass.js");
65
+ const hosted_boot_js_1 = require("./hosted-boot.js");
66
+ const token_budget_js_1 = require("./token-budget.js");
67
+ const paths_js_1 = require("./paths.js");
62
68
  const state_js_1 = require("./state.js");
63
69
  const embedder_js_1 = require("./embedder.js");
64
70
  const storage = __importStar(require("./storage.js"));
@@ -403,9 +409,94 @@ function createMcpServer() {
403
409
  // ---------------------------------------------------------------------------
404
410
  // HTTP server with SSE transport
405
411
  // ---------------------------------------------------------------------------
412
+ /**
413
+ * Resolve the request body-size limit in MB (#7). Pure — exported for tests.
414
+ * Precedence: an explicit config value > hosted-mode default (5) > self-hosted
415
+ * default (25, the historical fixed value → no regression). A finite positive
416
+ * config value wins; invalid/absent falls through.
417
+ */
418
+ function resolveBodyLimitMb(configVal, hostedMode) {
419
+ const cfg = Number(configVal);
420
+ if (Number.isFinite(cfg) && cfg > 0)
421
+ return cfg;
422
+ return hostedMode ? 5 : 25;
423
+ }
424
+ /**
425
+ * Express error middleware (#7): translate express.json's default HTML 413
426
+ * (entity.too.large) into a consistent JSON response. Catches body-parser
427
+ * errors only — which express.json emits BEFORE any route runs — so by
428
+ * registration order (this sits ahead of the routes) it never intercepts an
429
+ * error thrown inside a route handler; those reach Express's default handler.
430
+ * The `status === 413 || type === "entity.too.large"` check is defense-in-depth
431
+ * on top of that ordering. Exported so tests exercise the real handler.
432
+ */
433
+ function makeBodyLimitErrorHandler(limitMb) {
434
+ return (err, _req, res, next) => {
435
+ const status = err.status;
436
+ const type = err.type;
437
+ if (status === 413 || type === "entity.too.large") {
438
+ res.status(413).json({ error: "request body too large", limit_mb: limitMb });
439
+ return;
440
+ }
441
+ next(err);
442
+ };
443
+ }
406
444
  async function startServer(options = {}) {
407
445
  const port = options.port ?? 8787;
408
446
  const host = options.host ?? "0.0.0.0";
447
+ // ---------------------------------------------------------------------------
448
+ // Hosted-mode boot gate (#110 §1-§2, #271 — Phase 0B).
449
+ //
450
+ // MUST run BEFORE resolveDbPath/initDb: in hosted mode with HICORTEX_DB_PATH
451
+ // set, the server must refuse the attacker-chosen DB location WITHOUT first
452
+ // touching it. The hosted signals (hostedMode from config, bypassMarkerPresent
453
+ // from the marker file) do NOT depend on the DB, so reading them now is safe.
454
+ // CR warning 3: this block was previously after initDb, letting a hostile
455
+ // HICORTEX_DB_PATH create/touch a file at the chosen path before the gate.
456
+ //
457
+ // CR warning 1: the marker is a HOME-level file (like config.json, written by
458
+ // init to HICORTEX_HOME). Read it from hicortexHome() — NOT stateDir, which
459
+ // is dirname(dbPath) and drifts when HICORTEX_DB_PATH relocates the DB. The
460
+ // config key hostedMode likewise lives at <hicortexHome>/config.json.
461
+ //
462
+ // Decision logic lives in hosted-boot.ts (pure, unit-tested); the side-effect
463
+ // (console.error + process.exit) is local to boot. The marker state is
464
+ // captured once here and reused below to gate the localhost bypass in
465
+ // createAuthMiddleware (no per-request stat). CR warning 4: the upgrade-path
466
+ // warning is decided by the pure shouldEmitBypassWarning helper (behavior-
467
+ // tested), not an inline branch.
468
+ const bootConfig = readConfigFile((0, paths_js_1.hicortexHome)());
469
+ const hostedMode = (0, config_read_js_1.readStrictBoolean)(bootConfig ?? {}, "hostedMode") === true;
470
+ let bypassMarkerPresent = (0, localhost_bypass_js_1.localhostBypassEnabled)();
471
+ const bootDecision = (0, hosted_boot_js_1.checkHostedBoot)({
472
+ hostedMode,
473
+ dbPathEnvSet: !!process.env.HICORTEX_DB_PATH,
474
+ bypassMarkerPresent,
475
+ });
476
+ if (!bootDecision.ok) {
477
+ console.error(bootDecision.message);
478
+ process.exit(1);
479
+ }
480
+ // Upgrade migration (CR S1): self-hosted server-mode CC MCP registration
481
+ // carries NO bearer token (init.ts:192 — only client-mode adds the header),
482
+ // so it relies entirely on the localhost bypass. An existing install that
483
+ // upgrades without re-running init has no marker → the bypass silently
484
+ // disappears → every server-mode CC MCP call 401s. Auto-write the marker on
485
+ // first post-upgrade boot in self-hosted mode to preserve the prior
486
+ // unconditional-bypass behaviour. Hosted mode is untouched: checkHostedBoot
487
+ // refuses to start with a marker present, so this block — gated on
488
+ // !hostedMode — never runs for a hosted tenant. bypassMarkerPresent is
489
+ // reassigned so createAuthMiddleware below gates the bypass for THIS boot
490
+ // too (the file write and the in-memory flag stay in sync).
491
+ if (!hostedMode && !bypassMarkerPresent) {
492
+ (0, localhost_bypass_js_1.writeLocalhostBypassMarker)((0, paths_js_1.hicortexHome)());
493
+ bypassMarkerPresent = true;
494
+ console.log("[hicortex] Localhost auth-bypass marker written (upgrade migration).");
495
+ }
496
+ const bypassWarning = (0, hosted_boot_js_1.shouldEmitBypassWarning)(hostedMode, bypassMarkerPresent);
497
+ if (bypassWarning) {
498
+ console.warn(bypassWarning);
499
+ }
409
500
  // Initialize core
410
501
  const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
411
502
  console.log(`[hicortex] Initializing database at ${dbPath}`);
@@ -431,6 +522,17 @@ async function startServer(options = {}) {
431
522
  const { agentId } = (0, init_js_1.ensureAndPersistAgentId)((0, node_path_1.join)(stateDir, "config.json"));
432
523
  savedConfig.agentId = agentId;
433
524
  }
525
+ // #5: token-budget enforcement. Mode-agnostic — gates on cap > 0. Self-hosted
526
+ // uses config llmTokensPerMonth (default 0 = off); hosted uses HICORTEX_TOKEN_CAP
527
+ // env (provider-set, tenant-immutable) which takes precedence. Initialised here
528
+ // (after stateDir + savedConfig are known) so the warn-dedup can seed from state.
529
+ (0, token_budget_js_1.initTokenBudget)(stateDir, savedConfig?.llmTokensPerMonth);
530
+ // #7: request body-size limit. Config key wins; else 5 MB hosted / 25 MB
531
+ // self-hosted (the prior fixed value → no regression). Guards the OOM vector
532
+ // (the body is fully parsed into memory before the distiller truncates to 80K
533
+ // chars). Legitimate capture segments are ≤60K chars (~200KB), so this never
534
+ // constrains real flow — it's an abuse/backstop. Oversized → 413.
535
+ const bodyLimitMb = resolveBodyLimitMb(savedConfig?.distillBodyLimitMb, hostedMode);
434
536
  if (savedConfig?.llmBackend === "claude-cli") {
435
537
  const claudePath = (0, llm_js_1.findClaudeBinary)();
436
538
  if (claudePath) {
@@ -574,7 +676,11 @@ async function startServer(options = {}) {
574
676
  // Express app
575
677
  const app = (0, express_1.default)();
576
678
  // Raise the body limit — whole-session denoised transcripts exceed the 100 kB default.
577
- app.use(express_1.default.json({ limit: "25mb" }));
679
+ app.use(express_1.default.json({ limit: `${bodyLimitMb}mb` }));
680
+ // #7: JSON 413 on body-limit exceed (see makeBodyLimitErrorHandler). Server-side
681
+ // only — the client capture loop treats 413 like any non-2xx (holds cursor);
682
+ // it never fires for legitimate capture (segments ≤200KB ≪ the limit).
683
+ app.use(makeBodyLimitErrorHandler(bodyLimitMb));
578
684
  // CORS: reflect ONLY explicitly-allowlisted origins (config.corsAllowedOrigins),
579
685
  // and never send Access-Control-Allow-Credentials. Reflecting any origin with
580
686
  // credentials — combined with the localhost auth bypass and the default 0.0.0.0
@@ -615,7 +721,7 @@ async function startServer(options = {}) {
615
721
  // /dashboard has its own shell-exemption pattern. Gives the console one entry
616
722
  // point: http://<host>:8787/ → /dashboard.
617
723
  app.get("/", (_req, res) => res.redirect("/dashboard"));
618
- app.use((0, viz_js_1.createAuthMiddleware)(authToken, authTokenPrevious));
724
+ app.use((0, viz_js_1.createAuthMiddleware)(authToken, authTokenPrevious, bypassMarkerPresent));
619
725
  // SSE transport management — each connection gets its own McpServer instance
620
726
  const transports = new Map();
621
727
  // Health endpoint — PUBLIC minimal probe. Unauthenticated (the auth
@@ -935,7 +1041,8 @@ async function startServer(options = {}) {
935
1041
  });
936
1042
  // REST /distill — canonical capture endpoint (0.9.0+).
937
1043
  // Every machine (including the server itself) POSTs denoised session text here.
938
- // The server distills, embeds, stores. Body limit: 25 MB (raised at app init).
1044
+ // The server distills, embeds, stores. Body limit: see `distillBodyLimitMb`
1045
+ // (default 25 MB self-hosted / 5 MB hosted); oversized → 413 (#7).
939
1046
  //
940
1047
  // Accepts text (string, preferred nightly path) OR messages (array, legacy).
941
1048
  // Performs session-level dedup when session_id is present without segment_id.
@@ -1027,12 +1134,27 @@ async function startServer(options = {}) {
1027
1134
  const sourcePrefix = session_id
1028
1135
  ? `${session_id}${segment_id ? `#${segment_id}` : ""}`
1029
1136
  : undefined;
1137
+ // #5: declared outside the try so the finally can record tokens spent even
1138
+ // when distillSession throws partway through (the LLM calls already happened).
1139
+ let distillUsage = { prompt: 0, completion: 0, total: 0 };
1030
1140
  try {
1141
+ // #5: token-budget gate — refuse (429) BEFORE the LLM call if the tenant is
1142
+ // already at/over the monthly cap. Placed after the dedup short-circuits so
1143
+ // a skipped duplicate neither trips the gate nor consumes budget. The client
1144
+ // capture loop holds its cursor on 429 (dup-over-loss, capture.ts:303).
1145
+ if ((0, token_budget_js_1.isTokenBudgetExceeded)(stateDir)) {
1146
+ res.status(429).json({ error: "token budget exceeded", retry: "next billing period" });
1147
+ return;
1148
+ }
1031
1149
  // Collect gate-dropped entries so they can ride back in the response and
1032
1150
  // land in the caller's file-persisted nightly log (#156 audit trail); the
1033
1151
  // server-side per-entry console.log in distillChunk stays as well.
1034
1152
  const dropped = [];
1035
- const entries = await (0, distiller_js_1.distillSession)(llm, conversationText, project ?? "unknown", date, chunkSize, dropped);
1153
+ const entries = await (0, distiller_js_1.distillSession)(llm, conversationText, project ?? "unknown", date, chunkSize, dropped, (u) => {
1154
+ distillUsage.prompt += u.prompt_tokens ?? 0;
1155
+ distillUsage.completion += u.completion_tokens ?? 0;
1156
+ distillUsage.total += u.total_tokens ?? 0;
1157
+ });
1036
1158
  // Phase 1 — embed every chunk up front (async). If ANY embed fails we
1037
1159
  // never reach the insert, so nothing is stored.
1038
1160
  const createdAt = new Date(date).toISOString();
@@ -1084,12 +1206,26 @@ async function startServer(options = {}) {
1084
1206
  ids,
1085
1207
  distilled: ids.length,
1086
1208
  dropped: dropped.map((d) => (d.length > 120 ? `${d.slice(0, 120)}…` : d)),
1209
+ // #287: this segment's metered usage — the same breakdown
1210
+ // recordDistillUsage accrues below. Lets the capturing nightly
1211
+ // attribute distill tokens in its dashboard snapshot
1212
+ // (new_this_run.tokens_by_stage.distill). Always present (zeros when
1213
+ // no chunk reached an LLM call); pre-#287 clients ignore it.
1214
+ usage: distillUsage,
1087
1215
  });
1088
1216
  }
1089
1217
  catch (err) {
1090
1218
  res.status(500).json({ error: "Distillation failed" });
1091
1219
  console.error(`[hicortex] /distill: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`);
1092
1220
  }
1221
+ finally {
1222
+ // #5: record tokens spent against the monthly budget — in finally so a
1223
+ // mid-distil throw (some chunks' LLM calls already happened) still counts.
1224
+ // No-op when cap=0 (enforcement off) or distillUsage.total=0 (gate refused
1225
+ // / no chunk reached an LLM call).
1226
+ if (distillUsage.total > 0)
1227
+ (0, token_budget_js_1.recordDistillUsage)(stateDir, distillUsage);
1228
+ }
1093
1229
  });
1094
1230
  // -------------------------------------------------------------------------
1095
1231
  // REST /update — update a memory (and re-embed when content changes).
@@ -1367,6 +1503,13 @@ async function startServer(options = {}) {
1367
1503
  // express adapter that injects the live db + config. STRICTLY view-only —
1368
1504
  // no mutation endpoints on the dashboard surface.
1369
1505
  app.get("/dashboard/data", (0, dashboard_js_1.dashboardDataHandler)(() => db, () => readConfigFile(stateDir)));
1506
+ // GET /account — account identity for the console nav (name/org/plan from
1507
+ // config). The LIGHTWEIGHT twin of the account block inside /dashboard/data:
1508
+ // the /viz and /identity/ui pages need only this, not the metric payload;
1509
+ // also the natural whoami for the future OAuth session (#292). Bearer-only
1510
+ // (standard auth middleware, no shell exemption — it carries data); localhost
1511
+ // bypass applies. Handler lives in src/dashboard.ts next to its twin.
1512
+ app.get("/account", (0, dashboard_js_1.accountHandler)(() => readConfigFile(stateDir)));
1370
1513
  // SSE endpoint — each connection gets its own McpServer + transport
1371
1514
  app.get("/sse", async (req, res) => {
1372
1515
  const transport = new sse_js_1.SSEServerTransport("/messages", res);
@@ -30,7 +30,7 @@ exports.MEMORY_SECTION_NAME = "memory";
30
30
  * injected once per session into every agent on the fleet. */
31
31
  function renderMemoryInstructions() {
32
32
  return [
33
- "Your long-term memory is Hicortex — shared across all agents and sessions.",
33
+ "Hicortex is your persistent identity and long-term memory: what you learn, decide, and correct survives every session, compaction, and model switch one memory shared by all your agents.",
34
34
  "- A `## Memory recall (auto)` index may arrive with prompts: it is a MENU, not content. Fetch a full memory with `hicortex_get(id)` when the entry could change how you handle the current task.",
35
35
  "- Recall before assuming: `hicortex_search` for prior decisions/facts/preferences, `hicortex_recent` to catch up on a project.",
36
36
  "- Cite any memory you rely on by id + date, and mark it `FETCHED` (you read the full memory via `hicortex_get`) or `SNIPPET` (the one-line entry only). Don't present a SNIPPET citation as established. On conflicts, newer memories supersede older.",
package/dist/nightly.js CHANGED
@@ -75,6 +75,7 @@ const capture_js_1 = require("./capture.js");
75
75
  const dashboard_js_1 = require("./dashboard.js");
76
76
  const telemetry_js_1 = require("./telemetry.js");
77
77
  const init_js_1 = require("./init.js");
78
+ const backup_js_1 = require("./backup.js");
78
79
  const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
79
80
  function readNightlyConfig(stateDir) {
80
81
  const configPath = (0, node_path_1.join)(stateDir, "config.json");
@@ -168,7 +169,16 @@ function makeRemotePost(serverUrl, authToken) {
168
169
  async function normalizePostResult(resp) {
169
170
  if (resp.status === 201) {
170
171
  const data = (await resp.json().catch(() => ({})));
171
- return { status: 201, distilled: data.distilled ?? 0, dropped: data.dropped ?? [] };
172
+ // #287: the daemon reports the segment's metered usage. Shape-validated
173
+ // so a partial payload can't NaN the run's totals; a pre-#287 daemon
174
+ // simply omits it → capture sums zero (snapshot stays consolidation-only).
175
+ const usage = parseUsage(data.usage);
176
+ return {
177
+ status: 201,
178
+ distilled: data.distilled ?? 0,
179
+ dropped: data.dropped ?? [],
180
+ ...(usage ? { usage } : {}),
181
+ };
172
182
  }
173
183
  if (resp.status === 200) {
174
184
  const data = (await resp.json().catch(() => ({})));
@@ -177,6 +187,18 @@ async function normalizePostResult(resp) {
177
187
  const data = (await resp.json().catch(() => ({})));
178
188
  return { status: resp.status, error: data.error ?? "unknown error" };
179
189
  }
190
+ /** Strict {prompt, completion, total} parser for /distill's usage field (#287);
191
+ * undefined on anything malformed — the caller then treats it as unmetered. */
192
+ function parseUsage(u) {
193
+ if (typeof u !== "object" || u === null)
194
+ return undefined;
195
+ const { prompt, completion, total } = u;
196
+ const num = (n) => typeof n === "number" && Number.isFinite(n) && n >= 0 ? n : null;
197
+ const p = num(prompt), c = num(completion), t = num(total);
198
+ if (p === null || c === null || t === null)
199
+ return undefined;
200
+ return { prompt: p, completion: c, total: t };
201
+ }
180
202
  function writeLastRun(stateDir = HICORTEX_HOME) {
181
203
  (0, state_js_1.updateState)((s) => {
182
204
  s.lastNightly = new Date().toISOString();
@@ -347,6 +369,13 @@ async function runNightly(options = {}) {
347
369
  let batches = [];
348
370
  let memoriesIngested = 0;
349
371
  let hadTransientFailure = false;
372
+ // #287: distill tokens metered by the daemon across this run's segment
373
+ // POSTs (summed from the /distill responses by the capture loop).
374
+ // Forwarded to the dashboard snapshot so new_this_run.tokens is the run's
375
+ // TRUE total (distill + consolidation) and the distill share lands in
376
+ // tokens_by_stage. Zero when the daemon predates the usage field — the
377
+ // snapshot writer gates on total > 0, so old servers keep today's shape.
378
+ let distillUsage;
350
379
  // consolidateOnly (hosted service): skip capture entirely. The hosted
351
380
  // consolidation timer uses this so per-tenant nightly runs don't ingest the
352
381
  // operator's local sessions into the tenant's DB.
@@ -423,6 +452,7 @@ async function runNightly(options = {}) {
423
452
  sourceDomain: savedConfig?.sourceDomain,
424
453
  });
425
454
  memoriesIngested = result.memoriesIngested;
455
+ distillUsage = result.distillUsage;
426
456
  // A 429/401 stop must hold the watermark too (fix 1): the loop abandoned
427
457
  // the remaining sessions, and mtime discovery would never re-find them.
428
458
  hadTransientFailure = result.hadTransientFailure || result.stopped !== undefined;
@@ -430,7 +460,12 @@ async function runNightly(options = {}) {
430
460
  finally {
431
461
  releaseLock();
432
462
  }
433
- console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories`);
463
+ console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories` +
464
+ // #287: distill tokens the daemon metered for those segments (absent
465
+ // when the daemon predates the usage field or nothing distilled).
466
+ (distillUsage && distillUsage.total > 0
467
+ ? ` · ${distillUsage.total.toLocaleString()} distill tokens`
468
+ : ""));
434
469
  // Prune aged-out cursors (90d) — only on a clean run so a transient
435
470
  // failure doesn't drop a still-needed cursor.
436
471
  if (!dryRun && !hadTransientFailure) {
@@ -630,6 +665,53 @@ async function runNightly(options = {}) {
630
665
  }
631
666
  }
632
667
  console.log(`[hicortex] Nightly pipeline complete.`);
668
+ // Backup stage (#6, Phase 0B) — a transactionally-consistent snapshot of
669
+ // the irreplaceable data (DB + identity + state), packaged as one tar.gz
670
+ // the operator ships offsite via the optional `backupCommand` hook. Runs
671
+ // ONLY on a full nightly (capture-only is frequent + stateless; dry-run
672
+ // writes nothing). Backup failure must NOT fail the nightly — capture +
673
+ // consolidation have already succeeded; the snapshot is on disk and the
674
+ // failure surfaces as `backupOk:false` in the dashboard snapshot + telemetry
675
+ // for alerting (the operator's hook owns active alerting; no in-product
676
+ // channel yet — Phase 3).
677
+ let backupPath;
678
+ let backupBytes;
679
+ let backupOk;
680
+ if (!dryRun && !captureOnly) {
681
+ try {
682
+ const backupDir = typeof savedConfig?.backupDir === "string" && savedConfig.backupDir.trim()
683
+ ? savedConfig.backupDir
684
+ : undefined;
685
+ const bRes = await (0, backup_js_1.createBackup)({ db, home: stateDir, outDir: backupDir });
686
+ backupPath = bRes.path;
687
+ backupBytes = bRes.bytes;
688
+ backupOk = true;
689
+ console.log(`[hicortex] Backup: ${bRes.files} files, ${bRes.bytes.toLocaleString()} bytes -> ${bRes.path}`);
690
+ const cmd = typeof savedConfig?.backupCommand === "string" && savedConfig.backupCommand.trim()
691
+ ? savedConfig.backupCommand
692
+ : undefined;
693
+ if (cmd && backupPath) {
694
+ const hook = await (0, backup_js_1.runBackupHook)(backupPath, cmd);
695
+ if (!hook.ok) {
696
+ // The artifact is on disk; only the offsite copy failed. Keep
697
+ // backupPath/backupBytes (the snapshot records what was produced)
698
+ // but flip backupOk so the aggregate can alert.
699
+ backupOk = false;
700
+ console.error(`[hicortex] Backup hook failed (exit ${hook.exitCode ?? "n/a"}). ` +
701
+ `Artifact is on disk; offsite copy did NOT complete.`);
702
+ }
703
+ else {
704
+ console.log(`[hicortex] Backup hook ok (exit 0).`);
705
+ }
706
+ }
707
+ }
708
+ catch (err) {
709
+ // The whole backup stage failed (snapshot, tar, or write). Do NOT
710
+ // propagate — capture/consolidation already succeeded. Surface + continue.
711
+ backupOk = false;
712
+ console.error(`[hicortex] Backup FAILED: ${err instanceof Error ? err.message : String(err)}`);
713
+ }
714
+ }
633
715
  // Dashboard snapshot (#224) — full nightly only. The snapshot reflects
634
716
  // corpus state regardless of whether consolidation/LLM ran, so it is
635
717
  // ALWAYS written here (the use case is history; an LLM-less install still
@@ -673,6 +755,12 @@ async function runNightly(options = {}) {
673
755
  // when consolidation didn't run or made no metered calls).
674
756
  tokensThisRun,
675
757
  tokensByStage,
758
+ // #287: the capture phase's distill tokens (from the /distill
759
+ // responses). The writer merges them into `tokens` +
760
+ // `tokens_by_stage.distill` so the customer-facing total is the
761
+ // run's TRUE spend; zero (old daemon / nothing distilled) is a no-op
762
+ // and the row keeps its consolidation-only shape.
763
+ distillUsage,
676
764
  // #255 CR: always-on budget usage — undefined when consolidation
677
765
  // didn't run (capture-only / no_llm / throttled). Forwarded whenever
678
766
  // consolidation ran so the digest renders a continuous used/max bar.
@@ -682,6 +770,13 @@ async function runNightly(options = {}) {
682
770
  // (capture-only / no_llm / throttled) or didn't exhaust.
683
771
  budgetExhausted,
684
772
  budgetDeferredByStage,
773
+ // #6 backup stage — hoisted from the block above. Present whenever
774
+ // the backup stage ran (full nightly); undefined on capture-only /
775
+ // dry-run. backupOk flips to false on snapshot OR hook failure so the
776
+ // dashboard digest can flag a night the offsite copy didn't complete.
777
+ backupPath,
778
+ backupBytes,
779
+ backupOk,
685
780
  }, memorySoftCapResolved);
686
781
  }
687
782
  catch (snapErr) {
@@ -728,6 +823,13 @@ async function runNightly(options = {}) {
728
823
  // exhausted (false is omitted to keep the ping minimal; the aggregate
729
824
  // treats absent as "not exhausted / not measurable").
730
825
  ...(budgetExhausted ? { budget_exhausted: true } : {}),
826
+ // #6 backup stage outcome — forwarded only when the backup stage ran
827
+ // (full nightly). `ok` is false on snapshot OR hook failure; the fleet
828
+ // aggregate surfaces a sustained drop in backup_ok as a data-loss risk.
829
+ // Absent on capture-only / dry-run / client runs (no backup ran).
830
+ ...(backupOk !== undefined
831
+ ? { backup: { ok: backupOk === true, bytes: backupBytes ?? 0 } }
832
+ : {}),
731
833
  sessions: batches.length,
732
834
  ok: !hadTransientFailure,
733
835
  shown: adoption.shown,
package/dist/prompts.js CHANGED
@@ -120,6 +120,7 @@ EXTRACT into this markdown format:
120
120
 
121
121
  ### Decisions Made
122
122
  - [D] [SUBJECT]: [decision] — [reasoning] (${date})
123
+ (ONLY decisions the user explicitly made or confirmed — never an AI proposal)
123
124
 
124
125
  ### Knowledge Learned
125
126
  - [K] [SUBJECT]: [knowledge] — [context/source] (${date})
@@ -145,10 +146,19 @@ TYPE TAG (critical — prefix EVERY bullet with exactly one letter + space):
145
146
  - [K] KNOWLEDGE — a durable truth that will hold across sessions: "the API is at
146
147
  :8787", "uv is used for packages", "config lives in ~/.hicortex/". Not tied
147
148
  to a single moment.
148
- - [D] DECISIONS — a choice made that future work builds on, and that a later
149
- decision can SUPERSEDE: "switched from gemma4 to qwen3.5", "adopted the
150
- graded-schema tag model". Not knowledge (it can change) and not experience
151
- (it persists and constrains).
149
+ - [D] DECISIONS — a choice the USER explicitly made or confirmed in the
150
+ transcript: they said "do X / agreed / ok / go ahead", approved the plan, or
151
+ the transcript shows the change actually being carried out. It must be a
152
+ choice future work builds on and that a later decision can SUPERSEDE:
153
+ "switched from gemma4 to qwen3.5", "adopted the graded-schema tag model".
154
+ Not knowledge (it can change) and not experience (it persists and constrains).
155
+ - An AI recommendation or proposal is NEVER a decision, however detailed or
156
+ well-reasoned — even if the user seemed receptive. If the user declined,
157
+ deferred ("hold", "later", "wait for X"), or did not answer: record it as
158
+ [E] EXPERIENCE with proposal framing ("AI proposed X → user declined/held
159
+ because Y"), or under Corrections & Rejections.
160
+ - SELF-CHECK: if an item's text says "AI recommended/proposed" or the user
161
+ "has not (yet) confirmed" it, that item is NOT [D] — re-tag it [E].
152
162
  - NEVER use [L] (learnings). Learnings are extracted by a SEPARATE reflection stage,
153
163
  not here. If the model emits [L], it is wrong — re-tag as experience/knowledge/decisions.
154
164
  The type tag goes BEFORE the subject, never as a section/category bracket.
@@ -161,6 +171,8 @@ entity. The subject is what a future reader would search for.
161
171
  - NOT: "[E] User rejected AI's bundling of unknown loads"
162
172
  - Write: "[K] Nightly capture (Hermes): cron sessions are excluded — source='cron' is skipped before distillation"
163
173
  - NOT: "[K] Discovered that cron sessions are filtered out"
174
+ - Write: "[E] Qwen3.8-27B swap: AI proposed switching → user held on Qwen3.6-35B-A3B until an MoE variant ships"
175
+ - NOT: "[D] Qwen3.8-27B: switch from Qwen3.6-27B — drop-in upgrade"
164
176
  Reason: each item's first words (after the type tag) become the memory's one-line
165
177
  index entry AND dominate its search embedding. An item that opens with a category
166
178
  label, a sentiment ("Strong Negative"), or "User rejected…" is unfindable — it
@@ -169,6 +181,9 @@ the subject; put reaction, intensity and reasoning AFTER it.
169
181
 
170
182
  RULES:
171
183
  - Extract MAX 20 items total (quality over quantity)
184
+ - Use EXACT names/versions/paths/numbers as they appear in the transcript —
185
+ never substitute what seems current or more standard (writing "Qwen3.6-27B"
186
+ when the transcript says "Qwen3.6-35B-A3B" is a fabrication)
172
187
  - Each must be useful if recalled in a future session
173
188
  - Skip: routine code edits, standard tool usage, trivial fixes
174
189
  - Include: architectural decisions, debugging breakthroughs, user preferences,
@@ -116,6 +116,20 @@ export interface TelemetryPayload {
116
116
  * the dashboard snapshot, not on the wire (mirrors `tokens_this_run`).
117
117
  */
118
118
  budget_exhausted?: boolean;
119
+ /**
120
+ * Backup stage outcome (#6, Phase 0B). Present on every full nightly (absent
121
+ * on capture-only / dry-run / client runs — no backup stage runs there).
122
+ * `ok` is false when EITHER the snapshot write OR the operator's
123
+ * `backupCommand` hook failed — the aggregate data-loss-risk signal (a
124
+ * sustained drop in `backup.ok` means offsite copies are silently not
125
+ * landing). `bytes` is the compressed artifact size (0 when the snapshot
126
+ * itself failed before producing a file). The operator's hook owns active
127
+ * alerting (email/Discord); this field is the passive fleet-health signal.
128
+ */
129
+ backup?: {
130
+ ok: boolean;
131
+ bytes: number;
132
+ };
119
133
  }
120
134
  /**
121
135
  * Check if telemetry is enabled. Disabled by:
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Resolve the effective cap: env (HICORTEX_TOKEN_CAP) takes precedence over the
3
+ * config key. A positive, finite env wins; otherwise the config value (0/absent
4
+ * = unlimited). Pure — exported for tests.
5
+ */
6
+ export declare function resolveTokenCap(configCap: unknown): number;
7
+ /**
8
+ * Initialise at server boot (after stateDir is known). Resolves + caches the cap
9
+ * and seeds the 80%-warn dedup so a restart mid-period doesn't re-warn.
10
+ */
11
+ export declare function initTokenBudget(stateDir: string, configCap: unknown): void;
12
+ /** The resolved monthly cap (0 = unlimited / enforcement off). */
13
+ export declare function getTokenCap(): number;
14
+ /**
15
+ * Pre-call check for /distill: refuse (429) when the tenant is already at/over
16
+ * the monthly cap. Reuses `shouldThrottleTokens(cap, period, 0)` — lastRunTokens
17
+ * is 0 because we cannot predict a call's cost before making it, so this refuses
18
+ * only when already over (a tenant exactly at the cap is refused on the next
19
+ * call). Reads state.json fresh so the nightly process's writes are reflected.
20
+ */
21
+ export declare function isTokenBudgetExceeded(stateDir: string): boolean;
22
+ /**
23
+ * After a successful distill, add the consumed tokens to the monthly counter and
24
+ * emit the 80% warning once per period. Synchronous read-modify-write via
25
+ * `updateState` (serializes concurrent in-process /distill; picks up the nightly
26
+ * process's writes via the fresh read). Accumulates the full breakdown
27
+ * (prompt/completion/total) so the dashboard's prompt+completion stays
28
+ * consistent with total (distill + consolidation).
29
+ */
30
+ export declare function recordDistillUsage(stateDir: string, usage: {
31
+ prompt: number;
32
+ completion: number;
33
+ total: number;
34
+ }): void;