@gamaze/hicortex 0.18.3 → 0.19.1

package/dist/nightly.js

@@ -75,6 +75,7 @@ const capture_js_1 = require("./capture.js");
 const dashboard_js_1 = require("./dashboard.js");
 const telemetry_js_1 = require("./telemetry.js");
 const init_js_1 = require("./init.js");
+const backup_js_1 = require("./backup.js");
 const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
 function readNightlyConfig(stateDir) {
     const configPath = (0, node_path_1.join)(stateDir, "config.json");
@@ -168,7 +169,16 @@ function makeRemotePost(serverUrl, authToken) {
 async function normalizePostResult(resp) {
     if (resp.status === 201) {
         const data = (await resp.json().catch(() => ({})));
-        return { status: 201, distilled: data.distilled ?? 0, dropped: data.dropped ?? [] };
+        // #287: the daemon reports the segment's metered usage. Shape-validated
+        // so a partial payload can't NaN the run's totals; a pre-#287 daemon
+        // simply omits it → capture sums zero (snapshot stays consolidation-only).
+        const usage = parseUsage(data.usage);
+        return {
+            status: 201,
+            distilled: data.distilled ?? 0,
+            dropped: data.dropped ?? [],
+            ...(usage ? { usage } : {}),
+        };
     }
     if (resp.status === 200) {
         const data = (await resp.json().catch(() => ({})));
@@ -177,6 +187,18 @@ async function normalizePostResult(resp) {
     const data = (await resp.json().catch(() => ({})));
     return { status: resp.status, error: data.error ?? "unknown error" };
 }
+/** Strict {prompt, completion, total} parser for /distill's usage field (#287);
+ *  undefined on anything malformed — the caller then treats it as unmetered. */
+function parseUsage(u) {
+    if (typeof u !== "object" || u === null)
+        return undefined;
+    const { prompt, completion, total } = u;
+    const num = (n) => typeof n === "number" && Number.isFinite(n) && n >= 0 ? n : null;
+    const p = num(prompt), c = num(completion), t = num(total);
+    if (p === null || c === null || t === null)
+        return undefined;
+    return { prompt: p, completion: c, total: t };
+}
 function writeLastRun(stateDir = HICORTEX_HOME) {
     (0, state_js_1.updateState)((s) => {
         s.lastNightly = new Date().toISOString();
@@ -347,6 +369,13 @@ async function runNightly(options = {}) {
         let batches = [];
         let memoriesIngested = 0;
         let hadTransientFailure = false;
+        // #287: distill tokens metered by the daemon across this run's segment
+        // POSTs (summed from the /distill responses by the capture loop).
+        // Forwarded to the dashboard snapshot so new_this_run.tokens is the run's
+        // TRUE total (distill + consolidation) and the distill share lands in
+        // tokens_by_stage. Zero when the daemon predates the usage field — the
+        // snapshot writer gates on total > 0, so old servers keep today's shape.
+        let distillUsage;
         // consolidateOnly (hosted service): skip capture entirely. The hosted
         // consolidation timer uses this so per-tenant nightly runs don't ingest the
         // operator's local sessions into the tenant's DB.
@@ -423,6 +452,7 @@ async function runNightly(options = {}) {
                         sourceDomain: savedConfig?.sourceDomain,
                     });
                     memoriesIngested = result.memoriesIngested;
+                    distillUsage = result.distillUsage;
                     // A 429/401 stop must hold the watermark too (fix 1): the loop abandoned
                     // the remaining sessions, and mtime discovery would never re-find them.
                     hadTransientFailure = result.hadTransientFailure || result.stopped !== undefined;
@@ -430,7 +460,12 @@ async function runNightly(options = {}) {
                 finally {
                     releaseLock();
                 }
-                console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories`);
+                console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories` +
+                    // #287: distill tokens the daemon metered for those segments (absent
+                    // when the daemon predates the usage field or nothing distilled).
+                    (distillUsage && distillUsage.total > 0
+                        ? ` · ${distillUsage.total.toLocaleString()} distill tokens`
+                        : ""));
                 // Prune aged-out cursors (90d) — only on a clean run so a transient
                 // failure doesn't drop a still-needed cursor.
                 if (!dryRun && !hadTransientFailure) {
@@ -630,6 +665,53 @@ async function runNightly(options = {}) {
             }
         }
         console.log(`[hicortex] Nightly pipeline complete.`);
+        // Backup stage (#6, Phase 0B) — a transactionally-consistent snapshot of
+        // the irreplaceable data (DB + identity + state), packaged as one tar.gz
+        // the operator ships offsite via the optional `backupCommand` hook. Runs
+        // ONLY on a full nightly (capture-only is frequent + stateless; dry-run
+        // writes nothing). Backup failure must NOT fail the nightly — capture +
+        // consolidation have already succeeded; the snapshot is on disk and the
+        // failure surfaces as `backupOk:false` in the dashboard snapshot + telemetry
+        // for alerting (the operator's hook owns active alerting; no in-product
+        // channel yet — Phase 3).
+        let backupPath;
+        let backupBytes;
+        let backupOk;
+        if (!dryRun && !captureOnly) {
+            try {
+                const backupDir = typeof savedConfig?.backupDir === "string" && savedConfig.backupDir.trim()
+                    ? savedConfig.backupDir
+                    : undefined;
+                const bRes = await (0, backup_js_1.createBackup)({ db, home: stateDir, outDir: backupDir });
+                backupPath = bRes.path;
+                backupBytes = bRes.bytes;
+                backupOk = true;
+                console.log(`[hicortex] Backup: ${bRes.files} files, ${bRes.bytes.toLocaleString()} bytes -> ${bRes.path}`);
+                const cmd = typeof savedConfig?.backupCommand === "string" && savedConfig.backupCommand.trim()
+                    ? savedConfig.backupCommand
+                    : undefined;
+                if (cmd && backupPath) {
+                    const hook = await (0, backup_js_1.runBackupHook)(backupPath, cmd);
+                    if (!hook.ok) {
+                        // The artifact is on disk; only the offsite copy failed. Keep
+                        // backupPath/backupBytes (the snapshot records what was produced)
+                        // but flip backupOk so the aggregate can alert.
+                        backupOk = false;
+                        console.error(`[hicortex] Backup hook failed (exit ${hook.exitCode ?? "n/a"}). ` +
+                            `Artifact is on disk; offsite copy did NOT complete.`);
+                    }
+                    else {
+                        console.log(`[hicortex] Backup hook ok (exit 0).`);
+                    }
+                }
+            }
+            catch (err) {
+                // The whole backup stage failed (snapshot, tar, or write). Do NOT
+                // propagate — capture/consolidation already succeeded. Surface + continue.
+                backupOk = false;
+                console.error(`[hicortex] Backup FAILED: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
         // Dashboard snapshot (#224) — full nightly only. The snapshot reflects
         // corpus state regardless of whether consolidation/LLM ran, so it is
         // ALWAYS written here (the use case is history; an LLM-less install still
@@ -673,6 +755,12 @@ async function runNightly(options = {}) {
                     // when consolidation didn't run or made no metered calls).
                     tokensThisRun,
                     tokensByStage,
+                    // #287: the capture phase's distill tokens (from the /distill
+                    // responses). The writer merges them into `tokens` +
+                    // `tokens_by_stage.distill` so the customer-facing total is the
+                    // run's TRUE spend; zero (old daemon / nothing distilled) is a no-op
+                    // and the row keeps its consolidation-only shape.
+                    distillUsage,
                     // #255 CR: always-on budget usage — undefined when consolidation
                     // didn't run (capture-only / no_llm / throttled). Forwarded whenever
                     // consolidation ran so the digest renders a continuous used/max bar.
@@ -682,6 +770,13 @@ async function runNightly(options = {}) {
                     // (capture-only / no_llm / throttled) or didn't exhaust.
                     budgetExhausted,
                     budgetDeferredByStage,
+                    // #6 backup stage — hoisted from the block above. Present whenever
+                    // the backup stage ran (full nightly); undefined on capture-only /
+                    // dry-run. backupOk flips to false on snapshot OR hook failure so the
+                    // dashboard digest can flag a night the offsite copy didn't complete.
+                    backupPath,
+                    backupBytes,
+                    backupOk,
                 }, memorySoftCapResolved);
             }
             catch (snapErr) {
@@ -728,6 +823,13 @@ async function runNightly(options = {}) {
                 // exhausted (false is omitted to keep the ping minimal; the aggregate
                 // treats absent as "not exhausted / not measurable").
                 ...(budgetExhausted ? { budget_exhausted: true } : {}),
+                // #6 backup stage outcome — forwarded only when the backup stage ran
+                // (full nightly). `ok` is false on snapshot OR hook failure; the fleet
+                // aggregate surfaces a sustained drop in backup_ok as a data-loss risk.
+                // Absent on capture-only / dry-run / client runs (no backup ran).
+                ...(backupOk !== undefined
+                    ? { backup: { ok: backupOk === true, bytes: backupBytes ?? 0 } }
+                    : {}),
                 sessions: batches.length,
                 ok: !hadTransientFailure,
                 shown: adoption.shown,

package/dist/prompts.js

@@ -120,6 +120,7 @@ EXTRACT into this markdown format:
 
 ### Decisions Made
 - [D] [SUBJECT]: [decision] — [reasoning] (${date})
+  (ONLY decisions the user explicitly made or confirmed — never an AI proposal)
 
 ### Knowledge Learned
 - [K] [SUBJECT]: [knowledge] — [context/source] (${date})
@@ -145,10 +146,19 @@ TYPE TAG (critical — prefix EVERY bullet with exactly one letter + space):
 - [K] KNOWLEDGE — a durable truth that will hold across sessions: "the API is at
   :8787", "uv is used for packages", "config lives in ~/.hicortex/". Not tied
   to a single moment.
-- [D] DECISIONS — a choice made that future work builds on, and that a later
-  decision can SUPERSEDE: "switched from gemma4 to qwen3.5", "adopted the
-  graded-schema tag model". Not knowledge (it can change) and not experience
-  (it persists and constrains).
+- [D] DECISIONS — a choice the USER explicitly made or confirmed in the
+  transcript: they said "do X / agreed / ok / go ahead", approved the plan, or
+  the transcript shows the change actually being carried out. It must be a
+  choice future work builds on and that a later decision can SUPERSEDE:
+  "switched from gemma4 to qwen3.5", "adopted the graded-schema tag model".
+  Not knowledge (it can change) and not experience (it persists and constrains).
+- An AI recommendation or proposal is NEVER a decision, however detailed or
+  well-reasoned — even if the user seemed receptive. If the user declined,
+  deferred ("hold", "later", "wait for X"), or did not answer: record it as
+  [E] EXPERIENCE with proposal framing ("AI proposed X → user declined/held
+  because Y"), or under Corrections & Rejections.
+- SELF-CHECK: if an item's text says "AI recommended/proposed" or the user
+  "has not (yet) confirmed" it, that item is NOT [D] — re-tag it [E].
 - NEVER use [L] (learnings). Learnings are extracted by a SEPARATE reflection stage,
   not here. If the model emits [L], it is wrong — re-tag as experience/knowledge/decisions.
 The type tag goes BEFORE the subject, never as a section/category bracket.
@@ -161,6 +171,8 @@ entity. The subject is what a future reader would search for.
 - NOT:    "[E] User rejected AI's bundling of unknown loads"
 - Write:  "[K] Nightly capture (Hermes): cron sessions are excluded — source='cron' is skipped before distillation"
 - NOT:    "[K] Discovered that cron sessions are filtered out"
+- Write:  "[E] Qwen3.8-27B swap: AI proposed switching → user held on Qwen3.6-35B-A3B until an MoE variant ships"
+- NOT:    "[D] Qwen3.8-27B: switch from Qwen3.6-27B — drop-in upgrade"
 Reason: each item's first words (after the type tag) become the memory's one-line
 index entry AND dominate its search embedding. An item that opens with a category
 label, a sentiment ("Strong Negative"), or "User rejected…" is unfindable — it
@@ -169,6 +181,9 @@ the subject; put reaction, intensity and reasoning AFTER it.
 
 RULES:
 - Extract MAX 20 items total (quality over quantity)
+- Use EXACT names/versions/paths/numbers as they appear in the transcript —
+  never substitute what seems current or more standard (writing "Qwen3.6-27B"
+  when the transcript says "Qwen3.6-35B-A3B" is a fabrication)
 - Each must be useful if recalled in a future session
 - Skip: routine code edits, standard tool usage, trivial fixes
 - Include: architectural decisions, debugging breakthroughs, user preferences,

package/dist/telemetry.d.ts

@@ -116,6 +116,20 @@ export interface TelemetryPayload {
      * the dashboard snapshot, not on the wire (mirrors `tokens_this_run`).
      */
     budget_exhausted?: boolean;
+    /**
+     * Backup stage outcome (#6, Phase 0B). Present on every full nightly (absent
+     * on capture-only / dry-run / client runs — no backup stage runs there).
+     * `ok` is false when EITHER the snapshot write OR the operator's
+     * `backupCommand` hook failed — the aggregate data-loss-risk signal (a
+     * sustained drop in `backup.ok` means offsite copies are silently not
+     * landing). `bytes` is the compressed artifact size (0 when the snapshot
+     * itself failed before producing a file). The operator's hook owns active
+     * alerting (email/Discord); this field is the passive fleet-health signal.
+     */
+    backup?: {
+        ok: boolean;
+        bytes: number;
+    };
 }
 /**
  * Check if telemetry is enabled. Disabled by:

package/dist/token-budget.js

@@ -92,7 +92,14 @@ function isTokenBudgetExceeded(stateDir) {
  * consistent with total (distill + consolidation).
  */
 function recordDistillUsage(stateDir, usage) {
-    if (cap <= 0 || usage.total <= 0)
+    // Record ALWAYS; the cap only governs ENFORCEMENT (isTokenBudgetExceeded)
+    // and the 80% warning below. Without this, an uncapped install (self-hosted
+    // default) never accrues distill tokens into the period meter, so the
+    // dashboard's monthly headline read consolidation-only while the per-run
+    // chart next to it showed true (distill-inclusive) totals — numbers on the
+    // same card telling different stories (#287 CR). Always-recording makes the
+    // meter the honest "what did this install spend" number everywhere.
+    if (usage.total <= 0)
         return;
     let newTotal = 0;
     let periodStart = "";
@@ -123,7 +130,8 @@ function recordDistillUsage(stateDir, usage) {
         periodStart = s.llmTokensThisPeriod.periodStart;
     }, stateDir);
     // 80% warning — dedup per period (once per month per threshold crossing).
-    if (periodStart && warnedPeriod !== periodStart && newTotal >= cap * 0.8) {
+    // Cap-gated by design: no cap → no percentage to warn about.
+    if (cap > 0 && periodStart && warnedPeriod !== periodStart && newTotal >= cap * 0.8) {
         warnedPeriod = periodStart;
         const pct = Math.round((newTotal / cap) * 100);
         console.warn(`[hicortex] Token usage at ${pct}% of monthly cap (${newTotal.toLocaleString()}/${cap.toLocaleString()}).`);

package/dist/type-classify.js

@@ -78,10 +78,13 @@ function buildTypeClassifyPrompt(content) {
         `occurrence ("tried X, failed because Y", a correction, a debugging session).\n` +
         `- knowledge: a durable truth that holds across sessions, not tied to a ` +
         `single moment ("the API is at :8787", "uv is used for packages").\n` +
-        `- decisions: a choice made that future work builds on and a later ` +
-        `decision can supersede ("switched from gemma4 to qwen3.5", "adopted the ` +
-        `graded-schema tag model"). Not knowledge (it can change) and not an ` +
-        `experience (it persists).\n\n` +
+        `- decisions: a choice the user explicitly made or confirmed (or one the ` +
+        `memory records as actually carried out/applied) that future work builds ` +
+        `on and a later decision can supersede ("switched from gemma4 to qwen3.5", ` +
+        `"adopted the graded-schema tag model"). Not knowledge (it can change) and ` +
+        `not an experience (it persists). A bare AI recommendation or proposal is ` +
+        `NEVER a decision — "AI proposed X → user declined/held" is experience ` +
+        `(#290).\n\n` +
         `IMPORTANCE (0.0–1.0):\n` +
         `- 0.8–1.0: load-bearing — a core piece of knowledge or a decision the ` +
         `agent must know.\n` +

package/dist/types.d.ts

@@ -434,6 +434,46 @@ export interface HicortexConfig {
      * set it. Period resets monthly (state.json `llmTokensThisPeriod.periodStart`).
      */
     llmTokensPerMonth?: number;
+    /**
+     * Max request body size in MB accepted by the REST/MCP server (#7). The body
+     * is fully JSON-parsed into memory before the distiller truncates content to
+     * 80K chars, so an unbounded body is an OOM vector — a tenant (hosted) or a
+     * misbehaving client could POST a huge payload to exhaust RAM. Default: 25 MB
+     * self-hosted (unchanged), 5 MB hosted (the spec §6.3 figure — ~25× the 200KB
+     * segment max, so legitimate capture never approaches it; it's a pure
+     * abuse/OOM backstop). Oversized bodies get HTTP 413.
+     */
+    distillBodyLimitMb?: number;
+    /**
+     * Output directory for backup artifacts (#6). Default `<HICORTEX_HOME>/backups`.
+     * Both the CLI (`hicortex backup`) and the nightly backup stage resolve this
+     * before calling createBackup; an unset value falls back to the home dir.
+     * Operator-owned: point at a mounted backup volume, a tmpfs, etc.
+     */
+    backupDir?: string;
+    /**
+     * Post-backup offsite hook (#6). When set, `hicortex backup` and the nightly
+     * backup stage invoke this command with the artifact path appended as the LAST
+     * arg (e.g. `"rclone copyto"` → `rclone copyto <path> remote:bucket/`). Cloud
+     * credentials + active alerting (email/Discord) stay in the operator's wrapper
+     * script, out of the product. The hook is split on whitespace (no shell); a
+     * command with quoted args containing spaces should be a wrapper script. A
+     * failing/missing/timed-out hook reports `{ok:false}` and never throws —
+     * capture/consolidation have already succeeded, so a backup-hook failure must
+     * not fail the nightly. 5 min timeout.
+     */
+    backupCommand?: string;
+    /**
+     * Account identity shown in the dashboard header (hosted). When ALL three
+     * are absent the header renders no account element (self-hosted default —
+     * nothing changes). Strings only; read via readStringConfig, null when
+     * absent/not a string. Set per-tenant by provision-tenant.sh.
+     */
+    displayName?: string;
+    /** Organization name — rendered alongside displayName as "Name · Org". */
+    orgName?: string;
+    /** Plan/tier label rendered as a small badge (e.g. "Cloud · Early bird"). */
+    planLabel?: string;
 }
 /** A config-owned life-sphere domain (see HicortexConfig.domains). */
 export interface DomainDef {

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "hicortex",
   "name": "Hicortex — Long-term Memory That Learns",
   "description": "Your agents remember past decisions, avoid repeated mistakes, and get smarter every day. Nightly reflection generates actionable lessons that automatically update agent behavior.",
-  "version": "0.10.0",
+  "version": "0.19.1",
   "kind": "lifecycle",
   "skills": ["./skills/hicortex-memory"],
   "configSchema": {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gamaze/hicortex",
-  "version": "0.18.3",
-  "description": "Persistent agent identity for AI agents — a hand-edited identity layer, nightly-distilled experience, and lessons injected every session, shared across your whole fleet. Works with Hermes, OpenClaw, Claude Code, and Pi.",
+  "version": "0.19.1",
+  "description": "Persistent agent identity for AI agents \u2014 a hand-edited identity layer, nightly-distilled experience, and lessons injected every session, shared across your whole fleet. Works with Hermes, OpenClaw, Claude Code, and Pi.",
   "main": "dist/index.js",
   "bin": {
     "hicortex": "dist/cli.js"
@@ -47,6 +47,7 @@
     "@types/better-sqlite3": "^7.6.0",
     "@types/express": "^5.0.6",
     "@types/node": "^22.0.0",
+    "@types/tar-stream": "^3.1.4",
     "typescript": "^5.4.0",
     "vitest": "^3.0.0"
   },
@@ -69,6 +70,7 @@
     "@modelcontextprotocol/sdk": "^1.28.0",
     "better-sqlite3": "^12.11.1",
     "express": "^4.21.0",
-    "sqlite-vec": "^0.1.7"
+    "sqlite-vec": "^0.1.7",
+    "tar-stream": "^2.2.0"
   }
-}
+}