@gamaze/hicortex 0.16.9 → 0.17.0

package/dist/nightly.js

@@ -71,6 +71,7 @@ const retrieval_js_1 = require("./retrieval.js");
 const state_js_1 = require("./state.js");
 const capture_cursors_js_1 = require("./capture-cursors.js");
 const capture_js_1 = require("./capture.js");
+const dashboard_js_1 = require("./dashboard.js");
 const telemetry_js_1 = require("./telemetry.js");
 const init_js_1 = require("./init.js");
 const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
@@ -211,7 +212,8 @@ function rotateNightlyLog(stateDir = HICORTEX_HOME) {
 }
 async function runNightly(options = {}) {
     const dryRun = options.dryRun ?? false;
-    const captureOnly = options.captureOnly ?? false;
+    let captureOnly = options.captureOnly ?? false;
+    const watchdog = options.watchdog ?? false;
     const stateDir = options.stateDir ?? HICORTEX_HOME;
     const recaptureWindowDays = options.recaptureWindowDays;
     rotateNightlyLog(stateDir);
@@ -219,6 +221,8 @@ async function runNightly(options = {}) {
     (0, state_js_1.migrateLegacyState)(stateDir);
     // Check mode: client or server
     const savedConfig = readNightlyConfig(stateDir);
+    // 0.16.8 upgrade guard: warn if ignored per-stage keys are still present.
+    (0, config_read_js_1.warnIgnoredConfigKeys)(savedConfig);
     // 0.16.2 activation gap: pre-0.16.2 installs never re-run init, so their
     // config has no agentId → capture sent source_agent_id: null forever (the
     // provenance feature was inert for the whole existing fleet). Self-heal on
@@ -230,14 +234,59 @@ async function runNightly(options = {}) {
         const { agentId } = (0, init_js_1.ensureAndPersistAgentId)((0, node_path_1.join)(stateDir, "config.json"));
         savedConfig.agentId = agentId;
     }
+    const port = savedConfig?.port ?? 8787;
+    // Watchdog gate (before the client/server branch so it is uniform). See the
+    // option doc above. On skip it returns early; on proceed it forces
+    // capture-only and falls through to the normal capture path (the capture
+    // lock with waitMs=0 provides single-flight; writeLastRun advances the
+    // cooldown marker on success).
+    if (watchdog && !dryRun) {
+        captureOnly = true; // the watchdog captures only — never consolidates.
+        // Success-cooldown: lastNightly is advanced ONLY on a clean capture
+        // (writeLastRun is success-gated), so reusing it gives SUCCESS-based
+        // cooldown — a FAILED preflight/capture retries on the next tick (minutes),
+        // a SUCCESS waits the cooldown. This is the better semantics the custom
+        // server watchdog (trigger-based) got wrong.
+        // readNonNegativeConfig (not readPositiveConfig) so 0 is honoured: 0 = no
+        // cooldown = capture every poll (a valid opt-in for a wired/high-frequency
+        // source), not a silent fallback to the default.
+        const cooldownH = (0, config_read_js_1.readNonNegativeConfig)(savedConfig ?? {}, "captureCooldownHours", 6);
+        const last = (0, state_js_1.loadState)(stateDir).lastNightly;
+        if (last) {
+            const ageH = (Date.now() - new Date(last).getTime()) / 3_600_000;
+            if (ageH < cooldownH) {
+                console.log(`[hicortex] watchdog: last capture ${ageH.toFixed(1)}h ago (< ${cooldownH}h cooldown) — skipping`);
+                return;
+            }
+        }
+        // Quick reachability preflight (5s) — a cheap gate so a 20-min poll against
+        // a down link costs one short fetch, not the full in-run preflight. Client
+        // → remote server; server/co-located → localhost daemon.
+        const target = (savedConfig?.mode === "client"
+            ? savedConfig.serverUrl
+            : `http://127.0.0.1:${port}`).replace(/\/+$/, "");
+        try {
+            const resp = await fetch(`${target}/health`, { signal: AbortSignal.timeout(5_000) });
+            if (!resp.ok) {
+                console.log(`[hicortex] watchdog: ${target}/health not ok (${resp.status}) — skipping (retry next tick)`);
+                return;
+            }
+        }
+        catch (e) {
+            console.log(`[hicortex] watchdog: ${target} unreachable (` +
+                `${e instanceof Error ? e.message : String(e)}) — skipping (retry next tick)`);
+            return;
+        }
+        console.log(`[hicortex] watchdog: cooldown elapsed + ${target} reachable — capturing`);
+    }
     if (savedConfig?.mode === "client") {
         // --capture-only is accepted in client mode but irrelevant: client nightly
-        // is already capture-only (no consolidation step).
-        await runClientNightly(savedConfig, dryRun, stateDir, recaptureWindowDays);
+        // is already capture-only (no consolidation step). (If watchdog-gated, the
+        // gate above already ran.)
+        await runClientNightly(savedConfig, dryRun, stateDir, recaptureWindowDays, watchdog);
         return;
     }
     const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
-    const port = savedConfig?.port ?? 8787;
     // #192: consolidation's decay/prune stage must score with the same clock as
     // the server's retrieval path (config decayHalfLifeDays, default 365).
     (0, retrieval_js_1.configureDecay)({ halfLifeDays: savedConfig?.decayHalfLifeDays });
@@ -367,9 +416,14 @@ async function runNightly(options = {}) {
         // of the HTTP capture path). Full nightly only — capture-only runs are
         // intended to run more frequently than once daily.
         let lessonsGenerated; // hoisted for telemetry; undefined when reflection didn't run (skipped) — bucketed apart from a real 0
+        // Consolidation outcome for telemetry (0.17). undefined on capture-only runs
+        // (which send no nightly ping). "skipped" = runConsolidation's built-in
+        // nothing-to-do short-circuit (zero LLM calls), NOT a failure.
+        let consolidationStatus;
         if (!dryRun && !captureOnly) {
             if (!llm || !llmConfig) {
                 console.error("[hicortex] consolidation skipped: no LLM configured — run npx @gamaze/hicortex init");
+                consolidationStatus = "no_llm";
             }
             else {
                 // One model serves all phases (#231) — there is no separate endpoint to
@@ -388,9 +442,12 @@ async function runNightly(options = {}) {
                 }, {
                     minSimilarity: savedConfig?.supersessionMinSimilarity,
                     maxCalls: savedConfig?.supersessionMaxCalls,
-                });
+                }, 
+                // #241: config-driven total LLM-call ceiling (default 5000, was 200).
+                (0, config_read_js_1.readPositiveConfig)(savedConfig ?? {}, "consolidateMaxLlmCalls", consolidate_js_1.CONSOLIDATE_MAX_LLM_CALLS));
                 console.log(`[hicortex] Consolidation ${report.status} in ${report.elapsed_seconds}s` +
                     (report.stages.reflection ? ` (${report.stages.reflection.lessons_generated} lessons)` : ""));
+                consolidationStatus = report.status;
                 // Only set when reflection actually RAN (not skipped). A skipped stage
                 // (e.g. endpoint offline, #232 fail-soft) must NOT collapse to 0 — that
                 // would make "endpoint down" indistinguishable from "prompt too tight"
@@ -415,6 +472,53 @@ async function runNightly(options = {}) {
             }
         }
         console.log(`[hicortex] Nightly pipeline complete.`);
+        // Dashboard snapshot (#224) — full nightly only. The snapshot reflects
+        // corpus state regardless of whether consolidation/LLM ran, so it is
+        // ALWAYS written here (the use case is history; an LLM-less install still
+        // accrues memories). Capture-only runs SKIP it (above, the block guards
+        // on !captureOnly). On the first run after deploy the table is empty →
+        // backfill synthesizes one row per day from created_at so the growth/
+        // composition charts have real history on day one.
+        if (!dryRun && !captureOnly) {
+            try {
+                const backfilled = (0, dashboard_js_1.backfillSnapshots)(db);
+                if (backfilled > 0) {
+                    console.log(`[hicortex] Dashboard backfill: ${backfilled} day(s) synthesized from created_at`);
+                }
+                // Per-run deltas. `added` = this run's captured memory count (only
+                // available on the server path; client mode never reaches here — it
+                // POSTs to a remote /distill). dedup/supersession are derived from
+                // the dedup_log / superseded_by link tables: count rows created since
+                // the last snapshot (real OR backfilled — both carry valid ISO run_at,
+                // so a plain ORDER BY run_at DESC LIMIT 1 is the correct floor) so a
+                // manual `hicortex dedup` run between nightlies is still reflected,
+                // and a first-write after backfill counts only what landed AFTER the
+                // last backfilled day.
+                const lastSnap = db
+                    .prepare("SELECT run_at FROM dashboard_snapshots ORDER BY run_at DESC LIMIT 1")
+                    .get();
+                const sinceTs = lastSnap?.run_at ?? "1970-01-01T00:00:00.000Z";
+                const dedup = db
+                    .prepare("SELECT COUNT(*) AS c FROM dedup_log WHERE merged_at > ?")
+                    .get(sinceTs).c;
+                const supersession = db
+                    .prepare(`SELECT COUNT(*) AS c FROM memory_links
+                WHERE relationship = 'superseded_by' AND created_at > ?`)
+                    .get(sinceTs).c;
+                (0, dashboard_js_1.writeSnapshot)(db, new Date().toISOString(), {
+                    added: memoriesIngested,
+                    lessonsGenerated,
+                    dedup,
+                    supersession,
+                });
+            }
+            catch (snapErr) {
+                // The snapshot is a monitoring side-effect — a failure here must NOT
+                // advance to a telemetry gap or move the watermark. Surface + continue.
+                console.warn(`[hicortex] Dashboard snapshot write failed: ` +
+                    `${snapErr instanceof Error ? snapErr.message : String(snapErr)}`);
+            }
+        }
         // Anonymous telemetry (fire-and-forget, full nightly only).
         // Capture-only runs are excluded to avoid inflating install pings.
         if (!dryRun && !captureOnly && (0, telemetry_js_1.isTelemetryEnabled)(savedConfig)) {
@@ -444,6 +548,7 @@ async function runNightly(options = {}) {
                 mem: storage.countMemories(db),
                 lessons: storage.getLessons(db, 365).length,
                 lessonsGenerated,
+                consolidation: consolidationStatus,
                 sessions: batches.length,
                 ok: !hadTransientFailure,
                 shown: adoption.shown,
@@ -459,7 +564,7 @@ async function runNightly(options = {}) {
 // ---------------------------------------------------------------------------
 // Client Mode Nightly — denoise locally, POST to remote server's /distill
 // ---------------------------------------------------------------------------
-async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recaptureWindowDays) {
+async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recaptureWindowDays, watchdog = false) {
     const serverUrl = config.serverUrl.replace(/\/+$/, "");
     const authToken = config.authToken;
     console.log(`[hicortex] Client nightly starting${dryRun ? " (dry run)" : ""}`);
@@ -469,16 +574,20 @@ async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recapt
     // the whole run — the pre-flight only needs the link back, which can take
     // ~1 min after wake.
     //
-    // Config-overridable (#163): a wired Pi vs a sleeping laptop want different
-    // values. Defaults: 15s per-attempt timeout, 3 attempts, 60s gap.
+    // Config-overridable (#163): a wired Pi vs a sleeping/roaming laptop want
+    // different values. Defaults: 20s per-attempt timeout, 3 attempts, 60s gap.
+    // The 20s per-attempt (bumped from 15s in 0.17) absorbs a cold Tailscale
+    // handshake to a home server (DERP relay + NAT traversal can take 5–30s on a
+    // roaming laptop). For a genuinely DOWN link (`fetch failed`) no timeout
+    // length helps — the capture watchdog's frequent retry handles that (#239).
     //
     // WALL-CLOCK NOTE: setTimeout and AbortSignal.timeout do NOT advance while
-    // macOS is asleep, so the ~2m45s worst case (3×15s + 2×60s) is wall-clock-
+    // macOS is asleep, so the ~3m worst case (3×20s + 2×60s) is wall-clock-
     // optimistic — a sleeping laptop can straddle sleep cycles and the real
     // elapsed time can exceed it. Not a defect: the capture lock isn't held
     // during the retry and the cursor design is dup-over-loss, so a late success
-    // is harmless. Just don't treat 2m45s as a hard wall-clock bound.
-    const PREFLIGHT_TIMEOUT_MS = (0, config_read_js_1.readPositiveConfig)(config, "preflightTimeoutMs", 15_000);
+    // is harmless. Just don't treat 3m as a hard wall-clock bound.
+    const PREFLIGHT_TIMEOUT_MS = (0, config_read_js_1.readPositiveConfig)(config, "preflightTimeoutMs", 20_000);
     const PREFLIGHT_ATTEMPTS = Math.max(1, Math.floor((0, config_read_js_1.readPositiveConfig)(config, "preflightAttempts", 3)));
     const PREFLIGHT_RETRY_GAP_MS = (0, config_read_js_1.readPositiveConfig)(config, "preflightRetryGapMs", 60_000);
     let reachable = false;
@@ -511,7 +620,12 @@ async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recapt
         // no loop), and fire the telemetry ping (ok=false) so the abort is
         // distinguishable from "powered off / uninstalled" in the activity aggregate.
         process.exitCode = 1;
-        if (!dryRun && (0, telemetry_js_1.isTelemetryEnabled)(config)) {
+        // In watchdog mode, suppress this failed-preflight ping: the watchdog
+        // retries every ~20 min, so a flaky link would otherwise emit up to ~72
+        // ok:false pings/day and distort the fleet health ratio (#239 CR). A
+        // sustained outage is still visible — as the ABSENCE of success pings, and
+        // a non-watchdog (manual) run still emits ok:false.
+        if (!dryRun && !watchdog && (0, telemetry_js_1.isTelemetryEnabled)(config)) {
             await (0, telemetry_js_1.sendTelemetry)({
                 id: (0, telemetry_js_1.getTelemetryId)(stateDir),
                 v: VERSION,

package/dist/telemetry.d.ts

@@ -85,6 +85,17 @@ export interface TelemetryPayload {
      * and can't reveal a sudden drop in reflection output. 0.16.9+.
      */
     lessonsGenerated?: number;
+    /**
+     * Consolidation outcome for THIS full nightly (server mode only —
+     * capture-only runs send no nightly ping, so the field is absent there).
+     * `runConsolidation`'s status: "completed" | "skipped" | "failed", plus
+     * "no_llm" when consolidation was skipped because no LLM was configured.
+     * "skipped" = the built-in nothing-to-do short-circuit (no new + no unscored
+     * memories → zero LLM calls), NOT a failure. Lets the fleet aggregate tell a
+     * real consolidation run from a no-op without repurposing `ok` (which is the
+     * capture-health signal). 0.17+.
+     */
+    consolidation?: "completed" | "skipped" | "failed" | "no_llm";
 }
 /**
  * Check if telemetry is enabled. Disabled by:

package/dist/types.d.ts

@@ -215,6 +215,35 @@ export interface HicortexConfig {
      * "Unsorted" — if configured — is just a normal domain.
      */
     domains?: DomainDef[];
+    /**
+     * Success-cooldown (hours) for the CAPTURE watchdog (0.17). The capture
+     * timer fires `nightly --watchdog` on a short interval (~20 min); the
+     * watchdog captures only if MORE than this many hours have passed since the
+     * last SUCCESSFUL capture (state `lastNightly`). A failed preflight retries
+     * on the next tick (~20 min) — so a transient fire-instant network miss
+     * costs minutes, not a day (#239). Default 6 (≈4 captures/day). Read at
+     * runtime by the watchdog, not by `init`.
+     */
+    captureCooldownHours?: number;
+    /**
+     * Hours (0–23, local time) for the CONSOLIDATION timer — the full nightly
+     * (capture + distill + score + reflect + link), installed by `init` for
+     * server/co-located mode ONLY (0.17). Default [10, 22]: the 22:00 evening
+     * slot runs after the day's capture waves (same-day results); the 10:00
+     * morning slot runs AFTER the morning's wake-up capture so it catches those
+     * pushes. Omitted on client installs (no local DB → no timer). Validated by
+     * parseHours.
+     */
+    consolidationHours?: number[];
+    /**
+     * Ceiling on total LLM calls across all classify-tier consolidation stages
+     * (content-domain, link discovery, supersession) per nightly run (0.17, #241).
+     * A runaway backstop, not a throughput throttle — on a free local model the
+     * binding constraint is the nightly unit's wall-clock timeout, not call count.
+     * Default `5000` (was a hard-coded 200 that starved link/supersession during a
+     * classification backlog and drained large backlogs at ~cap/night).
+     */
+    consolidateMaxLlmCalls?: number;
     /**
      * Minimum cosine(memory embedding, best domain prototype) for a no-fit
      * memory to earn a WEAK primary instead of decaying (see nofit.ts).

package/dist/uninstall.js

@@ -43,11 +43,14 @@ async function runUninstall() {
         return;
     }
     console.log();
-    // 1. Stop and remove daemon + nightly timer (both units, or the timer
-    //    keeps firing against a half-removed install)
+    // 1. Stop and remove daemon + capture/nightly timers (all units, or a timer
+    //    keeps firing against a half-removed install). Since 0.17 there are two
+    //    scheduled jobs: the capture timer (hicortex-capture) and the
+    //    consolidation timer (hicortex-nightly); remove both either way (a client
+    //    install has no consolidation timer, a server install has both).
     const os = (0, node_os_1.platform)();
     if (os === "darwin") {
-        for (const name of ["com.gamaze.hicortex.plist", "com.gamaze.hicortex-nightly.plist"]) {
+        for (const name of ["com.gamaze.hicortex.plist", "com.gamaze.hicortex-nightly.plist", "com.gamaze.hicortex-capture.plist"]) {
             const plistPath = (0, node_path_1.join)((0, node_os_1.homedir)(), "Library", "LaunchAgents", name);
             if ((0, node_fs_1.existsSync)(plistPath)) {
                 try {
@@ -68,8 +71,12 @@ async function runUninstall() {
             (0, node_child_process_1.execSync)("systemctl --user disable --now hicortex-nightly.timer 2>/dev/null");
         }
         catch { /* not installed */ }
+        try {
+            (0, node_child_process_1.execSync)("systemctl --user disable --now hicortex-capture.timer 2>/dev/null");
+        }
+        catch { /* not installed */ }
         const unitDir = (0, node_path_1.join)((0, node_os_1.homedir)(), ".config", "systemd", "user");
-        for (const name of ["hicortex.service", "hicortex-nightly.timer", "hicortex-nightly.service"]) {
+        for (const name of ["hicortex.service", "hicortex-nightly.timer", "hicortex-nightly.service", "hicortex-capture.timer", "hicortex-capture.service"]) {
             const unitPath = (0, node_path_1.join)(unitDir, name);
             try {
                 if ((0, node_fs_1.existsSync)(unitPath))
@@ -81,7 +88,7 @@ async function runUninstall() {
             (0, node_child_process_1.execSync)("systemctl --user daemon-reload 2>/dev/null");
         }
         catch { /* fine */ }
-        console.log("  ✓ Removed systemd service + nightly timer");
+        console.log("  ✓ Removed systemd service + capture/nightly timers");
     }
     // 2. Remove MCP from CC
     try {

package/dist/viz.d.ts

@@ -66,6 +66,22 @@ export declare function readContextHtml(): string;
  * cannot be read, exactly like vizHandler.
  */
 export declare function contextUiHandler(): express.RequestHandler;
+/**
+ * Resolve the on-disk path of the dashboard page. Throws (fail explicitly)
+ * when the asset is missing — same contract as resolveVizHtmlPath and
+ * resolveContextHtmlPath. assets/ sits next to both dist/ and src/ (the
+ * sibling layout the other resolvers rely on).
+ */
+export declare function resolveDashboardHtmlPath(): string;
+/** Read the dashboard page. Read at request time so a reinstall is picked up live. */
+export declare function readDashboardHtml(): string;
+/**
+ * Express handler for GET /dashboard — the view-only analytics page (#224).
+ * 503 with the usual {error} shape when the asset cannot be read, exactly like
+ * vizHandler and contextUiHandler. The page SHELL is public (exempted in
+ * createAuthMiddleware); all data comes from GET /dashboard/data (bearer-only).
+ */
+export declare function dashboardHandler(): express.RequestHandler;
 /**
  * Resolve the on-disk path of an allowlisted vendor bundle, or null when the
  * requested name is not on the allowlist. The filesystem path is built ONLY

package/dist/viz.js

@@ -34,6 +34,9 @@ exports.vizHandler = vizHandler;
 exports.resolveContextHtmlPath = resolveContextHtmlPath;
 exports.readContextHtml = readContextHtml;
 exports.contextUiHandler = contextUiHandler;
+exports.resolveDashboardHtmlPath = resolveDashboardHtmlPath;
+exports.readDashboardHtml = readDashboardHtml;
+exports.dashboardHandler = dashboardHandler;
 exports.resolveVizVendorPath = resolveVizVendorPath;
 exports.vizVendorHandler = vizVendorHandler;
 const node_fs_1 = require("node:fs");
@@ -81,6 +84,14 @@ function createAuthMiddleware(authToken) {
         // sends it as a normal Authorization header on its /context fetches.
         if (req.method === "GET" && req.path === "/context/ui")
             return next();
+        // The /dashboard page SHELL is public for the same reason as /viz and
+        // /context/ui: a self-contained view-only analytics page, no data and no
+        // secrets (it ships verbatim in the npm tarball). All metric data comes
+        // from GET /dashboard/data, which stays bearer-only (localhost bypass) like
+        // every other data route; the page collects the token client-side and
+        // sends it as a normal Authorization header on its /dashboard/data fetch.
+        if (req.method === "GET" && req.path === "/dashboard")
+            return next();
         // The pinned renderer bundles the /viz page loads (#139) are public for
         // the same reason as the shell: static third-party code shipped verbatim
         // in the npm tarball, zero data. Kept tight: GET only, and ONLY names on
@@ -184,6 +195,44 @@ function contextUiHandler() {
         }
     };
 }
+// ---------------------------------------------------------------------------
+// Dashboard page (/dashboard, #224 — view-only memory analytics)
+// ---------------------------------------------------------------------------
+/**
+ * Resolve the on-disk path of the dashboard page. Throws (fail explicitly)
+ * when the asset is missing — same contract as resolveVizHtmlPath and
+ * resolveContextHtmlPath. assets/ sits next to both dist/ and src/ (the
+ * sibling layout the other resolvers rely on).
+ */
+function resolveDashboardHtmlPath() {
+    const candidates = [(0, node_path_1.join)(__dirname, "..", "assets", "dashboard.html")];
+    for (const candidate of candidates) {
+        if ((0, node_fs_1.existsSync)(candidate))
+            return candidate;
+    }
+    throw new Error(`dashboard.html asset not found — looked in: ${candidates.join(", ")}. ` +
+        `The package install is incomplete (assets/ missing).`);
+}
+/** Read the dashboard page. Read at request time so a reinstall is picked up live. */
+function readDashboardHtml() {
+    return (0, node_fs_1.readFileSync)(resolveDashboardHtmlPath(), "utf-8");
+}
+/**
+ * Express handler for GET /dashboard — the view-only analytics page (#224).
+ * 503 with the usual {error} shape when the asset cannot be read, exactly like
+ * vizHandler and contextUiHandler. The page SHELL is public (exempted in
+ * createAuthMiddleware); all data comes from GET /dashboard/data (bearer-only).
+ */
+function dashboardHandler() {
+    return (_req, res) => {
+        try {
+            res.type("html").send(readDashboardHtml());
+        }
+        catch (err) {
+            res.status(503).json({ error: err instanceof Error ? err.message : String(err) });
+        }
+    };
+}
 /**
  * Resolve the on-disk path of an allowlisted vendor bundle, or null when the
  * requested name is not on the allowlist. The filesystem path is built ONLY

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gamaze/hicortex",
-  "version": "0.16.9",
+  "version": "0.17.0",
   "description": "Self-learning memory for AI agents — experience captured automatically, distilled into lessons overnight, shared across your whole fleet. Works with Hermes, OpenClaw, Claude Code, and Pi.",
   "main": "dist/index.js",
   "bin": {