@gamaze/hicortex 0.20.9 → 0.21.0

package/dist/capture-pause.js

@@ -0,0 +1,127 @@
+"use strict";
+/**
+ * Operator capture pause (#423 phase 3, D3) — the server-side 200-skip.
+ *
+ * A pause is a row in `capture_pauses` (migration v18): machine × harness →
+ * paused_at. A row EXISTS = paused for that bundle; the /distill handler
+ * reads the table per POST, so a pause takes effect on the very next post —
+ * no restart — and answers 200 {skipped: true, paused: true}. The 200 is the
+ * whole point: capture.ts treats every 200 as confirmed and advances its
+ * cursor, so sessions that arrive while paused are deliberately NOT captured
+ * and are never re-sent or backfilled. Zero client changes.
+ *
+ * THE BUNDLE KEY. The pause's (machine, harness) must derive EXACTLY like
+ * the traffic it gates: machine via storage.sanitizeSourceMachine ('' when
+ * absent) and harness via harnessOfAgent below — the same normalization
+ * recordDistillActivity applies when it writes distill_activity, and the same
+ * "harness/profile" → "harness" split the console groups its bundles on. A
+ * key derived any other way would never match and the pause would silently
+ * not fire.
+ *
+ * LAST-SEEN (presence dots) derives ONLY from /distill activity — the one
+ * per-agent-identified traffic the server sees. Recall traffic (/search,
+ * /recall-index, /memory) carries no agent/machine identity on the wire, so
+ * attributing it would need new client fields — the heartbeats the spec
+ * forbids. Thresholds (green ≤36h — a nightly poster reads online through
+ * the following day; amber ≤7d — the distill_activity retention window; none
+ * beyond or with no rows) are page-side presentation; this module just
+ * reports the newest ts per bundle.
+ *
+ * Pure, unit-testable without express (the capture-health.ts layering):
+ * mcp-server.ts and dashboard.ts wire these functions to the live db.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.harnessOfAgent = harnessOfAgent;
+exports.capturePauseKey = capturePauseKey;
+exports.isCapturePaused = isCapturePaused;
+exports.setCapturePause = setCapturePause;
+exports.listCapturePauses = listCapturePauses;
+exports.readFleetLastSeen = readFleetLastSeen;
+const storage_js_1 = require("./storage.js");
+/**
+ * Derive the harness from a source_agent wire value — the bundle split the
+ * console already uses ("claude-code/main" → "claude-code"): the part before
+ * the first '/' when a slash is present at index > 0, else the whole trimmed
+ * string, capped at 128. Non-string/blank → "unknown" (mirrors how
+ * recordDistillActivity stores the agent when absent).
+ */
+function harnessOfAgent(sourceAgent) {
+    if (typeof sourceAgent !== "string")
+        return "unknown";
+    const t = sourceAgent.trim();
+    if (t.length === 0)
+        return "unknown";
+    const slash = t.indexOf("/");
+    return (slash > 0 ? t.slice(0, slash) : t).slice(0, 128);
+}
+/**
+ * Normalize the raw /distill wire fields into the pause key. MUST match the
+ * recordDistillActivity normalization (machine '' when absent, agent
+ * 'unknown' when absent) and the console's bundle grouping
+ * ((machine||'')+'|'+harness) — see the module doc.
+ */
+function capturePauseKey(machine, sourceAgent) {
+    return {
+        machine: (0, storage_js_1.sanitizeSourceMachine)(machine) ?? "",
+        harness: harnessOfAgent(sourceAgent),
+    };
+}
+/** True when a pause row exists for the (machine, harness) bundle. */
+function isCapturePaused(db, machine, harness) {
+    return (db
+        .prepare("SELECT 1 FROM capture_pauses WHERE machine = ? AND harness = ?")
+        .get(machine, harness) !== undefined);
+}
+/**
+ * Pause (upsert the row, timestamp now) or resume (delete it). Returns the
+ * persisted paused_at when pausing, null when resuming. No pruning, ever —
+ * see migration v18's provenance comment.
+ */
+function setCapturePause(db, machine, harness, paused) {
+    if (!paused) {
+        db.prepare("DELETE FROM capture_pauses WHERE machine = ? AND harness = ?").run(machine, harness);
+        return null;
+    }
+    const pausedAt = new Date().toISOString();
+    db.prepare("INSERT OR REPLACE INTO capture_pauses (machine, harness, paused_at) VALUES (?, ?, ?)").run(machine, harness, pausedAt);
+    return pausedAt;
+}
+/** Every paused bundle (newest first) — the dashboard fleet.pauses block. */
+function listCapturePauses(db) {
+    return db
+        .prepare("SELECT machine, harness, paused_at FROM capture_pauses ORDER BY paused_at DESC")
+        .all();
+}
+/**
+ * The newest /distill activity per bundle: for each (machine, agent) take
+ * MAX(ts) with that latest row's outcome, derive the harness per agent, then
+ * merge same-bundle agents keeping the newest ts (one dot per bundle, not
+ * per profile). Reads only distill_activity, which the recorder prunes to
+ * 7 days — older-than-window bundles simply have no rows and no dot.
+ */
+function readFleetLastSeen(db) {
+    const rows = db
+        .prepare(`SELECT a.machine, a.agent, a.ts, a.outcome
+         FROM distill_activity a
+         JOIN (
+           SELECT machine, agent, MAX(ts) AS max_ts
+             FROM distill_activity
+            GROUP BY machine, agent
+         ) latest
+           ON a.machine = latest.machine AND a.agent = latest.agent AND a.ts = latest.max_ts`)
+        .all();
+    const byBundle = new Map();
+    for (const r of rows) {
+        const key = `${r.machine}|${harnessOfAgent(r.agent)}`;
+        const prev = byBundle.get(key);
+        if (!prev || r.ts > prev.last_seen) {
+            byBundle.set(key, {
+                machine: r.machine,
+                harness: harnessOfAgent(r.agent),
+                last_seen: r.ts,
+                last_outcome: r.outcome,
+            });
+        }
+    }
+    return [...byBundle.values()].sort((a, b) => (a.last_seen < b.last_seen ? 1 : -1));
+}

package/dist/capture.d.ts

@@ -57,6 +57,9 @@ export interface DistillBody {
     source_agent_id?: string | null;
     /** Client-declared topic/domain of the capturing agent. Provenance only. */
     source_domain?: string | null;
+    /** Machine this capture ran on (#421 machine × harness): config
+     *  `machineName` ?? os.hostname(), stamped by the nightly. */
+    source_machine?: string | null;
     project: string;
     session_id: string;
     segment_id: string;
@@ -106,6 +109,12 @@ export interface CaptureOptions {
      * `source_domain` provenance. Null when undeclared.
      */
     sourceDomain?: string | null;
+    /**
+     * Machine stamp on every segment (#421 machine × harness): config
+     * `machineName` when set, else os.hostname() — resolved by the nightly
+     * caller. Null disables stamping.
+     */
+    sourceMachine?: string | null;
     /**
      * The run-wide pipeline deadline (#405), checked BETWEEN segment POSTs —
      * a boundary the per-session cursor discipline already guarantees is safe

package/dist/capture.js

@@ -204,7 +204,7 @@ async function postWithRateRetry(post, body) {
  * re-paying the Retry-After ladder (#327).
  */
 async function captureBatches(batches, opts) {
-    const { post, cursorStore, dryRun = false, segmentMaxChars = exports.SEGMENT_MAX_CHARS, sourceAgentId, sourceDomain, deadline } = opts;
+    const { post, cursorStore, dryRun = false, segmentMaxChars = exports.SEGMENT_MAX_CHARS, sourceAgentId, sourceDomain, sourceMachine, deadline } = opts;
     let memoriesIngested = 0;
     let sessionsSent = 0;
     let hadTransientFailure = false;
@@ -276,6 +276,7 @@ async function captureBatches(batches, opts) {
                 source_agent: batch.sourceAgent ?? `claude-code/${batch.projectName}`,
                 source_agent_id: sourceAgentId ?? null,
                 source_domain: sourceDomain ?? null,
+                source_machine: sourceMachine ?? null,
                 project: batch.projectName,
                 session_id: batch.sessionId,
                 segment_id: `${genPrefix}${seg.segStart}-${seg.segEnd}${seg.idSuffix}`,

package/dist/cli.js

@@ -189,6 +189,36 @@ switch (command) {
         });
         break;
     }
+    case "rescore-importance": {
+        // #425 — one-shot LLM backfill: re-judge the corpus under the
+        // re-anchored importance rubric. classify-domains shape (resumable
+        // cursor, --batch, --reset) + dedup discipline (dry-run default,
+        // --apply, DB backup before any write).
+        const args = process.argv.slice(3);
+        const intFlag = (name) => {
+            const idx = args.indexOf(name);
+            if (idx === -1)
+                return undefined;
+            const val = parseInt(args[idx + 1], 10);
+            if (isNaN(val)) {
+                console.error(`[hicortex] rescore-importance: ${name} requires an integer value`);
+                process.exit(1);
+            }
+            return val;
+        };
+        const rescoreOptions = {
+            apply: args.includes("--apply"),
+            reset: args.includes("--reset"),
+            batchSize: intFlag("--batch"),
+        };
+        import("./rescore-importance.js").then(({ runRescoreImportance }) => {
+            runRescoreImportance(rescoreOptions).catch((err) => {
+                console.error(err instanceof Error ? err.message : `[hicortex] rescore-importance failed: ${err}`);
+                process.exit(1);
+            });
+        });
+        break;
+    }
     case "classify-types": {
         const args = process.argv.slice(3);
         const intFlag = (name) => {
@@ -421,6 +451,8 @@ Commands:
   backup          Snapshot the DB + identity + state to a tar.gz (online, WAL-safe)
   classify-domains  Backfill content-based domain tags over the corpus (server mode, needs config.domains)
   classify-types    Backfill episode→fact/decision type tags over the corpus (server mode)
+  rescore-importance  Re-judge all memories' importance under the current rubric
+                      (server mode; dry run by default — --apply executes; resumable)
   learnings-identity  Fetch identity + lessons and print Markdown to stdout (CC SessionStart hook)
                       (alias: lessons-context — the pre-#264 name, kept for backcompat)
   recall-hook    Pushed recall index for the current prompt (CC UserPromptSubmit/SessionStart hook)
@@ -458,6 +490,10 @@ Options:
   classify-types --all      Reclassify every memory (default: only episodes)
   classify-types --batch <n>  Memories per batch (default: 200)
   classify-types --reset    Restart from the beginning (ignore saved cursor)
+  rescore-importance --apply  Execute the importance backfill (default: dry run, report only)
+                             Takes a DB backup first; resumable via a state.json cursor
+  rescore-importance --batch <n>  Rows per invocation (default: 500; LLM calls are 10 rows each)
+  rescore-importance --reset    Restart from the beginning (ignore saved cursor)
   identity show [name]      Print all identity sections, or just <name> (raw, pipeable)
   identity edit <name>      Edit a section in $EDITOR; PUT only if changed
   identity … --agent <id>   Target a per-agent scope instead of the global set

package/dist/consolidate.d.ts

@@ -120,6 +120,18 @@ export declare class BudgetTracker {
     } | undefined): void;
     summary(): NonNullable<ConsolidationReport["budget"]>;
 }
+/**
+ * #427 observability: warn when a consolidation run made LLM CALLS but
+ * metered ZERO tokens — the endpoint returned no usage objects on its
+ * completions (recordUsage skips undefined by design, never fabricates a
+ * zero). Such a run still spends budget calls but its snapshot carries token
+ * nulls, which read as a mystery on the dashboard. The warn is a structured
+ * event in the same journald-greppable style as `event=budget_exhausted`
+ * (grep `event=tokens_unmetered`), so the blind spot is visible instead of
+ * silent. Returns true when it warned (for tests); no fabrication either
+ * way — the numbers stay exactly what the endpoint reported.
+ */
+export declare function warnUnmeteredTokensRun(budget: NonNullable<ConsolidationReport["budget"]>): boolean;
 /**
  * True when a token-period start stamp is ABSENT or sits in a previous UTC
  * calendar month than `now` — the monthly-reset staleness check. #405: ONE
@@ -155,6 +167,29 @@ export declare function shouldThrottleTokens(cap: number, period: {
  * Parse JSON from LLM output, tolerating markdown fences and indexed formats.
  */
 export declare function parseJsonLenient<T>(text: string, fallback: T): T;
+/**
+ * The shared importance-scoring loop (#425 extraction): one LLM call per
+ * 10-memory batch through the production `importanceScoring` prompt, each
+ * written score clamped at IMPORTANCE_CEILING and stamped with the
+ * importance_scored_at watermark. Used by the nightly's stageImportance AND
+ * `hicortex rescore-importance` — there is exactly one scoring code path
+ * (no forked backfill logic; cap + watermark write identically everywhere).
+ *
+ * Failure semantics: a batch whose LLM call THROWS writes nothing (counted
+ * in `failed` — retried naturally later); a batch whose reply parses to a
+ * non-array falls back to 0.5 per memory (written + watermarked — the
+ * endpoint answered, the answer was unusable).
+ */
+export declare function scoreMemoriesImportance(db: Database.Database, memories: Memory[], llm: LlmClient, opts?: {
+    budget?: BudgetTracker;
+    deadline?: RunDeadline;
+    dryRun?: boolean;
+    onBatch?: (written: number, failed: number) => void;
+}): Promise<{
+    scored: number;
+    failed: number;
+    skipped_budget: number;
+}>;
 /**
  * Rebuild moduleIndex from the configured domain set + live DB counts, and
  * persist it. Shared by the nightly stage and `hicortex classify-domains`.

package/dist/consolidate.js

@@ -41,9 +41,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DEFAULT_MEMORY_SOFT_CAP = exports.DEFAULT_SUPERSESSION_MIN_SIMILARITY = exports.BudgetTracker = exports.REFLECTION_CONTRADICTION_MIN_COSINE = exports.l2ToCosine = exports.CROSS_PROJECT_LINK_THRESHOLD = exports.CONSOLIDATE_LINK_TOP_K = exports.CONSOLIDATE_LINK_THRESHOLD = exports.DEFAULT_NIGHTLY_LLM_CALL_BUDGET = void 0;
 exports.resolveNightlyLlmCallBudget = resolveNightlyLlmCallBudget;
 exports.isContradictionCandidate = isContradictionCandidate;
+exports.warnUnmeteredTokensRun = warnUnmeteredTokensRun;
 exports.isStaleTokenPeriod = isStaleTokenPeriod;
 exports.shouldThrottleTokens = shouldThrottleTokens;
 exports.parseJsonLenient = parseJsonLenient;
+exports.scoreMemoriesImportance = scoreMemoriesImportance;
 exports.rebuildContentModuleIndex = rebuildContentModuleIndex;
 exports.discoverLinkCandidates = discoverLinkCandidates;
 exports.classifyLinkCandidates = classifyLinkCandidates;
@@ -240,6 +242,27 @@ class BudgetTracker {
     }
 }
 exports.BudgetTracker = BudgetTracker;
+/**
+ * #427 observability: warn when a consolidation run made LLM CALLS but
+ * metered ZERO tokens — the endpoint returned no usage objects on its
+ * completions (recordUsage skips undefined by design, never fabricates a
+ * zero). Such a run still spends budget calls but its snapshot carries token
+ * nulls, which read as a mystery on the dashboard. The warn is a structured
+ * event in the same journald-greppable style as `event=budget_exhausted`
+ * (grep `event=tokens_unmetered`), so the blind spot is visible instead of
+ * silent. Returns true when it warned (for tests); no fabrication either
+ * way — the numbers stay exactly what the endpoint reported.
+ */
+function warnUnmeteredTokensRun(budget) {
+    const calls = budget.calls_used ?? 0;
+    const tokens = budget.tokens_total?.total ?? 0;
+    if (calls <= 0 || tokens > 0)
+        return false;
+    console.warn(`[hicortex] event=tokens_unmetered calls_used=${calls} — the LLM endpoint ` +
+        `returned no usage objects on its completions; this run's snapshot carries ` +
+        `no token metering (budget calls were still counted).`);
+    return true;
+}
 // ---------------------------------------------------------------------------
 // Token fair-use throttle decision (#246)
 // ---------------------------------------------------------------------------
@@ -351,13 +374,29 @@ function stagePrecheck(db, stateDir) {
 // ---------------------------------------------------------------------------
 // Stage 2: Importance Scoring
 // ---------------------------------------------------------------------------
-async function stageImportance(db, memories, llm, budget, dryRun, deadline) {
+/**
+ * The shared importance-scoring loop (#425 extraction): one LLM call per
+ * 10-memory batch through the production `importanceScoring` prompt, each
+ * written score clamped at IMPORTANCE_CEILING and stamped with the
+ * importance_scored_at watermark. Used by the nightly's stageImportance AND
+ * `hicortex rescore-importance` — there is exactly one scoring code path
+ * (no forked backfill logic; cap + watermark write identically everywhere).
+ *
+ * Failure semantics: a batch whose LLM call THROWS writes nothing (counted
+ * in `failed` — retried naturally later); a batch whose reply parses to a
+ * non-array falls back to 0.5 per memory (written + watermarked — the
+ * endpoint answered, the answer was unusable).
+ */
+async function scoreMemoriesImportance(db, memories, llm, opts = {}) {
     const batchSize = 10;
+    const budget = opts.budget;
+    const deadline = opts.deadline;
+    const dryRun = opts.dryRun ?? false;
     let scored = 0;
     let failed = 0;
     let skippedBudget = 0;
     for (let i = 0; i < memories.length; i += batchSize) {
-        if (budget.exhausted) {
+        if (budget?.exhausted) {
             skippedBudget += memories.length - i;
             break;
         }
@@ -372,13 +411,13 @@ async function stageImportance(db, memories, llm, budget, dryRun, deadline) {
         const prompt = (0, prompts_js_1.importanceScoring)(memoriesBlock);
         if (dryRun)
             continue;
-        if (!budget.use("importance")) {
+        if (budget && !budget.use("importance")) {
             skippedBudget += memories.length - i;
             break;
         }
         try {
             const r = await llm.complete(prompt);
-            budget.recordUsage("importance", r.usage);
+            budget?.recordUsage("importance", r.usage);
             let scores = parseJsonLenient(r.text, null);
             if (!Array.isArray(scores)) {
                 scores = new Array(batch.length).fill(0.5);
@@ -386,6 +425,8 @@ async function stageImportance(db, memories, llm, budget, dryRun, deadline) {
             while (scores.length < batch.length)
                 scores.push(0.5);
             scores = scores.slice(0, batch.length);
+            let batchWritten = 0;
+            let batchFailed = 0;
             for (let j = 0; j < batch.length; j++) {
                 let scoreVal = 0.5;
                 try {
@@ -396,21 +437,37 @@ async function stageImportance(db, memories, llm, budget, dryRun, deadline) {
                 catch {
                     scoreVal = 0.5;
                 }
+                // #425: the write cap — importance exactly 1.0 has decay rate exactly
+                // 1.0 and never decays, so no row is ever born immortal. The scored-at
+                // watermark lands in the SAME update, taking the row out of the
+                // nightly's unscored pool however it scored (the pre-#425 0.5-sentinel
+                // re-rolled genuinely-0.5 rows every night).
+                scoreVal = Math.min(scoreVal, CALIBRATION.IMPORTANCE_CEILING);
                 try {
-                    storage.updateMemory(db, batch[j].id, { base_strength: scoreVal });
+                    storage.updateMemory(db, batch[j].id, {
+                        base_strength: scoreVal,
+                        importance_scored_at: new Date().toISOString(),
+                    });
                     scored++;
+                    batchWritten++;
                 }
                 catch {
                     failed++;
+                    batchFailed++;
                 }
             }
+            opts.onBatch?.(batchWritten, batchFailed);
         }
         catch {
             failed += batch.length;
+            opts.onBatch?.(0, batch.length);
         }
     }
     return { scored, failed, skipped_budget: skippedBudget };
 }
+async function stageImportance(db, memories, llm, budget, dryRun, deadline) {
+    return scoreMemoriesImportance(db, memories, llm, { budget, deadline, dryRun });
+}
 // ---------------------------------------------------------------------------
 // Stage 2.5: Reflection
 // ---------------------------------------------------------------------------
@@ -964,7 +1021,9 @@ function classifyRelationship(source, target, similarity) {
 // Stage 3.5: Hub Detection & Strength Boost
 // ---------------------------------------------------------------------------
 const HUB_BOOST = 0.1;
-const HUB_STRENGTH_CAP = 1.0;
+// #425: the release-managed importance ceiling (calibration.ts) — hub boosts
+// can no longer push a row to base 1.0 (decay rate exactly 1.0 = immortal).
+const HUB_STRENGTH_CAP = CALIBRATION.IMPORTANCE_CEILING;
 function stageHubBoost(db, dryRun) {
     const hubs = (0, graph_js_1.detectHubs)(db);
     if (hubs.length === 0)
@@ -1362,7 +1421,15 @@ function stageMemoryCapEviction(db, dryRun, cap) {
     // rejects negatives at the boundary, but this stage is callable directly).
     if (cap <= 0)
         return { cap, evicted: 0 };
-    const count = storage.countMemories(db);
+    // #422 (#317 discipline): the cap keys off LIVE (non-absorbed) rows on BOTH
+    // the count and the victim SELECT — absorbed rows are invisible evidence
+    // (no vector, no FTS, recall never serves them); they must neither consume
+    // cap headroom nor be picked as eviction victims. The DISPLAYED headroom
+    // (dashboard.ts headline live_memories vs memory_soft_cap) reads the same
+    // predicate, so the enforced and displayed caps cannot disagree.
+    const count = db
+        .prepare("SELECT COUNT(*) AS c FROM memories WHERE COALESCE(status, '') != 'absorbed'")
+        .get().c;
     if (count <= cap)
         return { cap, evicted: 0 };
     const surplus = count - cap;
@@ -1370,10 +1437,12 @@ function stageMemoryCapEviction(db, dryRun, cap) {
     // NOT NULL after scoring; the `?? 0.5` mirrors stageDecayPrune's defensive
     // default for unscored rows (inserts at 0.5). last_accessed is NULL until
     // first /recall-index exposure — COALESCE to created_at for the tiebreak so
-    // never-shown memories sort by when they entered the corpus.
+    // never-shown memories sort by when they entered the corpus. Same
+    // non-absorbed predicate as the count above.
     const rows = db
         .prepare(`SELECT id, base_strength, last_accessed, access_count, created_at
-         FROM memories`)
+         FROM memories
+        WHERE COALESCE(status, '') != 'absorbed'`)
         .all();
     const linkCounts = storage.getAllLinkCounts(db);
     const now = new Date();
@@ -1469,6 +1538,13 @@ async function skippedRunResolutionReport(db, dryRun, stateDir, options = {}) {
         explicit_verified: 0,
         explicit_divergent: 0,
         cursor: (0, state_js_1.loadState)(stateDir).reconsolidationCursor ?? 0,
+        // #439 fields: zeros on a quiet night (no scan ran — nothing re-judged,
+        // new, skipped, or deferred; the type carries them so the report surface
+        // stays uniform).
+        pairs_reevaluated: 0,
+        pairs_new: 0,
+        skipped_absorbed: 0,
+        merge_pairs_deferred: 0,
         merges,
         merge_pairs_applied: 0,
         merge_below_gate: 0,