@gamaze/hicortex 0.20.7 → 0.20.10

package/dist/capture-pause.js

@@ -0,0 +1,127 @@
+"use strict";
+/**
+ * Operator capture pause (#423 phase 3, D3) — the server-side 200-skip.
+ *
+ * A pause is a row in `capture_pauses` (migration v18): machine × harness →
+ * paused_at. A row EXISTS = paused for that bundle; the /distill handler
+ * reads the table per POST, so a pause takes effect on the very next post —
+ * no restart — and answers 200 {skipped: true, paused: true}. The 200 is the
+ * whole point: capture.ts treats every 200 as confirmed and advances its
+ * cursor, so sessions that arrive while paused are deliberately NOT captured
+ * and are never re-sent or backfilled. Zero client changes.
+ *
+ * THE BUNDLE KEY. The pause's (machine, harness) must derive EXACTLY like
+ * the traffic it gates: machine via storage.sanitizeSourceMachine ('' when
+ * absent) and harness via harnessOfAgent below — the same normalization
+ * recordDistillActivity applies when it writes distill_activity, and the same
+ * "harness/profile" → "harness" split the console groups its bundles on. A
+ * key derived any other way would never match and the pause would silently
+ * not fire.
+ *
+ * LAST-SEEN (presence dots) derives ONLY from /distill activity — the one
+ * per-agent-identified traffic the server sees. Recall traffic (/search,
+ * /recall-index, /memory) carries no agent/machine identity on the wire, so
+ * attributing it would need new client fields — the heartbeats the spec
+ * forbids. Thresholds (green ≤36h — a nightly poster reads online through
+ * the following day; amber ≤7d — the distill_activity retention window; none
+ * beyond or with no rows) are page-side presentation; this module just
+ * reports the newest ts per bundle.
+ *
+ * Pure, unit-testable without express (the capture-health.ts layering):
+ * mcp-server.ts and dashboard.ts wire these functions to the live db.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.harnessOfAgent = harnessOfAgent;
+exports.capturePauseKey = capturePauseKey;
+exports.isCapturePaused = isCapturePaused;
+exports.setCapturePause = setCapturePause;
+exports.listCapturePauses = listCapturePauses;
+exports.readFleetLastSeen = readFleetLastSeen;
+const storage_js_1 = require("./storage.js");
+/**
+ * Derive the harness from a source_agent wire value — the bundle split the
+ * console already uses ("claude-code/main" → "claude-code"): the part before
+ * the first '/' when a slash is present at index > 0, else the whole trimmed
+ * string, capped at 128. Non-string/blank → "unknown" (mirrors how
+ * recordDistillActivity stores the agent when absent).
+ */
+function harnessOfAgent(sourceAgent) {
+    if (typeof sourceAgent !== "string")
+        return "unknown";
+    const t = sourceAgent.trim();
+    if (t.length === 0)
+        return "unknown";
+    const slash = t.indexOf("/");
+    return (slash > 0 ? t.slice(0, slash) : t).slice(0, 128);
+}
+/**
+ * Normalize the raw /distill wire fields into the pause key. MUST match the
+ * recordDistillActivity normalization (machine '' when absent, agent
+ * 'unknown' when absent) and the console's bundle grouping
+ * ((machine||'')+'|'+harness) — see the module doc.
+ */
+function capturePauseKey(machine, sourceAgent) {
+    return {
+        machine: (0, storage_js_1.sanitizeSourceMachine)(machine) ?? "",
+        harness: harnessOfAgent(sourceAgent),
+    };
+}
+/** True when a pause row exists for the (machine, harness) bundle. */
+function isCapturePaused(db, machine, harness) {
+    return (db
+        .prepare("SELECT 1 FROM capture_pauses WHERE machine = ? AND harness = ?")
+        .get(machine, harness) !== undefined);
+}
+/**
+ * Pause (upsert the row, timestamp now) or resume (delete it). Returns the
+ * persisted paused_at when pausing, null when resuming. No pruning, ever —
+ * see migration v18's provenance comment.
+ */
+function setCapturePause(db, machine, harness, paused) {
+    if (!paused) {
+        db.prepare("DELETE FROM capture_pauses WHERE machine = ? AND harness = ?").run(machine, harness);
+        return null;
+    }
+    const pausedAt = new Date().toISOString();
+    db.prepare("INSERT OR REPLACE INTO capture_pauses (machine, harness, paused_at) VALUES (?, ?, ?)").run(machine, harness, pausedAt);
+    return pausedAt;
+}
+/** Every paused bundle (newest first) — the dashboard fleet.pauses block. */
+function listCapturePauses(db) {
+    return db
+        .prepare("SELECT machine, harness, paused_at FROM capture_pauses ORDER BY paused_at DESC")
+        .all();
+}
+/**
+ * The newest /distill activity per bundle: for each (machine, agent) take
+ * MAX(ts) with that latest row's outcome, derive the harness per agent, then
+ * merge same-bundle agents keeping the newest ts (one dot per bundle, not
+ * per profile). Reads only distill_activity, which the recorder prunes to
+ * 7 days — older-than-window bundles simply have no rows and no dot.
+ */
+function readFleetLastSeen(db) {
+    const rows = db
+        .prepare(`SELECT a.machine, a.agent, a.ts, a.outcome
+         FROM distill_activity a
+         JOIN (
+           SELECT machine, agent, MAX(ts) AS max_ts
+             FROM distill_activity
+            GROUP BY machine, agent
+         ) latest
+           ON a.machine = latest.machine AND a.agent = latest.agent AND a.ts = latest.max_ts`)
+        .all();
+    const byBundle = new Map();
+    for (const r of rows) {
+        const key = `${r.machine}|${harnessOfAgent(r.agent)}`;
+        const prev = byBundle.get(key);
+        if (!prev || r.ts > prev.last_seen) {
+            byBundle.set(key, {
+                machine: r.machine,
+                harness: harnessOfAgent(r.agent),
+                last_seen: r.ts,
+                last_outcome: r.outcome,
+            });
+        }
+    }
+    return [...byBundle.values()].sort((a, b) => (a.last_seen < b.last_seen ? 1 : -1));
+}

package/dist/capture.d.ts

@@ -14,6 +14,7 @@
  */
 import type { TranscriptBatch } from "./transcript-reader.js";
 import type { CursorStore } from "./capture-cursors.js";
+import type { RunDeadline } from "./run-deadline.js";
 /**
  * Max denoised chars per segment. Kept below the server's 80K distill cap
  * (distiller.ts MAX_TRANSCRIPT_CHARS) with ~20K headroom so NO capture path can
@@ -56,6 +57,9 @@ export interface DistillBody {
     source_agent_id?: string | null;
     /** Client-declared topic/domain of the capturing agent. Provenance only. */
     source_domain?: string | null;
+    /** Machine this capture ran on (#421 machine × harness): config
+     *  `machineName` ?? os.hostname(), stamped by the nightly. */
+    source_machine?: string | null;
     project: string;
     session_id: string;
     segment_id: string;
@@ -105,6 +109,21 @@ export interface CaptureOptions {
      * `source_domain` provenance. Null when undeclared.
      */
     sourceDomain?: string | null;
+    /**
+     * Machine stamp on every segment (#421 machine × harness): config
+     * `machineName` when set, else os.hostname() — resolved by the nightly
+     * caller. Null disables stamping.
+     */
+    sourceMachine?: string | null;
+    /**
+     * The run-wide pipeline deadline (#405), checked BETWEEN segment POSTs —
+     * a boundary the per-session cursor discipline already guarantees is safe
+     * (the cursor only advances past server-confirmed segments, so a deadline
+     * stop holds every unconfirmed segment for the next run; dup-over-loss).
+     * Full and consolidate-only nightlies pass it; capture-only/watchdog runs
+     * keep their 30-min unit backstop instead.
+     */
+    deadline?: RunDeadline;
 }
 export interface CaptureResult {
     memoriesIngested: number;
@@ -113,10 +132,12 @@ export interface CaptureResult {
     /**
      * Set when the loop stopped early on a terminal server response: "limit"
      * (token-budget 429, mcp-server.ts's `"token budget exceeded"` gate) or
-     * "auth" (401). A rate-limit 429 never sets this — it is transient (#327).
-     * The caller decides watermark handling.
+     * "auth" (401); "deadline" (#405) when the run-wide pipeline deadline fired
+     * between segments (transient — the watermark holds, everything unconfirmed
+     * retries next run). A rate-limit 429 never sets this — it is transient
+     * (#327). The caller decides watermark handling.
      */
-    stopped?: "limit" | "auth";
+    stopped?: "limit" | "auth" | "deadline";
     /**
      * Run-global rate-429 latch (#327 CR): true when at least one session
      * SURRENDERED to a rate-limit 429 (the transient kind — postWithRateRetry

package/dist/capture.js

@@ -204,7 +204,7 @@ async function postWithRateRetry(post, body) {
  * re-paying the Retry-After ladder (#327).
  */
 async function captureBatches(batches, opts) {
-    const { post, cursorStore, dryRun = false, segmentMaxChars = exports.SEGMENT_MAX_CHARS, sourceAgentId, sourceDomain } = opts;
+    const { post, cursorStore, dryRun = false, segmentMaxChars = exports.SEGMENT_MAX_CHARS, sourceAgentId, sourceDomain, sourceMachine, deadline } = opts;
     let memoriesIngested = 0;
     let sessionsSent = 0;
     let hadTransientFailure = false;
@@ -249,6 +249,15 @@ async function captureBatches(batches, opts) {
         let sessionPosted = false;
         for (let s = 0; s < segments.length; s++) {
             const seg = segments[s];
+            // #405: stop BETWEEN segments — a safe boundary by construction (the
+            // cursor below only advances past server-confirmed segments). The whole
+            // session loop breaks on `stopped` at the bottom; unconfirmed segments
+            // hold and retry next run.
+            if (!dryRun && deadline?.hit("capture")) {
+                console.warn(`[hicortex]     Run deadline reached — capture stops after the last confirmed segment`);
+                stopped = "deadline";
+                break;
+            }
             // A segment advances the cursor to its segEnd only when it is the LAST
             // segment ending at that boundary. Hard-split pieces (.p0,.p1,…) of one
             // entry share the same segEnd; confirming an earlier piece must NOT move
@@ -267,6 +276,7 @@ async function captureBatches(batches, opts) {
                 source_agent: batch.sourceAgent ?? `claude-code/${batch.projectName}`,
                 source_agent_id: sourceAgentId ?? null,
                 source_domain: sourceDomain ?? null,
+                source_machine: sourceMachine ?? null,
                 project: batch.projectName,
                 session_id: batch.sessionId,
                 segment_id: `${genPrefix}${seg.segStart}-${seg.segEnd}${seg.idSuffix}`,

package/dist/classify-domains.d.ts

@@ -50,6 +50,12 @@ export interface ClassifyDomainsOptions {
     llm?: LlmClient;
     /** Config override (tests). Defaults to reading stateDir/config.json. */
     config?: Record<string, unknown> | null;
+    /**
+     * Weak-primary floor (#408): release-managed default (calibration.ts via
+     * nofit's DEFAULT_WEAK_PRIMARY_FLOOR); this field is the eval/test seam —
+     * the config key is gone from the surface. Invalid → default.
+     */
+    weakPrimaryFloor?: number;
     /**
      * Embedder override (tests). Used only for domain-description prototype
      * seeds; defaults to the local ONNX embedder, loaded lazily on first need

package/dist/classify-domains.js

@@ -118,7 +118,13 @@ async function runClassifyDomains(options = {}) {
             '{ "name": "Boating", "description": "..." }] } and re-run. ' +
             "No fallback bucket is needed — no-fit memories are handled automatically.");
     }
-    const weakPrimaryFloor = (0, nofit_js_1.resolveWeakPrimaryFloor)(config);
+    // #408: the floor is a release-managed calibration constant; the Options
+    // field is the eval/test seam (invalid values keep the default, the stage
+    // knob-validation style — silent fallback, no warn).
+    const floorRaw = Number(options.weakPrimaryFloor);
+    const weakPrimaryFloor = Number.isFinite(floorRaw) && floorRaw > 0 && floorRaw < 1
+        ? floorRaw
+        : nofit_js_1.DEFAULT_WEAK_PRIMARY_FLOOR;
     // Resolve the LLM (one model serves all phases — #231).
     let llm;
     if (options.llm) {

package/dist/cli.js

@@ -189,6 +189,36 @@ switch (command) {
         });
         break;
     }
+    case "rescore-importance": {
+        // #425 — one-shot LLM backfill: re-judge the corpus under the
+        // re-anchored importance rubric. classify-domains shape (resumable
+        // cursor, --batch, --reset) + dedup discipline (dry-run default,
+        // --apply, DB backup before any write).
+        const args = process.argv.slice(3);
+        const intFlag = (name) => {
+            const idx = args.indexOf(name);
+            if (idx === -1)
+                return undefined;
+            const val = parseInt(args[idx + 1], 10);
+            if (isNaN(val)) {
+                console.error(`[hicortex] rescore-importance: ${name} requires an integer value`);
+                process.exit(1);
+            }
+            return val;
+        };
+        const rescoreOptions = {
+            apply: args.includes("--apply"),
+            reset: args.includes("--reset"),
+            batchSize: intFlag("--batch"),
+        };
+        import("./rescore-importance.js").then(({ runRescoreImportance }) => {
+            runRescoreImportance(rescoreOptions).catch((err) => {
+                console.error(err instanceof Error ? err.message : `[hicortex] rescore-importance failed: ${err}`);
+                process.exit(1);
+            });
+        });
+        break;
+    }
     case "classify-types": {
         const args = process.argv.slice(3);
         const intFlag = (name) => {
@@ -421,6 +451,8 @@ Commands:
   backup          Snapshot the DB + identity + state to a tar.gz (online, WAL-safe)
   classify-domains  Backfill content-based domain tags over the corpus (server mode, needs config.domains)
   classify-types    Backfill episode→fact/decision type tags over the corpus (server mode)
+  rescore-importance  Re-judge all memories' importance under the current rubric
+                      (server mode; dry run by default — --apply executes; resumable)
   learnings-identity  Fetch identity + lessons and print Markdown to stdout (CC SessionStart hook)
                       (alias: lessons-context — the pre-#264 name, kept for backcompat)
   recall-hook    Pushed recall index for the current prompt (CC UserPromptSubmit/SessionStart hook)
@@ -445,9 +477,8 @@ Options:
   dedup --apply             Execute the merge (default: dry run, report only)
                             Losers are absorbed — hidden from recall, kept as
                             evidence (fetchable by id; dedup_log audit) — not deleted
-  dedup --threshold <t>     Override the threshold for one run (default: config
-                            dedupAutoMergeThreshold, legacy dedupMergeThreshold
-                            still honored; else 0.92)
+  dedup --threshold <t>     Override the threshold for one run (default: the
+                            release-managed calibration ceiling, 0.92)
   dedup --db <path>         DB path override (defaults to the configured DB)
   history --rollback <row>  Roll back history row <row>: restores prior content/status, un-absorbs triggers
   history --db <path>       DB path override (defaults to the configured DB)
@@ -459,6 +490,10 @@ Options:
   classify-types --all      Reclassify every memory (default: only episodes)
   classify-types --batch <n>  Memories per batch (default: 200)
   classify-types --reset    Restart from the beginning (ignore saved cursor)
+  rescore-importance --apply  Execute the importance backfill (default: dry run, report only)
+                             Takes a DB backup first; resumable via a state.json cursor
+  rescore-importance --batch <n>  Rows per invocation (default: 500; LLM calls are 10 rows each)
+  rescore-importance --reset    Restart from the beginning (ignore saved cursor)
   identity show [name]      Print all identity sections, or just <name> (raw, pipeable)
   identity edit <name>      Edit a section in $EDITOR; PUT only if changed
   identity … --agent <id>   Target a per-agent scope instead of the global set

package/dist/config-read.d.ts

@@ -27,7 +27,7 @@ export declare function readStrictBoolean(config: Record<string, unknown>, key:
 /**
  * Read a non-negative finite number (allows 0, unlike readPositiveConfig).
  * Returns `def` when absent OR invalid. Used for keys where 0 is a valid "off"
- * value (e.g. ollamaFlushEvery).
+ * value (e.g. memorySoftCap's eviction opt-out, timerJitterSeconds).
  */
 export declare function readNonNegativeConfig(config: Record<string, unknown>, key: string, def: number): number;
 /**

package/dist/config-read.js

@@ -52,7 +52,7 @@ function readStrictBoolean(config, key) {
 /**
  * Read a non-negative finite number (allows 0, unlike readPositiveConfig).
  * Returns `def` when absent OR invalid. Used for keys where 0 is a valid "off"
- * value (e.g. ollamaFlushEvery).
+ * value (e.g. memorySoftCap's eviction opt-out, timerJitterSeconds).
  */
 function readNonNegativeConfig(config, key, def) {
     const v = config[key];
@@ -152,6 +152,63 @@ const IGNORED_CONFIG_KEYS = [
     "classifyModel", "classifyBaseUrl", "classifyApiKey", "classifyProvider",
     "distillFallback",
 ];
+/**
+ * Config keys REMOVED by the #405 budget simplification, each mapped to its
+ * replacement (or "removed" when there is none). The old single 0.16.8-style
+ * message does not fit — every key here needs to name where its job went.
+ * Warned at the config boundary alongside the 0.16.8 keys above.
+ */
+const REMOVED_CONFIG_KEYS = {
+    reconsolidationMaxMinutes: "removed — the ONE run deadline (nightlyTimeBudgetMinutes) bounds the stage",
+    reconsolidationMaxCalls: "removed — the ONE call budget (nightlyLlmCallBudget) caps all stages",
+    supersessionMaxCalls: "removed — the ONE call budget (nightlyLlmCallBudget) caps all stages",
+    dedupNightlyMaxMerges: "removed — the run deadline (nightlyTimeBudgetMinutes) bounds nightly merges",
+    classifyMaxTokens: "removed — maxTokens is the single output ceiling for every call",
+    llmBreakerThreshold: "removed — the breaker threshold is a constant (3)",
+    llmBreakerCooldownMs: "removed — the breaker cooldown is a constant (10 min)",
+    llmSingleFlightWaitMs: "removed — the wait derives from llmTimeoutMs",
+    preflightTimeoutMs: "removed — a constant (20 s per attempt)",
+    preflightAttempts: "removed — a constant (3 attempts)",
+    preflightRetryGapMs: "removed — a constant (60 s gap)",
+    moduleIndexTokenBudget: "removed — was documentation-only",
+};
+/**
+ * Config keys that became RELEASE-MANAGED CALIBRATION constants (#408): the
+ * ~35 tuning keys of the 0.15–0.20 era. Their values are ignored — the
+ * constants ship with each release (src/calibration.ts) and change only in
+ * releases with eval evidence linked in the changelog. Like the #405 list,
+ * this is deliberately WIDER than the `HicortexConfig` type ever was: several
+ * keys (supersessionPenalty, the rrf/bm25 knob set, recallTitleChars, ...)
+ * were README-documented but never typed. Don't trim the list to match the
+ * interface — the warn exists precisely for the untyped surface.
+ */
+const RELEASE_MANAGED_CONFIG_KEYS = [
+    "decayHalfLifeDays", "searchLimit", "recentLimit", "recentWindowDays",
+    "coldExposureSlots", "recallMaxItems", "recallMinSimilarity",
+    "recallReshowTurns", "recallMinPromptChars", "recallTitleChars",
+    "sessionIntentWeight", "noveltyFloorSlots", "scoreSimilarityWeight",
+    "scoreStrengthWeight", "scoreConnectionsWeight", "scoreRecencyWeight",
+    "freshnessBoostDays", "freshnessBoostWeight", "supersededDemotion",
+    "projectAffinityWeight", "domainAffinityWeight", "rrfK",
+    "rrfCompositeWeight", "rrfFtsWeight", "rrfVectorWeight",
+    "bm25WeightBody", "bm25WeightProject", "bm25WeightDomain",
+    "dedupAutoMergeThreshold", "dedupMergeThreshold",
+    "supersessionMinSimilarity", "supersessionPenalty",
+    "correctionMinSimilarity", "correctionRewriteMinConfidence",
+    "weakPrimaryFloor",
+];
+/**
+ * Config keys MOVED to the diagnostic env tier (#408): the ollama-operational
+ * family left config.json and now resolves from environment variables
+ * (calibration.ts resolvers; env > constant, invalid env warns + falls back).
+ * Each entry maps the old config key to its env replacement so the warning
+ * can name the exact variable.
+ */
+const ENV_MOVED_CONFIG_KEYS = {
+    numCtx: "HICORTEX_NUM_CTX",
+    ollamaFlushEvery: "HICORTEX_OLLAMA_FLUSH_EVERY",
+    ollamaFlushWaitMs: "HICORTEX_OLLAMA_FLUSH_WAIT_MS",
+};
 /**
  * Warn if the saved config carries keys that 0.16.8+ ignores. Call at every
  * config read (daemon boot + nightly). The warning clears once the keys are
@@ -163,12 +220,42 @@ function warnIgnoredConfigKeys(savedConfig) {
         return;
     const present = IGNORED_CONFIG_KEYS.filter((k) => savedConfig[k] !== undefined);
     const hasModelsBlock = savedConfig.models !== undefined;
-    if (present.length === 0 && !hasModelsBlock)
-        return;
-    const detail = [...present, ...(hasModelsBlock ? ["models"] : [])].join(", ");
-    console.warn(`[hicortex] config has keys IGNORED since 0.16.8 (${detail}). They have no effect now. ` +
-        `Per-stage model keys / the \`models\` block: one model serves all phases — set ` +
-        `llmModel/llmBaseUrl/llmProvider (+ llmApiKey) to your intended model. ` +
-        `distillFallback: removed (strict mode is default — a failed distill retries next run). ` +
-        `Remove these keys to clear this warning. See the 0.16.8 changelog.`);
+    if (present.length > 0 || hasModelsBlock) {
+        const detail = [...present, ...(hasModelsBlock ? ["models"] : [])].join(", ");
+        console.warn(`[hicortex] config has keys IGNORED since 0.16.8 (${detail}). They have no effect now. ` +
+            `Per-stage model keys / the \`models\` block: one model serves all phases — set ` +
+            `llmModel/llmBaseUrl/llmProvider (+ llmApiKey) to your intended model. ` +
+            `distillFallback: removed (strict mode is default — a failed distill retries next run). ` +
+            `Remove these keys to clear this warning. See the 0.16.8 changelog.`);
+    }
+    // #405: keys removed by the budget simplification — one line naming each
+    // present key and its replacement (or "removed").
+    const removed = Object.entries(REMOVED_CONFIG_KEYS)
+        .filter(([k]) => savedConfig[k] !== undefined);
+    if (removed.length > 0) {
+        const detail = removed.map(([k, v]) => `${k} (${v})`).join(", ");
+        console.warn(`[hicortex] config has keys REMOVED by the budget simplification (#405): ${detail}. ` +
+            `They have no effect. Remove them to clear this warning.`);
+    }
+    // #408: tuning keys that became release-managed calibration constants —
+    // one line naming every present key (their VALUES are ignored; the
+    // constants ship with each release and change only with published eval
+    // evidence linked in the changelog).
+    const releaseManaged = RELEASE_MANAGED_CONFIG_KEYS
+        .filter((k) => savedConfig[k] !== undefined);
+    if (releaseManaged.length > 0) {
+        console.warn(`[hicortex] config has keys that are now RELEASE-MANAGED CALIBRATION ` +
+            `(${releaseManaged.join(", ")}): their values are ignored — calibration ` +
+            `ships with each release and changes only with published eval evidence. ` +
+            `Remove them to clear this warning.`);
+    }
+    // #408: the diagnostic-tier keys that moved to environment variables —
+    // one line naming each old key AND its env replacement.
+    const envMoved = Object.entries(ENV_MOVED_CONFIG_KEYS)
+        .filter(([k]) => savedConfig[k] !== undefined);
+    if (envMoved.length > 0) {
+        const detail = envMoved.map(([k, env]) => `${k} → set ${env} instead`).join(", ");
+        console.warn(`[hicortex] config has keys that MOVED to environment variables (#408): ${detail}. ` +
+            `The config keys are ignored. Remove them to clear this warning.`);
+    }
 }