@gamaze/hicortex 0.19.2 → 0.19.4

package/dist/nightly.js

@@ -45,6 +45,8 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeSince = computeSince;
+exports.parseRetryAfterMs = parseRetryAfterMs;
 exports.runNightly = runNightly;
 const paths_js_1 = require("./paths.js");
 const node_fs_1 = require("node:fs");
@@ -77,6 +79,17 @@ const telemetry_js_1 = require("./telemetry.js");
 const init_js_1 = require("./init.js");
 const backup_js_1 = require("./backup.js");
 const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
+/**
+ * Consolidate-only backup gate (#327 CR blocker). Hosted tenants run ONLY
+ * `nightly --consolidate-only` several times a day (hicortex-consolidate@.timer)
+ * and the provisioner has no backup job of its own, so those runs DO run the
+ * backup stage — but at most once per day, keyed on the newest existing
+ * artifact's age (the artifact IS the marker; no extra state file). 20h is
+ * slightly under a day so a timer firing at a drifting clock hour still gets
+ * exactly one backup per day; with `backupRetention` (default 7) the artifact
+ * count stays bounded.
+ */
+const CONSOLIDATE_ONLY_BACKUP_MIN_AGE_MS = 20 * 60 * 60 * 1000;
 function readNightlyConfig(stateDir) {
     const configPath = (0, node_path_1.join)(stateDir, "config.json");
     let loaded;
@@ -125,14 +138,30 @@ function readLastRun(stateDir = HICORTEX_HOME) {
  * skip (then, via writeLastRun, permanently lose) the 8-to-N-day-old sessions
  * (#189 review, fix 3). Per-session cursors keep the wide re-scan cheap: an
  * already-captured session yields an empty delta.
+ *
+ * Clock-jump clamp (#327): a FUTURE-dated lastNightly (client clock error —
+ * NTP not yet synced at write time) would, once the clock corrects, sit ahead
+ * of every session mtime and permanently skip quiet sessions (their mtimes
+ * never re-cross a future watermark). Clamped to `now` with a warn; the warn
+ * fires once per affected run (this function runs once per nightly).
+ * `now` is injectable for tests.
  */
-function computeSince(stateDir, recaptureWindowDays) {
+function computeSince(stateDir, recaptureWindowDays, now = new Date()) {
     const lastRun = readLastRun(stateDir);
+    let effective = lastRun;
+    if (lastRun.getTime() > now.getTime()) {
+        console.warn(`[hicortex] state lastNightly (${lastRun.toISOString()}) is ahead of the clock ` +
+            `(${now.toISOString()}) — clamping discovery to now. A future watermark permanently ` +
+            `skips quiet sessions once the clock corrects; check the machine's clock/NTP. ` +
+            `Run \`hicortex nightly --recapture-window <days>\` to recover sessions missed ` +
+            `while the clock was wrong.`);
+        effective = now;
+    }
     if (recaptureWindowDays && recaptureWindowDays > 0) {
-        const windowStart = new Date(Date.now() - recaptureWindowDays * 24 * 60 * 60 * 1000);
-        return windowStart < lastRun ? windowStart : lastRun;
+        const windowStart = new Date(now.getTime() - recaptureWindowDays * 24 * 60 * 60 * 1000);
+        return windowStart < effective ? windowStart : effective;
     }
-    return lastRun;
+    return effective;
 }
 /** POST /distill transport for server mode — localhost. Sends authToken so
  *  self-capture works regardless of the localhost-bypass marker (#271 root-cause fix). */
@@ -185,7 +214,32 @@ async function normalizePostResult(resp) {
         return { status: 200, skipped: Boolean(data.skipped) };
     }
     const data = (await resp.json().catch(() => ({})));
-    return { status: resp.status, error: data.error ?? "unknown error" };
+    const retryAfterMs = parseRetryAfterMs(resp);
+    return {
+        status: resp.status,
+        error: data.error ?? "unknown error",
+        // #327: surface Retry-After so a rate-limit 429 backs off as the server
+        // asked (absent on the tenant's terminal budget-429, which ignores it).
+        ...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
+    };
+}
+/**
+ * Parse a `Retry-After` header into ms (#327). Handles both RFC forms —
+ * delay-seconds (`"30"`) and HTTP-date — and returns undefined for anything
+ * unparseable (the caller then falls back to its own backoff schedule).
+ * Exported for unit tests (pure on the header value).
+ */
+function parseRetryAfterMs(resp) {
+    const v = resp.headers.get("retry-after");
+    if (!v)
+        return undefined;
+    const secs = Number(v);
+    if (Number.isFinite(secs) && secs >= 0)
+        return secs * 1000;
+    const at = new Date(v).getTime();
+    if (Number.isFinite(at))
+        return Math.max(0, at - Date.now());
+    return undefined;
 }
 /** Strict {prompt, completion, total} parser for /distill's usage field (#287);
  *  undefined on anything malformed — the caller then treats it as unmetered. */
@@ -296,7 +350,15 @@ async function runNightly(options = {}) {
         const cooldownH = (0, config_read_js_1.readNonNegativeConfig)(savedConfig ?? {}, "captureCooldownHours", 6);
         const last = (0, state_js_1.loadState)(stateDir).lastNightly;
         if (last) {
-            const ageH = (Date.now() - new Date(last).getTime()) / 3_600_000;
+            // #327 clamp: a FUTURE-dated stamp (clock error at write time) reads as
+            // a NEGATIVE age raw, and `negative < cooldownH` is true even at the
+            // cooldown-0 opt-in ("capture every poll") — the watchdog would stay
+            // silent until real time passed the future stamp, exactly when catch-up
+            // ticks are most needed. Clamped, the worst honest reading is "captured
+            // just now", which the normal cooldown handles (and discovery applies
+            // the same clamp in computeSince).
+            const lastMs = Math.min(new Date(last).getTime(), Date.now());
+            const ageH = (Date.now() - lastMs) / 3_600_000;
             if (ageH < cooldownH) {
                 console.log(`[hicortex] watchdog: last capture ${ageH.toFixed(1)}h ago (< ${cooldownH}h cooldown) — skipping`);
                 return;
@@ -668,21 +730,52 @@ async function runNightly(options = {}) {
         // Backup stage (#6, Phase 0B) — a transactionally-consistent snapshot of
         // the irreplaceable data (DB + identity + state), packaged as one tar.gz
         // the operator ships offsite via the optional `backupCommand` hook. Runs
-        // ONLY on a full nightly (capture-only is frequent + stateless; dry-run
-        // writes nothing). Backup failure must NOT fail the nightly — capture +
-        // consolidation have already succeeded; the snapshot is on disk and the
-        // failure surfaces as `backupOk:false` in the dashboard snapshot + telemetry
-        // for alerting (the operator's hook owns active alerting; no in-product
+        // on every full nightly and on consolidate-only runs (capture-only is
+        // frequent + stateless; dry-run writes nothing). Consolidate-only MUST
+        // back up (#327 CR blocker): hosted tenants run ONLY --consolidate-only
+        // several times a day (hicortex-consolidate@.timer) and the provisioner
+        // has no backup job of its own — skipping the stage left them with NO
+        // recurring backup. Their cadence is bounded by the artifact-age gate
+        // below (~1/day) instead, so `backupRetention` keeps the dir bounded.
+        // Backup failure must NOT fail the nightly — capture + consolidation
+        // have already succeeded; the snapshot is on disk and the failure
+        // surfaces as `backupOk:false` in the dashboard snapshot + telemetry for
+        // alerting (the operator's hook owns active alerting; no in-product
         // channel yet — Phase 3).
         let backupPath;
         let backupBytes;
         let backupOk;
-        if (!dryRun && !captureOnly) {
+        // Artifact-age gate for consolidate-only runs: skip only while the newest
+        // existing artifact is younger than ~20h. 20h (not 24) so a timer firing
+        // at a drifting clock hour still gets exactly one backup per day, and
+        // robust to an hour of clock skew either way. Keyed on the artifact, not
+        // a state timestamp — no new state file to drift out of sync with the
+        // disk it describes.
+        const effectiveBackupDir = typeof savedConfig?.backupDir === "string" && savedConfig.backupDir.trim()
+            ? savedConfig.backupDir
+            : (0, node_path_1.join)(stateDir, "backups");
+        let skipBackup = false;
+        let newestArtifactMs;
+        if (consolidateOnly) {
+            newestArtifactMs = (0, backup_js_1.newestBackupArtifactMs)(effectiveBackupDir);
+            skipBackup =
+                newestArtifactMs !== undefined &&
+                    Date.now() - newestArtifactMs < CONSOLIDATE_ONLY_BACKUP_MIN_AGE_MS;
+        }
+        if (!dryRun && !captureOnly && !skipBackup) {
             try {
-                const backupDir = typeof savedConfig?.backupDir === "string" && savedConfig.backupDir.trim()
-                    ? savedConfig.backupDir
-                    : undefined;
-                const bRes = await (0, backup_js_1.createBackup)({ db, home: stateDir, outDir: backupDir });
+                const bRes = await (0, backup_js_1.createBackup)({
+                    db,
+                    home: stateDir,
+                    // undefined falls back to <home>/backups inside createBackup — the
+                    // same resolution effectiveBackupDir above uses for the age gate.
+                    outDir: typeof savedConfig?.backupDir === "string" && savedConfig.backupDir.trim()
+                        ? savedConfig.backupDir
+                        : undefined,
+                    // Same reader + default as the CLI (#327): keep the N newest
+                    // artifacts, 0 = keep all.
+                    retention: (0, config_read_js_1.readNonNegativeConfig)(savedConfig ?? {}, "backupRetention", backup_js_1.DEFAULT_BACKUP_RETENTION),
+                });
                 backupPath = bRes.path;
                 backupBytes = bRes.bytes;
                 backupOk = true;
@@ -712,6 +805,15 @@ async function runNightly(options = {}) {
                 console.error(`[hicortex] Backup FAILED: ${err instanceof Error ? err.message : String(err)}`);
             }
         }
+        else if (consolidateOnly && !dryRun) {
+            // #327: explicit log line so a hosted consolidation timer's log doesn't
+            // read as a silently-missing backup stage — and names WHY (age gate), so
+            // an operator reading "skipped" can tell a healthy cadence gate from a
+            // dead one.
+            const ageH = newestArtifactMs !== undefined ? (Date.now() - newestArtifactMs) / 3_600_000 : -1;
+            console.log(`[hicortex] Backup stage skipped — consolidate-only run: newest backup is ` +
+                `${ageH.toFixed(1)}h old (< ${Math.round(CONSOLIDATE_ONLY_BACKUP_MIN_AGE_MS / 3_600_000)}h gate).`);
+        }
         // Dashboard snapshot (#224) — full nightly only. The snapshot reflects
         // corpus state regardless of whether consolidation/LLM ran, so it is
         // ALWAYS written here (the use case is history; an LLM-less install still
@@ -771,9 +873,11 @@ async function runNightly(options = {}) {
                     budgetExhausted,
                     budgetDeferredByStage,
                     // #6 backup stage — hoisted from the block above. Present whenever
-                    // the backup stage ran (full nightly); undefined on capture-only /
-                    // dry-run. backupOk flips to false on snapshot OR hook failure so the
-                    // dashboard digest can flag a night the offsite copy didn't complete.
+                    // the backup stage ran (full nightly, and consolidate-only runs past
+                    // the artifact-age gate); undefined on capture-only / dry-run / a
+                    // gated-skip consolidate-only run. backupOk flips to false on
+                    // snapshot OR hook failure so the dashboard digest can flag a night
+                    // the offsite copy didn't complete.
                     backupPath,
                     backupBytes,
                     backupOk,
@@ -826,7 +930,8 @@ async function runNightly(options = {}) {
                 // #6 backup stage outcome — forwarded only when the backup stage ran
                 // (full nightly). `ok` is false on snapshot OR hook failure; the fleet
                 // aggregate surfaces a sustained drop in backup_ok as a data-loss risk.
-                // Absent on capture-only / dry-run / client runs (no backup ran).
+                // Absent on capture-only / consolidate-only / dry-run / client runs
+                // (no backup ran).
                 ...(backupOk !== undefined
                     ? { backup: { ok: backupOk === true, bytes: backupBytes ?? 0 } }
                     : {}),

package/dist/prompts.d.ts

@@ -18,6 +18,21 @@ export declare function importanceScoring(memoriesBlock: string): string;
 export declare function reflection(memoriesBlock: string, recentLessons?: string): string;
 /**
  * Distillation prompt. Extracts knowledge from a session transcript.
+ *
+ * LAYOUT (#329 item 6): the ~6.3KB of static instructions come FIRST and the
+ * transcript LAST, so every distill call shares a byte-identical instruction
+ * prefix and provider-side prompt prefix caching can actually hit (per-session
+ * calls with the same project/date share everything up to the transcript; a
+ * multi-segment session — the common capture shape — re-uses the cached prefix
+ * for every segment after the first). The static block is the transcript-first
+ * block MOVED plus ONE deliberate addition in the same change: the #329 item-5
+ * [D]-override sentence in the NEVER-RECORD section ("a version bump, merge,
+ * or count is NEVER [D]"). Anyone diffing distill behavior across this change
+ * must baseline against BOTH the reorder and that wording addition. NOTE the
+ * prefix is only
+ * fully shared while project/date agree: "# Session Memory: ${date} -
+ * ${projectName}" and the (${date}) format examples interpolate inside the
+ * static block by design (the model needs the real date in its output format).
  */
 export declare function distillation(projectName: string, date: string, transcript: string): string;
 /**

package/dist/prompts.js

@@ -106,14 +106,26 @@ Respond with a JSON array. Empty array [] is a valid response.`;
 }
 /**
  * Distillation prompt. Extracts knowledge from a session transcript.
+ *
+ * LAYOUT (#329 item 6): the ~6.3KB of static instructions come FIRST and the
+ * transcript LAST, so every distill call shares a byte-identical instruction
+ * prefix and provider-side prompt prefix caching can actually hit (per-session
+ * calls with the same project/date share everything up to the transcript; a
+ * multi-segment session — the common capture shape — re-uses the cached prefix
+ * for every segment after the first). The static block is the transcript-first
+ * block MOVED plus ONE deliberate addition in the same change: the #329 item-5
+ * [D]-override sentence in the NEVER-RECORD section ("a version bump, merge,
+ * or count is NEVER [D]"). Anyone diffing distill behavior across this change
+ * must baseline against BOTH the reorder and that wording addition. NOTE the
+ * prefix is only
+ * fully shared while project/date agree: "# Session Memory: ${date} -
+ * ${projectName}" and the (${date}) format examples interpolate inside the
+ * static block by design (the model needs the real date in its output format).
  */
 function distillation(projectName, date, transcript) {
     return `You are a memory extraction agent. Analyze this AI session transcript and extract
 knowledge worth remembering long-term.
 
-SESSION TRANSCRIPT (project: ${projectName}, date: ${date}):
-${transcript}
-
 EXTRACT into this markdown format:
 
 # Session Memory: ${date} - ${projectName}
@@ -195,6 +207,9 @@ do not score it lower and write it anyway: OMIT it. A closed category of never-r
 The durable part of the same event may still qualify — the CHOICE a change
 embodies ("standardize on model X", user-confirmed) is [D], a configuration
 that holds going forward is [K]; the version bump, merge, or count itself never is.
+Override for the [D] "actually carried out" test: even if carried out by the
+user, a version bump, merge, or count is NEVER [D] — only the durable
+user-confirmed standardization it embodies qualifies.
 If EVERY item in the transcript is never-record ephemera, output ONLY:
 "NO_EXTRACT" — zero memories is the correct result for a pure-status segment.
 
@@ -216,6 +231,9 @@ RULES:
   "[Strong Negative] User rejected per-agent billing"). The subject always comes first.
 - Omit any section that has zero items (don't include empty sections)
 - If nothing worth extracting, output ONLY: "NO_EXTRACT"
+
+SESSION TRANSCRIPT (project: ${projectName}, date: ${date}):
+${transcript}
 `;
 }
 /**

package/dist/recall-index.d.ts

@@ -18,9 +18,37 @@
  * per-session TURN-based dedup (SessionRecallRegistry), short-prompt skip,
  * and a hard item cap. On a prompt with no relevant memories the block is
  * null and the hook prints nothing.
+ *
+ * Novelty floor (#324): the session-intent blend (#192 session-intent keying)
+ * can dilute a topic-switching prompt below the relevance floor — the live
+ * failure was a technically-primed session asking about "my Sargo" and getting
+ * ZERO relevant memories while a fresh session with the identical prompt got
+ * the perfect top hit. So a second, PURE-prompt search (no centroid blend,
+ * SAME candidate window as the blended search) runs alongside the blended
+ * one, and its top hit(s) that pass the floor are GUARANTEED slots in the
+ * index (dedup by id against the blended picks, capped by
+ * `noveltyFloorSlots`; rendered first). When the pure top hits are already
+ * among the blended picks — the common continuing-intent case — the output is
+ * unchanged. Turn suppression still wins: a recently shown novelty pick is
+ * suppressed like any other (the guarantee is about candidate inclusion, not
+ * forcing re-shows).
+ *
+ * #329 item 3 — the pure search is SKIPPED when it would be byte-identical
+ * to the blended one: turn 1 (no centroid yet — nothing to blend) or
+ * sessionIntentWeight 0 (blend disabled). The blended result IS the pure
+ * result there, so the floor is trivially satisfied by the blended picks and
+ * the second search (embeds aside, its whole DB + FTS half) is pure waste.
+ *
+ * #329 item 4 — novelty backfill: when the blended picks are empty/short,
+ * unclaimed maxItems slots are filled from the remaining filtered
+ * pure-prompt tail (gate + suppression already applied). Without it the
+ * topic-switch turn — the one the floor exists for — got the MOST truncated
+ * menu: novelty slots + a diluted remainder, while further pure candidates
+ * that had already passed every gate sat unused.
  */
 import type Database from "better-sqlite3";
 import type { MemorySearchResult } from "./types.js";
+import * as storage from "./storage.js";
 import { SessionRecallRegistry } from "./recall-registry.js";
 export interface RecallIndexOptions {
     /** Minimum measured cosine for vector-only candidates (config
@@ -50,11 +78,40 @@ export interface RecallIndexOptions {
      *  identical (0.6pts apart, N=40, full CI overlap); 100 saves ~13% tokens
      *  per block. */
     titleChars?: number;
+    /** Slots of `maxItems` guaranteed to the pure-prompt (unblended) search's
+     *  top passing hit(s) — the #324 novelty floor. Config `noveltyFloorSlots`,
+     *  default 2 (mirrors coldExposureSlots sizing: small, a floor not a
+     *  takeover). 0 disables the pure-prompt search entirely (the kill-switch).
+     *  Clamped to [0, maxItems]. */
+    noveltyFloorSlots?: number;
 }
+/** Default #324 novelty-floor slots (config `noveltyFloorSlots`). 2 mirrors
+ *  coldExposureSlots sizing — enough to guarantee the pure-prompt top hit
+ *  plus a runner-up, never a takeover of the index. The floor only SPENDS
+ *  slots when a pure-prompt hit differs from the blended picks (topic
+ *  switch); continuing-intent sessions pay nothing. Exported for the boot
+ *  log's knob line (mcp-server resolves config-vs-default here, once). */
+export declare const DEFAULT_NOVELTY_FLOOR_SLOTS = 2;
+/** Resolve the EFFECTIVE novelty floor (raw ?? default, clamped to
+ *  [0, maxItems]) — one definition shared by the handler and the boot knob
+ *  line so the logged value is what handleRecallIndex actually uses.
+ *  maxItems may be the handler's already-resolved number OR raw config
+ *  (boot-log site) — raw is resolved with the handler's exact constants. */
+export declare function resolveNoveltyFloorSlots(rawSlots: unknown, rawMaxItems: unknown): number;
 export interface RecallIndexResult {
     status: number;
     body: Record<string, unknown>;
 }
+/**
+ * Hard cap on `session_id` length (#328 item 2a). The id is retained as a Map
+ * key by SessionRecallRegistry for the process lifetime (maxSessions=500 LRU
+ * + a per-session shown-set + intent centroid), so an unbounded id is an OOM
+ * vector: ~4.9MB ids × 500 sessions ≈ 2.4GB of retained keys from an
+ * authenticated-but-hostile tenant. Real session ids (CC UUIDs, plugin
+ * session keys) are ≤64 chars — 128 is generous headroom. Longer → 400 with
+ * a clear error; the client treats it like any bad request.
+ */
+export declare const MAX_SESSION_ID_CHARS = 128;
 /** First content line, de-markdowned and truncated — the index line title. */
 export declare function memoryTitle(content: string, maxLen?: number): string;
 /**
@@ -97,15 +154,59 @@ export interface RecallFilters {
      *  affinity in computeScore via max overlapping memory_tags.weight. */
     mission_domains?: string[];
 }
+/** The search-closure contract handleRecallIndex consumes (see
+ *  RecallIndexDeps.retrieveFn). Named so the production factory
+ *  (createRecallRetrieveFn) and test doubles share one type. */
+export type RecallRetrieveFn = (query: string, limit: number, filters: RecallFilters | undefined, sessionId: string, purePrompt?: boolean) => Promise<MemorySearchResult[]>;
 export interface RecallIndexDeps {
     db: Database.Database;
     registry: SessionRecallRegistry;
-    /** Search closure. `sessionId` is forwarded so the closure (in mcp-server)
-     *  can resolve/update the session-intent centroid and pass a blended query
-     *  vector into retrieve() — see #192 session-intent keying (0.15.3). */
-    retrieveFn: (query: string, limit: number, filters: RecallFilters | undefined, sessionId: string) => Promise<MemorySearchResult[]>;
+    /** Search closure. `sessionId` is forwarded so the closure resolves/updates
+     *  the session-intent centroid and passes a blended query vector into
+     *  retrieve() — see #192 session-intent keying (0.15.3).
+     *
+     *  `purePrompt` (#324 novelty floor): request the PURE-prompt search — the
+     *  closure must search with the prompt embedding UNBLENDED (no session
+     *  centroid) and must NOT fold the prompt into the centroid a second time
+     *  (the blended call owns this turn's EMA update). Older closures that
+     *  ignore the flag degrade to blended-only recall — no novelty floor, but
+     *  no breakage. */
+    retrieveFn: RecallRetrieveFn;
     options?: RecallIndexOptions;
 }
+/**
+ * The PRODUCTION /recall-index retrieveFn (what mcp-server wires into
+ * handleRecallIndex), extracted from the route handler so the #324 path is
+ * testable without HTTP — same precedent as blendQueryVector/recallQueryVector
+ * ("extracted from the /recall-index closure so the exact decision is
+ * unit-testable").
+ *
+ * Per call:
+ *   - embed the prompt ONCE per request — a single-entry promise memo keyed
+ *     on the query text. The blended and pure-prompt searches of one request
+ *     carry the same prompt, so they share one embed; the factory is built
+ *     per request, so the memo never outlives it.
+ *   - resolve the search vector via retrieval.recallQueryVector (blend + EMA
+ *     fold, or the pure prompt with NO centroid state for #324);
+ *   - retrieve() with noStrengthen (exposure is recorded by
+ *     handleRecallIndex via touchMemoriesShown, never here).
+ *
+ * #329 CR finding 1b: the FTS candidate list is ALSO computed once per
+ * request (ftsOnce, keyed on query + candidate window) and threaded into both
+ * retrieve() calls via the ftsCandidates provider — the blended and pure
+ * searches of one request carry identical query text and window, so their FTS
+ * halves were byte-identical SQL executed twice. `ftsFn` is the DI seam for
+ * tests (production: storage.searchFts); a throwing FTS computation memoizes
+ * to an empty shared list — the same vector-only degradation retrieve()'s
+ * catch always produced, never an error.
+ */
+export declare function createRecallRetrieveFn(deps: {
+    db: Database.Database;
+    registry: SessionRecallRegistry;
+    embedFn: (text: string) => Promise<Float32Array>;
+    /** FTS resolution override (tests). Defaults to storage.searchFts. */
+    ftsFn?: typeof storage.searchFts;
+}): RecallRetrieveFn;
 /** Normalize a request-supplied string-list param: array of strings or a CSV
  *  string → string[] | undefined. Anything else (or an empty result) means
  *  "absent" — never a partial guess. Used by `mission_domains` (#203) so it