@gamaze/hicortex 0.20.6 → 0.20.9

package/dist/cli.js

@@ -445,9 +445,8 @@ Options:
   dedup --apply             Execute the merge (default: dry run, report only)
                             Losers are absorbed — hidden from recall, kept as
                             evidence (fetchable by id; dedup_log audit) — not deleted
-  dedup --threshold <t>     Override the threshold for one run (default: config
-                            dedupAutoMergeThreshold, legacy dedupMergeThreshold
-                            still honored; else 0.92)
+  dedup --threshold <t>     Override the threshold for one run (default: the
+                            release-managed calibration ceiling, 0.92)
   dedup --db <path>         DB path override (defaults to the configured DB)
   history --rollback <row>  Roll back history row <row>: restores prior content/status, un-absorbs triggers
   history --db <path>       DB path override (defaults to the configured DB)

package/dist/config-read.d.ts

@@ -27,7 +27,7 @@ export declare function readStrictBoolean(config: Record<string, unknown>, key:
 /**
  * Read a non-negative finite number (allows 0, unlike readPositiveConfig).
  * Returns `def` when absent OR invalid. Used for keys where 0 is a valid "off"
- * value (e.g. ollamaFlushEvery).
+ * value (e.g. memorySoftCap's eviction opt-out, timerJitterSeconds).
  */
 export declare function readNonNegativeConfig(config: Record<string, unknown>, key: string, def: number): number;
 /**

package/dist/config-read.js

@@ -52,7 +52,7 @@ function readStrictBoolean(config, key) {
 /**
  * Read a non-negative finite number (allows 0, unlike readPositiveConfig).
  * Returns `def` when absent OR invalid. Used for keys where 0 is a valid "off"
- * value (e.g. ollamaFlushEvery).
+ * value (e.g. memorySoftCap's eviction opt-out, timerJitterSeconds).
  */
 function readNonNegativeConfig(config, key, def) {
     const v = config[key];
@@ -152,6 +152,63 @@ const IGNORED_CONFIG_KEYS = [
     "classifyModel", "classifyBaseUrl", "classifyApiKey", "classifyProvider",
     "distillFallback",
 ];
+/**
+ * Config keys REMOVED by the #405 budget simplification, each mapped to its
+ * replacement (or "removed" when there is none). The old single 0.16.8-style
+ * message does not fit — every key here needs to name where its job went.
+ * Warned at the config boundary alongside the 0.16.8 keys above.
+ */
+const REMOVED_CONFIG_KEYS = {
+    reconsolidationMaxMinutes: "removed — the ONE run deadline (nightlyTimeBudgetMinutes) bounds the stage",
+    reconsolidationMaxCalls: "removed — the ONE call budget (nightlyLlmCallBudget) caps all stages",
+    supersessionMaxCalls: "removed — the ONE call budget (nightlyLlmCallBudget) caps all stages",
+    dedupNightlyMaxMerges: "removed — the run deadline (nightlyTimeBudgetMinutes) bounds nightly merges",
+    classifyMaxTokens: "removed — maxTokens is the single output ceiling for every call",
+    llmBreakerThreshold: "removed — the breaker threshold is a constant (3)",
+    llmBreakerCooldownMs: "removed — the breaker cooldown is a constant (10 min)",
+    llmSingleFlightWaitMs: "removed — the wait derives from llmTimeoutMs",
+    preflightTimeoutMs: "removed — a constant (20 s per attempt)",
+    preflightAttempts: "removed — a constant (3 attempts)",
+    preflightRetryGapMs: "removed — a constant (60 s gap)",
+    moduleIndexTokenBudget: "removed — was documentation-only",
+};
+/**
+ * Config keys that became RELEASE-MANAGED CALIBRATION constants (#408): the
+ * ~35 tuning keys of the 0.15–0.20 era. Their values are ignored — the
+ * constants ship with each release (src/calibration.ts) and change only in
+ * releases with eval evidence linked in the changelog. Like the #405 list,
+ * this is deliberately WIDER than the `HicortexConfig` type ever was: several
+ * keys (supersessionPenalty, the rrf/bm25 knob set, recallTitleChars, ...)
+ * were README-documented but never typed. Don't trim the list to match the
+ * interface — the warn exists precisely for the untyped surface.
+ */
+const RELEASE_MANAGED_CONFIG_KEYS = [
+    "decayHalfLifeDays", "searchLimit", "recentLimit", "recentWindowDays",
+    "coldExposureSlots", "recallMaxItems", "recallMinSimilarity",
+    "recallReshowTurns", "recallMinPromptChars", "recallTitleChars",
+    "sessionIntentWeight", "noveltyFloorSlots", "scoreSimilarityWeight",
+    "scoreStrengthWeight", "scoreConnectionsWeight", "scoreRecencyWeight",
+    "freshnessBoostDays", "freshnessBoostWeight", "supersededDemotion",
+    "projectAffinityWeight", "domainAffinityWeight", "rrfK",
+    "rrfCompositeWeight", "rrfFtsWeight", "rrfVectorWeight",
+    "bm25WeightBody", "bm25WeightProject", "bm25WeightDomain",
+    "dedupAutoMergeThreshold", "dedupMergeThreshold",
+    "supersessionMinSimilarity", "supersessionPenalty",
+    "correctionMinSimilarity", "correctionRewriteMinConfidence",
+    "weakPrimaryFloor",
+];
+/**
+ * Config keys MOVED to the diagnostic env tier (#408): the ollama-operational
+ * family left config.json and now resolves from environment variables
+ * (calibration.ts resolvers; env > constant, invalid env warns + falls back).
+ * Each entry maps the old config key to its env replacement so the warning
+ * can name the exact variable.
+ */
+const ENV_MOVED_CONFIG_KEYS = {
+    numCtx: "HICORTEX_NUM_CTX",
+    ollamaFlushEvery: "HICORTEX_OLLAMA_FLUSH_EVERY",
+    ollamaFlushWaitMs: "HICORTEX_OLLAMA_FLUSH_WAIT_MS",
+};
 /**
  * Warn if the saved config carries keys that 0.16.8+ ignores. Call at every
  * config read (daemon boot + nightly). The warning clears once the keys are
@@ -163,12 +220,42 @@ function warnIgnoredConfigKeys(savedConfig) {
         return;
     const present = IGNORED_CONFIG_KEYS.filter((k) => savedConfig[k] !== undefined);
     const hasModelsBlock = savedConfig.models !== undefined;
-    if (present.length === 0 && !hasModelsBlock)
-        return;
-    const detail = [...present, ...(hasModelsBlock ? ["models"] : [])].join(", ");
-    console.warn(`[hicortex] config has keys IGNORED since 0.16.8 (${detail}). They have no effect now. ` +
-        `Per-stage model keys / the \`models\` block: one model serves all phases — set ` +
-        `llmModel/llmBaseUrl/llmProvider (+ llmApiKey) to your intended model. ` +
-        `distillFallback: removed (strict mode is default — a failed distill retries next run). ` +
-        `Remove these keys to clear this warning. See the 0.16.8 changelog.`);
+    if (present.length > 0 || hasModelsBlock) {
+        const detail = [...present, ...(hasModelsBlock ? ["models"] : [])].join(", ");
+        console.warn(`[hicortex] config has keys IGNORED since 0.16.8 (${detail}). They have no effect now. ` +
+            `Per-stage model keys / the \`models\` block: one model serves all phases — set ` +
+            `llmModel/llmBaseUrl/llmProvider (+ llmApiKey) to your intended model. ` +
+            `distillFallback: removed (strict mode is default — a failed distill retries next run). ` +
+            `Remove these keys to clear this warning. See the 0.16.8 changelog.`);
+    }
+    // #405: keys removed by the budget simplification — one line naming each
+    // present key and its replacement (or "removed").
+    const removed = Object.entries(REMOVED_CONFIG_KEYS)
+        .filter(([k]) => savedConfig[k] !== undefined);
+    if (removed.length > 0) {
+        const detail = removed.map(([k, v]) => `${k} (${v})`).join(", ");
+        console.warn(`[hicortex] config has keys REMOVED by the budget simplification (#405): ${detail}. ` +
+            `They have no effect. Remove them to clear this warning.`);
+    }
+    // #408: tuning keys that became release-managed calibration constants —
+    // one line naming every present key (their VALUES are ignored; the
+    // constants ship with each release and change only with published eval
+    // evidence linked in the changelog).
+    const releaseManaged = RELEASE_MANAGED_CONFIG_KEYS
+        .filter((k) => savedConfig[k] !== undefined);
+    if (releaseManaged.length > 0) {
+        console.warn(`[hicortex] config has keys that are now RELEASE-MANAGED CALIBRATION ` +
+            `(${releaseManaged.join(", ")}): their values are ignored — calibration ` +
+            `ships with each release and changes only with published eval evidence. ` +
+            `Remove them to clear this warning.`);
+    }
+    // #408: the diagnostic-tier keys that moved to environment variables —
+    // one line naming each old key AND its env replacement.
+    const envMoved = Object.entries(ENV_MOVED_CONFIG_KEYS)
+        .filter(([k]) => savedConfig[k] !== undefined);
+    if (envMoved.length > 0) {
+        const detail = envMoved.map(([k, env]) => `${k} → set ${env} instead`).join(", ");
+        console.warn(`[hicortex] config has keys that MOVED to environment variables (#408): ${detail}. ` +
+            `The config keys are ignored. Remove them to clear this warning.`);
+    }
 }

package/dist/consolidate.d.ts

@@ -9,18 +9,27 @@ import type { LlmClient } from "./llm.js";
 import type { EmbedFn } from "./retrieval.js";
 import { type DomainDef } from "./domain-classify.js";
 import { type ReconsolidationOptions } from "./reconsolidation.js";
+import type { RunDeadline } from "./run-deadline.js";
 /**
- * Default ceiling on total LLM calls across all classify-tier consolidation
- * stages (content-domain, link discovery, supersession) per run. This is a
- * runaway BACKSTOP, not a throughput throttle — on a free local model there is
- * no per-call cost to defend against; the binding constraint is the nightly
- * unit's wall-clock timeout (TimeoutStartSec), not call count. 5000 clears a
- * one-time classification backlog (a ~2000-memory batch drains in ~1-2 runs
- * instead of ~11 nights at the old 200) with margin for link/supersession, and
- * ~5000 calls x ~1-3s/call ≈ 1.4-4.2h fits the 6h consolidation backstop.
- * Config-overridable as `consolidateMaxLlmCalls` (#241).
+ * Default ceiling on LLM calls across the WHOLE nightly pipeline (#405; the
+ * #241 consolidateMaxLlmCalls mechanism, renamed and widened). A runaway
+ * BACKSTOP that bounds money/load INDEPENDENT OF LATENCY — a fast metered or
+ * capacity-limited endpoint permits thousands of calls inside the wall-clock
+ * budget, so time alone cannot protect it (owner ruling 2026-09-12). Consumed
+ * in run order: a stage that exhausts it defers its remainder via its cursor.
+ * 5000 clears a one-time classification backlog (a ~2000-memory batch drains
+ * in ~1-2 runs) with margin. Config: `nightlyLlmCallBudget` (#405); the old
+ * `consolidateMaxLlmCalls` key is a deprecated alias honored one release.
  */
-export declare const CONSOLIDATE_MAX_LLM_CALLS = 5000;
+export declare const DEFAULT_NIGHTLY_LLM_CALL_BUDGET = 5000;
+/**
+ * Resolve the per-run LLM call budget from config (#405):
+ *  - `nightlyLlmCallBudget` present (positive finite) → it wins;
+ *  - else `consolidateMaxLlmCalls` present → used as a DEPRECATED ALIAS with
+ *    a warn naming the replacement (honored one release);
+ *  - absent/invalid → the 5000 default.
+ */
+export declare function resolveNightlyLlmCallBudget(config: Record<string, unknown> | null | undefined): number;
 /**
  * Minimum COSINE similarity for a link candidate.
  *
@@ -66,14 +75,12 @@ export declare class BudgetTracker {
     callsByStage: Record<string, number>;
     /**
      * Per-stage count of LLM-call REQUESTS refused because the budget was
-     * exhausted (#255). Keys are the same stage labels passed to `use()`. The
-     * value is the SUM of the `count` args passed to each refused `use()` call
-     * in that stage (in production every `use()` call passes count=1, so each
-     * refused call adds 1 — but the API accepts a batch count, so a single
-     * refused batch request accrues its full count). Stages break on the first
-     * refusal, so a stage's value is the count of the one request that crossed
-     * the boundary. For item-level skip counts (how many memories or pairs were
-     * left unprocessed), see the per-stage reports — e.g.
+     * exhausted (#255). Keys are the same stage labels passed to `use()`; each
+     * refused call adds 1 (the dead batch `count` param is gone — #405 —
+     * production always passed 1 anyway). Stages break on the first refusal,
+     * so a stage's value is the count of requests that crossed the boundary.
+     * For item-level skip counts (how many memories or pairs were left
+     * unprocessed), see the per-stage reports — e.g.
      * `stages.importance.skipped_budget` — which count MEMORIES, not call
      * requests. Surfaced in summary() and ConsolidationReport as
      * `deferred_by_stage`.
@@ -99,7 +106,7 @@ export declare class BudgetTracker {
     constructor(maxCalls: number);
     get exhausted(): boolean;
     get remaining(): number;
-    use(stage: string, count?: number): boolean;
+    use(stage: string): boolean;
     /**
      * Record token usage from one LLM call (#246). Called by the consolidation
      * stages after each metered completion. `undefined` usage (claude-cli path,
@@ -113,6 +120,14 @@ export declare class BudgetTracker {
     } | undefined): void;
     summary(): NonNullable<ConsolidationReport["budget"]>;
 }
+/**
+ * True when a token-period start stamp is ABSENT or sits in a previous UTC
+ * calendar month than `now` — the monthly-reset staleness check. #405: ONE
+ * shared helper — the check was triplicated (the nightly's throttle branch,
+ * the nightly's accrual write, token-budget.ts recordDistillUsage) and each
+ * copy re-derived the year+month comparison by hand.
+ */
+export declare function isStaleTokenPeriod(periodStart: string | undefined, now?: Date): boolean;
 /**
  * Decide whether consolidation should be throttled this run based on the
  * `llmTokensPerMonth` fair-use cap. Pure (no I/O) so it can be unit-tested
@@ -125,8 +140,8 @@ export declare class BudgetTracker {
  *
  * `cap = 0` (the self-hosted default) → never throttle (unlimited).
  * `periodStart` in a previous calendar month → period resets to 0 first
- * (mirrors the reset logic in nightly.ts; both sides agree because both read
- * the same state + clock).
+ * (isStaleTokenPeriod — the same helper every monthly-reset site uses, so
+ * the sides agree because they read the same state + clock).
  */
 export declare function shouldThrottleTokens(cap: number, period: {
     total: number;
@@ -176,19 +191,16 @@ export declare function discoverLinkCandidates(db: Database.Database, mem: Memor
  * 672-link audit (see the Stage 3 header) found the LLM-classified UPPERCASE
  * types near-useless (CONTRADICTS 4% acceptable). Every candidate now takes its
  * pre-computed `heuristicType` (only `extends` or `relates_to` — see
- * classifyRelationship). No LLM call is made.
- *
- * Signature stability: `llm` and `budget` are RETAINED but intentionally
- * ignored so the callers (nightly `stageLinks`, `hicortex relink`) and the
- * tests that import this need no change to their call sites. The return shape
- * is unchanged; `llmClassified` is always 0 now and `heuristicFallback` counts
- * every candidate. Do NOT re-add an LLM path here without a classifier that
- * passes the audit harness at >= 70% acceptable.
+ * classifyRelationship). No LLM call is made. #405: the ignored `llm`/`budget`
+ * params are deleted — the signature now tells the truth.
+ * The return shape is unchanged; `llmClassified` is always 0 and
+ * `heuristicFallback` counts every candidate. Do NOT re-add an LLM path here
+ * without a classifier that passes the audit harness at >= 70% acceptable.
  *
  * Shared between the nightly `stageLinks` and `hicortex relink`.
  * Returns one relationship type per candidate (same order as input).
  */
-export declare function classifyLinkCandidates(candidates: LinkCandidate[], _llm: LlmClient | null, _budget: BudgetTracker): Promise<{
+export declare function classifyLinkCandidates(candidates: LinkCandidate[]): Promise<{
     types: string[];
     llmClassified: number;
     heuristicFallback: number;
@@ -210,22 +222,17 @@ export declare function classifyLinkCandidates(candidates: LinkCandidate[], _llm
  * boundary from the l2ToCosine calibration is preserved.
  */
 export declare function classifyRelationship(source: Memory, target: Memory, similarity: number): string;
-/** Default minimum COSINE similarity for a supersession candidate pair. */
+/** Default minimum COSINE similarity for a supersession candidate pair —
+ *  RELEASE-MANAGED since #408 (calibration.ts SUPERSESSION_MIN_SIMILARITY). */
 export declare const DEFAULT_SUPERSESSION_MIN_SIMILARITY = 0.8;
-/**
- * Default max classify-tier LLM calls (pairs evaluated) spent per nightly run.
- * 0 = no separate cap — supersession shares the consolidation budget
- * (CONSOLIDATE_MAX_LLM_CALLS, default 5000) like every other stage. The old
- * default of 30 was set when the corpus had 14 decisions; with the distiller
- * now classifying types correctly (#216), decisions are common and the cap
- * was throttling supersession to a crawl. On a local free model there is no
- * per-call cost to defend against — the binding constraint is the wall-clock
- * timeout (TimeoutStartSec), not call count.
- */
-export declare const DEFAULT_SUPERSESSION_MAX_CALLS = 0;
 export interface SupersessionOptions {
+    /** Candidate-pair cosine floor. Release-managed default (calibration.ts);
+     *  this field is the eval/test seam. Invalid → default. */
     minSimilarity?: number;
-    maxCalls?: number;
+    /** The run-wide pipeline deadline (#405) — checked at each candidate
+     *  boundary; on expiry the scan stops and the cursor holds at the last
+     *  fully-considered candidate (resumed next run). */
+    deadline?: RunDeadline;
 }
 export interface SupersessionStageResult {
     scanned: number;
@@ -265,6 +272,13 @@ export declare function parseSupersessionReply(reply: string): boolean | null;
  * candidacy. It only stops SHORT of a candidate when the budget is already
  * exhausted before that candidate starts, so the cursor never skips a
  * candidate that was never looked at.
+ *
+ * #405: the cursor persists after EVERY fully-considered candidate (the
+ * post-#404 reconsolidation pattern), not at stage end — a run killed or
+ * deadline-deferred mid-stage loses at most the candidate in flight. No
+ * orphan clamp is needed (unlike reconsolidation): supersession applies each
+ * verdict's link immediately, so `cursor = candidate.__rowid` always sits
+ * after all of that candidate's writes.
  */
 export declare function stageSupersession(db: Database.Database, llm: LlmClient, budget: BudgetTracker, embedFn: EmbedFn, dryRun: boolean, stateDir: string | undefined, options?: SupersessionOptions): Promise<SupersessionStageResult>;
 /**
@@ -320,7 +334,7 @@ export declare function stageMemoryCapEviction(db: Database.Database, dryRun: bo
  * When `domains` is a non-empty list, the pipeline uses content-based
  * classification (config-owned) INSTEAD of project grouping. The single
  * model serves all phases; if it's unavailable, `complete()` retries
- * internally (30s/60s/120s) and the phase fails soft on persistence —
+ * internally (one 60 s retry, #405) and the phase fails soft on persistence —
  * the nightly retries on the next run. No pre-flight health checks; the
  * phase either answers or is skipped until the next scheduled run. When
  * `domains` is absent/empty, the legacy project-grouping curation runs
@@ -330,38 +344,35 @@ export interface DomainStageOptions {
     domains?: DomainDef[] | null;
     contentDomainsReady?: boolean;
     /**
-     * Weak-primary floor for the no-fit path (see nofit.ts). Resolved by the
-     * caller from config (`weakPrimaryFloor`); defaults to
-     * DEFAULT_WEAK_PRIMARY_FLOOR when absent.
+     * Weak-primary floor for the no-fit path (see nofit.ts). Release-managed
+     * default (#408 — calibration.ts WEAK_PRIMARY_FLOOR via nofit's
+     * DEFAULT_WEAK_PRIMARY_FLOOR); this field stays as the eval/test seam.
      */
     weakPrimaryFloor?: number;
 }
 export declare function runConsolidation(db: Database.Database, llm: LlmClient, embedFn: EmbedFn, dryRun?: boolean, skipReflection?: boolean, stateDir?: string, domainOptions?: DomainStageOptions, supersessionOptions?: SupersessionOptions, 
-/** Total LLM-call ceiling across classify-tier stages (#241). The caller
- *  reads `consolidateMaxLlmCalls` from config and passes it; unset → the
- *  exported `CONSOLIDATE_MAX_LLM_CALLS` default (5000). */
+/** The ONE per-run LLM-call ceiling (#405/#241). The caller resolves
+ *  `nightlyLlmCallBudget` from config (consolidateMaxLlmCalls is a
+ *  deprecated alias — resolveNightlyLlmCallBudget) and passes it; unset →
+ *  the exported DEFAULT_NIGHTLY_LLM_CALL_BUDGET (5000). */
 budgetMaxCalls?: number, 
 /** Soft cap on the corpus (#245). Nightly.ts reads `memorySoftCap` from
  *  config and passes it; unset → `DEFAULT_MEMORY_SOFT_CAP` (10000). `0`
  *  disables eviction (indefinite growth). */
 memorySoftCap?: number, 
-/** Reconsolidation-stage knobs (#384), threaded from config by nightly.ts
- *  (correctionMinSimilarity / correctionRewriteMinConfidence) exactly like
- *  supersessionOptions above; unset fields → the stage's defaults.
- *  Appended AFTER the pre-#384 params so every existing positional caller
- *  (tests, hosted nightly) keeps its argument meaning. */
-reconsolidationOptions?: ReconsolidationOptions): Promise<ConsolidationReport>;
+/** Reconsolidation-stage knobs (#384) — eval/test seams since #408 (the
+ *  values are release-managed calibration constants; nightly.ts threads
+ *  NOTHING), exactly like supersessionOptions above; unset fields → the
+ *  stage's calibration defaults. Appended AFTER the pre-#384 params so
+ *  every existing positional caller (tests, hosted nightly) keeps its
+ *  argument meaning. */
+reconsolidationOptions?: ReconsolidationOptions, 
 /**
- * Calculate milliseconds until the next occurrence of a given hour (local time).
- */
-export declare function msUntilHour(hour: number): number;
-/**
- * Schedule the consolidation pipeline to run nightly.
- * Returns a cleanup function to cancel the timer.
- *
- * NOTE: currently unused (nightly.ts drives consolidation directly). Any future
- * caller MUST read config.domains and thread `domainOptions` into runConsolidation
- * when content domains are configured — otherwise it silently falls back to the
- * legacy project-grouping path even when a domain list is set.
+ * The run-wide pipeline deadline (#405), created at nightly start and
+ * shared by capture + every consolidation stage. Absent = no deadline
+ * (tests, evict-only paths, pre-#405 callers). When it fires, every
+ * not-yet-run stage defers (logs event=deadline_deferred stage=<name>) and
+ * the report status becomes "deferred" — which keeps lastConsolidated
+ * un-advanced so the next run re-finds the pending work.
  */
-export declare function scheduleConsolidation(db: Database.Database, llm: LlmClient, embedFn: EmbedFn, hour?: number): () => void;
+deadline?: RunDeadline): Promise<ConsolidationReport>;