@gamaze/hicortex 0.20.7 → 0.20.10

package/dist/consolidate.d.ts

@@ -9,18 +9,27 @@ import type { LlmClient } from "./llm.js";
 import type { EmbedFn } from "./retrieval.js";
 import { type DomainDef } from "./domain-classify.js";
 import { type ReconsolidationOptions } from "./reconsolidation.js";
+import type { RunDeadline } from "./run-deadline.js";
 /**
- * Default ceiling on total LLM calls across all classify-tier consolidation
- * stages (content-domain, link discovery, supersession) per run. This is a
- * runaway BACKSTOP, not a throughput throttle — on a free local model there is
- * no per-call cost to defend against; the binding constraint is the nightly
- * unit's wall-clock timeout (TimeoutStartSec), not call count. 5000 clears a
- * one-time classification backlog (a ~2000-memory batch drains in ~1-2 runs
- * instead of ~11 nights at the old 200) with margin for link/supersession, and
- * ~5000 calls x ~1-3s/call ≈ 1.4-4.2h fits the 6h consolidation backstop.
- * Config-overridable as `consolidateMaxLlmCalls` (#241).
+ * Default ceiling on LLM calls across the WHOLE nightly pipeline (#405; the
+ * #241 consolidateMaxLlmCalls mechanism, renamed and widened). A runaway
+ * BACKSTOP that bounds money/load INDEPENDENT OF LATENCY — a fast metered or
+ * capacity-limited endpoint permits thousands of calls inside the wall-clock
+ * budget, so time alone cannot protect it (owner ruling 2026-09-12). Consumed
+ * in run order: a stage that exhausts it defers its remainder via its cursor.
+ * 5000 clears a one-time classification backlog (a ~2000-memory batch drains
+ * in ~1-2 runs) with margin. Config: `nightlyLlmCallBudget` (#405); the old
+ * `consolidateMaxLlmCalls` key is a deprecated alias honored one release.
  */
-export declare const CONSOLIDATE_MAX_LLM_CALLS = 5000;
+export declare const DEFAULT_NIGHTLY_LLM_CALL_BUDGET = 5000;
+/**
+ * Resolve the per-run LLM call budget from config (#405):
+ *  - `nightlyLlmCallBudget` present (positive finite) → it wins;
+ *  - else `consolidateMaxLlmCalls` present → used as a DEPRECATED ALIAS with
+ *    a warn naming the replacement (honored one release);
+ *  - absent/invalid → the 5000 default.
+ */
+export declare function resolveNightlyLlmCallBudget(config: Record<string, unknown> | null | undefined): number;
 /**
  * Minimum COSINE similarity for a link candidate.
  *
@@ -66,14 +75,12 @@ export declare class BudgetTracker {
     callsByStage: Record<string, number>;
     /**
      * Per-stage count of LLM-call REQUESTS refused because the budget was
-     * exhausted (#255). Keys are the same stage labels passed to `use()`. The
-     * value is the SUM of the `count` args passed to each refused `use()` call
-     * in that stage (in production every `use()` call passes count=1, so each
-     * refused call adds 1 — but the API accepts a batch count, so a single
-     * refused batch request accrues its full count). Stages break on the first
-     * refusal, so a stage's value is the count of the one request that crossed
-     * the boundary. For item-level skip counts (how many memories or pairs were
-     * left unprocessed), see the per-stage reports — e.g.
+     * exhausted (#255). Keys are the same stage labels passed to `use()`; each
+     * refused call adds 1 (the dead batch `count` param is gone — #405 —
+     * production always passed 1 anyway). Stages break on the first refusal,
+     * so a stage's value is the count of requests that crossed the boundary.
+     * For item-level skip counts (how many memories or pairs were left
+     * unprocessed), see the per-stage reports — e.g.
      * `stages.importance.skipped_budget` — which count MEMORIES, not call
      * requests. Surfaced in summary() and ConsolidationReport as
      * `deferred_by_stage`.
@@ -99,7 +106,7 @@ export declare class BudgetTracker {
     constructor(maxCalls: number);
     get exhausted(): boolean;
     get remaining(): number;
-    use(stage: string, count?: number): boolean;
+    use(stage: string): boolean;
     /**
      * Record token usage from one LLM call (#246). Called by the consolidation
      * stages after each metered completion. `undefined` usage (claude-cli path,
@@ -113,6 +120,26 @@ export declare class BudgetTracker {
     } | undefined): void;
     summary(): NonNullable<ConsolidationReport["budget"]>;
 }
+/**
+ * #427 observability: warn when a consolidation run made LLM CALLS but
+ * metered ZERO tokens — the endpoint returned no usage objects on its
+ * completions (recordUsage skips undefined by design, never fabricates a
+ * zero). Such a run still spends budget calls but its snapshot carries token
+ * nulls, which read as a mystery on the dashboard. The warn is a structured
+ * event in the same journald-greppable style as `event=budget_exhausted`
+ * (grep `event=tokens_unmetered`), so the blind spot is visible instead of
+ * silent. Returns true when it warned (for tests); no fabrication either
+ * way — the numbers stay exactly what the endpoint reported.
+ */
+export declare function warnUnmeteredTokensRun(budget: NonNullable<ConsolidationReport["budget"]>): boolean;
+/**
+ * True when a token-period start stamp is ABSENT or sits in a previous UTC
+ * calendar month than `now` — the monthly-reset staleness check. #405: ONE
+ * shared helper — the check was triplicated (the nightly's throttle branch,
+ * the nightly's accrual write, token-budget.ts recordDistillUsage) and each
+ * copy re-derived the year+month comparison by hand.
+ */
+export declare function isStaleTokenPeriod(periodStart: string | undefined, now?: Date): boolean;
 /**
  * Decide whether consolidation should be throttled this run based on the
  * `llmTokensPerMonth` fair-use cap. Pure (no I/O) so it can be unit-tested
@@ -125,8 +152,8 @@ export declare class BudgetTracker {
  *
  * `cap = 0` (the self-hosted default) → never throttle (unlimited).
  * `periodStart` in a previous calendar month → period resets to 0 first
- * (mirrors the reset logic in nightly.ts; both sides agree because both read
- * the same state + clock).
+ * (isStaleTokenPeriod — the same helper every monthly-reset site uses, so
+ * the sides agree because they read the same state + clock).
  */
 export declare function shouldThrottleTokens(cap: number, period: {
     total: number;
@@ -140,6 +167,29 @@ export declare function shouldThrottleTokens(cap: number, period: {
  * Parse JSON from LLM output, tolerating markdown fences and indexed formats.
  */
 export declare function parseJsonLenient<T>(text: string, fallback: T): T;
+/**
+ * The shared importance-scoring loop (#425 extraction): one LLM call per
+ * 10-memory batch through the production `importanceScoring` prompt, each
+ * written score clamped at IMPORTANCE_CEILING and stamped with the
+ * importance_scored_at watermark. Used by the nightly's stageImportance AND
+ * `hicortex rescore-importance` — there is exactly one scoring code path
+ * (no forked backfill logic; cap + watermark write identically everywhere).
+ *
+ * Failure semantics: a batch whose LLM call THROWS writes nothing (counted
+ * in `failed` — retried naturally later); a batch whose reply parses to a
+ * non-array falls back to 0.5 per memory (written + watermarked — the
+ * endpoint answered, the answer was unusable).
+ */
+export declare function scoreMemoriesImportance(db: Database.Database, memories: Memory[], llm: LlmClient, opts?: {
+    budget?: BudgetTracker;
+    deadline?: RunDeadline;
+    dryRun?: boolean;
+    onBatch?: (written: number, failed: number) => void;
+}): Promise<{
+    scored: number;
+    failed: number;
+    skipped_budget: number;
+}>;
 /**
  * Rebuild moduleIndex from the configured domain set + live DB counts, and
  * persist it. Shared by the nightly stage and `hicortex classify-domains`.
@@ -176,19 +226,16 @@ export declare function discoverLinkCandidates(db: Database.Database, mem: Memor
  * 672-link audit (see the Stage 3 header) found the LLM-classified UPPERCASE
  * types near-useless (CONTRADICTS 4% acceptable). Every candidate now takes its
  * pre-computed `heuristicType` (only `extends` or `relates_to` — see
- * classifyRelationship). No LLM call is made.
- *
- * Signature stability: `llm` and `budget` are RETAINED but intentionally
- * ignored so the callers (nightly `stageLinks`, `hicortex relink`) and the
- * tests that import this need no change to their call sites. The return shape
- * is unchanged; `llmClassified` is always 0 now and `heuristicFallback` counts
- * every candidate. Do NOT re-add an LLM path here without a classifier that
- * passes the audit harness at >= 70% acceptable.
+ * classifyRelationship). No LLM call is made. #405: the ignored `llm`/`budget`
+ * params are deleted — the signature now tells the truth.
+ * The return shape is unchanged; `llmClassified` is always 0 and
+ * `heuristicFallback` counts every candidate. Do NOT re-add an LLM path here
+ * without a classifier that passes the audit harness at >= 70% acceptable.
  *
  * Shared between the nightly `stageLinks` and `hicortex relink`.
  * Returns one relationship type per candidate (same order as input).
  */
-export declare function classifyLinkCandidates(candidates: LinkCandidate[], _llm: LlmClient | null, _budget: BudgetTracker): Promise<{
+export declare function classifyLinkCandidates(candidates: LinkCandidate[]): Promise<{
     types: string[];
     llmClassified: number;
     heuristicFallback: number;
@@ -210,22 +257,17 @@ export declare function classifyLinkCandidates(candidates: LinkCandidate[], _llm
  * boundary from the l2ToCosine calibration is preserved.
  */
 export declare function classifyRelationship(source: Memory, target: Memory, similarity: number): string;
-/** Default minimum COSINE similarity for a supersession candidate pair. */
+/** Default minimum COSINE similarity for a supersession candidate pair —
+ *  RELEASE-MANAGED since #408 (calibration.ts SUPERSESSION_MIN_SIMILARITY). */
 export declare const DEFAULT_SUPERSESSION_MIN_SIMILARITY = 0.8;
-/**
- * Default max classify-tier LLM calls (pairs evaluated) spent per nightly run.
- * 0 = no separate cap — supersession shares the consolidation budget
- * (CONSOLIDATE_MAX_LLM_CALLS, default 5000) like every other stage. The old
- * default of 30 was set when the corpus had 14 decisions; with the distiller
- * now classifying types correctly (#216), decisions are common and the cap
- * was throttling supersession to a crawl. On a local free model there is no
- * per-call cost to defend against — the binding constraint is the wall-clock
- * timeout (TimeoutStartSec), not call count.
- */
-export declare const DEFAULT_SUPERSESSION_MAX_CALLS = 0;
 export interface SupersessionOptions {
+    /** Candidate-pair cosine floor. Release-managed default (calibration.ts);
+     *  this field is the eval/test seam. Invalid → default. */
     minSimilarity?: number;
-    maxCalls?: number;
+    /** The run-wide pipeline deadline (#405) — checked at each candidate
+     *  boundary; on expiry the scan stops and the cursor holds at the last
+     *  fully-considered candidate (resumed next run). */
+    deadline?: RunDeadline;
 }
 export interface SupersessionStageResult {
     scanned: number;
@@ -265,6 +307,13 @@ export declare function parseSupersessionReply(reply: string): boolean | null;
  * candidacy. It only stops SHORT of a candidate when the budget is already
  * exhausted before that candidate starts, so the cursor never skips a
  * candidate that was never looked at.
+ *
+ * #405: the cursor persists after EVERY fully-considered candidate (the
+ * post-#404 reconsolidation pattern), not at stage end — a run killed or
+ * deadline-deferred mid-stage loses at most the candidate in flight. No
+ * orphan clamp is needed (unlike reconsolidation): supersession applies each
+ * verdict's link immediately, so `cursor = candidate.__rowid` always sits
+ * after all of that candidate's writes.
  */
 export declare function stageSupersession(db: Database.Database, llm: LlmClient, budget: BudgetTracker, embedFn: EmbedFn, dryRun: boolean, stateDir: string | undefined, options?: SupersessionOptions): Promise<SupersessionStageResult>;
 /**
@@ -320,7 +369,7 @@ export declare function stageMemoryCapEviction(db: Database.Database, dryRun: bo
  * When `domains` is a non-empty list, the pipeline uses content-based
  * classification (config-owned) INSTEAD of project grouping. The single
  * model serves all phases; if it's unavailable, `complete()` retries
- * internally (30s/60s/120s) and the phase fails soft on persistence —
+ * internally (one 60 s retry, #405) and the phase fails soft on persistence —
  * the nightly retries on the next run. No pre-flight health checks; the
  * phase either answers or is skipped until the next scheduled run. When
  * `domains` is absent/empty, the legacy project-grouping curation runs
@@ -330,38 +379,35 @@ export interface DomainStageOptions {
     domains?: DomainDef[] | null;
     contentDomainsReady?: boolean;
     /**
-     * Weak-primary floor for the no-fit path (see nofit.ts). Resolved by the
-     * caller from config (`weakPrimaryFloor`); defaults to
-     * DEFAULT_WEAK_PRIMARY_FLOOR when absent.
+     * Weak-primary floor for the no-fit path (see nofit.ts). Release-managed
+     * default (#408 — calibration.ts WEAK_PRIMARY_FLOOR via nofit's
+     * DEFAULT_WEAK_PRIMARY_FLOOR); this field stays as the eval/test seam.
      */
     weakPrimaryFloor?: number;
 }
 export declare function runConsolidation(db: Database.Database, llm: LlmClient, embedFn: EmbedFn, dryRun?: boolean, skipReflection?: boolean, stateDir?: string, domainOptions?: DomainStageOptions, supersessionOptions?: SupersessionOptions, 
-/** Total LLM-call ceiling across classify-tier stages (#241). The caller
- *  reads `consolidateMaxLlmCalls` from config and passes it; unset → the
- *  exported `CONSOLIDATE_MAX_LLM_CALLS` default (5000). */
+/** The ONE per-run LLM-call ceiling (#405/#241). The caller resolves
+ *  `nightlyLlmCallBudget` from config (consolidateMaxLlmCalls is a
+ *  deprecated alias — resolveNightlyLlmCallBudget) and passes it; unset →
+ *  the exported DEFAULT_NIGHTLY_LLM_CALL_BUDGET (5000). */
 budgetMaxCalls?: number, 
 /** Soft cap on the corpus (#245). Nightly.ts reads `memorySoftCap` from
  *  config and passes it; unset → `DEFAULT_MEMORY_SOFT_CAP` (10000). `0`
  *  disables eviction (indefinite growth). */
 memorySoftCap?: number, 
-/** Reconsolidation-stage knobs (#384), threaded from config by nightly.ts
- *  (correctionMinSimilarity / correctionRewriteMinConfidence) exactly like
- *  supersessionOptions above; unset fields → the stage's defaults.
- *  Appended AFTER the pre-#384 params so every existing positional caller
- *  (tests, hosted nightly) keeps its argument meaning. */
-reconsolidationOptions?: ReconsolidationOptions): Promise<ConsolidationReport>;
-/**
- * Calculate milliseconds until the next occurrence of a given hour (local time).
- */
-export declare function msUntilHour(hour: number): number;
+/** Reconsolidation-stage knobs (#384) — eval/test seams since #408 (the
+ *  values are release-managed calibration constants; nightly.ts threads
+ *  NOTHING), exactly like supersessionOptions above; unset fields → the
+ *  stage's calibration defaults. Appended AFTER the pre-#384 params so
+ *  every existing positional caller (tests, hosted nightly) keeps its
+ *  argument meaning. */
+reconsolidationOptions?: ReconsolidationOptions, 
 /**
- * Schedule the consolidation pipeline to run nightly.
- * Returns a cleanup function to cancel the timer.
- *
- * NOTE: currently unused (nightly.ts drives consolidation directly). Any future
- * caller MUST read config.domains and thread `domainOptions` into runConsolidation
- * when content domains are configured — otherwise it silently falls back to the
- * legacy project-grouping path even when a domain list is set.
+ * The run-wide pipeline deadline (#405), created at nightly start and
+ * shared by capture + every consolidation stage. Absent = no deadline
+ * (tests, evict-only paths, pre-#405 callers). When it fires, every
+ * not-yet-run stage defers (logs event=deadline_deferred stage=<name>) and
+ * the report status becomes "deferred" — which keeps lastConsolidated
+ * un-advanced so the next run re-finds the pending work.
  */
-export declare function scheduleConsolidation(db: Database.Database, llm: LlmClient, embedFn: EmbedFn, hour?: number): () => void;
+deadline?: RunDeadline): Promise<ConsolidationReport>;