@sema-agent/core 5.57.0 → 5.59.0

package/dist/core/memory-engine/distiller.d.ts

@@ -0,0 +1,550 @@
+/**
+ * design/376 片① — the LLM distillation DRIVER's single source (the "driver half" of design/339
+ * §1.1: the driver proposes, the engine adjudicates). Everything here is the product code the
+ * memory benchmark measures — the bench adapters import THESE exports, so every compare run is an
+ * end-to-end regression of the shipped driver (the true-load-path law: the bytes the rig scores
+ * are the bytes the npm package ships).
+ *
+ * Structure, and why each piece lives here:
+ *  - the PROMPT CONTRACT (frozen, versioned): the prompt is a behavior surface. v1 of the product
+ *    contract is the dl-3 contract BYTE-IDENTICAL — the version whose readings the benchmark
+ *    archived. Any change to these strings is a NEW contract version and must produce bench
+ *    readings before it ships (no-readings prompt edits are unreviewed behavior changes).
+ *  - `parseJsonAnswer` / `sanitizeLlmGroups` / `planParseRepairs`: the structural adapter between
+ *    a model answer and a protocol-shaped proposal, every repair COUNTED (a silent repair lets a
+ *    sloppy plan read as a clean one).
+ *  - `scheduleUnderFuse`: the driver-side plan selection under the engine's supersession fuse
+ *    (next-fit decreasing, no backfill). The ceiling itself is NEVER re-derived here — callers
+ *    compute it with the engine's own exported `supersessionFuseCeiling` (a second implementation
+ *    would be a drift seam).
+ *  - `mintLlmConsolidationPlan`: the live half (grouping + writing passes), archiving everything
+ *    the model was told and answered.
+ *  - `llmPlanDistiller`: a minted plan replayed as a per-cycle distiller (the run-cache form: mint
+ *    once, filter per cycle — never re-mint inside one run).
+ *  - `driveConsolidationToFixpoint`: the host loop over the four engine verbs (snapshot → commit →
+ *    reconcile, repeated), counting exclusively STORE deltas.
+ *
+ * The engine holds no distillation power and no model seat (design/339 §1.1 boundary): this module
+ * calls the engine's HOST verbs and nothing else writes.
+ */
+import type { MemoryEntry } from "./types.js";
+import type { ConsolidationProposal } from "./consolidation.js";
+export declare const DISTILLER_DEFAULT_MAX_INPUTS_PER_PRODUCT = 32;
+export interface LlmDistillerContract {
+    /** The contract's registered version id (a NEW id for any byte change, with bench readings). */
+    version: string;
+    grouping: {
+        system: string;
+        instructionLines: readonly string[];
+        instructions: string;
+    };
+    writing: {
+        system: string;
+        instructions: string;
+    };
+    sampling: {
+        temperature: number;
+        maxTokens: {
+            grouping: number;
+            writing: number;
+        };
+    };
+    /** Format-failure retries per call. Frozen INTO the contract: the archived readings were taken
+     *  under this value, so changing it moves the thing under measurement. */
+    retries: number;
+}
+export declare const LLM_DISTILLER_CONTRACT: LlmDistillerContract;
+/** dl-2: the SAME contract with the retrievability clause disambiguated, nothing else touched. */
+export declare const LLM_DISTILLER_CONTRACT_DL2: LlmDistillerContract;
+/** dl-3: dl-2 with the SIZE clause substituted, nothing else touched. */
+export declare const LLM_DISTILLER_CONTRACT_DL3: LlmDistillerContract;
+export declare const LLM_DISTILLER_CONTRACTS: Record<string, LlmDistillerContract>;
+/**
+ * The PRODUCT contract, v1 = dl-3 byte-identical (the version whose benchmark readings are
+ * archived). The shipping rule this alias exists to make auditable: a change to the product prompt
+ * is a NEW `version` (pd-2, …) and must produce rig readings BEFORE it ships — the contract-hash
+ * pin in the test suite reds on any silent byte drift under an unchanged version id.
+ */
+export declare const MEMORY_DISTILLER_CONTRACT_V1: LlmDistillerContract;
+/**
+ * The A/B witness: which grouping lines differ between two contracts, by index — published beside
+ * readings so "only the wording moved" is a checkable statement about the prompts actually sent.
+ */
+export declare function contractGroupingDiff(a: LlmDistillerContract | undefined, b: LlmDistillerContract | undefined): {
+    sameLineCount: boolean;
+    changedClauseIndexes: number[];
+    before: Array<string | null>;
+    after: Array<string | null>;
+    otherClausesIdentical: boolean;
+    writingPromptIdentical: boolean;
+    systemPromptsIdentical: boolean;
+    samplingIdentical: boolean;
+};
+/** The snapshot row shape the engine's `snapshotConsolidationCandidates` answers. */
+export interface DistillerCandidate {
+    entry: MemoryEntry;
+    marked: boolean;
+}
+/** One JSON-only completion request (no streaming, no tools — the contract's whole call shape). */
+export interface DistillerChatRequest {
+    system: string;
+    user: string;
+    maxTokens: number;
+    temperature: number;
+}
+export interface DistillerChatAnswer {
+    content: string;
+    /** The provider's finish reason; `"length"` marks a token-ceiling cut-off (retried, disclosed). */
+    finishReason?: string | null;
+    usage?: {
+        promptTokens?: number;
+        completionTokens?: number;
+    } | null;
+    /** The model id the provider REPORTS having served (audit anchor beside the requested id). */
+    model?: string | null;
+}
+export type DistillerChatFn = (req: DistillerChatRequest) => Promise<DistillerChatAnswer>;
+/** True when the id is an ALIAS spelling (re-pointable upstream), refused wherever a model id is
+ *  taken. A conservative SCREEN over the common alias grammars, not a full catalog — the honest
+ *  direction is one-way: `true` ⇒ alias, while a `false` only means "not a spelling this screen
+ *  knows". Screened: any id whose FINAL dash segment is `latest`/`default` regardless of how many
+ *  segments the family name has (`chatgpt-4o-latest`, `claude-3-5-sonnet-latest`,
+ *  `gemini-1.5-pro-latest`), plus the single-family `-chat` form (`deepseek-chat`, `qwen-chat` —
+ *  the alias family upstream re-points have actually ridden). Multi-segment `-chat` ids are NOT
+ *  screened on purpose: that suffix also names pinned chat variants of fixed weights
+ *  (`llama-2-7b-chat`), and refusing those would reject explicit ids. */
+export declare function isAliasModelId(model: string): boolean;
+/**
+ * An OpenAI-compatible chat seat over an explicit endpoint. Screened at CONSTRUCTION (the
+ * bad-value loudness law): a missing key or an alias model id refuses with a code before any call
+ * is made — an alias can be re-pointed upstream and would move the driven model with no trace in
+ * the archived plan.
+ */
+export declare function openAiCompatChatSeat(seat: {
+    baseUrl: string;
+    apiKey: string;
+    model: string;
+}): DistillerChatFn;
+export type ParsedJsonAnswer = {
+    ok: true;
+    value: unknown;
+    repaired: boolean;
+} | {
+    ok: false;
+    error: string;
+    sample: string;
+};
+/**
+ * Parse ONE JSON object out of a model answer. Fenced and prefixed answers are tolerated (a
+ * formatting failure, not a distillation one) and the tolerance is COUNTED. A truncated answer is
+ * never salvaged — half a plan scored (or committed) as a whole plan is the failure mode this
+ * function exists to refuse.
+ */
+export declare function parseJsonAnswer(text: unknown): ParsedJsonAnswer;
+export interface PlanParseRepairs {
+    grouping: number;
+    writing: number;
+    groupingCalls: number;
+    writingCalls: number;
+    /** Calls whose answer was cut off at the token ceiling — a HARNESS parameter biting, named per
+     *  call and never folded into a repair count it is not. */
+    lengthCapped: string[];
+    retries: number;
+}
+/**
+ * Format repairs per PHASE, re-derived from a plan's own archived transcript — one derivation
+ * produces the number (the grouping stage cannot be counted while the writing stage silently is
+ * not), and any reader can re-run it against the archive and get the same answer.
+ */
+export declare function planParseRepairs(plan: {
+    minting?: {
+        transcript?: Array<{
+            label?: unknown;
+            answer?: unknown;
+            finishReason?: unknown;
+        }>;
+    };
+} | undefined | null): PlanParseRepairs;
+export interface SanitizedLlmGroups {
+    groups: Array<{
+        key: string;
+        intent: string;
+        members: Array<{
+            n: number;
+            id: string;
+        }>;
+    }>;
+    repairs: {
+        hallucinatedMembers: number;
+        duplicateMembers: number;
+        emptyGroups: number;
+        oversizeGroups: Array<{
+            key: string;
+            members: number;
+            cap: number;
+        }>;
+        nonIntegerMembers: number;
+    };
+    claimedIds: string[];
+}
+/**
+ * Turn the model's groups into a protocol-shaped product list, repairing only STRUCTURE and
+ * counting every repair:
+ *
+ *   hallucinatedMembers — a number that is not in the candidate list at all;
+ *   duplicateMembers    — a number claimed by a second group (first claim, in the model's own
+ *                         group order, wins; the later claim is dropped);
+ *   emptyGroups         — a group left with no members after the two above;
+ *   oversizeGroups      — a group over `maxInputsPerProduct`. NOT split: chunking a group is a
+ *                         distillation decision, and making it on the model's behalf would put
+ *                         this module in the driver's seat. They ride on and the schedule reports
+ *                         them as structurally infeasible;
+ *   nonIntegerMembers   — a member reference that is not a strict integer. STRICT because
+ *                         `parseInt` is a silent truncator and the thing truncated is a ROW
+ *                         reference: "2foo"→2 and "1.5"→1 would each fold a DIFFERENT entry than
+ *                         the model named while every repair counter stayed at zero.
+ */
+export declare function sanitizeLlmGroups(groups: unknown, { candidateIds, maxInputsPerProduct }: {
+    candidateIds: readonly string[];
+    maxInputsPerProduct?: number;
+}): SanitizedLlmGroups;
+export interface LlmConsolidationPlanProduct {
+    key: string;
+    intent: string;
+    name: string;
+    description: string;
+    type: "project" | "reference";
+    body: string;
+    inputIds: string[];
+    /** The committed rev of each input AT MINT TIME (id → rev). The replay filter refuses to fold an
+     *  input whose committed rev has moved since the model read it — the product's body was written
+     *  over the OLD bytes, and superseding the edited entry under it would suppress a newer value
+     *  behind stale prose. Absent on plans minted before this field existed: those replay by id
+     *  alone, exactly as they always did (the archived-readings compatibility arm). */
+    inputRevs?: Record<string, string>;
+}
+export interface LlmConsolidationPlan {
+    kind: "llm-consolidation-plan";
+    contractVersion: string;
+    model: string;
+    /** The endpoint the mint ran against, when the caller supplied one (audit locator only). */
+    baseUrl?: string;
+    mintedAt: string;
+    maxInputsPerProduct: number;
+    candidateCount: number;
+    products: LlmConsolidationPlanProduct[];
+    minting: {
+        calls: number;
+        groupingAttempts: number;
+        groupingParseRepairs: number;
+        parseRepairs: PlanParseRepairs;
+        repairs: SanitizedLlmGroups["repairs"];
+        writeFailures: Array<{
+            key: string;
+            members: number;
+            reason: string | null;
+        }>;
+        usage: {
+            promptTokens: number;
+            completionTokens: number;
+        };
+        prompts: {
+            grouping: string;
+            writingTemplate: string;
+            systems: {
+                grouping: string;
+                writing: string;
+            };
+        };
+        transcript: Array<{
+            label: string;
+            ms: number;
+            usage: DistillerChatAnswer["usage"];
+            finishReason: string | null;
+            model: string | null;
+            answer: string;
+        }>;
+    };
+}
+export type MintLlmConsolidationPlanResult = {
+    ok: true;
+    plan: LlmConsolidationPlan;
+} | {
+    ok: false;
+    reason: string;
+    calls: LlmConsolidationPlan["minting"]["transcript"];
+};
+/**
+ * Mint a fold plan from a live model over one candidate snapshot. `candidates` is the snapshot
+ * shape the engine hands the driver (`{ entry, marked }`). Everything the model saw and everything
+ * it answered is returned inside the plan so the run can be re-read, re-scored and disputed from
+ * the archive alone.
+ */
+export declare function mintLlmConsolidationPlan({ candidates, chat, model, baseUrl, maxInputsPerProduct, contract, onProgress, }: {
+    candidates: readonly DistillerCandidate[];
+    chat: DistillerChatFn;
+    /** The requested model id, recorded into the plan (audit anchor). Alias spellings refuse. */
+    model: string;
+    baseUrl?: string;
+    maxInputsPerProduct?: number;
+    contract?: LlmDistillerContract;
+    onProgress?: ((p: {
+        phase: "grouping";
+        groups: number;
+        claimed: number;
+        of: number;
+    } | {
+        phase: "writing";
+        done: number;
+        of: number;
+    }) => void) | null;
+}): Promise<MintLlmConsolidationPlanResult>;
+/** The distiller signature the drive consumes: `(candidates, snapshot, scope) -> proposal`.
+ *  At runtime only `entry.id` is load-bearing for the replay filter — test/bench stubs that hand
+ *  partial entry shapes are tolerated by the reads, the type states the engine's real snapshot.
+ *  The drive consumes the proposal's FOLD face only (`products`; `scope` is the drive's own): a
+ *  proposal carrying directed `intents` is REFUSED by the drive rather than silently narrowed —
+ *  directed update/delete intents go through `commitConsolidationPlan` directly (see the drive's
+ *  own doc for why a per-cycle forward is structurally impossible). */
+export type ConsolidationDistillFn = (candidates: readonly DistillerCandidate[], snapshot: unknown, scope: string) => ConsolidationProposal | Promise<ConsolidationProposal>;
+/**
+ * A minted plan replayed as a per-cycle distiller. Per cycle it filters each product's inputs down
+ * to the ids the snapshot is still offering — a product whose inputs have all been folded is empty
+ * and is the drive's termination signal. `supersede: true` rides on every input: an input
+ * attributed without the edge is provenance only, the row would stay active, and the run would
+ * read as "consolidation did nothing" for a reason on this side of the seam.
+ *
+ * This is the RUN-CACHE form the cost model depends on: one mint per run, this filter per cycle —
+ * re-minting per cycle would multiply the model bill by the cycle count for the same answer.
+ */
+export declare function llmPlanDistiller(plan: {
+    products?: ReadonlyArray<Pick<LlmConsolidationPlanProduct, "name" | "description" | "type" | "body" | "inputIds"> & {
+        inputRevs?: Record<string, string>;
+    }>;
+} | undefined | null): ConsolidationDistillFn;
+export interface FuseSchedule<P> {
+    schedule: "next-fit-decreasing";
+    picked: P[];
+    targets: number;
+    ceiling: number;
+    deferred: number;
+    oversize: Array<{
+        name: string;
+        inputs: number;
+        cap: number;
+    }>;
+}
+/**
+ * One cycle's plan selection under the engine's supersession fuse.
+ *
+ * NEXT-FIT DECREASING: products in descending target count, stop at the first that does not fit.
+ * The missing backfill is the point, not an oversight: the ceiling is a FRACTION of the active set
+ * and every fold shrinks the active set, so a cycle that spends leftover budget on a small fold —
+ * one that would still fit later, at a lower ceiling — permanently lowers the ceiling every
+ * remaining large fold has to clear. Measured with the shipped 0.25 cap: a backfilling schedule
+ * dead-ends with two large digests permanently unfoldable while the same products converge when
+ * the large folds simply go first.
+ *
+ * `maxProductsPerRun` is honoured here too; a product whose input count exceeds
+ * `maxInputsPerProduct` can never be frozen at all and is reported structurally infeasible rather
+ * than retried forever.
+ */
+export declare function scheduleUnderFuse<P extends {
+    name?: string;
+    inputs?: Array<{
+        id: string;
+        supersede?: boolean;
+    }>;
+}>(products: readonly P[] | undefined, ceiling: number, { maxProductsPerRun, maxInputsPerProduct }?: {
+    maxProductsPerRun?: number;
+    maxInputsPerProduct?: number;
+}): FuseSchedule<P>;
+/** The engine face the drive consumes (MemoryEngine satisfies it structurally; tests may stub). */
+export interface ConsolidationDriveEngine {
+    snapshotConsolidationCandidates(scope: string, opts?: {
+        force?: {
+            requestId: string;
+        };
+        full?: boolean;
+    }): Promise<{
+        cycleToken: string;
+        watermark?: unknown;
+        candidates: Array<{
+            entry: MemoryEntry;
+            marked: boolean;
+        }>;
+    }>;
+    commitConsolidationPlan(cycleToken: string, proposal: ConsolidationProposal, opts: {
+        requestId: string;
+    }): Promise<ConsolidationDriveReceipt>;
+    reconcileConsolidation(scope: string): Promise<{
+        replayed?: {
+            planId: string;
+            state: string;
+        } | null;
+    } | undefined | void>;
+}
+/** The commit receipt fields the drive reads (the engine's receipt carries more). */
+export interface ConsolidationDriveReceipt {
+    planId?: string;
+    state?: string;
+    products?: {
+        frozen?: number;
+        applied?: number | null;
+        conflicts?: number | null;
+        refused?: Array<{
+            index: number;
+            reason: string;
+        }>;
+    };
+}
+export interface ConsolidationFoldState {
+    active: number;
+    superseded: number;
+    products: number;
+}
+export interface ConsolidationDriveCycleRow {
+    cycle: number;
+    activeSetSize: number;
+    ceiling: number;
+    candidates?: number;
+    /** How many of the cycle's served candidates carried a committed external-origin marker — the
+     *  engine-served fact the fold law and the instruction hard gate key on (design/376 §2.1: a
+     *  reading about the marked treatment must show the snapshot actually saw marked content, and a
+     *  zero-marked run must show the zero). Absent only on the row of a snapshot that refused. */
+    markedCandidates?: number;
+    deferred?: number;
+    planId?: string;
+    state?: string;
+    applied?: number | null;
+    conflicts?: number | null;
+    refusedProducts?: Array<{
+        index: number;
+        reason: string;
+    }>;
+    reconciled?: {
+        planId: string;
+        state: string;
+    } | null;
+    note?: string;
+    refused?: string;
+    recovery?: {
+        attempted: boolean;
+        ok: boolean;
+        replayed: {
+            planId: string;
+            state: string;
+        } | null;
+        summary: string;
+        report?: unknown;
+    };
+    attempted: {
+        products: number;
+        targets: number;
+    };
+    committed: {
+        products: number;
+        superseded: number;
+    };
+    /** `committed`'s two flat mirrors — the drive-report projections read the row under these names
+     *  as well as under `committed.*`; all three spellings are one store-delta fact. */
+    products: number;
+    targets: number;
+}
+export interface ConsolidationDriveResult {
+    /** A STORE fact: did anything actually fold. A refused first cycle answers false. */
+    ran: boolean;
+    schedule: "next-fit-decreasing";
+    cycles: ConsolidationDriveCycleRow[];
+    cycleCount: number;
+    productsCommitted: number;
+    entriesSuperseded: number;
+    productsAttempted: number;
+    engineRefusedProducts: Array<{
+        index: number;
+        reason: string;
+    }>;
+    converged: boolean;
+    foldState: {
+        before: ConsolidationFoldState;
+        after: ConsolidationFoldState;
+    };
+    stop?: {
+        reason: string;
+        refused?: boolean;
+        stalled?: boolean;
+        parked?: boolean;
+        exhausted?: boolean;
+        planId?: string;
+        fuseBlocked?: Array<{
+            name: string;
+            targets: number;
+        }>;
+        oversize?: Array<{
+            name: string;
+            inputs: number;
+            cap: number;
+        }>;
+        recovery?: unknown;
+    };
+}
+/** The engine's refusals carry a `code` and a `reasons` array; a bare `.message` throws half away. */
+export declare function codedReason(err: unknown): string;
+export interface DriveConsolidationOptions {
+    distill: ConsolidationDistillFn;
+    /** The fuse denominator AND every published count are store facts, never driver assumptions. */
+    readFoldState: () => Promise<ConsolidationFoldState> | ConsolidationFoldState;
+    options: {
+        supersedeRatioCap: number;
+        supersedeAbsoluteFloor: number;
+        maxProductsPerRun: number;
+        maxInputsPerProduct: number;
+    };
+    /** The ceiling arithmetic — pass the engine's `supersessionFuseCeiling` (single source). */
+    fuseCeilingOf: (activeSetSize: number, opts: {
+        supersedeRatioCap: number;
+        supersedeAbsoluteFloor: number;
+    }) => number;
+    requestIdPrefix?: string;
+    maxCycles?: number;
+    /**
+     * The snapshot step, injectable so a HOST scheduling policy (throttle waits, run-scoped force,
+     * busy recovery) wraps the verb without forking the loop. Default = the plain full snapshot.
+     * Anything it throws takes the loop's refusal arm exactly as the plain call would.
+     */
+    takeSnapshot?: (cycle: number) => ReturnType<ConsolidationDriveEngine["snapshotConsolidationCandidates"]>;
+    /** Observation hook after each cycle row closes against the store (run persistence seat). */
+    onCycle?: (row: ConsolidationDriveCycleRow) => void | Promise<void>;
+}
+/**
+ * Drive consolidation to its FIXPOINT: repeat snapshot → commit → reconcile until nothing is left
+ * to fold, the fuse can no longer admit the smallest remaining product, or the engine refuses.
+ *
+ * Why a loop is the honest shape: the supersession fuse caps ONE plan at a fraction of the active
+ * set (0.25 shipped), so a library that accumulated for many sessions cannot be folded by a single
+ * plan — the engine refuses that by design, and a driver that fired one plan and stopped would be
+ * converting a safety property into a failure report.
+ *
+ * Every snapshot asks `full: true`: snapshots are CHANGE-DRIVEN by default, so cycle 2 of a stable
+ * library would answer an EMPTY candidate set and strand most of the library unfolded. `full` is
+ * the protocol's own name for "re-distill what is already there", which is exactly what every
+ * cycle of a first consolidation is.
+ *
+ * WHAT WAS PROPOSED IS NEVER WHAT IS COUNTED: every per-cycle figure is a STORE delta —
+ * `readFoldState()` before and after each cycle, `committed` is the difference. Receipts are not
+ * enough (a `completed` receipt can carry per-product freeze refusals; a `conflict` receipt says
+ * how many applied but not which); the store is the only witness that cannot be off by a refusal.
+ */
+export declare function driveConsolidationToFixpoint(engine: ConsolidationDriveEngine, scope: string, opts: DriveConsolidationOptions): Promise<ConsolidationDriveResult>;
+/** A host snapshot policy (the injectable `takeSnapshot`) may classify its own stop as PARKED by
+ *  throwing an error carrying this marker — e.g. a busy seat whose reconcile answered a conflict
+ *  plan awaiting the host valve. The drive maps it onto the parked stop family. */
+export declare const CONSOLIDATION_SNAPSHOT_PARKED: unique symbol;
+/**
+ * A host ABORT marker: an error the injectable snapshot policy tags with this symbol is NOT an
+ * engine refusal — it is the host tearing the run down (a cancelled wait, a shutdown hook). The
+ * drive RETHROWS it instead of composing a refusal stop, so the caller's run account stays
+ * PENDING and the paid mint survives to the resume — folding a host abort into a terminal
+ * `refused` would silently discard a whole mint over a shutdown.
+ */
+export declare const CONSOLIDATION_DRIVE_ABORT: unique symbol;