@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/structured-output.d.ts

@@ -0,0 +1,311 @@
+/**
+ * Structured-output / forced-span / prefill plumbing for the local-inference
+ * engine path.
+ *
+ * The canonical contract lives in `@elizaos/core` `GenerateTextParams`
+ * (`prefill`, `responseSkeleton`, `grammar`, `streamStructured`) and is
+ * threaded through `useModel` → router. This module is the
+ * local-inference-layer mirror of the relevant subset plus the GBNF
+ * compilation that turns a `ResponseSkeleton` into a *lazy* grammar so the
+ * model only ever samples the free positions of the response envelope
+ * (single-value enums collapse to literals — no tokens spent on the scaffold).
+ *
+ * Nothing here is local-model-specific in shape; cloud adapters never read
+ * these fields. There is no fallback path — adapters that can't honour
+ * `grammar` / `prefill` / `responseSkeleton` ignore them, full stop.
+ */
+import type { JSONSchema, ResponseSkeleton, ResponseSkeletonSpan, SpanSamplerPlan } from "@elizaos/core";
+export { repairStructuredOutput, type StructuredOutputRepairOptions, type StructuredOutputRepairResult, type StructuredOutputRepairStatus, StructuredOutputRepairStream, } from "./structured-output/deterministic-repair";
+export type { ResponseSkeleton, ResponseSkeletonSpan, SpanSamplerPlan };
+/**
+ * GBNF grammar fragment ready for a llama-server request body. `lazy` grammars
+ * only kick in once a trigger word/sequence appears in the stream
+ * (llama.cpp's `grammar_lazy` + `grammar_triggers`) — that lets the model
+ * free-run the prose `replyText` and only constrain the structured scaffold
+ * once the envelope boundary is reached.
+ */
+export interface GbnfGrammar {
+    /** GBNF source. */
+    source: string;
+    /** When true, the server applies the grammar lazily (`grammar_lazy: true`). */
+    lazy?: boolean;
+    /** Trigger words that activate a lazy grammar (`grammar_triggers`). */
+    triggers?: ReadonlyArray<string>;
+}
+/**
+ * Local-inference mirror of the structured-output extensions on
+ * `GenerateTextParams`. Threaded `useModel` → router → local handler →
+ * engine → FFI runtime.
+ */
+export interface StructuredGenerateParams {
+    /**
+     * Assistant-turn prefill — a partial assistant message the model should
+     * *continue* rather than start fresh. On llama-server this is sent as a
+     * trailing assistant message with `continue_final_message` / the
+     * `assistant` chat-template prefix; the capacitor-llama path seeds the
+     * prompt text and re-prepends the prefill to the result.
+     */
+    prefill?: string;
+    /**
+     * Forced response skeleton. When set the engine compiles it to a lazy GBNF
+     * (single-value enums → literals) so the model only samples the free
+     * positions of the envelope.
+     */
+    responseSkeleton?: ResponseSkeleton;
+    /** Optional whole-response JSON schema from `GenerateTextParams`. */
+    responseSchema?: JSONSchema;
+    /**
+     * Explicit GBNF grammar string. When both `grammar` and `responseSkeleton`
+     * are present, the explicit `grammar` wins.
+     */
+    grammar?: string;
+    /**
+     * When true, the engine streams per-token chunks back via `onTextChunk`
+     * (and structured-field events) instead of returning the whole string in
+     * one shot.
+     */
+    streamStructured?: boolean;
+    /**
+     * The eliza harness schema for this call — the compact descriptor bundling
+     * the response skeleton, a pre-built grammar (optional), the derived
+     * deterministic-token {@link ElizaPrefillPlan}, and the short/long name maps.
+     * When present, guided structured decode is *on* for this call: the engine
+     * sends the grammar AND the prefill plan, and seeds the leading literal run
+     * as an assistant-turn prefill. Absent → guided decode is off (the engine
+     * may still honour a bare `grammar` / `responseSkeleton`, but never emits a
+     * prefill plan). This is the off-by-default switch for the deterministic
+     * short-circuit.
+     */
+    elizaSchema?: ElizaHarnessSchema;
+    /**
+     * Per-span sampler overrides for the {@link responseSkeleton}. When set,
+     * the engine emits `eliza_span_samplers` on the llama-server request body so
+     * the fork-side server swaps to argmax (`llama_sampler_init_greedy()`) at
+     * the indicated enum / number / boolean positions. Stock llama-server
+     * ignores the field — the grammar still constrains the same tokens, we
+     * just lose the argmax determinism guarantee on that path.
+     *
+     * Producer: `@elizaos/core` `buildSpanSamplerPlan(skeleton)`.
+     */
+    spanSamplerPlan?: SpanSamplerPlan;
+    /**
+     * Per-request chat-template thinking control for reasoning-capable local
+     * models. `off` maps to `chat_template_kwargs.enable_thinking=false` for
+     * response-handler/direct-reply calls that must emit user-visible text, while
+     * planner/action calls can omit this and use the catalog/server default.
+     */
+    thinking?: "auto" | "on" | "off";
+}
+/**
+ * Collapse a skeleton: `enum` spans with exactly one value (or zero values)
+ * become `literal` spans (C4). Adjacent literals stay separate spans — the
+ * compiler merges them in the root rule.
+ */
+export declare function collapseSkeleton(skeleton: ResponseSkeleton): ResponseSkeleton;
+/**
+ * Compile a `ResponseSkeleton` to a *lazy* GBNF grammar. The grammar's `root`
+ * rule is the concatenation of every span:
+ *   - `literal` spans → GBNF string literals (the JSON key/glue scaffold),
+ *   - `enum` spans (≥2 values) → an alternation of quoted-string literals,
+ *   - `free-string` spans → a quoted JSON string rule,
+ *   - `free-json` spans → the recursive JSON-value rule.
+ *
+ * The grammar runs *lazily* when the skeleton opens with a literal (the
+ * trigger word) — generation free-runs until that literal is seen, then the
+ * grammar pins the rest of the envelope. That keeps the prose prefix
+ * unconstrained while forcing the JSON scaffold.
+ *
+ * Returns `null` when the skeleton has no free spans (nothing for the model to
+ * sample — the caller should just emit the literal text and skip generation).
+ */
+export declare function compileSkeletonToGbnf(skeletonInput: ResponseSkeleton): GbnfGrammar | null;
+/**
+ * Resolve the GBNF grammar to apply for a generation call. Precedence: an
+ * explicit `grammar` string on the params, then a compiled `responseSkeleton`.
+ * Returns null when neither is set.
+ */
+export declare function resolveGrammarForParams(params: StructuredGenerateParams | undefined): GbnfGrammar | null;
+/**
+ * Build the OpenAI-/llama-server-compatible request-body fragment for a
+ * grammar. Returns `grammar` + (when lazy) `grammar_lazy` / `grammar_triggers`.
+ * Recent llama.cpp accepts these on both `/v1/chat/completions` and
+ * `/completion`.
+ */
+export declare function grammarRequestFields(grammar: GbnfGrammar): Record<string, unknown>;
+/**
+ * Split a skeleton's leading literal run off as an assistant-turn prefill
+ * candidate, returning that prefix plus the remaining spans. Used by the
+ * multi-call infill fallback (emit prefix as a prefill, generate the first
+ * free span, then loop).
+ */
+export declare function splitSkeletonAtFirstFree(skeleton: ResponseSkeleton): {
+    prefixLiteral: string;
+    rest: ResponseSkeletonSpan[];
+};
+/**
+ * One deterministically-forced byte run in an {@link ElizaPrefillPlan}. The
+ * runs alternate with the free (sampled) spans, so a run is unambiguously
+ * anchored by *position* in that alternation rather than by an absolute byte
+ * offset (the sampled spans have unknown length at plan time):
+ *
+ *   run[0]  free[0]  run[1]  free[1]  …  run[n]   (n = number of free spans)
+ *
+ * `afterFreeSpan` is `-1` for the leading run (before any free span — the
+ * assistant-turn prefill), then `0, 1, 2, …` for the run that follows free
+ * span 0, 1, 2, … . The server resumes sampling after writing each run; once
+ * the matching free span is sampled it writes the next run's token ids without
+ * a forward pass and advances the decoder to the next free span.
+ */
+export interface PrefillRun {
+    /**
+     * Index of the free span this run *follows*. `-1` = the leading run (the
+     * prefill); `k >= 0` = the run after free span `k`. The last run (`n`) is the
+     * tail scaffold (closing braces) after the final free span.
+     */
+    afterFreeSpan: number;
+    /** The deterministically-forced bytes. */
+    text: string;
+    /**
+     * Optional pre-tokenized token IDs for this run. When provided at compile time
+     * via a tokenizer callback, the FFI runtime can use these directly without
+     * re-tokenizing, improving latency.
+     */
+    tokenIds?: number[];
+}
+/**
+ * Compact descriptor of the deterministic structure of a constrained decode:
+ * the ordered runs of bytes that are fixed (so the server can prefill their
+ * token ids and skip the forward passes) interleaved with the count of free
+ * positions, plus the leading literal run that should be seeded as an
+ * assistant-turn prefill (`prefix`). Sent on the request as `eliza_prefill_plan`.
+ *
+ * Purely a speedup hint — a server that ignores it produces the identical
+ * output because the lazy GBNF already forces the same bytes.
+ */
+export interface ElizaPrefillPlan {
+    /**
+     * The leading deterministic run — emitted as an assistant-turn prefill so
+     * the model never samples it. Empty when the skeleton opens with a free span.
+     */
+    prefix: string;
+    /**
+     * Deterministic byte runs alternating with the free spans (see
+     * {@link PrefillRun}), in output order, including the prefix run when
+     * non-empty.
+     */
+    runs: PrefillRun[];
+    /** Number of free (sampled) spans in the skeleton. `runs.length` is `freeCount + 1` minus the leading run when the skeleton starts free. */
+    freeCount: number;
+    /**
+     * Opaque cache key (mirrors the skeleton's `id`) so the server can cache the
+     * tokenised form of the runs across turns when the structure is unchanged.
+     */
+    id?: string;
+}
+/**
+ * Compute the {@link ElizaPrefillPlan} for a response skeleton: walk the spans,
+ * accumulating consecutive `literal` spans (and single-value enums collapsed to
+ * literals) into deterministic byte runs and counting the free spans. Adjacent
+ * literals merge into one run. Returns `null` when the skeleton has no
+ * deterministic runs at all (nothing to prefill).
+ *
+ * Invariant the consumer relies on: concatenating the runs interleaved with the
+ * (eventually-sampled) free-span values, in order, reproduces a byte-identical
+ * JSON document to what the lazy GBNF from {@link compileSkeletonToGbnf} would
+ * have produced. The tests assert this.
+ */
+export declare function compilePrefillPlan(skeletonInput: ResponseSkeleton, tokenize?: (text: string) => number[]): ElizaPrefillPlan | null;
+/**
+ * Build the request-body fragment carrying the prefill plan. The server reads
+ * `eliza_prefill_plan` (a tolerant extension — old binaries ignore it and the
+ * grammar still forces the same bytes). Returns `{}` when there is no plan.
+ */
+export declare function prefillPlanRequestFields(plan: ElizaPrefillPlan | null): Record<string, unknown>;
+/**
+ * Build the request-body fragment carrying per-span sampler overrides. The
+ * fork-side llama-server reads `eliza_span_samplers` (a tolerant extension —
+ * old binaries ignore it; the grammar still constrains the same tokens, we
+ * just lose the per-span argmax determinism guarantee on the legacy path).
+ *
+ * Wire schema (snake_case for OpenAI body conventions):
+ *   {
+ *     overrides: [
+ *       { span_index: number, temperature: number, top_k?: number, top_p?: number }
+ *     ],
+ *     strict?: boolean
+ *   }
+ *
+ * Returns `{}` when there is no plan or no overrides — keep the wire surface
+ * narrow so a stock server never has to skip past empty fork extensions.
+ */
+export declare function spanSamplerPlanRequestFields(plan: SpanSamplerPlan | undefined | null): Record<string, unknown>;
+/**
+ * The compact, engine-facing descriptor for a structured output the agent loop
+ * wants forced. It is the bundle of (a) a {@link ResponseSkeleton} (which
+ * compiles to a lazy GBNF for the constrained-decode path), (b) the derived
+ * {@link ElizaPrefillPlan} (the deterministic-token short-circuit), and (c) the
+ * short-name ↔ long-name maps so the on-wire/decoded form uses canonical short
+ * action ids / enum values and the runtime expands them for the caller.
+ *
+ * Producers: `@elizaos/core` `buildPlannerActionGrammar` / `buildResponseGrammar`
+ * wrapped by {@link elizaHarnessSchemaFromSkeleton}. Consumer: the local engine
+ * (`ffi-streaming-backend.ts` / `engine.ts`).
+ */
+export interface ElizaHarnessSchema {
+    /** Structure-forcing description; compiles to a lazy GBNF. */
+    skeleton: ResponseSkeleton;
+    /** Pre-built GBNF (wins over compiling the skeleton), when the producer made one. */
+    grammar?: string;
+    /** Deterministic-token short-circuit derived from the skeleton. */
+    prefillPlan: ElizaPrefillPlan | null;
+    /**
+     * Canonical short id → human-facing long name (display label), for any
+     * closed enum the descriptor pins (action ids, known enum values). The wire
+     * form is the short id; callers that want the long name look it up here.
+     * Empty when nothing needs expanding.
+     */
+    longNames: Record<string, string>;
+    /** Cache key (the skeleton's id). */
+    id?: string;
+}
+/**
+ * Wrap a {@link ResponseSkeleton} (+ optional pre-built grammar + name map)
+ * into an {@link ElizaHarnessSchema}, computing the prefill plan. This is the
+ * single place the prefill plan is derived so producers don't each reimplement
+ * it.
+ */
+export declare function elizaHarnessSchemaFromSkeleton(input: {
+    skeleton: ResponseSkeleton;
+    grammar?: string;
+    longNames?: Record<string, string>;
+    tokenize?: (text: string) => number[];
+}): ElizaHarnessSchema;
+/**
+ * Expand a canonical short id decoded out of a constrained generation back to
+ * its human-facing long name (display label), using the descriptor's
+ * {@link ElizaHarnessSchema.longNames} map (sourced from the action catalog).
+ * Identity when there is no mapping — the canonical action ids
+ * (`normalizeActionName` results, e.g. `SEND_MESSAGE`) are already the on-wire
+ * form, so this is only meaningful when a producer registered a separate
+ * display label.
+ */
+export declare function expandShortName(schema: ElizaHarnessSchema | undefined, shortId: string): string;
+/**
+ * Invert {@link expandShortName}: given a (possibly long) name the caller
+ * supplied, return the canonical short id the wire form expects. Identity when
+ * the name is already a known short id or no mapping matches.
+ */
+export declare function canonicalizeShortName(schema: ElizaHarnessSchema | undefined, name: string): string;
+/**
+ * Resolve the GBNF + prefill plan + assistant-turn prefill to apply for a
+ * generation call given the structured params. Precedence for the grammar:
+ * an explicit `grammar` string, then a harness schema's `grammar`, then
+ * compiling the harness schema's / params' `responseSkeleton`. The prefill plan
+ * is only present when a harness schema is supplied (off by default).
+ */
+export declare function resolveGuidedDecodeForParams(params: StructuredGenerateParams | undefined): {
+    grammar: GbnfGrammar | null;
+    prefillPlan: ElizaPrefillPlan | null;
+    prefill: string | null;
+};
+//# sourceMappingURL=structured-output.d.ts.map

package/src/services/structured-output.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-output.d.ts","sourceRoot":"","sources":["structured-output.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EACX,UAAU,EACV,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,EACf,MAAM,eAAe,CAAC;AAEvB,OAAO,EACN,sBAAsB,EACtB,KAAK,6BAA6B,EAClC,KAAK,4BAA4B,EACjC,KAAK,4BAA4B,EACjC,4BAA4B,GAC5B,MAAM,0CAA0C,CAAC;AAClD,YAAY,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,eAAe,EAAE,CAAC;AAExE;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC3B,mBAAmB;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uEAAuE;IACvE,QAAQ,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;CACjC;AAED;;;;GAIG;AACH,MAAM,WAAW,wBAAwB;IACxC;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,qEAAqE;IACrE,cAAc,CAAC,EAAE,UAAU,CAAC;IAC5B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;OAUG;IACH,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,KAAK,CAAC;CACjC;AAiCD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,gBAAgB,CAe7E;AAoBD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CACpC,aAAa,EAAE,gBAAgB,GAC7B,WAAW,GAAG,IAAI,CA+EpB;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CACtC,MAAM,EAAE,wBAAwB,GAAG,SAAS,GAC1C,WAAW,GAAG,IAAI,CASpB;AA+BD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CACnC,OAAO,EAAE,WAAW,GAClB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAYzB;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,gBAAgB,GAAG;IACrE,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,oBAAoB,EAAE,CAAC;CAC7B,CAWA;AAsBD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,UAAU;IAC1B;;;;OAIG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,gBAAgB;IAChC;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,4IAA4I;IAC5I,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;CACZ;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CACjC,aAAa,EAAE,gBAAgB,EAC/B,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,GACnC,gBAAgB,GAAG,IAAI,CA0CzB;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACvC,IAAI,EAAE,gBAAgB,GAAG,IAAI,GAC3B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAmBzB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,4BAA4B,CAC3C,IAAI,EAAE,eAAe,GAAG,SAAS,GAAG,IAAI,GACtC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAczB;AAMD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,kBAAkB;IAClC,8DAA8D;IAC9D,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,qFAAqF;IACrF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mEAAmE;IACnE,WAAW,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC;;;;;OAKG;IACH,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,qCAAqC;IACrC,EAAE,CAAC,EAAE,MAAM,CAAC;CACZ;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,KAAK,EAAE;IACrD,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CACtC,GAAG,kBAAkB,CAQrB;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC9B,MAAM,EAAE,kBAAkB,GAAG,SAAS,EACtC,OAAO,EAAE,MAAM,GACb,MAAM,CAGR;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACpC,MAAM,EAAE,kBAAkB,GAAG,SAAS,EACtC,IAAI,EAAE,MAAM,GACV,MAAM,CAOR;AAED;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAC3C,MAAM,EAAE,wBAAwB,GAAG,SAAS,GAC1C;IACF,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CA8BA"}