@qvac/sdk 0.10.2 → 0.12.0

package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.js

@@ -1,114 +1,31 @@
 /**
- * Pure state + decision helpers for kv-cache bookkeeping used by
- * `completion-stream.ts`.
+ * Pure decision helper for kv-cache history slicing used by
+ * `completion-stream.ts` (via the slice computed from
+ * `TurnHandle.savedCount`).
  *
  * This module intentionally has **no** `bare-*` imports so it can be
- * exercised directly from unit tests running under `bun` without pulling
- * in the Bare runtime (which is not available in that environment).
- * The file-system-dependent pieces (e.g. `recordCacheSaveCount`) live in
- * `completion-stream.ts` and consume the state exported here.
- */
-/**
- * Number of chat messages the kv-cache file on disk is known to cover, keyed
- * by cache path. Written after a successful completion records a save, read
- * by `prepareMessagesForCache` to slice the history on the next turn so a
- * consumer can stage multiple messages between completions (e.g. an
- * `[assistant, user]` recovery sequence) without resending the whole history.
- *
- * INVARIANT: an entry is only present if the corresponding kv-cache file is
- * considered trustworthy. On any turn where the SDK cannot prove the saved
- * count reflects the on-disk state (cancellation mid-decode, zero-token
- * reply, cache file missing after a save attempt), the entry MUST be
- * deleted; a stale entry causes the next turn to slice its history down to
- * an empty payload and the model returns zero tokens.
- *
- * Mode notes:
- *   - Static-mode turns are the only readers of this map. `clearStaleCount`
- *     in `decideCachedHistorySlice` exists so a stale-but-non-zero entry
- *     can be detected and dropped on the next read.
- *   - Dynamic-mode turns do NOT consume this map — the addon trims tools
- *     and the chain output from the kv-cache after each round, so the
- *     SDK falls back to role-based dispatch in `prepareMessagesForCache`.
- *     Writes still happen on dynamic-mode turns, but the recorded count
- *     reflects the messages the SDK shipped, not the (possibly trimmed)
- *     on-disk cache shape. The map is internally consistent within a
- *     single mode; a `kvCache` key should not be reused across modes.
- */
-export const cachedMessageCounts = new Map();
-/**
- * Monotonic counter of cancel requests per model. `completion()` snapshots
- * the value before running the model and compares afterward to decide
- * whether the turn was cancelled mid-decode. A counter (not a boolean) is
- * used deliberately: a cancel that lands near a turn boundary cannot bleed
- * into the next turn because we always compare snapshots, not values.
- */
-const modelCancelCounters = new Map();
-/**
- * Clear bookkeeping entries. With no argument, clears the whole map. With a
- * `prefix`, removes any entry whose path is equal to it OR sits beneath it
- * as a directory (i.e. `key.startsWith(prefix + sep)`).
+ * exercised directly from unit tests running under `bun` without
+ * pulling in the Bare runtime (which is not available in that
+ * environment).
  *
- * Runtime callers MUST pass the platform path separator (e.g. `path.sep`
- * from `bare-path`) so directory-prefix matches are correct on every
- * target. The "/" default exists only for unit tests under `bun`, where
- * cache paths are POSIX-shaped and importing `bare-path` would pull in
- * the Bare runtime. The exported wrapper in `completion-stream.ts`
- * injects the real separator for in-process use.
+ * Cancel detection flows through the per-request `AbortSignal` from
+ * `RequestRegistry`; the `cachedMessageCounts` map lives in
+ * `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
+ * layers (saved counts, init flags, on-disk files). Only the pure
+ * slice-decision helper remains here.
  */
-export function clearCachedMessageCounts(prefix, sep = "/") {
-    if (!prefix) {
-        cachedMessageCounts.clear();
-        return;
-    }
-    for (const key of cachedMessageCounts.keys()) {
-        if (key === prefix) {
-            cachedMessageCounts.delete(key);
-            continue;
-        }
-        if (!key.startsWith(prefix + sep))
-            continue;
-        cachedMessageCounts.delete(key);
-    }
-}
-/**
- * Called from `server/bare/ops/cancel.ts` right before `addon.cancel()` so
- * that the in-flight `completion()` for this model can detect that its
- * current turn is being cancelled and skip poisoning `cachedMessageCounts`
- * with a `history.length + 1` entry that does not correspond to a real
- * assistant reply.
- */
-export function noteCancelRequested(modelId) {
-    modelCancelCounters.set(modelId, (modelCancelCounters.get(modelId) ?? 0) + 1);
-}
-export function snapshotCancelCount(modelId) {
-    return modelCancelCounters.get(modelId) ?? 0;
-}
-/** Test-only. */
-export function _resetCancelCountersForTest() {
-    modelCancelCounters.clear();
-}
-/**
- * A completion's `savedCount` should only be recorded when the turn ran to
- * completion AND produced at least one token. Any other outcome —
- * cancelled mid-decode, legitimate zero-token reply, or an early EOS — must
- * clear the entry instead, because there is no guarantee that the kv-cache
- * file on disk matches the `history.length + 1` boundary the SDK would
- * otherwise record.
- */
-export function shouldRecordSavedCount(wasCancelled, producedTokens) {
-    return !wasCancelled && producedTokens;
-}
 /**
  * Pure slice decision for `prepareMessagesForCache`.
  *
  * Mirrors the shape of the logic in `completion-stream.ts` but without
- * calling `transformMessages` (which depends on `bare-fs` for attachment
- * probing). Kept here so the decision can be unit-tested in isolation.
+ * calling `transformMessages` (which depends on `bare-fs` for
+ * attachment probing). Kept here so the decision can be unit-tested in
+ * isolation.
  *
- * The key regression guard: when a non-zero `savedCount` would slice the
- * history down to an empty array, it is treated as stale — the caller
- * falls back to sending the system-stripped full history rather than
- * handing the model an empty payload.
+ * The key regression guard: when a non-zero `savedCount` would slice
+ * the history down to an empty array, it is treated as stale — the
+ * caller falls back to sending the system-stripped full history rather
+ * than handing the model an empty payload.
  */
 export function decideCachedHistorySlice(savedCount, cacheExists, history) {
     if (!cacheExists || history.length === 0) {

package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts

@@ -28,7 +28,18 @@ export declare const llmPlugin: {
         }>>;
         "cache-type-k": import("zod").ZodOptional<import("zod").ZodString>;
         "cache-type-v": import("zod").ZodOptional<import("zod").ZodString>;
+        "main-gpu": import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodEnum<{
+            integrated: "integrated";
+            dedicated: "dedicated";
+        }>]>>;
+        "split-mode": import("zod").ZodOptional<import("zod").ZodEnum<{
+            none: "none";
+            layer: "layer";
+            row: "row";
+        }>>;
+        "tensor-split": import("zod").ZodOptional<import("zod").ZodString>;
         openclCacheDir: import("zod").ZodOptional<import("zod").ZodString>;
+        reasoning_budget: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodLiteral<-1>, import("zod").ZodLiteral<0>]>>;
         projectionModelSrc: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
             src: import("zod").ZodString;
             name: import("zod").ZodOptional<import("zod").ZodString>;
@@ -42,21 +53,26 @@ export declare const llmPlugin: {
             sha256Checksum: import("zod").ZodOptional<import("zod").ZodString>;
             addon: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodEnum<{
                 embeddings: "embeddings";
+                diffusion: "diffusion";
+                tts: "tts";
+                ocr: "ocr";
+                vla: "vla";
                 "llamacpp-completion": "llamacpp-completion";
                 "whispercpp-transcription": "whispercpp-transcription";
                 "llamacpp-embedding": "llamacpp-embedding";
                 "nmtcpp-translation": "nmtcpp-translation";
                 "onnx-tts": "onnx-tts";
+                "tts-ggml": "tts-ggml";
                 "parakeet-transcription": "parakeet-transcription";
                 "onnx-ocr": "onnx-ocr";
                 "sdcpp-generation": "sdcpp-generation";
+                "ggml-vla": "ggml-vla";
+                "ggml-classification": "ggml-classification";
                 llm: "llm";
                 whisper: "whisper";
                 nmt: "nmt";
                 parakeet: "parakeet";
-                tts: "tts";
-                ocr: "ocr";
-                diffusion: "diffusion";
+                classification: "classification";
             }>, import("zod").ZodLiteral<"vad">]>>;
         }, import("zod/v4/core").$strip>]>>;
     }, import("zod/v4/core").$strip>;
@@ -83,7 +99,11 @@ export declare const llmPlugin: {
             toolsMode?: "static" | "dynamic" | undefined;
             "cache-type-k"?: string | undefined;
             "cache-type-v"?: string | undefined;
+            "main-gpu"?: number | "integrated" | "dedicated" | undefined;
+            "split-mode"?: "none" | "layer" | "row" | undefined;
+            "tensor-split"?: string | undefined;
             openclCacheDir?: string | undefined;
+            reasoning_budget?: 0 | -1 | undefined;
         };
         artifacts?: never;
     } | {
@@ -109,7 +129,11 @@ export declare const llmPlugin: {
             toolsMode?: "static" | "dynamic" | undefined;
             "cache-type-k"?: string | undefined;
             "cache-type-v"?: string | undefined;
+            "main-gpu"?: number | "integrated" | "dedicated" | undefined;
+            "split-mode"?: "none" | "layer" | "row" | undefined;
+            "tensor-split"?: string | undefined;
             openclCacheDir?: string | undefined;
+            reasoning_budget?: 0 | -1 | undefined;
         };
         artifacts: {
             projectionModelPath: string;
@@ -158,6 +182,7 @@ export declare const llmPlugin: {
                 frequency_penalty: import("zod").ZodOptional<import("zod").ZodNumber>;
                 presence_penalty: import("zod").ZodOptional<import("zod").ZodNumber>;
                 repeat_penalty: import("zod").ZodOptional<import("zod").ZodNumber>;
+                reasoning_budget: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodLiteral<-1>, import("zod").ZodLiteral<0>]>>;
             }, import("zod/v4/core").$strict>>;
             captureThinking: import("zod").ZodOptional<import("zod").ZodBoolean>;
             emitRawDeltas: import("zod").ZodOptional<import("zod").ZodBoolean>;
@@ -166,6 +191,8 @@ export declare const llmPlugin: {
                 pythonic: "pythonic";
                 json: "json";
                 harmony: "harmony";
+                qwen35: "qwen35";
+                gemma4: "gemma4";
             }>>;
             responseFormat: import("zod").ZodOptional<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
                 type: import("zod").ZodLiteral<"text">;
@@ -180,6 +207,7 @@ export declare const llmPlugin: {
                     strict: import("zod").ZodOptional<import("zod").ZodBoolean>;
                 }, import("zod/v4/core").$strict>;
             }, import("zod/v4/core").$strict>], "type">>;
+            requestId: import("zod").ZodOptional<import("zod").ZodString>;
             type: import("zod").ZodLiteral<"completionStream">;
         }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
             type: import("zod").ZodLiteral<"completionStream">;
@@ -224,6 +252,7 @@ export declare const llmPlugin: {
                     timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
                     tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
                     cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
+                    promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
                     generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
                     backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
                         cpu: "cpu";
@@ -244,6 +273,7 @@ export declare const llmPlugin: {
                 type: import("zod").ZodLiteral<"completionDone">;
                 seq: import("zod").ZodNumber;
                 stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
+                    cancelled: "cancelled";
                     eos: "eos";
                     length: "length";
                     stopSequence: "stopSequence";
@@ -315,6 +345,7 @@ export declare const llmPlugin: {
                 resume: "resume";
             }>>;
             withProgress: import("zod").ZodOptional<import("zod").ZodBoolean>;
+            requestId: import("zod").ZodOptional<import("zod").ZodString>;
         }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
             modelId: import("zod").ZodString;
             options: import("zod").ZodObject<{
@@ -412,6 +443,7 @@ export declare const llmPlugin: {
                 nmt: "nmt";
             }>;
             type: import("zod").ZodLiteral<"translate">;
+            requestId: import("zod").ZodOptional<import("zod").ZodString>;
         }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
             modelId: import("zod").ZodString;
             text: import("zod").ZodString;
@@ -424,6 +456,7 @@ export declare const llmPlugin: {
             to: import("zod").ZodString;
             context: import("zod").ZodOptional<import("zod").ZodString>;
             type: import("zod").ZodLiteral<"translate">;
+            requestId: import("zod").ZodOptional<import("zod").ZodString>;
         }, import("zod/v4/core").$strip>]>, import("zod").ZodObject<{
             type: import("zod").ZodLiteral<"translate">;
             token: import("zod").ZodString;

package/dist/server/bare/plugins/llamacpp-completion/plugin.js

@@ -1,41 +1,73 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
 import LlmLlamacpp from "@qvac/llm-llamacpp";
 import llmAddonLogging from "@qvac/llm-llamacpp/addonLogging.js";
-import { definePlugin, defineHandler, finetuneRequestSchema, completionStreamRequestSchema, completionStreamResponseSchema, finetuneResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM, TOOLS_MODE, } from "../../../../schemas/index.js";
+import { definePlugin, defineHandler, finetuneRequestSchema, completionStreamRequestSchema, completionStreamResponseSchema, finetuneResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM, } from "../../../../schemas/index.js";
 import { createStreamLogger, registerAddonLogger } from "../../../../logging/index.js";
 import { expandGGUFIntoShards } from "../../../utils/index.js";
 import { completion } from "./ops/completion-stream.js";
 import { finetune } from "./ops/finetune.js";
 import { translate } from "../../ops/translate.js";
+import { transformLlmConfig } from "./transform.js";
 import { attachModelExecutionMs } from "../../../../profiling/model-execution.js";
 import { getModelConfig } from "../../registry/model-registry.js";
 import { createCompletionNormalizer } from "../../../utils/completion-normalizer.js";
 import { detectToolDialect } from "../../../utils/tool-integration.js";
-function transformLlmConfig(llmConfig) {
-    const transformed = JSON.parse(JSON.stringify(llmConfig, (key, v) => key === "modelType"
-        ? undefined
-        : key === "stop_sequences"
-            ? Array.isArray(v)
-                ? v.join(", ")
-                : v
-            : typeof v === "number" || typeof v === "boolean"
-                ? String(v)
-                : v).replace(/"([a-z][A-Za-z]*)":/g, (_, key) => `"${key.replace(/[A-Z]/g, (l) => `_${l.toLowerCase()}`)}":`));
-    if ("stop_sequences" in transformed) {
-        transformed["reverse_prompt"] = transformed["stop_sequences"];
-        delete transformed["stop_sequences"];
-    }
-    if ("opencl_cache_dir" in transformed) {
-        transformed["openclCacheDir"] = transformed["opencl_cache_dir"];
-        delete transformed["opencl_cache_dir"];
-    }
-    if ("tools_mode" in transformed) {
-        if (transformed["tools_mode"] === TOOLS_MODE.dynamic) {
-            transformed["tools_compact"] = "true";
-        }
-        delete transformed["tools_mode"];
-    }
-    return transformed;
-}
+import { getRequestRegistry, withRequestContext, } from "../../runtime/index.js";
+import { generateServerRequestId } from "../../runtime/request-id.js";
+import { getServerLogger } from "../../../../logging/index.js";
+import { ContextOverflowError } from "../../../../utils/errors-server.js";
+import { isAddonContextOverflowError, parseContextOverflowMessage, } from "./ops/context-overflow.js";
 function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
     const logger = createStreamLogger(modelId, ModelType.llamacppCompletion);
     registerAddonLogger(modelId, ModelType.llamacppCompletion, logger);
@@ -78,71 +110,119 @@ export const llmPlugin = definePlugin({
             requestSchema: completionStreamRequestSchema,
             responseSchema: completionStreamResponseSchema,
             streaming: true,
+            cancel: { scope: "model", hard: true },
             handler: async function* (request) {
-                const filteredHistory = request.history.map(({ role, content, attachments }) => ({
-                    role,
-                    content,
-                    attachments: attachments ?? [],
-                }));
-                const modelCfg = getModelConfig(request.modelId);
-                const toolsActive = (request.tools?.length ?? 0) > 0 &&
-                    modelCfg.tools === true;
-                const capabilities = {
-                    toolCalling: toolsActive ? "textParse" : "none",
-                    thinkingFraming: request.captureThinking ? "thinkTags" : "none",
-                };
-                // Dialect runs regardless of tool availability — thinking/content
-                // stripping is needed even on plain completions.
-                const dialect = request.toolDialect ?? detectToolDialect(request.modelId);
-                const normalizer = createCompletionNormalizer({
-                    capabilities,
-                    tools: request.tools ?? [],
-                    captureThinking: request.captureThinking ?? false,
-                    emitRawDeltas: request.emitRawDeltas ?? false,
-                    toolDialect: dialect,
-                });
-                const stream = completion({
-                    history: filteredHistory,
-                    modelId: request.modelId,
-                    kvCache: request.kvCache,
-                    ...(toolsActive && request.tools && { tools: request.tools }),
-                    ...(request.generationParams && { generationParams: request.generationParams }),
-                    ...(toolsActive && { toolDialect: dialect }),
-                    ...(request.responseFormat && { responseFormat: request.responseFormat }),
-                });
+                const env_1 = { stack: [], error: void 0, hasError: false };
                 try {
-                    const batchedEvents = [];
-                    let result = await stream.next();
-                    while (!result.done) {
-                        const events = normalizer.push(result.value.token);
-                        if (request.stream) {
-                            yield {
-                                type: "completionStream",
-                                events,
-                            };
+                    const filteredHistory = request.history.map(({ role, content, attachments }) => ({
+                        role,
+                        content,
+                        attachments: attachments ?? [],
+                    }));
+                    const modelCfg = getModelConfig(request.modelId);
+                    const toolsActive = (request.tools?.length ?? 0) > 0 &&
+                        modelCfg.tools === true;
+                    const capabilities = {
+                        toolCalling: toolsActive ? "textParse" : "none",
+                        thinkingFraming: request.captureThinking ? "thinkTags" : "none",
+                    };
+                    // Dialect runs regardless of tool availability — thinking/content
+                    // stripping is needed even on plain completions.
+                    const dialect = request.toolDialect ?? detectToolDialect(request.modelId);
+                    const normalizer = createCompletionNormalizer({
+                        capabilities,
+                        tools: request.tools ?? [],
+                        captureThinking: request.captureThinking ?? false,
+                        emitRawDeltas: request.emitRawDeltas ?? false,
+                        toolDialect: dialect,
+                    });
+                    // Open a request-scoped lifecycle. The registry is the single
+                    // source of truth for "is this turn cancelled?" — we plumb the
+                    // signal into `completion()` and expose `requestId` so the
+                    // client can target this run with `cancel({ requestId })`.
+                    // Falls back to a server-generated id if the client (e.g. an
+                    // older release) didn't send one.
+                    const ctx = __addDisposableResource(env_1, getRequestRegistry().begin({
+                        requestId: request.requestId ?? generateServerRequestId(),
+                        kind: "completion",
+                        modelId: request.modelId,
+                    }), true);
+                    const requestLogger = withRequestContext(getServerLogger(), ctx);
+                    const stream = completion({
+                        history: filteredHistory,
+                        modelId: request.modelId,
+                        kvCache: request.kvCache,
+                        ...(toolsActive && request.tools && { tools: request.tools }),
+                        ...(request.generationParams && { generationParams: request.generationParams }),
+                        ...(toolsActive && { toolDialect: dialect }),
+                        ...(request.responseFormat && { responseFormat: request.responseFormat }),
+                    }, { signal: ctx.signal, scope: ctx.scope, logger: requestLogger });
+                    try {
+                        const batchedEvents = [];
+                        let result = await stream.next();
+                        while (!result.done) {
+                            const events = normalizer.push(result.value.token);
+                            if (request.stream) {
+                                yield {
+                                    type: "completionStream",
+                                    events,
+                                };
+                            }
+                            else {
+                                batchedEvents.push(...events);
+                            }
+                            result = await stream.next();
                         }
-                        else {
-                            batchedEvents.push(...events);
+                        const { modelExecutionMs, stats, toolCalls } = result.value;
+                        // `stopReason: "cancelled"` rides the success-done path: the
+                        // events stream ends normally, the cancellation is observable
+                        // via the last event's `stopReason`, and the client-side
+                        // `CompletionRun` aggregates (`final` / `text` / `toolCalls` /
+                        // `stats`) reject with `InferenceCancelledError` carrying the
+                        // partial state.
+                        const cancelled = ctx.signal.aborted;
+                        const terminalEvents = normalizer.finish({
+                            ...(stats && { stats }),
+                            ...(toolCalls.length > 0 && { toolCalls }),
+                            ...(cancelled && { stopReason: "cancelled" }),
+                        });
+                        if (!request.stream) {
+                            batchedEvents.push(...terminalEvents);
                         }
-                        result = await stream.next();
+                        const finalEvents = request.stream ? terminalEvents : batchedEvents;
+                        yield attachModelExecutionMs({
+                            type: "completionStream",
+                            done: true,
+                            events: finalEvents,
+                        }, modelExecutionMs);
                     }
-                    const { modelExecutionMs, stats, toolCalls } = result.value;
-                    const terminalEvents = normalizer.finish({
-                        ...(stats && { stats }),
-                        ...(toolCalls.length > 0 && { toolCalls }),
-                    });
-                    if (!request.stream) {
-                        batchedEvents.push(...terminalEvents);
+                    catch (err) {
+                        // The llama.cpp addon emits a structured `ContextOverflow` status
+                        // (LlmErrors.hpp::ContextOverflow = 14) when the prompt exceeds
+                        // the model's `ctx_size`. Bare's `js_throw_error(env, code, msg)`
+                        // surfaces it as a JS Error with `.code = "[ <addonId> :: ContextOverflow ]"`
+                        // and `.message` carrying the C++-formatted detail. Rethrow as
+                        // a typed `ContextOverflowError` so consumers can switch on the
+                        // class (and `err.code === SDK_SERVER_ERROR_CODES.CONTEXT_OVERFLOW`)
+                        // instead of substring-matching on the raw addon message.
+                        if (isAddonContextOverflowError(err)) {
+                            const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : "");
+                            throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
+                        }
+                        throw err;
                     }
-                    const finalEvents = request.stream ? terminalEvents : batchedEvents;
-                    yield attachModelExecutionMs({
-                        type: "completionStream",
-                        done: true,
-                        events: finalEvents,
-                    }, modelExecutionMs);
+                    finally {
+                        await stream.return?.(undefined);
+                    }
+                }
+                catch (e_1) {
+                    env_1.error = e_1;
+                    env_1.hasError = true;
                 }
                 finally {
-                    await stream.return?.(undefined);
+                    const result_1 = __disposeResources(env_1);
+                    if (result_1)
+                        await result_1;
                 }
             },
         }),
@@ -150,6 +230,14 @@ export const llmPlugin = definePlugin({
             requestSchema: finetuneRequestSchema,
             responseSchema: finetuneResponseSchema,
             streaming: false,
+            // Reality matches addon: llama.cpp exposes `model.cancel()` for
+            // the running finetune job, so we flip from `scope: "none"` to
+            // `{ scope: "model", hard: true }`. The `startFinetune` op
+            // forwards the registry's abort signal to that call; the broad
+            // `cancel({ modelId, kind: "finetune" })` and legacy
+            // `cancelFinetune(modelId)` paths both flow through the
+            // registry.
+            cancel: { scope: "model", hard: true },
             handler: function (request) {
                 return finetune(request);
             },
@@ -158,8 +246,9 @@ export const llmPlugin = definePlugin({
             requestSchema: translateRequestSchema,
             responseSchema: translateResponseSchema,
             streaming: true,
+            cancel: { scope: "model", hard: true },
             handler: async function* (request) {
-                const stream = translate(request);
+                const stream = translate(request, request.requestId);
                 try {
                     let result = await stream.next();
                     while (!result.done) {
@@ -177,6 +266,15 @@ export const llmPlugin = definePlugin({
                         ...(stats && { stats }),
                     }, modelExecutionMs);
                 }
+                catch (err) {
+                    // Same addon, same overflow path as `completionStream`. Wrap so
+                    // translate consumers can `instanceof ContextOverflowError` too.
+                    if (isAddonContextOverflowError(err)) {
+                        const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : "");
+                        throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
+                    }
+                    throw err;
+                }
                 finally {
                     await stream.return?.(undefined);
                 }

package/dist/server/bare/plugins/llamacpp-completion/transform.d.ts

@@ -0,0 +1,10 @@
+import { type LlmConfig } from "../../../../schemas/index";
+/**
+ * Converts an LlmConfig into the flat string-keyed map the C++ addon expects.
+ *
+ * JS-only fields excluded from the output (must NOT be forwarded to the addon):
+ *   - modelType   (schema discriminant, meaningless at C++ level)
+ *   - system_prompt  (JS-side history seeding only; C++ removed --system-prompt in 8189)
+ */
+export declare function transformLlmConfig(llmConfig: LlmConfig): Record<string, string>;
+//# sourceMappingURL=transform.d.ts.map

package/dist/server/bare/plugins/llamacpp-completion/transform.js

@@ -0,0 +1,34 @@
+import { TOOLS_MODE } from "../../../../schemas/index.js";
+/**
+ * Converts an LlmConfig into the flat string-keyed map the C++ addon expects.
+ *
+ * JS-only fields excluded from the output (must NOT be forwarded to the addon):
+ *   - modelType   (schema discriminant, meaningless at C++ level)
+ *   - system_prompt  (JS-side history seeding only; C++ removed --system-prompt in 8189)
+ */
+export function transformLlmConfig(llmConfig) {
+    const transformed = JSON.parse(JSON.stringify(llmConfig, (key, v) => key === "modelType" || key === "system_prompt"
+        ? undefined
+        : key === "stop_sequences"
+            ? Array.isArray(v)
+                ? v.join(", ")
+                : v
+            : typeof v === "number" || typeof v === "boolean"
+                ? String(v)
+                : v).replace(/"([a-z][A-Za-z]*)":/g, (_, key) => `"${key.replace(/[A-Z]/g, (l) => `_${l.toLowerCase()}`)}":`));
+    if ("stop_sequences" in transformed) {
+        transformed["reverse_prompt"] = transformed["stop_sequences"];
+        delete transformed["stop_sequences"];
+    }
+    if ("opencl_cache_dir" in transformed) {
+        transformed["openclCacheDir"] = transformed["opencl_cache_dir"];
+        delete transformed["opencl_cache_dir"];
+    }
+    if ("tools_mode" in transformed) {
+        if (transformed["tools_mode"] === TOOLS_MODE.dynamic) {
+            transformed["tools_compact"] = "true";
+        }
+        delete transformed["tools_mode"];
+    }
+    return transformed;
+}