@qvac/sdk 0.10.1 → 0.11.0

package/dist/server/bare/ops/transcribe.js

@@ -1,3 +1,55 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
 import { getModel, getModelConfig, getModelEntry, } from "../registry/model-registry.js";
 import { ModelType, } from "../../../schemas/index.js";
 import { createAudioStream } from "../utils/audio-input.js";
@@ -6,8 +58,9 @@ import { TranscriptionFailedError } from "../../../utils/errors-server.js";
 import { nowMs } from "../../../profiling/index.js";
 import { buildStreamResult } from "../../../profiling/model-execution.js";
 import { assertMetadataSupported, toTranscribeSegment, } from "../utils/transcribe-metadata.js";
+import { getRequestRegistry, withRequestContext, } from "../runtime/index.js";
+import { generateServerRequestId } from "../runtime/request-id.js";
 export { assertMetadataSupported, toTranscribeSegment, };
-const logger = getServerLogger();
 function hasRunStreaming(model) {
     return "runStreaming" in model && typeof model.runStreaming === "function";
 }
@@ -56,22 +109,59 @@ async function restorePrompt(modelId, originalConfig) {
         ...(miscConfig && { miscConfig }),
     });
 }
-export async function* transcribe(params) {
-    const { modelId, metadata } = params;
-    const engineType = getEngineModelType(modelId);
-    assertMetadataSupported(modelId, engineType, metadata);
-    const silenceMarker = SILENCE_MARKERS[engineType] ?? "";
-    const audioFormat = getAudioFormat(modelId, engineType);
-    const originalConfig = await applyPrompt(modelId, params.prompt, engineType);
-    let modelExecutionMs = 0;
-    let response;
+export async function* transcribe(params, requestId) {
+    const env_1 = { stack: [], error: void 0, hasError: false };
     try {
+        const { modelId, metadata } = params;
+        // Open a request-scoped lifecycle. The registry routes
+        // `cancel({ requestId })` and `cancel({ modelId, kind: "transcribe" })`
+        // through this context. Falls back to a server-generated id if the
+        // client didn't send one.
+        const ctx = __addDisposableResource(env_1, getRequestRegistry().begin({
+            requestId: requestId ?? generateServerRequestId(),
+            kind: "transcribe",
+            modelId,
+        }), true);
+        const requestLogger = withRequestContext(getServerLogger(), ctx);
+        const engineType = getEngineModelType(modelId);
+        assertMetadataSupported(modelId, engineType, metadata);
+        const silenceMarker = SILENCE_MARKERS[engineType] ?? "";
+        const audioFormat = getAudioFormat(modelId, engineType);
+        const originalConfig = await applyPrompt(modelId, params.prompt, engineType);
+        if (originalConfig) {
+            // `restorePrompt` runs on every exit path — happy, throw, cancel —
+            // via the scope. LIFO unwinding pairs with the addon-cancel detach
+            // below.
+            ctx.scope.defer(() => restorePrompt(modelId, originalConfig));
+        }
         const model = getModel(modelId);
+        // Hard-cancel wiring: whisper.cpp / parakeet expose model-wide
+        // `addon.cancel()`. The listener forwards an abort so the
+        // currently-running transcription stops decoding ASAP. The
+        // `if (ctx.signal.aborted) break` guard inside the iterate loop is
+        // the soft-cancel safety net for the case where the abort fires
+        // between the addon flag flipping and the iterator's next pull.
+        const onAbort = () => {
+            const addon = model.addon;
+            if (addon?.cancel) {
+                addon.cancel.call(addon).catch((err) => {
+                    requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
+                });
+            }
+        };
+        ctx.signal.addEventListener("abort", onAbort, { once: true });
+        if (ctx.signal.aborted)
+            onAbort();
+        ctx.scope.defer(() => {
+            ctx.signal.removeEventListener("abort", onAbort);
+        });
         const audioStream = await createAudioStream(params.audioChunk, audioFormat);
         const modelStart = nowMs();
-        response = (await model.run(audioStream));
+        const response = (await model.run(audioStream));
         for await (const output of response.iterate()) {
-            logger.debug("Streaming Transcription Update:", output);
+            if (ctx.signal.aborted)
+                break;
+            requestLogger.debug("Streaming Transcription Update:", output);
             const chunks = output;
             if (metadata) {
                 for (const chunk of chunks) {
@@ -91,41 +181,102 @@ export async function* transcribe(params) {
                 yield text;
             }
         }
-        modelExecutionMs = nowMs() - modelStart;
+        const modelExecutionMs = nowMs() - modelStart;
+        const stats = {
+            ...(response.stats?.audioDurationMs !== undefined && { audioDuration: response.stats.audioDurationMs }),
+            ...(response.stats?.realTimeFactor !== undefined && { realTimeFactor: response.stats.realTimeFactor }),
+            ...(response.stats?.tokensPerSecond !== undefined && { tokensPerSecond: response.stats.tokensPerSecond }),
+            ...(response.stats?.totalTokens !== undefined && { totalTokens: response.stats.totalTokens }),
+            ...(response.stats?.totalSegments !== undefined && { totalSegments: response.stats.totalSegments }),
+            ...(response.stats?.whisperEncodeMs !== undefined && { whisperEncodeTime: response.stats.whisperEncodeMs }),
+            ...(response.stats?.whisperDecodeMs !== undefined && { whisperDecodeTime: response.stats.whisperDecodeMs }),
+            ...(response.stats?.encoderMs !== undefined && { encoderTime: response.stats.encoderMs }),
+            ...(response.stats?.decoderMs !== undefined && { decoderTime: response.stats.decoderMs }),
+            ...(response.stats?.melSpecMs !== undefined && { melSpecTime: response.stats.melSpecMs }),
+        };
+        return buildStreamResult(modelExecutionMs, stats);
+    }
+    catch (e_1) {
+        env_1.error = e_1;
+        env_1.hasError = true;
     }
     finally {
-        if (originalConfig) {
-            await restorePrompt(modelId, originalConfig);
-        }
+        const result_1 = __disposeResources(env_1);
+        if (result_1)
+            await result_1;
     }
-    const stats = {
-        ...(response?.stats?.audioDurationMs !== undefined && { audioDuration: response.stats.audioDurationMs }),
-        ...(response?.stats?.realTimeFactor !== undefined && { realTimeFactor: response.stats.realTimeFactor }),
-        ...(response?.stats?.tokensPerSecond !== undefined && { tokensPerSecond: response.stats.tokensPerSecond }),
-        ...(response?.stats?.totalTokens !== undefined && { totalTokens: response.stats.totalTokens }),
-        ...(response?.stats?.totalSegments !== undefined && { totalSegments: response.stats.totalSegments }),
-        ...(response?.stats?.whisperEncodeMs !== undefined && { whisperEncodeTime: response.stats.whisperEncodeMs }),
-        ...(response?.stats?.whisperDecodeMs !== undefined && { whisperDecodeTime: response.stats.whisperDecodeMs }),
-        ...(response?.stats?.encoderMs !== undefined && { encoderTime: response.stats.encoderMs }),
-        ...(response?.stats?.decoderMs !== undefined && { decoderTime: response.stats.decoderMs }),
-        ...(response?.stats?.melSpecMs !== undefined && { melSpecTime: response.stats.melSpecMs }),
-    };
-    return buildStreamResult(modelExecutionMs, stats);
 }
-export async function* transcribeStream(modelId, audioInputStream, prompt, metadata) {
-    const engineType = getEngineModelType(modelId);
-    assertMetadataSupported(modelId, engineType, metadata);
-    const silenceMarker = SILENCE_MARKERS[engineType] ?? "";
-    const originalConfig = await applyPrompt(modelId, prompt, engineType);
+export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId) {
+    const env_2 = { stack: [], error: void 0, hasError: false };
     try {
+        // Same `kind: "transcribe"` as the unary variant — the registry
+        // doesn't distinguish streaming vs non-streaming variants of the same
+        // operation, so `cancel({ modelId, kind: "transcribe" })` cancels
+        // either shape.
+        const ctx = __addDisposableResource(env_2, getRequestRegistry().begin({
+            requestId: requestId ?? generateServerRequestId(),
+            kind: "transcribe",
+            modelId,
+        }), true);
+        const requestLogger = withRequestContext(getServerLogger(), ctx);
+        const engineType = getEngineModelType(modelId);
+        assertMetadataSupported(modelId, engineType, metadata);
+        const silenceMarker = SILENCE_MARKERS[engineType] ?? "";
+        const originalConfig = await applyPrompt(modelId, prompt, engineType);
+        if (originalConfig) {
+            ctx.scope.defer(() => restorePrompt(modelId, originalConfig));
+        }
         const model = getModel(modelId);
         if (!hasRunStreaming(model)) {
             throw new TranscriptionFailedError(`Model ${modelId} does not support streaming transcription`);
         }
-        const response = await model.runStreaming(audioInputStream);
-        for await (const segments of response.iterate()) {
-            logger.debug("Live Transcription Update:", segments);
-            for (const segment of segments) {
+        const onAbort = () => {
+            const addon = model.addon;
+            if (addon?.cancel) {
+                addon.cancel.call(addon).catch((err) => {
+                    requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
+                });
+            }
+        };
+        ctx.signal.addEventListener("abort", onAbort, { once: true });
+        if (ctx.signal.aborted)
+            onAbort();
+        ctx.scope.defer(() => {
+            ctx.signal.removeEventListener("abort", onAbort);
+        });
+        const runOpts = {};
+        if (opts?.emitVadEvents)
+            runOpts.emitVadEvents = true;
+        if (opts?.endOfTurnSilenceMs !== undefined) {
+            runOpts.endOfTurnSilenceMs = opts.endOfTurnSilenceMs;
+        }
+        if (opts?.vadRunIntervalMs !== undefined) {
+            runOpts.vadRunIntervalMs = opts.vadRunIntervalMs;
+        }
+        const response = await model.runStreaming(audioInputStream, runOpts);
+        for await (const output of response.iterate()) {
+            if (ctx.signal.aborted)
+                break;
+            requestLogger.debug("Live Transcription Update:", output);
+            if (!Array.isArray(output)) {
+                if (output.type === "vad") {
+                    yield {
+                        type: "vad",
+                        speaking: output.speaking,
+                        probability: output.probability,
+                    };
+                    continue;
+                }
+                if (output.type === "endOfTurn") {
+                    yield {
+                        type: "endOfTurn",
+                        silenceDurationMs: output.silenceDurationMs,
+                    };
+                    continue;
+                }
+                continue;
+            }
+            for (const segment of output) {
                 if (!segment.text)
                     continue;
                 if (silenceMarker && segment.text.includes(silenceMarker))
@@ -140,9 +291,13 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
             }
         }
     }
+    catch (e_2) {
+        env_2.error = e_2;
+        env_2.hasError = true;
+    }
     finally {
-        if (originalConfig) {
-            await restorePrompt(modelId, originalConfig);
-        }
+        const result_2 = __disposeResources(env_2);
+        if (result_2)
+            await result_2;
     }
 }

package/dist/server/bare/ops/translate.d.ts

@@ -1,7 +1,7 @@
 import { type TranslateParams, type TranslationStats } from "../../../schemas/index";
 export declare function getLanguage(code: string | undefined): string;
 export declare function isAfrican(code: string | undefined): boolean;
-export declare function translate(params: TranslateParams): AsyncGenerator<string, {
+export declare function translate(params: TranslateParams, requestId?: string): AsyncGenerator<string, {
     modelExecutionMs: number;
     stats?: TranslationStats;
 }, unknown>;

package/dist/server/bare/ops/translate.js

@@ -1,9 +1,64 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
 import { getModelEntry } from "../registry/model-registry.js";
 import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LANGUAGES_MAP, } from "../../../schemas/index.js";
 import { getLangName } from "@qvac/langdetect-text";
 import { nowMs } from "../../../profiling/index.js";
 import { buildStreamResult } from "../../../profiling/model-execution.js";
 import { ModelIsDelegatedError, ModelNotFoundError, ModelTypeMismatchError, } from "../../../utils/errors-server.js";
+import { getRequestRegistry, withRequestContext, } from "../runtime/index.js";
+import { generateServerRequestId } from "../runtime/request-id.js";
+import { getServerLogger } from "../../../logging/index.js";
 export function getLanguage(code) {
     if (!code)
         return "";
@@ -26,104 +81,158 @@ const LLM_TRANSLATE_GENERATION_PARAMS = {
 function shouldSkipPerCallSampling(modelName) {
     return !!modelName && modelName.startsWith("AFRICAN_");
 }
-export async function* translate(params) {
-    const { modelId, text, modelType: inputModelType } = params;
-    const entry = getModelEntry(modelId);
-    if (!entry) {
-        throw new ModelNotFoundError(modelId);
-    }
-    if (entry.isDelegated) {
-        throw new ModelIsDelegatedError(modelId);
-    }
-    const canonicalModelType = entry.local.modelType;
-    const model = entry.local.model;
-    if (inputModelType !== undefined) {
-        const requestedCanonical = normalizeModelType(inputModelType);
-        if (requestedCanonical !== canonicalModelType) {
-            throw new ModelTypeMismatchError(canonicalModelType, requestedCanonical);
+export async function* translate(params, requestId) {
+    const env_1 = { stack: [], error: void 0, hasError: false };
+    try {
+        const { modelId, text, modelType: inputModelType } = params;
+        const entry = getModelEntry(modelId);
+        if (!entry) {
+            throw new ModelNotFoundError(modelId);
         }
-    }
-    const isLlm = canonicalModelType === ModelType.llamacppCompletion;
-    const from = isLlm ? params.from : undefined;
-    const to = isLlm ? params.to : undefined;
-    const context = isLlm ? params.context : undefined;
-    const afriquePrompt = isLlm && (isAfrican(from) || isAfrican(to));
-    translateServerParamsSchema.parse(params);
-    const fromLanguage = getLanguage(from);
-    const toLanguage = getLanguage(to);
-    // Check if input is an array and model type is NMT
-    if (Array.isArray(text) &&
-        canonicalModelType === ModelType.nmtcppTranslation) {
-        // Use runBatch for batch processing
+        if (entry.isDelegated) {
+            throw new ModelIsDelegatedError(modelId);
+        }
+        const canonicalModelType = entry.local.modelType;
+        const model = entry.local.model;
+        if (inputModelType !== undefined) {
+            const requestedCanonical = normalizeModelType(inputModelType);
+            if (requestedCanonical !== canonicalModelType) {
+                throw new ModelTypeMismatchError(canonicalModelType, requestedCanonical);
+            }
+        }
+        const isLlm = canonicalModelType === ModelType.llamacppCompletion;
+        const from = isLlm ? params.from : undefined;
+        const to = isLlm ? params.to : undefined;
+        const context = isLlm ? params.context : undefined;
+        const afriquePrompt = isLlm && (isAfrican(from) || isAfrican(to));
+        translateServerParamsSchema.parse(params);
+        const fromLanguage = getLanguage(from);
+        const toLanguage = getLanguage(to);
+        // Open a request-scoped lifecycle for both engine branches. LLM-
+        // translate inherits llamacpp-completion's `{ scope: "model",
+        // hard: true }` cancel surface; NMT-translate inherits nmtcpp's
+        // `{ scope: "none" }` — so the addon-cancel wiring below only
+        // engages on the LLM path. NMT cancel is purely soft: the loop
+        // exits on `signal.aborted`, scope unwinds, and the addon may run
+        // to completion in the background — acceptable because the result
+        // is dropped either way.
+        const ctx = __addDisposableResource(env_1, getRequestRegistry().begin({
+            requestId: requestId ?? generateServerRequestId(),
+            kind: "translate",
+            modelId,
+        }), true);
+        const requestLogger = withRequestContext(getServerLogger(), ctx);
+        if (isLlm) {
+            const onAbort = () => {
+                const addon = model.addon;
+                if (addon?.cancel) {
+                    addon.cancel.call(addon).catch((err) => {
+                        requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
+                    });
+                }
+            };
+            ctx.signal.addEventListener("abort", onAbort, { once: true });
+            if (ctx.signal.aborted)
+                onAbort();
+            ctx.scope.defer(() => {
+                ctx.signal.removeEventListener("abort", onAbort);
+            });
+        }
+        // Check if input is an array and model type is NMT
+        if (Array.isArray(text) &&
+            canonicalModelType === ModelType.nmtcppTranslation) {
+            // Use runBatch for batch processing
+            const modelStart = nowMs();
+            const translations = await model.runBatch(text);
+            const modelExecutionMs = nowMs() - modelStart;
+            // Soft-cancel boundary: if `cancel({ requestId })` landed while
+            // the addon was running the batch, bail before yielding anything.
+            // The addon may have completed its work; the result is dropped.
+            if (ctx.signal.aborted) {
+                return { modelExecutionMs };
+            }
+            // Yield each translation with a newline separator
+            for (let i = 0; i < translations.length; i++) {
+                if (ctx.signal.aborted)
+                    break;
+                const translation = translations[i];
+                yield translation;
+                if (i < translations.length - 1) {
+                    yield "\n";
+                }
+            }
+            return { modelExecutionMs };
+        }
+        // Single text processing (for NMT or LLM)
+        const singleText = Array.isArray(text) ? text[0] : text;
+        // Prepare input based on model type
+        const input = canonicalModelType === ModelType.nmtcppTranslation
+            ? singleText
+            : [
+                {
+                    role: afriquePrompt ? "user" : "system",
+                    content: afriquePrompt
+                        ? `Translate ${fromLanguage} to ${toLanguage}.\n${fromLanguage}: ${singleText}\n${toLanguage}:`
+                        : `${context ? `${context}. ` : ""}Translate the following text from ${fromLanguage} into ${toLanguage}. Only output the translation, nothing else.\n\n${fromLanguage}: ${singleText}\n${toLanguage}:`,
+                },
+            ];
         const modelStart = nowMs();
-        const translations = await model.runBatch(text);
-        const modelExecutionMs = nowMs() - modelStart;
-        // Yield each translation with a newline separator
-        for (let i = 0; i < translations.length; i++) {
-            const translation = translations[i];
-            yield translation;
-            if (i < translations.length - 1) {
-                yield "\n";
+        let response;
+        if (canonicalModelType === ModelType.llamacppCompletion &&
+            !shouldSkipPerCallSampling(entry.local.name)) {
+            // AnyModel.run is intentionally erased to a single-arg signature in the
+            // registry layer (Omit<BaseInference, "addon">). Re-narrow to the engine
+            // shape so we get the same typing as @qvac/llm-llamacpp.run().
+            const llmRun = model.run.bind(model);
+            response = await llmRun(input, {
+                generationParams: LLM_TRANSLATE_GENERATION_PARAMS,
+            });
+        }
+        else {
+            response = await model.run(input);
+        }
+        // Check if the response has an iterate method (like LLM models)
+        if (canonicalModelType === ModelType.llamacppCompletion &&
+            typeof response.iterate === "function") {
+            const llmResponse = response;
+            for await (const token of llmResponse.iterate()) {
+                if (ctx.signal.aborted)
+                    break;
+                yield token;
             }
+            const modelExecutionMs = nowMs() - modelStart;
+            const stats = {
+                ...(llmResponse.stats?.TPS !== undefined && { tokensPerSecond: llmResponse.stats.TPS }),
+                ...(llmResponse.stats?.TTFT !== undefined && { timeToFirstToken: llmResponse.stats.TTFT }),
+                ...(llmResponse.stats?.CacheTokens !== undefined && { cacheTokens: llmResponse.stats.CacheTokens }),
+                ...(llmResponse.stats?.generatedTokens !== undefined && { totalTokens: llmResponse.stats.generatedTokens }),
+            };
+            return buildStreamResult(modelExecutionMs, stats);
         }
-        return { modelExecutionMs };
-    }
-    // Single text processing (for NMT or LLM)
-    const singleText = Array.isArray(text) ? text[0] : text;
-    // Prepare input based on model type
-    const input = canonicalModelType === ModelType.nmtcppTranslation
-        ? singleText
-        : [
-            {
-                role: afriquePrompt ? "user" : "system",
-                content: afriquePrompt
-                    ? `Translate ${fromLanguage} to ${toLanguage}.\n${fromLanguage}: ${singleText}\n${toLanguage}:`
-                    : `${context ? `${context}. ` : ""}Translate the following text from ${fromLanguage} into ${toLanguage}. Only output the translation, nothing else.\n\n${fromLanguage}: ${singleText}\n${toLanguage}:`,
-            },
-        ];
-    const modelStart = nowMs();
-    let response;
-    if (canonicalModelType === ModelType.llamacppCompletion &&
-        !shouldSkipPerCallSampling(entry.local.name)) {
-        // AnyModel.run is intentionally erased to a single-arg signature in the
-        // registry layer (Omit<BaseInference, "addon">). Re-narrow to the engine
-        // shape so we get the same typing as @qvac/llm-llamacpp.run().
-        const llmRun = model.run.bind(model);
-        response = await llmRun(input, {
-            generationParams: LLM_TRANSLATE_GENERATION_PARAMS,
-        });
-    }
-    else {
-        response = await model.run(input);
-    }
-    // Check if the response has an iterate method (like LLM models)
-    if (canonicalModelType === ModelType.llamacppCompletion &&
-        typeof response.iterate === "function") {
-        const llmResponse = response;
-        for await (const token of llmResponse.iterate()) {
+        const nmtResponse = response;
+        for await (const token of nmtResponse.iterate()) {
+            if (ctx.signal.aborted)
+                break;
             yield token;
         }
         const modelExecutionMs = nowMs() - modelStart;
         const stats = {
-            ...(llmResponse.stats?.TPS !== undefined && { tokensPerSecond: llmResponse.stats.TPS }),
-            ...(llmResponse.stats?.TTFT !== undefined && { timeToFirstToken: llmResponse.stats.TTFT }),
-            ...(llmResponse.stats?.CacheTokens !== undefined && { cacheTokens: llmResponse.stats.CacheTokens }),
-            ...(llmResponse.stats?.generatedTokens !== undefined && { totalTokens: llmResponse.stats.generatedTokens }),
+            ...(nmtResponse.stats?.totalTime !== undefined && { totalTime: nmtResponse.stats.totalTime }),
+            ...(nmtResponse.stats?.totalTokens !== undefined && { totalTokens: nmtResponse.stats.totalTokens }),
+            ...(nmtResponse.stats?.decodeTime !== undefined && { decodeTime: nmtResponse.stats.decodeTime }),
+            ...(nmtResponse.stats?.encodeTime !== undefined && { encodeTime: nmtResponse.stats.encodeTime }),
+            ...(nmtResponse.stats?.TPS !== undefined && { tokensPerSecond: nmtResponse.stats.TPS }),
+            ...(nmtResponse.stats?.TTFT !== undefined && { timeToFirstToken: nmtResponse.stats.TTFT }),
         };
         return buildStreamResult(modelExecutionMs, stats);
     }
-    const nmtResponse = response;
-    for await (const token of nmtResponse.iterate()) {
-        yield token;
+    catch (e_1) {
+        env_1.error = e_1;
+        env_1.hasError = true;
+    }
+    finally {
+        const result_1 = __disposeResources(env_1);
+        if (result_1)
+            await result_1;
     }
-    const modelExecutionMs = nowMs() - modelStart;
-    const stats = {
-        ...(nmtResponse.stats?.totalTime !== undefined && { totalTime: nmtResponse.stats.totalTime }),
-        ...(nmtResponse.stats?.totalTokens !== undefined && { totalTokens: nmtResponse.stats.totalTokens }),
-        ...(nmtResponse.stats?.decodeTime !== undefined && { decodeTime: nmtResponse.stats.decodeTime }),
-        ...(nmtResponse.stats?.encodeTime !== undefined && { encodeTime: nmtResponse.stats.encodeTime }),
-        ...(nmtResponse.stats?.TPS !== undefined && { tokensPerSecond: nmtResponse.stats.TPS }),
-        ...(nmtResponse.stats?.TTFT !== undefined && { timeToFirstToken: nmtResponse.stats.TTFT }),
-    };
-    return buildStreamResult(modelExecutionMs, stats);
 }

package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.d.ts

@@ -1,17 +1,26 @@
+import type { AbortSignal } from "bare-abort-controller";
 import type { CompletionParams, CompletionStats, GenerationParams, ResponseFormat, Tool, ToolCall, ToolDialect } from "../../../../../schemas/index";
-import { noteCancelRequested as noteCancelRequestedFromState } from "./kv-cache-state";
+import type { DisposableScope } from "../../../runtime/disposable-scope";
+import type { Logger } from "../../../../../logging/types";
 interface CompletionResult {
     modelExecutionMs: number;
     stats?: CompletionStats;
     toolCalls: ToolCall[];
 }
-export declare function clearCachedMessageCounts(prefix?: string): void;
-export declare const noteCancelRequested: typeof noteCancelRequestedFromState;
 export declare function completion(params: CompletionParams & {
     tools?: Tool[];
     generationParams?: GenerationParams;
     toolDialect?: ToolDialect;
     responseFormat?: ResponseFormat;
+}, opts: {
+    signal: AbortSignal;
+    scope: DisposableScope;
+    /**
+     * Request-scoped logger forwarded to `createKvCacheSession` so
+     * kv-cache lines share the request's lifecycle prefix. Falls
+     * back to the module-level server logger when omitted.
+     */
+    logger?: Logger;
 }): AsyncGenerator<{
     token: string;
 }, CompletionResult, unknown>;