@abot-ai/runtime 1.9.0 → 2.0.0

package/dist/src/model-gateway/providers/llamacpp/connection.js

@@ -0,0 +1,73 @@
+import { resolveLlamaCppUrls } from "./settings.js";
+const MAX_RESPONSE_BYTES = 256 * 1024;
+const CONNECTION_TIMEOUT_MS = 5000;
+async function readBoundedJson(response) {
+    if (!response.ok)
+        throw new Error(`llamacpp_connection_http_${response.status}`);
+    if (!response.body)
+        throw new Error("llamacpp_connection_empty_response");
+    const reader = response.body.getReader();
+    const chunks = [];
+    let bytes = 0;
+    try {
+        while (true) {
+            const next = await reader.read();
+            if (next.done)
+                break;
+            bytes += next.value.byteLength;
+            if (bytes > MAX_RESPONSE_BYTES)
+                throw new Error("llamacpp_connection_response_too_large");
+            chunks.push(next.value);
+        }
+        return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+    }
+    finally {
+        void reader.cancel().catch(() => { });
+        reader.releaseLock();
+    }
+}
+function record(value) {
+    return !!value && typeof value === "object" && !Array.isArray(value);
+}
+/** Explicit read-only connection check; model capabilities remain user declarations. */
+export async function inspectLlamaCppConnection(params) {
+    const { serverUrl, apiUrl } = resolveLlamaCppUrls(params.baseUrl.trim());
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(new Error("llamacpp_connection_timeout")), CONNECTION_TIMEOUT_MS);
+    const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
+    const headers = { Accept: "application/json" };
+    if (params.apiKey)
+        headers.Authorization = `Bearer ${params.apiKey}`;
+    const fetchImpl = params.fetchImpl ?? fetch;
+    try {
+        const health = await readBoundedJson(await fetchImpl(`${serverUrl}/health`, { headers, signal }));
+        if (!record(health) || health.status !== "ok")
+            throw new Error("llamacpp_connection_not_ready");
+        const catalog = await readBoundedJson(await fetchImpl(`${apiUrl}/models`, { headers, signal }));
+        if (!record(catalog) || !Array.isArray(catalog.data) || catalog.data.length > 128) {
+            throw new Error("llamacpp_connection_models_invalid");
+        }
+        const models = catalog.data.map((entry) => {
+            if (!record(entry) || typeof entry.id !== "string" || !entry.id.trim() || entry.id.length > 512) {
+                throw new Error("llamacpp_connection_models_invalid");
+            }
+            return entry.id;
+        });
+        if (!models.length)
+            throw new Error("llamacpp_connection_no_models");
+        return { models: Object.freeze([...new Set(models)]) };
+    }
+    catch (error) {
+        if (controller.signal.aborted)
+            throw new Error("llamacpp_connection_timeout");
+        if (params.signal?.aborted)
+            throw params.signal.reason;
+        if (error instanceof Error && /^llamacpp_connection_/.test(error.message))
+            throw error;
+        // Never expose fetched content, credentials or a URL from a network error.
+        throw new Error("llamacpp_connection_failed");
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}

package/dist/src/model-gateway/providers/llamacpp/diagnostics.d.ts

@@ -0,0 +1,2 @@
+import type { ModelProviderInvocationParams } from "../contracts.js";
+export declare function emitLlamaCppStreamTermination(params: ModelProviderInvocationParams, startedAt: number, error?: unknown): void;

package/dist/src/model-gateway/providers/llamacpp/diagnostics.js

@@ -0,0 +1,17 @@
+import { traceDebug } from "../../../runtime/observability/debug-logger.js";
+export function emitLlamaCppStreamTermination(params, startedAt, error) {
+    const aborted = !!error && typeof error === "object" && "name" in error && error.name === "AbortError";
+    traceDebug("model-gateway.server", "llamacpp.stream.terminated", {
+        requestId: params.requestBody.debugRequestId ?? "",
+        endpoint: params.endpoint, modelStep: params.requestBody.modelStep ?? "",
+        profileId: params.invocation.profile.id, providerId: params.invocation.profile.providerId,
+        provider: "llamacpp", model: params.invocation.model, elapsedMs: Date.now() - startedAt,
+        outcome: error === undefined ? "completed" : aborted ? "aborted" : "failed",
+        abortRequested: aborted,
+        ...(params.invocation.think !== undefined ? { reasoningEffort: params.invocation.think } : {}),
+        ...(params.invocation.reasoningBudget ? { reasoningBudget: params.invocation.reasoningBudget } : {}),
+        ...(params.invocation.reasoningControlSource ? { reasoningControlSource: params.invocation.reasoningControlSource } : {}),
+        ...(error !== undefined ? { error: error instanceof Error
+                ? { name: error.name, message: error.message } : { name: "Error", message: String(error) } } : {}),
+    });
+}

package/dist/src/model-gateway/providers/llamacpp/embeddings.d.ts

@@ -0,0 +1,2 @@
+import type { ModelProviderEmbeddingParams, ModelProviderEmbeddingResult } from "../contracts.js";
+export declare function embedWithLlamaCpp(params: ModelProviderEmbeddingParams): Promise<ModelProviderEmbeddingResult>;

package/dist/src/model-gateway/providers/llamacpp/embeddings.js

@@ -0,0 +1,56 @@
+import { createEmbeddingModelFingerprint } from "../../embeddings/fingerprint.js";
+import { validateEmbeddingVectors } from "../../embeddings/validation.js";
+import { readOpenAiCompatibleEmbeddingVectors } from "../../protocol/openai-embedding-response.js";
+import { resolveLlamaCppProviderSettings } from "./settings.js";
+export async function embedWithLlamaCpp(params) {
+    let settings;
+    try {
+        settings = resolveLlamaCppProviderSettings(params.profile.providerConfig);
+    }
+    catch (error) {
+        return {
+            kind: "error",
+            statusCode: 500,
+            message: error instanceof Error ? error.message : "llamacpp_configuration_invalid",
+        };
+    }
+    const response = await params.fetchImpl(`${settings.apiUrl}/embeddings`, {
+        method: "POST",
+        headers: settings.headers,
+        body: JSON.stringify({
+            ...params.profile.options,
+            model: params.profile.model,
+            input: params.texts,
+            encoding_format: "float",
+        }),
+    });
+    if (!response.ok) {
+        return {
+            kind: "error",
+            statusCode: response.status || 502,
+            message: (await response.text()) || "llamacpp embedding error",
+        };
+    }
+    const body = await response.json().catch((error) => {
+        if (error instanceof SyntaxError)
+            return undefined;
+        throw error;
+    });
+    const vectors = readLlamaCppEmbeddingVectors(body, params.texts.length);
+    return Object.freeze({
+        kind: "embedded",
+        vectors: validateEmbeddingVectors({
+            vectors,
+            expectedCount: params.texts.length,
+        }),
+        modelFingerprint: createEmbeddingModelFingerprint(params.profile, settings.apiUrl),
+    });
+}
+function readLlamaCppEmbeddingVectors(value, expectedCount) {
+    if (value === null || typeof value !== "object" || Array.isArray(value))
+        return [];
+    const data = value.data;
+    if (!Array.isArray(data) || data.length !== expectedCount)
+        return [];
+    return readOpenAiCompatibleEmbeddingVectors(value, expectedCount);
+}

package/dist/src/model-gateway/providers/llamacpp/input-token-count.d.ts

@@ -0,0 +1,3 @@
+import type { ModelProviderInputTokenCountParams, ModelProviderInputTokenCountResult } from "../contracts.js";
+/** Counts the identical chat envelope through llama.cpp's non-generating native endpoint. */
+export declare function countLlamaCppInputTokens(params: ModelProviderInputTokenCountParams): Promise<ModelProviderInputTokenCountResult>;

package/dist/src/model-gateway/providers/llamacpp/input-token-count.js

@@ -0,0 +1,59 @@
+import { toolMediaSafeError, toolMediaSafeMessage } from "../../observability/tool-media-privacy.js";
+import { createProviderEnvelopeFingerprint } from "../envelope-budget.js";
+import { buildLlamaCppChatPayload } from "./payload.js";
+import { resolveLlamaCppProviderSettings } from "./settings.js";
+function readLlamaCppInputTokenCount(value) {
+    if (value === null || typeof value !== "object" || Array.isArray(value))
+        return undefined;
+    const count = value.input_tokens;
+    return isLlamaCppInputTokenCount(count) ? count : undefined;
+}
+function isLlamaCppInputTokenCount(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+/** Counts the identical chat envelope through llama.cpp's non-generating native endpoint. */
+export async function countLlamaCppInputTokens(params) {
+    let settings;
+    try {
+        settings = resolveLlamaCppProviderSettings(params.invocation.profile.providerConfig);
+    }
+    catch (error) {
+        return {
+            kind: "error",
+            statusCode: 500,
+            message: toolMediaSafeMessage(error instanceof Error ? error.message : "llamacpp_configuration_invalid", params.requestBody),
+        };
+    }
+    const payload = buildLlamaCppChatPayload(params);
+    const response = await params.fetchImpl(`${settings.apiUrl}/chat/completions/input_tokens`, {
+        method: "POST",
+        headers: settings.headers,
+        body: JSON.stringify(payload),
+    }).catch((error) => { throw toolMediaSafeError(error, params.requestBody); });
+    if (!response.ok) {
+        const message = await response.text();
+        return {
+            kind: "error",
+            statusCode: response.status || 502,
+            message: toolMediaSafeMessage(message || "llamacpp input token count error", params.requestBody),
+        };
+    }
+    const body = await response.json().catch((error) => {
+        if (error instanceof SyntaxError)
+            return undefined;
+        throw error;
+    });
+    const inputTokens = readLlamaCppInputTokenCount(body);
+    if (inputTokens === undefined) {
+        return {
+            kind: "error",
+            statusCode: 502,
+            message: "llamacpp input token count response invalid",
+        };
+    }
+    return {
+        kind: "counted",
+        inputTokens,
+        providerEnvelopeFingerprint: createProviderEnvelopeFingerprint(payload),
+    };
+}

package/dist/src/model-gateway/providers/llamacpp/message-projection.d.ts

@@ -0,0 +1,2 @@
+import type { ModelGatewayRequest, ResolvedModelInvocation } from "../../types.js";
+export declare function projectLlamaCppMessages(request: ModelGatewayRequest, invocation: ResolvedModelInvocation, endpoint: "chat" | "raw"): Record<string, unknown>[];

package/dist/src/model-gateway/providers/llamacpp/message-projection.js

@@ -0,0 +1,33 @@
+import { hasModelGatewayToolInteractionLane, isAssistantToolCallMessage, isTextChatMessage, isToolResultMessage, validateModelGatewayMessages, } from "../../protocol/message-contract.js";
+export function projectLlamaCppMessages(request, invocation, endpoint) {
+    validateModelGatewayMessages(request.messages);
+    if (endpoint === "raw" && hasModelGatewayToolInteractionLane(request.messages)) {
+        throw new TypeError("model_gateway_raw_tool_interaction_unsupported");
+    }
+    const input = Array.isArray(request.messages) && request.messages.length > 0
+        ? request.messages
+        : [{ role: "user", content: request.prompt ?? request.text ?? "" }];
+    const messages = input.map((message) => {
+        if (message?.attachments?.length)
+            throw new TypeError("llamacpp_image_input_unsupported");
+        if (isAssistantToolCallMessage(message)) {
+            return {
+                role: "assistant", content: "",
+                tool_calls: message.toolCalls.map((call) => ({
+                    id: call.callId, type: "function",
+                    function: { name: call.name, arguments: call.arguments },
+                })),
+            };
+        }
+        if (isToolResultMessage(message)) {
+            return { role: "tool", tool_call_id: message.toolCallId, name: message.toolName, content: message.content };
+        }
+        if (!isTextChatMessage(message))
+            throw new TypeError("llamacpp_message_invalid");
+        return { role: message.role, content: message.content };
+    });
+    const instructions = (invocation.instructions ?? []).map((line) => line.trim()).filter(Boolean);
+    return instructions.length
+        ? [{ role: "system", content: instructions.join("\n") }, ...messages]
+        : messages;
+}

package/dist/src/model-gateway/providers/llamacpp/payload.d.ts

@@ -0,0 +1,2 @@
+import type { ModelProviderInvocationParams } from "../contracts.js";
+export declare function buildLlamaCppChatPayload({ requestBody, invocation, endpoint }: Pick<ModelProviderInvocationParams, "requestBody" | "invocation" | "endpoint">): Record<string, unknown>;

package/dist/src/model-gateway/providers/llamacpp/payload.js

@@ -0,0 +1,44 @@
+import { isFiniteReasoningBudgetTokens } from "../../../shared/model-reasoning.js";
+import { resolveModelGatewayFormat } from "../../structured-output/format-contract.js";
+import { projectLlamaCppMessages } from "./message-projection.js";
+import { projectLlamaCppFormat } from "./schema-format.js";
+function nativeReasoningBudgetTokens(budget) {
+    if (budget.mode === "server-default")
+        return -1;
+    if (!isFiniteReasoningBudgetTokens(budget.tokens))
+        throw new TypeError("llamacpp_reasoning_budget_invalid");
+    return budget.tokens;
+}
+export function buildLlamaCppChatPayload({ requestBody, invocation, endpoint }) {
+    const payload = {
+        model: invocation.model,
+        messages: projectLlamaCppMessages(requestBody, invocation, endpoint),
+        stream: true,
+        stream_options: { include_usage: true },
+        reasoning_format: "deepseek",
+    };
+    const generation = invocation.profile.generation;
+    if (typeof generation.temperature === "number" && Number.isFinite(generation.temperature)) {
+        payload.temperature = generation.temperature;
+    }
+    if (typeof generation.topP === "number" && Number.isFinite(generation.topP)) {
+        payload.top_p = generation.topP;
+    }
+    const format = projectLlamaCppFormat(resolveModelGatewayFormat(requestBody.format, invocation.format));
+    if (format)
+        payload.response_format = format;
+    if (invocation.think !== undefined) {
+        payload.reasoning_effort = invocation.think;
+        payload.chat_template_kwargs = { enable_thinking: invocation.think !== "none" };
+    }
+    const thinkingIsDisabled = invocation.think === "none";
+    if (thinkingIsDisabled) {
+        payload.reasoning_budget_tokens = 0;
+        return payload;
+    }
+    if (invocation.reasoningBudget) {
+        payload.reasoning_budget_tokens = nativeReasoningBudgetTokens(invocation.reasoningBudget);
+        payload.chat_template_kwargs = { enable_thinking: true };
+    }
+    return payload;
+}

package/dist/src/model-gateway/providers/llamacpp/response-lifetime.d.ts

@@ -0,0 +1 @@
+export declare function bindLlamaCppResponseLifetime(response: Response): Response;

package/dist/src/model-gateway/providers/llamacpp/response-lifetime.js

@@ -0,0 +1,56 @@
+import { SseDataDecoder } from "../../protocol/sse-data.js";
+import { LlamaCppChatStreamNormalizer } from "./stream-normalizer.js";
+// End the upstream body before tracing tees it. A native finish chunk precedes
+// usage, so only the final marker/error ends transport; the normalizer validates.
+export function bindLlamaCppResponseLifetime(response) {
+    if (!response.ok || !response.body)
+        return response;
+    const reader = response.body.getReader();
+    const decoder = new SseDataDecoder();
+    const normalizer = new LlamaCppChatStreamNormalizer();
+    let stopped = false;
+    const cancel = (reason) => {
+        stopped = true;
+        void reader.cancel(reason).catch(() => { });
+        reader.releaseLock();
+    };
+    const body = new ReadableStream({
+        async pull(controller) {
+            try {
+                const { done, value } = await reader.read();
+                if (stopped)
+                    return;
+                if (done) {
+                    stopped = true;
+                    controller.close();
+                    reader.releaseLock();
+                    return;
+                }
+                let terminal = false;
+                for (const frame of decoder.accept(value)) {
+                    try {
+                        normalizer.consume(frame);
+                    }
+                    catch {
+                        terminal = true;
+                        break;
+                    }
+                }
+                terminal ||= normalizer.hasTerminated;
+                controller.enqueue(value);
+                if (terminal) {
+                    controller.close();
+                    cancel();
+                }
+            }
+            catch (error) {
+                if (stopped)
+                    return;
+                controller.error(error);
+                cancel(error);
+            }
+        },
+        cancel,
+    });
+    return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers });
+}

package/dist/src/model-gateway/providers/llamacpp/schema-format.d.ts

@@ -0,0 +1,2 @@
+import type { ModelGatewayFormat } from "../../types.js";
+export declare function projectLlamaCppFormat(format?: ModelGatewayFormat): Record<string, unknown> | undefined;

package/dist/src/model-gateway/providers/llamacpp/schema-format.js

@@ -0,0 +1,29 @@
+import { isModelGatewayJsonSchemaFormat, isPlainObject, rejectMisplacedProjectionMetadata, } from "../../structured-output/format-contract.js";
+// llama.cpp consumes the schema as grammar; do not inject it into model messages
+// or drop constraints just because another provider cannot represent them.
+export function projectLlamaCppFormat(format) {
+    if (format === undefined)
+        return undefined;
+    if (format === "json")
+        return { type: "json_object" };
+    if (isModelGatewayJsonSchemaFormat(format)) {
+        return {
+            type: "json_schema",
+            json_schema: {
+                name: format.name, schema: format.schema,
+                ...(format.description !== undefined ? { description: format.description } : {}),
+                ...(format.strict !== undefined ? { strict: format.strict } : {}),
+            },
+        };
+    }
+    if (!isPlainObject(format))
+        throw new TypeError("llamacpp_format_invalid");
+    rejectMisplacedProjectionMetadata(format);
+    if (format.type === "text")
+        return undefined;
+    if (format.type === "json_object")
+        return { type: "json_object" };
+    if (format.type === "json_schema")
+        throw new TypeError("llamacpp_json_schema_wrapper_invalid");
+    return { type: "json_schema", json_schema: { name: "runtime_output", schema: format } };
+}

package/dist/src/model-gateway/providers/llamacpp/settings.d.ts

@@ -0,0 +1,9 @@
+import type { ModelGatewayProviderConfig } from "../../types.js";
+export declare function resolveLlamaCppUrls(baseUrl: string): {
+    apiUrl: string;
+    serverUrl: string;
+};
+export declare function resolveLlamaCppProviderSettings(config?: ModelGatewayProviderConfig): {
+    apiUrl: string;
+    headers: Record<string, string>;
+};

package/dist/src/model-gateway/providers/llamacpp/settings.js

@@ -0,0 +1,25 @@
+export function resolveLlamaCppUrls(baseUrl) {
+    const url = new URL(baseUrl);
+    if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash) {
+        throw new TypeError("llamacpp_base_url_invalid");
+    }
+    const root = url.toString().replace(/\/+$/, "");
+    const serverUrl = root.endsWith("/v1") ? root.slice(0, -3) : root;
+    return { serverUrl, apiUrl: `${serverUrl}/v1` };
+}
+export function resolveLlamaCppProviderSettings(config) {
+    if (!config?.baseUrl?.trim())
+        throw new TypeError("llamacpp_base_url_required");
+    const { apiUrl } = resolveLlamaCppUrls(config.baseUrl.trim());
+    const headers = { "Content-Type": "application/json" };
+    if (config.apiKeyEnv !== undefined) {
+        if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(config.apiKeyEnv)) {
+            throw new TypeError("llamacpp_api_key_env_invalid");
+        }
+        const apiKey = process.env[config.apiKeyEnv]?.trim();
+        if (!apiKey)
+            throw new TypeError(`missing llama.cpp API key env: ${config.apiKeyEnv}`);
+        headers.Authorization = `Bearer ${apiKey}`;
+    }
+    return { apiUrl, headers };
+}

package/dist/src/model-gateway/providers/llamacpp/stream-normalizer.d.ts

@@ -0,0 +1,10 @@
+import type { ModelGatewayEvent } from "../../types.js";
+export declare class LlamaCppChatStreamNormalizer {
+    private finishReason;
+    private responseId;
+    private usage;
+    private terminated;
+    get hasTerminated(): boolean;
+    consume(data: string): ModelGatewayEvent[];
+    finish(): void;
+}

package/dist/src/model-gateway/providers/llamacpp/stream-normalizer.js

@@ -0,0 +1,85 @@
+import { buildModelTokenUsage } from "../../protocol/usage.js";
+function record(value) {
+    return !!value && typeof value === "object" && !Array.isArray(value);
+}
+function fail(code) { throw new Error(`llamacpp_stream_${code}`); }
+export class LlamaCppChatStreamNormalizer {
+    finishReason;
+    responseId;
+    usage;
+    terminated = false;
+    get hasTerminated() { return this.terminated; }
+    consume(data) {
+        if (this.terminated)
+            return [];
+        if (data === "[DONE]") {
+            if (!this.finishReason)
+                fail("completion_missing");
+            this.terminated = true;
+            return [{ type: "done", done: true, doneReason: this.finishReason, ...(this.usage ? { usage: this.usage } : {}) }];
+        }
+        let chunk;
+        try {
+            chunk = JSON.parse(data);
+        }
+        catch {
+            return fail("json_invalid");
+        }
+        if (!record(chunk))
+            fail("chunk_invalid");
+        if (chunk.error !== undefined)
+            fail("provider_error");
+        if (typeof chunk.id === "string") {
+            if (this.responseId !== undefined && this.responseId !== chunk.id)
+                fail("response_identity_changed");
+            this.responseId = chunk.id;
+        }
+        if (record(chunk.usage)) {
+            const promptDetails = record(chunk.usage.prompt_tokens_details) ? chunk.usage.prompt_tokens_details : {};
+            const completionDetails = record(chunk.usage.completion_tokens_details) ? chunk.usage.completion_tokens_details : {};
+            this.usage = buildModelTokenUsage({
+                inputTokens: chunk.usage.prompt_tokens, outputTokens: chunk.usage.completion_tokens,
+                totalTokens: chunk.usage.total_tokens, cachedInputTokens: promptDetails.cached_tokens,
+                reasoningTokens: completionDetails.reasoning_tokens,
+            });
+        }
+        if (!Array.isArray(chunk.choices))
+            fail("choices_invalid");
+        if (chunk.choices.length === 0)
+            return [];
+        if (chunk.choices.length !== 1)
+            fail("multiple_choices");
+        const choice = chunk.choices[0];
+        if (!record(choice) || choice.index !== 0 || !record(choice.delta))
+            fail("choice_invalid");
+        const delta = choice.delta;
+        if (delta.tool_calls !== undefined || delta.function_call !== undefined)
+            fail("unexpected_tool_call");
+        if (delta.refusal)
+            fail("refusal");
+        const events = [];
+        for (const [key, type] of [["reasoning_content", "thinking"], ["content", "content"]]) {
+            const value = delta[key];
+            if (value === undefined || value === null || value === "")
+                continue;
+            if (typeof value !== "string")
+                fail("delta_invalid");
+            if (this.finishReason)
+                fail("content_after_completion");
+            events.push({ type, text: value });
+        }
+        if (choice.finish_reason !== undefined && choice.finish_reason !== null) {
+            if (this.finishReason)
+                fail("duplicate_completion");
+            if (choice.finish_reason !== "stop" && choice.finish_reason !== "length")
+                fail("completion_reason_unsupported");
+            this.finishReason = choice.finish_reason;
+        }
+        return events;
+    }
+    finish() {
+        // A transport EOF or [DONE] alone must never authorize partial output.
+        if (!this.terminated)
+            fail("incomplete");
+    }
+}

package/dist/src/model-gateway/providers/llamacpp/stream.d.ts

@@ -0,0 +1,9 @@
+import type { ModelTokenUsage } from "../../types.js";
+import type { ModelProviderEventSink } from "../contracts.js";
+export declare function forwardLlamaCppChatStream(body: ReadableStream<Uint8Array>, writer: ModelProviderEventSink): Promise<void>;
+export declare function collectLlamaCppChatStream(body: ReadableStream<Uint8Array>): Promise<{
+    text: string;
+    thinking: string;
+    usage?: ModelTokenUsage;
+    doneReason?: string;
+}>;

package/dist/src/model-gateway/providers/llamacpp/stream.js

@@ -0,0 +1,33 @@
+import { readSseDataFrames } from "../../protocol/sse-data.js";
+import { LlamaCppChatStreamNormalizer } from "./stream-normalizer.js";
+export async function forwardLlamaCppChatStream(body, writer) {
+    const normalizer = new LlamaCppChatStreamNormalizer();
+    for await (const frame of readSseDataFrames(body)) {
+        for (const event of normalizer.consume(frame))
+            writer.emit(event);
+        if (normalizer.hasTerminated)
+            return;
+    }
+    normalizer.finish();
+}
+export async function collectLlamaCppChatStream(body) {
+    let text = "";
+    let thinking = "";
+    let usage;
+    let doneReason;
+    await forwardLlamaCppChatStream(body, {
+        emit(event) {
+            if (event.type === "content")
+                text += event.text;
+            else if (event.type === "thinking")
+                thinking += event.text;
+            else if (event.type === "error")
+                throw new Error(event.error);
+            else {
+                usage = event.usage;
+                doneReason = event.doneReason ?? undefined;
+            }
+        },
+    });
+    return { text, thinking, ...(usage ? { usage } : {}), ...(doneReason ? { doneReason } : {}) };
+}

package/dist/src/model-gateway/providers/openai/embeddings.js

@@ -1,5 +1,6 @@
 import { createEmbeddingModelFingerprint } from "../../embeddings/fingerprint.js";
 import { validateEmbeddingVectors } from "../../embeddings/validation.js";
+import { readOpenAiCompatibleEmbeddingVectors } from "../../protocol/openai-embedding-response.js";
 const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
 const DEFAULT_OPENAI_API_KEY_ENV = "OPENAI_API_KEY";
 const ENV_VAR_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
@@ -33,7 +34,7 @@ export async function embedWithOpenAI(params) {
         };
     }
     const body = await response.json().catch(() => undefined);
-    const vectors = readOrderedVectors(body, params.texts.length);
+    const vectors = readOpenAiCompatibleEmbeddingVectors(body, params.texts.length);
     return Object.freeze({
         kind: "embedded",
         vectors: validateEmbeddingVectors({
@@ -57,31 +58,3 @@ function resolveEmbeddingSettings(params) {
         ...(apiKey ? { apiKey } : {}),
     };
 }
-function readOrderedVectors(value, expectedCount) {
-    const body = value && typeof value === "object" && !Array.isArray(value)
-        ? value
-        : undefined;
-    if (!body || !Array.isArray(body.data)) {
-        return [];
-    }
-    const entries = body.data.filter(isRecord);
-    if (!hasContiguousIndices(entries, expectedCount)) {
-        return [];
-    }
-    return [...entries]
-        .sort((left, right) => Number(left.index) - Number(right.index))
-        .map((entry) => entry.embedding);
-}
-function hasContiguousIndices(entries, expectedCount) {
-    if (entries.length !== expectedCount) {
-        return false;
-    }
-    const indices = new Set(entries.map(({ index }) => index));
-    return (indices.size === expectedCount &&
-        [...indices].every((index) => Number.isSafeInteger(index) &&
-            Number(index) >= 0 &&
-            Number(index) < expectedCount));
-}
-function isRecord(value) {
-    return value !== null && typeof value === "object" && !Array.isArray(value);
-}

package/dist/src/model-gateway/providers/openai/sse-data.d.ts

@@ -1,9 +1 @@
-export declare class OpenAISseDataDecoder {
-    private readonly decoder;
-    private line;
-    private data;
-    private afterCr;
-    accept(bytes: Uint8Array): string[];
-    private acceptLine;
-}
-export declare function readOpenAIDataFrames(body: ReadableStream<Uint8Array>): AsyncGenerator<string>;
+export { SseDataDecoder as OpenAISseDataDecoder, readSseDataFrames as readOpenAIDataFrames } from "../../protocol/sse-data.js";