@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/http/openai-compat/embeddings.js

@@ -0,0 +1,86 @@
+import { estimateTokens } from '@bendyline/gezel';
+import { embedModelId } from '../../memory/embed-core.js';
+import { EmbeddingsDisabledError, embedBatch } from '../../memory/embeddings.js';
+import { resolveModelTarget } from './translate.js';
+/**
+ * Names the on-device text embedder whichever model it is configured to run.
+ * Persona ids are `gezel:<role>-<name>` and always contain a hyphen, so this
+ * can never collide with one.
+ */
+export const LOCAL_EMBEDDING_MODEL_ALIAS = 'gezel:embeddings';
+/** OpenAI's own per-request input limit; one caller must not monopolize the shared embed worker. */
+export const MAX_EMBEDDING_INPUTS = 2048;
+/**
+ * Whether `model` addresses the on-device embedder: the alias, the configured
+ * model id (`Xenova/bge-small-en-v1.5`), or its bare name (`bge-small-en-v1.5`,
+ * the form Ollama clients send).
+ */
+export function isLocalEmbeddingModel(model) {
+    const wanted = model.trim().toLowerCase();
+    const id = embedModelId().toLowerCase();
+    return (wanted === LOCAL_EMBEDDING_MODEL_ALIAS ||
+        wanted === id ||
+        wanted === id.slice(id.lastIndexOf('/') + 1));
+}
+/**
+ * Embed `input` for the OpenAI and Ollama embedding routes.
+ *
+ * The local path uses passage-side embedding (`embedBatch`): these APIs do not
+ * say whether a text is a query or a document, and bge needs no passage prefix.
+ * A chat model is never silently swapped for the embedder — vectors from a
+ * different model would corrupt the caller's index — so a provider without
+ * `createEmbedding` gets an error that names the alias instead.
+ */
+export async function createEmbeddings(ctx, model, input) {
+    if (isLocalEmbeddingModel(model)) {
+        const texts = typeof input === 'string' ? [input] : input;
+        try {
+            const vectors = await embedBatch(texts);
+            return {
+                ok: true,
+                vectors,
+                model: embedModelId(),
+                inputTokens: texts.reduce((sum, text) => sum + estimateTokens(text), 0),
+            };
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            return err instanceof EmbeddingsDisabledError
+                ? { ok: false, status: 503, code: 'embeddings_unavailable', message }
+                : { ok: false, status: 500, code: 'provider_error', message };
+        }
+    }
+    const target = resolveModelTarget(model);
+    if (!target) {
+        return {
+            ok: false,
+            status: 404,
+            code: 'model_not_found',
+            message: `Unknown model "${model}". Expected "<provider>:<model>", or "${LOCAL_EMBEDDING_MODEL_ALIAS}" for on-device embeddings.`,
+        };
+    }
+    const unsupported = {
+        ok: false,
+        status: 400,
+        code: 'embeddings_not_supported',
+        message: `Provider "${target.provider}" does not support embeddings. Use "${LOCAL_EMBEDDING_MODEL_ALIAS}" for on-device embeddings.`,
+    };
+    // Asking a local engine for a model can admit and start it; only a host
+    // that can look a provider up without binding a model asks it at all.
+    if (!ctx.chat.getProvider)
+        return unsupported;
+    try {
+        const provider = await ctx.chat.getProvider(target.provider);
+        if (!provider.createEmbedding)
+            return unsupported;
+        const result = await provider.createEmbedding({
+            input,
+            ...(target.model ? { model: target.model } : {}),
+        });
+        return { ok: true, vectors: result.vectors, model, inputTokens: result.usage.inputTokens };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 500, code: 'provider_error', message };
+    }
+}

package/dist/http/openai-compat/request-tuning.d.ts

@@ -0,0 +1,35 @@
+import type { ResolvedTuning } from '../../model-profile/tuning.js';
+import type { ChatCompletionRequest } from './translate.js';
+/**
+ * Overlay per-request knobs onto a model's resolved tuning as the
+ * TOPMOST layer — an explicit value in the request beats the gezel
+ * override, the install default, and the catalog recommendation, which
+ * is exactly what an OpenAI-shaped caller expects when it sends
+ * `temperature: 0.2`.
+ *
+ * The result flows through the normal `SessionOpts.tuning` path, so it
+ * reaches every tuning-consuming provider (llama-cpp, MLX, Ollama,
+ * ds4, OpenAI, Anthropic) via each one's tuning map. Copilot and the
+ * CLI providers ignore tuning — the same silent-no-op UI sessions have
+ * on those backends.
+ */
+export declare function overlayOpenAiRequestTuning(tuning: ResolvedTuning, req: ChatCompletionRequest): ResolvedTuning;
+/** OpenAI effort values that mean "answer without a reasoning phase". */
+export declare function disablesThinking(effort: string | undefined): boolean;
+/**
+ * Ollama's per-request `options` block, overlaid the same way. Only the
+ * sampling-shaped keys are honored; engine-level knobs (`num_ctx`,
+ * `num_thread`, …) are ignored — the engine pool owns those.
+ */
+export interface OllamaRequestOptions {
+    temperature?: number;
+    top_p?: number;
+    top_k?: number;
+    min_p?: number;
+    num_predict?: number;
+    seed?: number;
+    repeat_penalty?: number;
+    presence_penalty?: number;
+    frequency_penalty?: number;
+}
+export declare function overlayOllamaRequestTuning(tuning: ResolvedTuning, options: OllamaRequestOptions | undefined, format: 'json' | Record<string, unknown> | undefined): ResolvedTuning;

package/dist/http/openai-compat/request-tuning.js

@@ -0,0 +1,87 @@
+/**
+ * Overlay per-request knobs onto a model's resolved tuning as the
+ * TOPMOST layer — an explicit value in the request beats the gezel
+ * override, the install default, and the catalog recommendation, which
+ * is exactly what an OpenAI-shaped caller expects when it sends
+ * `temperature: 0.2`.
+ *
+ * The result flows through the normal `SessionOpts.tuning` path, so it
+ * reaches every tuning-consuming provider (llama-cpp, MLX, Ollama,
+ * ds4, OpenAI, Anthropic) via each one's tuning map. Copilot and the
+ * CLI providers ignore tuning — the same silent-no-op UI sessions have
+ * on those backends.
+ */
+export function overlayOpenAiRequestTuning(tuning, req) {
+    const sampling = { ...tuning.sampling };
+    if (req.temperature !== undefined)
+        sampling.temperature = req.temperature;
+    if (req.top_p !== undefined)
+        sampling.topP = req.top_p;
+    const maxTokens = req.max_completion_tokens ?? req.max_tokens;
+    if (maxTokens !== undefined)
+        sampling.maxTokens = maxTokens;
+    if (req.presence_penalty !== undefined)
+        sampling.presencePenalty = req.presence_penalty;
+    if (req.frequency_penalty !== undefined)
+        sampling.frequencyPenalty = req.frequency_penalty;
+    if (req.seed !== undefined)
+        sampling.seed = req.seed;
+    const output = { ...tuning.output };
+    if (req.response_format?.type === 'json_object') {
+        output.responseFormat = 'json_object';
+    }
+    else if (req.response_format?.type === 'json_schema') {
+        output.jsonSchema = req.response_format.json_schema.schema;
+    }
+    const toolChoice = typeof req.tool_choice === 'string' ? req.tool_choice : tuning.toolChoice;
+    // Only the off direction is mapped: turning thinking ON for a model whose
+    // catalog leaves it off could select a mode the model does not have.
+    const reasoning = disablesThinking(req.reasoning_effort)
+        ? { ...tuning.reasoning, enableThinking: false }
+        : tuning.reasoning;
+    return {
+        ...tuning,
+        sampling,
+        reasoning,
+        output,
+        ...(toolChoice !== undefined ? { toolChoice } : {}),
+    };
+}
+/** OpenAI effort values that mean "answer without a reasoning phase". */
+export function disablesThinking(effort) {
+    return effort === 'none' || effort === 'minimal';
+}
+export function overlayOllamaRequestTuning(tuning, options, format) {
+    const sampling = { ...tuning.sampling };
+    if (options) {
+        if (options.temperature !== undefined)
+            sampling.temperature = options.temperature;
+        if (options.top_p !== undefined)
+            sampling.topP = options.top_p;
+        if (options.top_k !== undefined)
+            sampling.topK = options.top_k;
+        if (options.min_p !== undefined)
+            sampling.minP = options.min_p;
+        if (options.num_predict !== undefined && options.num_predict > 0) {
+            sampling.maxTokens = options.num_predict;
+        }
+        if (options.seed !== undefined)
+            sampling.seed = options.seed;
+        if (options.repeat_penalty !== undefined)
+            sampling.repetitionPenalty = options.repeat_penalty;
+        if (options.presence_penalty !== undefined)
+            sampling.presencePenalty = options.presence_penalty;
+        if (options.frequency_penalty !== undefined) {
+            sampling.frequencyPenalty = options.frequency_penalty;
+        }
+    }
+    const output = { ...tuning.output };
+    if (format === 'json') {
+        output.responseFormat = 'json_object';
+    }
+    else if (format && typeof format === 'object') {
+        // Ollama's structured-output form: `format` IS the JSON schema.
+        output.jsonSchema = format;
+    }
+    return { ...tuning, sampling, output };
+}

package/dist/http/openai-compat/responses-streaming.d.ts

@@ -0,0 +1,351 @@
+import type { ExternalToolCall, ImageAttachment, LLMSession, TurnUsage } from '../../providers/types.js';
+export type ResponsesToolKind = 'function' | 'custom';
+/**
+ * Maps a provider-safe local tool name back to the identity advertised on
+ * the Responses wire. Namespaces are flattened before local inference, so
+ * the serializer needs this sidecar to restore them on output.
+ */
+export interface ResponsesToolBinding {
+    kind: ResponsesToolKind;
+    name: string;
+    namespace?: string;
+}
+export interface ResponsesOutputText {
+    type: 'output_text';
+    text: string;
+    annotations: [];
+}
+export interface ResponsesOutputMessage {
+    id: string;
+    type: 'message';
+    role: 'assistant';
+    status: 'in_progress' | 'completed' | 'incomplete';
+    content: ResponsesOutputText[];
+}
+export interface ResponsesFunctionCall {
+    id: string;
+    type: 'function_call';
+    call_id: string;
+    name: string;
+    namespace?: string;
+    arguments: string;
+    status: 'in_progress' | 'completed' | 'incomplete';
+}
+export interface ResponsesCustomToolCall {
+    id: string;
+    type: 'custom_tool_call';
+    call_id: string;
+    name: string;
+    namespace?: string;
+    input: string;
+    status: 'in_progress' | 'completed' | 'incomplete';
+}
+export type ResponsesOutputItem = ResponsesOutputMessage | ResponsesFunctionCall | ResponsesCustomToolCall;
+export interface ResponsesUsage {
+    input_tokens: number;
+    input_tokens_details: {
+        cached_tokens: number;
+        cache_write_tokens: number;
+    };
+    output_tokens: number;
+    output_tokens_details: {
+        reasoning_tokens: number;
+    };
+    total_tokens: number;
+}
+export type ResponsesStatus = 'in_progress' | 'completed' | 'incomplete' | 'failed' | 'cancelled';
+/**
+ * Stable subset of the OpenAI Response resource consumed by Codex. Fields
+ * that Gezel cannot truthfully populate are represented by their normal
+ * null/default values instead of being invented from provider internals.
+ */
+export interface ResponsesResponse {
+    id: string;
+    object: 'response';
+    created_at: number;
+    completed_at: number | null;
+    status: ResponsesStatus;
+    error: {
+        code: string;
+        message: string;
+    } | null;
+    incomplete_details: {
+        reason: 'max_output_tokens' | 'content_filter';
+    } | null;
+    instructions: string | null;
+    metadata: Record<string, string> | null;
+    model: string;
+    output: ResponsesOutputItem[];
+    output_text: string;
+    parallel_tool_calls: boolean;
+    previous_response_id: string | null;
+    reasoning: {
+        effort?: string | null;
+        summary?: string | null;
+    } | null;
+    store: boolean;
+    temperature: number | null;
+    text: {
+        format: {
+            type: 'text';
+        };
+    };
+    tool_choice: unknown;
+    tools: unknown[];
+    top_logprobs: number;
+    top_p: number | null;
+    truncation: 'auto' | 'disabled';
+    max_output_tokens: number | null;
+    usage: ResponsesUsage | null;
+}
+type WithSequence<T> = T & {
+    sequence_number: number;
+};
+export type ResponsesStreamEvent = WithSequence<{
+    type: 'response.created';
+    response: ResponsesResponse;
+}> | WithSequence<{
+    type: 'response.in_progress';
+    response: ResponsesResponse;
+}> | WithSequence<{
+    type: 'response.output_item.added';
+    output_index: number;
+    item: ResponsesOutputItem;
+}> | WithSequence<{
+    type: 'response.content_part.added';
+    item_id: string;
+    output_index: number;
+    content_index: 0;
+    part: ResponsesOutputText;
+}> | WithSequence<{
+    type: 'response.output_text.delta';
+    item_id: string;
+    output_index: number;
+    content_index: 0;
+    delta: string;
+    logprobs: [];
+}> | WithSequence<{
+    type: 'response.output_text.done';
+    item_id: string;
+    output_index: number;
+    content_index: 0;
+    text: string;
+    logprobs: [];
+}> | WithSequence<{
+    type: 'response.content_part.done';
+    item_id: string;
+    output_index: number;
+    content_index: 0;
+    part: ResponsesOutputText;
+}> | WithSequence<{
+    type: 'response.function_call_arguments.delta';
+    item_id: string;
+    output_index: number;
+    delta: string;
+}> | WithSequence<{
+    type: 'response.function_call_arguments.done';
+    item_id: string;
+    output_index: number;
+    name: string;
+    arguments: string;
+}> | WithSequence<{
+    type: 'response.custom_tool_call_input.delta';
+    item_id: string;
+    output_index: number;
+    delta: string;
+}> | WithSequence<{
+    type: 'response.custom_tool_call_input.done';
+    item_id: string;
+    output_index: number;
+    input: string;
+}> | WithSequence<{
+    type: 'response.output_item.done';
+    output_index: number;
+    item: ResponsesOutputItem;
+}> | WithSequence<{
+    type: 'response.completed';
+    response: ResponsesResponse;
+}> | WithSequence<{
+    type: 'response.incomplete';
+    response: ResponsesResponse;
+}> | WithSequence<{
+    type: 'response.failed';
+    response: ResponsesResponse;
+}> | WithSequence<{
+    type: 'error';
+    code: string | null;
+    message: string;
+    param: string | null;
+}>;
+/**
+ * Small source-event vocabulary shared by the serializer and route adapter.
+ * It deliberately mirrors LLMSession callbacks, rather than depending on
+ * the much larger UI ChatEvent union.
+ */
+export type ResponsesSourceEvent = {
+    type: 'text_delta';
+    delta: string;
+} | {
+    type: 'tool_arguments_delta';
+    name: string;
+    delta: string;
+} | {
+    type: 'usage';
+    usage: Pick<TurnUsage, 'inputTokens' | 'outputTokens' | 'cachedInputTokens'>;
+} | {
+    type: 'complete';
+    text: string;
+    toolCalls?: ExternalToolCall[];
+    incomplete?: boolean;
+} | {
+    type: 'error';
+    message: string;
+    code?: string;
+} | {
+    type: 'cancelled';
+};
+export interface ResponsesRequestEcho {
+    instructions?: string | null;
+    metadata?: Record<string, string> | null;
+    parallelToolCalls?: boolean;
+    previousResponseId?: string | null;
+    reasoning?: {
+        effort?: string | null;
+        summary?: string | null;
+    } | null;
+    store?: boolean;
+    temperature?: number | null;
+    toolChoice?: unknown;
+    tools?: unknown[];
+    topP?: number | null;
+    truncation?: 'auto' | 'disabled';
+    maxOutputTokens?: number | null;
+}
+export interface ResponsesSerializerLimits {
+    /** Maximum visible assistant text retained in one response. */
+    maxOutputTextChars?: number;
+    /** Maximum raw JSON argument text retained for one tool call. */
+    maxToolArgumentChars?: number;
+    /** Maximum output items retained in one response. */
+    maxOutputItems?: number;
+}
+export type ResponsesIdKind = 'response' | 'message' | 'function_call' | 'custom_tool_call' | 'call';
+export interface ResponsesSerializerOptions {
+    model: string;
+    responseId?: string;
+    createdAt?: number;
+    nowSeconds?: () => number;
+    idFactory?: (kind: ResponsesIdKind) => string;
+    /** Preferred identity map produced by the Responses request translator. */
+    toolBindings?: Record<string, ResponsesToolBinding>;
+    /** Backward-compatible projections for callers without full bindings. */
+    toolKinds?: Record<string, ResponsesToolKind>;
+    toolNamespaces?: Record<string, string>;
+    echo?: ResponsesRequestEcho;
+    limits?: ResponsesSerializerLimits;
+}
+/**
+ * Stateful Responses output serializer. Every returned event is an immutable
+ * snapshot: later deltas never mutate an earlier response.created payload.
+ */
+export declare class ResponsesStreamSerializer {
+    private readonly model;
+    private readonly responseId;
+    private readonly createdAt;
+    private readonly nowSeconds;
+    private readonly idFactory;
+    private readonly toolBindings;
+    private readonly toolKinds;
+    private readonly toolNamespaces;
+    private readonly echo;
+    private readonly limits;
+    private started;
+    private terminal;
+    private sequenceNumber;
+    private status;
+    private completedAt;
+    private responseError;
+    private incompleteDetails;
+    private usage;
+    private output;
+    private active;
+    private toolStates;
+    private unnamedToolArguments;
+    private visibleText;
+    constructor(options: ResponsesSerializerOptions);
+    get isTerminal(): boolean;
+    /** Emit the canonical opening pair exactly once. */
+    start(): ResponsesStreamEvent[];
+    push(source: ResponsesSourceEvent): ResponsesStreamEvent[];
+    /** Convenience method matching LLMSession.onDelta. */
+    textDelta(delta: string): ResponsesStreamEvent[];
+    /** Convenience method matching LLMSession.onToolArgsDelta. */
+    toolArgumentsDelta(name: string, delta: string): ResponsesStreamEvent[];
+    /** Convenience method matching LLMSession.onUsage. */
+    setUsage(usage: Pick<TurnUsage, 'inputTokens' | 'outputTokens' | 'cachedInputTokens'>): void;
+    /**
+     * Finalize the response after sendAndWait resolves. Captured tool calls
+     * supply stable final argument strings; raw argument callbacks are used
+     * for live deltas when the provider exposes them.
+     */
+    complete(text: string, toolCalls?: ExternalToolCall[], options?: {
+        incomplete?: boolean;
+        includeStart?: boolean;
+    }): ResponsesStreamEvent[];
+    fail(message: string, code?: string, includeStart?: boolean): ResponsesStreamEvent[];
+    cancel(includeStart?: boolean): ResponsesStreamEvent[];
+    /** Return an immutable non-streaming response snapshot. */
+    response(): ResponsesResponse;
+    private pushTextDelta;
+    private pushToolArgumentsDelta;
+    private openMessage;
+    private openTool;
+    private announceTool;
+    private appendToolArguments;
+    private finalizeActive;
+    private finalizeMessage;
+    private finalizeTool;
+    private reconcileToolCalls;
+    private reconcileFinalText;
+    private markActiveIncomplete;
+    private bindingFor;
+    private assertTextCapacity;
+    private assertToolCapacity;
+    private assertOutputCapacity;
+    private event;
+}
+export interface ResponsesSSEMessage {
+    event?: string;
+    data: string;
+}
+export interface ResponsesSSEWriter {
+    writeSSE(message: ResponsesSSEMessage): Promise<void>;
+}
+/** Responses streams name each SSE event after its JSON `type`. */
+export declare function serializeResponsesSseEvent(event: ResponsesStreamEvent): ResponsesSSEMessage;
+export interface RunResponsesOptions extends Omit<ResponsesSerializerOptions, 'model' | 'createdAt'> {
+    attachments?: ImageAttachment[];
+    /** Continue from a replayed external tool result without adding an empty user turn. */
+    continueFromToolResult?: boolean;
+    /** Abort queued or in-flight provider work when the Responses client disconnects. */
+    signal?: AbortSignal;
+    /** Receives the raw provider exception for server-side diagnostics only. */
+    onProviderError?: (error: unknown) => void;
+    /** Reaching this cap without a tool call yields response.incomplete. */
+    lengthCapTokens?: number;
+}
+/**
+ * High-level streaming adapter for the existing LLMSession surface. Writes
+ * every event in callback order. Responses streams terminate at the final
+ * JSON event followed by EOF; unlike Chat Completions, there is no `[DONE]`
+ * sentinel.
+ */
+export declare function runResponsesStreaming(session: LLMSession, prompt: string, echoModel: string, sink: ResponsesSSEWriter, nowSeconds: () => number, options?: RunResponsesOptions): Promise<ResponsesResponse>;
+/**
+ * High-level one-shot adapter. Provider failures are represented as a
+ * status=failed Response so callers can use the same envelope as streaming.
+ */
+export declare function runResponsesNonStreaming(session: LLMSession, prompt: string, echoModel: string, nowSeconds: () => number, options?: RunResponsesOptions): Promise<ResponsesResponse>;
+/** Serialize an already-materialized source-event sequence for tests/embedders. */
+export declare function serializeNonStreamingResponse(sourceEvents: Iterable<ResponsesSourceEvent>, options: ResponsesSerializerOptions): ResponsesResponse;
+export {};