@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/http/routes/v1-models.js

@@ -0,0 +1,189 @@
+import { externalGezelModelId, isRecommendedModel, resolveDistributionProfile, } from '@bendyline/gezel';
+import { resolveOnDeviceProvider } from '@bendyline/gezel/native';
+import { Hono } from 'hono';
+import { appleSystemModel } from '../openai-compat/apple-model.js';
+import { resolveFallbackGezelId } from '../openai-compat/chat-target.js';
+/**
+ * `/v1/models` — OpenAI-compatible model directory aggregated across
+ * the user's gezels and every configured provider, plus the retrieve
+ * form (`GET /v1/models/{id}`) OpenAI SDKs call after listing.
+ *
+ * Gezel entries use `gezel:<role>-<name>` ids and carry `name` +
+ * `role` metadata for richer clients. Raw model entries use qualified
+ * `<provider>:<model>` ids. Both shapes are accepted by
+ * `/v1/chat/completions`; the effective fallback gezel leads the list.
+ *
+ * Providers that haven't been credentialed (e.g. `openai` without an
+ * API key) throw on `listModels` — we swallow those silently and emit
+ * what we can. The Settings UI is where users discover that a provider
+ * needs setup.
+ */
+const PROVIDERS_TO_ENUMERATE = [
+    'copilot',
+    'openai',
+    'anthropic',
+    'anthropic-cli',
+    'codex-cli',
+    'ollama',
+    'llama-cpp',
+    'mlx',
+    'ds4',
+];
+const LOCAL_PROVIDERS = ['llama-cpp', 'mlx', 'ds4'];
+function catalogBackend(manifest) {
+    // Store hosts have no provisioned Python runtime. Only advertise models
+    // backed by bundled native engines until a frozen MLX runtime ships.
+    const store = resolveDistributionProfile().pythonProvisioning === 'frozen-only';
+    if (store)
+        return manifest.llamaCpp ? 'llama-cpp' : manifest.ds4 ? 'ds4' : null;
+    const preferred = resolveOnDeviceProvider(process.platform, process.arch);
+    if (preferred === 'mlx' && manifest.mlx && !manifest.mlx.disabledReason)
+        return 'mlx';
+    if (preferred === 'llama-cpp' && manifest.llamaCpp)
+        return 'llama-cpp';
+    if (manifest.llamaCpp)
+        return 'llama-cpp';
+    if (manifest.mlx && !manifest.mlx.disabledReason && process.platform === 'darwin')
+        return 'mlx';
+    if (manifest.ds4)
+        return 'ds4';
+    return null;
+}
+async function downloadableCatalogEntries(ctx, created, existingIds) {
+    const items = await ctx.catalog.list('chat-model').catch(() => []);
+    return items
+        .flatMap((item) => {
+        const manifest = item.manifest;
+        if (manifest.kind !== 'chat-model')
+            return [];
+        const provider = catalogBackend(manifest);
+        if (!provider)
+            return [];
+        const id = `${provider}:${manifest.id}`;
+        if (existingIds.has(id))
+            return [];
+        return [
+            {
+                manifest,
+                entry: {
+                    id,
+                    object: 'model',
+                    created,
+                    owned_by: provider,
+                    name: manifest.name,
+                    availability: 'download-required',
+                    preparation: 'app-download',
+                    reason_code: 'model_download_required',
+                    recovery_actions: ['prepare'],
+                    locality: 'on-device',
+                    download_bytes: manifest.approxSizeBytes,
+                    ...(manifest.contextWindow ? { context_window: manifest.contextWindow } : {}),
+                    capabilities: {
+                        text: true,
+                        tools: manifest.supportsTools,
+                        structuredOutput: false,
+                        images: false,
+                        foregroundOnly: false,
+                    },
+                },
+            },
+        ];
+    })
+        .sort((a, b) => {
+        const aRecommended = isRecommendedModel(a.manifest);
+        const bRecommended = isRecommendedModel(b.manifest);
+        if (aRecommended !== bRecommended)
+            return aRecommended ? -1 : 1;
+        const score = (b.manifest.recoScore ?? 0) - (a.manifest.recoScore ?? 0);
+        return score || (a.entry.name ?? a.entry.id).localeCompare(b.entry.name ?? b.entry.id);
+    })
+        .map(({ entry }) => entry);
+}
+async function buildModelEntries(ctx, created, options) {
+    // Advertise every gezel as a selectable OpenAI "model". Keep the
+    // effective fallback first because a number of generic clients pick
+    // the first entry by default. Role + name form the human-readable
+    // routing id; `gezel_id` remains stable metadata for richer integrations.
+    let gezelEntries = [];
+    if (!options.localOnly && ctx.store.listGezels) {
+        const config = await ctx.store.readConfig().catch(() => null);
+        const gezels = await ctx.store.listGezels().catch(() => []);
+        const fallbackGezelId = await resolveFallbackGezelId(ctx, config?.openaiEndpoints?.servingGezelId);
+        gezelEntries = [...gezels]
+            .sort((a, b) => {
+            if (a.id === fallbackGezelId)
+                return -1;
+            if (b.id === fallbackGezelId)
+                return 1;
+            return a.name.localeCompare(b.name);
+        })
+            .map((gezel) => ({
+            id: externalGezelModelId(gezel),
+            object: 'model',
+            created,
+            owned_by: 'gezel',
+            gezel_id: gezel.id,
+            name: gezel.name,
+            ...(gezel.role ? { role: gezel.role } : {}),
+            ...(gezel.id === fallbackGezelId ? { is_fallback: true } : {}),
+        }));
+    }
+    const providers = (options.localOnly ? LOCAL_PROVIDERS : PROVIDERS_TO_ENUMERATE).filter((provider) => provider !== 'mlx' || resolveDistributionProfile().pythonProvisioning !== 'frozen-only');
+    const buckets = await Promise.all(providers.map(async (provider) => {
+        try {
+            const models = await ctx.chat.listModelsForProvider(provider);
+            return models.map((m) => ({
+                id: `${provider}:${m.id}`,
+                object: 'model',
+                created,
+                owned_by: provider,
+                ...(provider === 'llama-cpp' || provider === 'mlx' || provider === 'ds4'
+                    ? {
+                        availability: 'available',
+                        locality: 'on-device',
+                        preparation: 'app-download',
+                    }
+                    : {}),
+                ...(m.contextWindow ? { context_window: m.contextWindow } : {}),
+                ...(m.supportsReasoning ? { supports_reasoning: true } : {}),
+            }));
+        }
+        catch {
+            // Provider not configured / not available on this host —
+            // skip silently rather than failing the whole listing.
+            return [];
+        }
+    }));
+    const availableEntries = buckets.flat();
+    const existingIds = new Set([...gezelEntries, ...availableEntries].map((entry) => entry.id));
+    const downloadable = await downloadableCatalogEntries(ctx, created, existingIds);
+    const apple = await appleSystemModel(created);
+    return [...gezelEntries, ...apple, ...availableEntries, ...downloadable];
+}
+export function v1ModelsRoutes(ctx, options = {}) {
+    const app = new Hono();
+    app.get('/', async (c) => {
+        const created = Math.floor(Date.now() / 1000);
+        const data = await buildModelEntries(ctx, created, options);
+        return c.json({ object: 'list', data });
+    });
+    // Retrieve form. The `{.+}` pattern keeps colons and slashes inside
+    // the id (model ids like `ollama:llama3.1:8b` carry both).
+    app.get('/:id{.+}', async (c) => {
+        const id = c.req.param('id');
+        const created = Math.floor(Date.now() / 1000);
+        const entries = await buildModelEntries(ctx, created, options);
+        const found = entries.find((m) => m.id === id);
+        if (!found) {
+            return c.json({
+                error: {
+                    message: `Model "${id}" not found — see GET /v1/models for the available ids.`,
+                    type: 'invalid_request_error',
+                    code: 'model_not_found',
+                },
+            }, 404);
+        }
+        return c.json(found);
+    });
+    return app;
+}

package/dist/http/routes/v1-responses.d.ts

@@ -0,0 +1,20 @@
+import { Hono } from 'hono';
+import { type ProviderName } from '../../providers/types.js';
+import type { InferenceContext } from '../inference-context.js';
+/**
+ * `POST /v1/responses` — the stateless Responses API subset used by Codex.
+ *
+ * Codex remains the agent harness: it owns its transcript, sandbox, tools,
+ * approvals, and filesystem changes. Gezel only performs one inference turn
+ * and returns assistant text/tool calls in Responses wire shapes. The route
+ * intentionally does not create a Gezel chat session or attach Gezel MCP.
+ *
+ * In the full service, auth is mounted in `server.ts`: the Connected Apps
+ * master switch, bearer authentication, and the `openai` scope all apply. Codex can therefore use
+ * an app-scoped token without ever learning the daemon's root credential.
+ */
+export interface V1ResponsesRouteOptions {
+    /** Refuse every other provider; the in-process inference host serves only on-device models. */
+    allowedProviders?: readonly ProviderName[];
+}
+export declare function v1ResponsesRoutes(ctx: InferenceContext, opts?: V1ResponsesRouteOptions): Hono;

package/dist/http/routes/v1-responses.js

@@ -0,0 +1,327 @@
+import { createLogger } from '@bendyline/gezel';
+import { Hono } from 'hono';
+import { bodyLimit } from 'hono/body-limit';
+import { streamSSE } from 'hono/streaming';
+import { ZodError } from 'zod';
+import { ExternalToolsUnsupportedError, ModelNotInstalledError, } from '../../providers/types.js';
+import { appendCallerOwnedActionLedger } from '../openai-compat/action-ledger.js';
+import { profileForCallerOwnedInference } from '../openai-compat/caller-owned-profile.js';
+import { resolveChatTarget } from '../openai-compat/chat-target.js';
+import { runResponsesNonStreaming, runResponsesStreaming, } from '../openai-compat/responses-streaming.js';
+import { ResponsesRequestSchema, translateResponsesRequest, } from '../openai-compat/responses-translate.js';
+import { flattenTranscriptIntoPrompt, parseGezelModelRef, resolveModelTarget, } from '../openai-compat/translate.js';
+const log = createLogger('v1-responses');
+const MAX_RESPONSES_BODY_BYTES = 16 * 1024 * 1024;
+const NATIVE_AGENT_LOOP_PROVIDERS = new Set(['anthropic-cli', 'codex-cli']);
+export function v1ResponsesRoutes(ctx, opts = {}) {
+    const app = new Hono();
+    app.post('/', bodyLimit({
+        maxSize: MAX_RESPONSES_BODY_BYTES,
+        onError: (c) => c.json({
+            error: {
+                message: 'Responses request body exceeds the 16 MiB limit.',
+                type: 'invalid_request_error',
+                code: 'request_too_large',
+            },
+        }, 413),
+    }), async (c) => {
+        let raw;
+        try {
+            raw = await c.req.json();
+        }
+        catch {
+            return invalidRequest(c, 'Request body is not valid JSON.', 'invalid_json');
+        }
+        const rawRecord = asRecord(raw);
+        const unsupportedTool = firstUnsupportedHostedTool(rawRecord?.tools);
+        if (unsupportedTool) {
+            return invalidRequest(c, `Responses tool "${unsupportedTool}" is provider-hosted and is not available through Gezel local inference. Disable it in Codex (web_search = "disabled") or use a caller-executed function/custom tool.`, 'unsupported_tool');
+        }
+        if (rawRecord?.previous_response_id != null ||
+            rawRecord?.conversation != null ||
+            rawRecord?.store === true ||
+            rawRecord?.background === true) {
+            return invalidRequest(c, 'Gezel Responses inference is stateless. Send store: false and replay input items instead of using previous_response_id, conversation, or background mode.', 'stored_state_not_supported');
+        }
+        let translated;
+        let parsed;
+        try {
+            parsed = ResponsesRequestSchema.parse(raw);
+            translated = translateResponsesRequest(parsed);
+        }
+        catch (err) {
+            const message = err instanceof ZodError
+                ? err.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; ')
+                : err instanceof Error
+                    ? err.message
+                    : String(err);
+            return invalidRequest(c, message, 'invalid_body');
+        }
+        const endpointsConfig = await ctx.store
+            .readConfig()
+            .then((config) => config.openaiEndpoints ?? {})
+            .catch(() => ({}));
+        let target;
+        try {
+            target = await resolveResponsesTarget(parsed.model, ctx);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            const code = parseGezelModelRef(parsed.model) !== null ? 'gezel_not_found' : 'model_not_found';
+            return c.json({ error: { message, type: 'invalid_request_error', code } }, 404);
+        }
+        if (opts.allowedProviders && !opts.allowedProviders.includes(target.provider)) {
+            return c.json({
+                error: {
+                    code: 'provider_not_allowed',
+                    message: 'This host only serves on-device models.',
+                },
+            }, 403);
+        }
+        if (NATIVE_AGENT_LOOP_PROVIDERS.has(target.provider)) {
+            return invalidRequest(c, `Provider "${target.provider}" owns a native coding-agent tool loop and cannot be nested behind the inference-only Responses facade. Select a raw local/API model instead.`, 'provider_not_inference_only');
+        }
+        let session;
+        const requestStartedAt = Date.now();
+        try {
+            translated.sessionInput = prependSystemPrefix(translated.sessionInput, target.systemPrefix);
+            const actionLedger = appendCallerOwnedActionLedger(translated.sessionInput);
+            translated.sessionInput = actionLedger.input;
+            const provider = await ctx.chat.getProviderForModel(target.provider, target.model);
+            let externalTools = translated.externalTools;
+            const pinnedToolName = 'name' in translated.toolChoice ? translated.toolChoice.name : undefined;
+            if (translated.toolChoice.mode === 'none') {
+                externalTools = [];
+            }
+            else if (pinnedToolName) {
+                externalTools = externalTools.filter((tool) => tool.name === pinnedToolName);
+            }
+            if (translated.toolChoice.mode === 'required' && externalTools.length === 0) {
+                return invalidRequest(c, 'tool_choice is required, but the request does not contain a matching executable tool.', 'invalid_tool_choice');
+            }
+            if (externalTools.length > 0 && provider.supportsExternalTools !== true) {
+                return invalidRequest(c, `Provider "${provider.name}" does not support caller-executed tool calls.`, 'tools_not_supported_for_provider');
+            }
+            if (provider.supportsPriorMessages !== true) {
+                translated.sessionInput = flattenTranscriptIntoPrompt(translated.sessionInput);
+            }
+            const { systemMessage, prompt, priorMessages, attachments } = translated.sessionInput;
+            const defaults = await ctx.chat
+                .resolveModelSessionDefaults(target.provider, target.model, target.tuningOverrides ?? {})
+                .catch(() => null);
+            const tuning = defaults ? overlayResponsesTuning(defaults.tuning, translated) : null;
+            const lengthCapTokens = tuning?.sampling.maxTokens ?? translated.maxOutputTokens;
+            const supportingBehaviors = endpointsConfig.supportingBehaviors !== false;
+            const continueFromToolResult = prompt.length === 0 && priorMessages.at(-1)?.role === 'tool';
+            const sessionOpts = {
+                systemMessage,
+                ...(target.model ? { model: target.model } : {}),
+                ...(priorMessages.length > 0 ? { priorMessages } : {}),
+                ...(externalTools.length > 0 ? { externalTools } : {}),
+                ...(tuning ? { tuning } : {}),
+                ...(typeof translated.reasoning?.effort === 'string'
+                    ? { reasoningEffort: translated.reasoning.effort }
+                    : target.tuningOverrides?.reasoningEffort
+                        ? { reasoningEffort: target.tuningOverrides.reasoningEffort }
+                        : {}),
+                // Codex owns this action loop. Keep model compatibility behavior,
+                // but never apply Gezel's must-use-a-tool ramble intervention.
+                ...(defaults && supportingBehaviors
+                    ? { profile: profileForCallerOwnedInference(defaults.profile) }
+                    : {}),
+            };
+            session = await provider.createSession(sessionOpts);
+            const usageRef = { value: null };
+            session.onUsage((usage) => {
+                usageRef.value = usage;
+                ctx.chat.recordExternalUsage?.(target.provider, usage);
+            });
+            const appId = c.get('auth')?.appId ?? 'root';
+            const logOutcome = (outcome) => {
+                void ctx.history
+                    ?.log({
+                    kind: 'v1.chat.completion',
+                    summary: `${appId} · Responses ${outcome} (${parsed.model})`,
+                    details: {
+                        appId,
+                        surface: 'responses',
+                        outcome,
+                        model: parsed.model,
+                        provider: target.provider,
+                        actionReceiptCount: actionLedger.receiptCount,
+                        ...(usageRef.value
+                            ? {
+                                inputTokens: usageRef.value.inputTokens,
+                                outputTokens: usageRef.value.outputTokens,
+                            }
+                            : {}),
+                    },
+                })
+                    .catch(() => { });
+                log.info(`caller-owned Responses iteration ${outcome}: ${JSON.stringify({
+                    appId,
+                    model: parsed.model,
+                    provider: target.provider,
+                    inputItemCount: Array.isArray(parsed.input) ? parsed.input.length : 1,
+                    toolCount: parsed.tools?.length ?? 0,
+                    actionReceiptCount: actionLedger.receiptCount,
+                    elapsedMs: Date.now() - requestStartedAt,
+                })}`);
+            };
+            const turnAbort = new AbortController();
+            const requestSignal = c.req.raw.signal;
+            const abortTurn = () => turnAbort.abort();
+            if (requestSignal.aborted)
+                abortTurn();
+            else
+                requestSignal.addEventListener('abort', abortTurn, { once: true });
+            const detachRequestAbort = () => {
+                requestSignal.removeEventListener('abort', abortTurn);
+            };
+            const runnerOptions = {
+                ...(attachments.length > 0 ? { attachments } : {}),
+                ...(lengthCapTokens !== undefined ? { lengthCapTokens } : {}),
+                toolBindings: translated.toolBindings,
+                toolKinds: translated.toolKinds,
+                continueFromToolResult,
+                signal: turnAbort.signal,
+                onProviderError: (error) => {
+                    const message = error instanceof Error ? error.message : String(error);
+                    log.warn(`Responses provider failed: ${message}`);
+                },
+                echo: {
+                    instructions: parsed.instructions ?? null,
+                    parallelToolCalls: translated.parallelToolCalls ?? true,
+                    reasoning: translated.reasoning ?? null,
+                    store: false,
+                    toolChoice: parsed.tool_choice,
+                    tools: parsed.tools,
+                    maxOutputTokens: translated.maxOutputTokens ?? null,
+                },
+            };
+            if (translated.stream === true) {
+                return streamSSE(c, async (stream) => {
+                    stream.onAbort(() => {
+                        abortTurn();
+                        void session.disconnect().catch(() => { });
+                    });
+                    try {
+                        const response = await runResponsesStreaming(session, prompt, parsed.model, stream, () => Math.floor(Date.now() / 1_000), runnerOptions);
+                        logOutcome(response.status === 'completed' || response.status === 'incomplete'
+                            ? response.status
+                            : 'failed');
+                    }
+                    catch (err) {
+                        const message = err instanceof Error ? err.message : String(err);
+                        log.warn(`Responses stream failed: ${message}`);
+                        logOutcome('failed');
+                        await stream
+                            .writeSSE({
+                            event: 'error',
+                            data: JSON.stringify({
+                                type: 'error',
+                                code: 'provider_error',
+                                message: 'The response stream ended unexpectedly.',
+                                param: null,
+                            }),
+                        })
+                            .catch(() => { });
+                    }
+                    finally {
+                        detachRequestAbort();
+                        await session.disconnect().catch(() => { });
+                    }
+                });
+            }
+            try {
+                const response = await runResponsesNonStreaming(session, prompt, parsed.model, () => Math.floor(Date.now() / 1_000), runnerOptions);
+                logOutcome(response.status === 'incomplete' ? 'incomplete' : 'completed');
+                return c.json(response);
+            }
+            finally {
+                detachRequestAbort();
+                await session.disconnect().catch(() => { });
+                session = undefined;
+            }
+        }
+        catch (err) {
+            if (session)
+                await session.disconnect().catch(() => { });
+            if (err instanceof ModelNotInstalledError) {
+                return c.json({
+                    error: {
+                        message: err.message,
+                        type: 'invalid_request_error',
+                        code: 'model_not_found',
+                    },
+                }, 404);
+            }
+            if (err instanceof ExternalToolsUnsupportedError) {
+                return invalidRequest(c, err.message, 'tools_not_supported_for_provider');
+            }
+            const message = err instanceof Error ? err.message : String(err);
+            if (message.includes('messages must') ||
+                message.includes('empty') ||
+                message.includes('cannot be an assistant turn')) {
+                return invalidRequest(c, message, 'empty_prompt');
+            }
+            log.warn(`Responses request failed: ${message}`);
+            return c.json({ error: { message, type: 'server_error', code: 'provider_error' } }, 500);
+        }
+    });
+    return app;
+}
+async function resolveResponsesTarget(model, ctx) {
+    const gezelRef = parseGezelModelRef(model);
+    if (gezelRef !== null) {
+        return resolveChatTarget({ kind: 'gezel', ref: gezelRef }, ctx);
+    }
+    const modelTarget = resolveModelTarget(model);
+    if (modelTarget) {
+        return {
+            provider: modelTarget.provider,
+            model: modelTarget.model,
+            systemPrefix: '',
+        };
+    }
+    throw new Error(`Unknown model "${model}". Use an explicit <provider>:<model> id or advertised gezel:<role>-<name> alias.`);
+}
+function prependSystemPrefix(input, prefix) {
+    if (!prefix)
+        return input;
+    return {
+        ...input,
+        systemMessage: `${prefix}${input.systemMessage ? '\n\n---\n\n' : ''}${input.systemMessage}`,
+    };
+}
+function overlayResponsesTuning(base, request) {
+    const sampling = { ...base.sampling };
+    if (request.maxOutputTokens !== undefined)
+        sampling.maxTokens = request.maxOutputTokens;
+    return {
+        ...base,
+        sampling,
+        toolChoice: request.toolChoice.mode,
+    };
+}
+function asRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value)
+        ? value
+        : null;
+}
+function firstUnsupportedHostedTool(value) {
+    if (!Array.isArray(value))
+        return null;
+    for (const entry of value) {
+        const record = asRecord(entry);
+        const type = record?.type;
+        if (typeof type === 'string' &&
+            !['function', 'custom', 'freeform', 'namespace'].includes(type)) {
+            return type;
+        }
+    }
+    return null;
+}
+function invalidRequest(c, message, code) {
+    return c.json({ error: { message, type: 'invalid_request_error', code } }, 400);
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { startService, type InferenceHttpErrorEvent, type InferenceServiceOptions, type RunningInferenceService, } from './inference-service.js';
+export { verifyNativeBinaries } from './engines/electron-native-reuse.js';

package/dist/index.js

@@ -0,0 +1,2 @@
+export { startService, } from './inference-service.js';
+export { verifyNativeBinaries } from './engines/electron-native-reuse.js';

package/dist/inference-service.d.ts

@@ -0,0 +1,28 @@
+import { type ProviderName } from '@bendyline/gezel';
+import type { LLMProvider } from './providers/types.js';
+import './http/auth-context.js';
+/** Same shape the full service reports, so one handler serves either host. */
+export interface InferenceHttpErrorEvent {
+    kind: 'unhandled_exception';
+    requestId: string;
+    method: string;
+    path: string;
+    status: number;
+    detail: string;
+}
+export interface InferenceServiceOptions {
+    home?: string;
+    /** Host-provided local providers, useful for testing without downloading weights. */
+    providers?: ReadonlyMap<ProviderName, LLMProvider>;
+    onUnexpectedHttpError?: (event: InferenceHttpErrorEvent) => void;
+}
+export interface RunningInferenceService {
+    readonly profile: 'embedded-inference';
+    readonly port: number;
+    readonly cert: null;
+    readonly clientToken: string;
+    readonly fetch: typeof fetch;
+    stop(): Promise<void>;
+}
+/** Direct Fetch host: no listener, product store, background downloads or daemon discovery. */
+export declare function startService(options?: InferenceServiceOptions): Promise<RunningInferenceService>;