@fusionkit/model-gateway 0.1.0

package/dist/fusion-backend.js

@@ -0,0 +1,521 @@
+/**
+ * The fusion front-door backend.
+ *
+ * This is the clean abstraction behind "the judge streams a trajectory the
+ * user's harness executes". It implements the gateway {@link Backend} contract
+ * (an OpenAI Chat Completions surface) so it slots into the existing
+ * `startGateway` server and reuses every dialect adapter (chat / responses /
+ * anthropic) — including their full tool-call, tool-result, and streaming
+ * support — for free.
+ *
+ * Per front-door turn it:
+ *   1. derives a stable session key from the conversation prefix,
+ *   2. runs the panel **once** per session (injected `runPanels`, so this
+ *      package keeps no dependency on `@fusionkit/ensemble`) to produce the
+ *      candidate trajectories,
+ *   3. forwards the live conversation + the harness tools + the candidate
+ *      trajectories to FusionKit's `trajectory:step`, whose response (an OpenAI
+ *      chat completion, optionally streamed, that may carry `tool_calls`) is
+ *      returned verbatim for the server to translate into the caller's dialect.
+ *
+ * There is no apply/verify/repair here: iteration is the user's harness's job.
+ *
+ * Failures are surfaced, never swallowed: a panel run that throws or yields no
+ * usable candidate, or a `trajectory:step` that errors, produces an explicit
+ * error (a non-2xx response when nothing has streamed yet, or a terminal error
+ * event with `finish_reason: "error"` once the SSE has started) and the failed
+ * session is evicted so the next turn retries instead of replaying the failure.
+ */
+import { createHash } from "node:crypto";
+import { emitTrace, getTraceEmitter, judgeFinalPayload, judgeRequestPayload, judgeThinkingPayload, newSpanId, newTraceId, TRACE_ID_HEADER } from "@fusionkit/protocol";
+const DEFAULT_SESSION_TTL_MS = 60 * 60 * 1000;
+const DEFAULT_PANEL_TIMEOUT_MS = 15 * 60 * 1000;
+const DEFAULT_STEP_TIMEOUT_MS = 10 * 60 * 1000;
+function textOfContent(content) {
+    if (typeof content === "string")
+        return content;
+    if (Array.isArray(content)) {
+        return content
+            .map((part) => {
+            if (typeof part === "string")
+                return part;
+            if (part !== null && typeof part === "object" && typeof part.text === "string") {
+                return part.text;
+            }
+            return "";
+        })
+            .join("");
+    }
+    return "";
+}
+function errorText(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+/** A candidate set is usable when at least one trajectory did not fail. */
+function hasUsableCandidates(candidates) {
+    return candidates.some((candidate) => candidate.status !== "failed");
+}
+/** Combine an optional client-abort signal with a wall-clock timeout. */
+function withDeadline(signal, timeoutMs) {
+    const timeout = AbortSignal.timeout(timeoutMs);
+    return signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
+}
+/** Reject if `promise` does not settle within `timeoutMs` (the work detaches). */
+async function withTimeout(promise, timeoutMs, label) {
+    let timer;
+    const timeout = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
+    });
+    try {
+        return await Promise.race([promise, timeout]);
+    }
+    finally {
+        if (timer !== undefined)
+            clearTimeout(timer);
+    }
+}
+function jsonError(status, message) {
+    return new Response(JSON.stringify({ error: { message, type: "fusion_error" } }), {
+        status,
+        headers: { "content-type": "application/json" }
+    });
+}
+/** Best-effort reassembly of an OpenAI chat SSE stream into content, usage,
+ *  tool-call deltas, and finish reason (used to tell terminal from intermediate). */
+function assembleSseContent(buffer) {
+    let content = "";
+    let usage;
+    let finishReason;
+    const toolCalls = [];
+    for (const line of buffer.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed.startsWith("data:"))
+            continue;
+        const data = trimmed.slice(5).trim();
+        if (data.length === 0 || data === "[DONE]")
+            continue;
+        try {
+            const json = JSON.parse(data);
+            const choice = json.choices?.[0];
+            const delta = choice?.delta?.content;
+            if (typeof delta === "string")
+                content += delta;
+            if (Array.isArray(choice?.delta?.tool_calls))
+                toolCalls.push(...choice.delta.tool_calls);
+            if (typeof choice?.finish_reason === "string")
+                finishReason = choice.finish_reason;
+            if (json.usage !== undefined && json.usage !== null)
+                usage = json.usage;
+        }
+        catch {
+            // ignore partial/non-JSON lines
+        }
+    }
+    return {
+        content,
+        toolCalls,
+        ...(usage !== undefined ? { usage } : {}),
+        ...(finishReason !== undefined ? { finishReason } : {})
+    };
+}
+/** A judge step is terminal (the real answer) only when it requests no tool calls. */
+function isTerminalJudgeStep(toolCalls, finishReason) {
+    const calls = Array.isArray(toolCalls) ? toolCalls : [];
+    return calls.length === 0 && finishReason !== "tool_calls";
+}
+/** A terminal SSE chunk that marks the turn as failed (not a normal stop). */
+function errorEvent(message) {
+    return (`data: ${JSON.stringify({
+        choices: [{ index: 0, delta: { content: message }, finish_reason: "error" }]
+    })}\n\n` + "data: [DONE]\n\n");
+}
+export class FusionBackend {
+    defaultModel;
+    #stepUrl;
+    #runPanels;
+    #judgeModel;
+    #ttlMs;
+    #panelTimeoutMs;
+    #stepTimeoutMs;
+    #mintTraceId;
+    #sessions = new Map();
+    constructor(options) {
+        this.#stepUrl = options.stepUrl;
+        this.#runPanels = options.runPanels;
+        this.defaultModel = options.defaultModel;
+        this.#judgeModel = options.judgeModel;
+        this.#ttlMs = options.sessionTtlMs ?? DEFAULT_SESSION_TTL_MS;
+        this.#panelTimeoutMs = options.panelTimeoutMs ?? DEFAULT_PANEL_TIMEOUT_MS;
+        this.#stepTimeoutMs = options.stepTimeoutMs ?? DEFAULT_STEP_TIMEOUT_MS;
+        this.#mintTraceId = options.mintTraceId ?? newTraceId;
+    }
+    async chat(body, signal, options = {}) {
+        const chat = (body ?? {});
+        const messages = Array.isArray(chat.messages) ? chat.messages : [];
+        const sessionKey = this.#sessionKey(messages);
+        const session = this.#ensureSession(sessionKey);
+        const streaming = chat.stream === true;
+        const buildStepBody = (candidates) => {
+            const stepBody = {
+                model: chat.model ?? this.defaultModel ?? "fusion-panel",
+                messages,
+                trajectories: candidates,
+                stream: streaming
+            };
+            if (chat.tools !== undefined)
+                stepBody.tools = chat.tools;
+            if (chat.tool_choice !== undefined)
+                stepBody.tool_choice = chat.tool_choice;
+            if (this.#judgeModel !== undefined)
+                stepBody.judge_model = this.#judgeModel;
+            return JSON.stringify(stepBody);
+        };
+        const headers = {
+            "content-type": "application/json",
+            [TRACE_ID_HEADER]: session.traceId
+        };
+        if (options.modelCallId)
+            headers["x-velum-model-call-id"] = options.modelCallId;
+        // The judge step is a child span of the session: emit the full prompt sent to
+        // the judge (the live conversation + candidate trajectories + tools) and the
+        // judge's final output, so the companion app can show exactly what the judge
+        // saw and produced.
+        const judgeSpan = newSpanId();
+        const traceEnabled = getTraceEmitter().isEnabled();
+        const sessionTraceId = session.traceId;
+        const sessionSpan = session.sessionSpan;
+        const judgeModel = this.#judgeModel;
+        // The user-turn index: a follow-up user message is a new turn, while the
+        // harness's internal tool-loop continuations (which append assistant/tool
+        // messages, not user ones) keep the same count and thus the same turn. The
+        // panel runs once per turn, so each new user request is fused over fresh
+        // candidates; the tool loop within a turn reuses them.
+        const turn = messages.filter((message) => message.role === "user").length;
+        const turnCandidates = this.#ensureTurnCandidates(session, sessionKey, turn, messages);
+        const emitJudgeRequest = (candidates) => {
+            if (!traceEnabled)
+                return;
+            emitTrace({
+                component: "judge",
+                event_type: "judge.request",
+                traceId: sessionTraceId,
+                spanId: judgeSpan,
+                parentSpanId: sessionSpan,
+                payload: judgeRequestPayload({
+                    ...(judgeModel !== undefined ? { judgeModel } : {}),
+                    messages,
+                    trajectories: candidates,
+                    ...(chat.tools !== undefined ? { tools: chat.tools } : {}),
+                    ...(chat.tool_choice !== undefined ? { toolChoice: chat.tool_choice } : {}),
+                    trajectoryIds: candidates.map((candidate) => candidate.trajectory_id),
+                    turn
+                })
+            });
+        };
+        const emitJudgeFinal = (input) => {
+            if (!traceEnabled)
+                return;
+            emitTrace({
+                component: "judge",
+                event_type: "judge.final",
+                traceId: sessionTraceId,
+                spanId: judgeSpan,
+                parentSpanId: sessionSpan,
+                payload: judgeFinalPayload({ ...input, turn })
+            });
+        };
+        // An intermediate tool-calling turn is NOT the final answer: the harness will
+        // execute the tool calls and call back. Emit it as `judge.thinking` so the
+        // companion app shows it as in-progress instead of marking the session done.
+        const emitJudgeStep = (input) => {
+            if (!traceEnabled)
+                return;
+            const toolCallCount = input.toolCalls?.length ?? 0;
+            const rawAnalysis = input.content !== undefined && input.content.length > 0
+                ? input.content
+                : `judge requested ${toolCallCount} tool call(s)`;
+            emitTrace({
+                component: "judge",
+                event_type: "judge.thinking",
+                traceId: sessionTraceId,
+                spanId: judgeSpan,
+                parentSpanId: sessionSpan,
+                payload: judgeThinkingPayload({
+                    rawAnalysis,
+                    ...(input.toolCalls !== undefined ? { toolCalls: input.toolCalls } : {}),
+                    ...(input.usage !== undefined ? { usage: input.usage } : {}),
+                    turn
+                })
+            });
+        };
+        // Resolve the panel candidates (bounded), failing loudly so a panel crash or
+        // an empty/all-failed candidate set never silently fuses into a blank answer.
+        const resolveCandidates = async () => {
+            const candidates = await withTimeout(turnCandidates, this.#panelTimeoutMs, "fusion panel");
+            if (!hasUsableCandidates(candidates)) {
+                throw new Error(candidates.length === 0
+                    ? "fusion panel produced no candidates"
+                    : "fusion panel produced no usable candidates (every model failed)");
+            }
+            return candidates;
+        };
+        // Non-streaming: the panel phase can block before the single JSON reply.
+        if (!streaming) {
+            let candidates;
+            try {
+                candidates = await resolveCandidates();
+            }
+            catch (error) {
+                this.#evictTurn(session, turn);
+                console.error(`fusion: panel phase failed: ${errorText(error)}`);
+                return jsonError(502, errorText(error));
+            }
+            emitJudgeRequest(candidates);
+            const response = await fetch(this.#stepUrl, {
+                method: "POST",
+                headers,
+                body: buildStepBody(candidates),
+                signal: withDeadline(signal, this.#stepTimeoutMs)
+            });
+            if (traceEnabled) {
+                // Capture the judge's output without consuming the piped response.
+                const clone = response.clone();
+                void (async () => {
+                    try {
+                        if (!clone.ok) {
+                            emitJudgeFinal({ httpStatus: clone.status, error: (await clone.text()).slice(0, 2000) });
+                            return;
+                        }
+                        const judged = (await clone.json());
+                        const choice = judged.choices?.[0];
+                        const message = choice?.message;
+                        const content = typeof message?.content === "string" ? message.content : undefined;
+                        const toolCalls = Array.isArray(message?.tool_calls) ? message.tool_calls : [];
+                        if (isTerminalJudgeStep(toolCalls, choice?.finish_reason)) {
+                            emitJudgeFinal({
+                                httpStatus: clone.status,
+                                ...(content !== undefined ? { content } : {}),
+                                ...(judged.usage !== undefined ? { usage: judged.usage } : {})
+                            });
+                        }
+                        else {
+                            emitJudgeStep({
+                                ...(content !== undefined ? { content } : {}),
+                                toolCalls,
+                                ...(judged.usage !== undefined ? { usage: judged.usage } : {})
+                            });
+                        }
+                    }
+                    catch {
+                        // best-effort judge.final
+                    }
+                })();
+            }
+            return response;
+        }
+        // Streaming: return immediately with a live SSE stream so the caller's HTTP
+        // client sees the response start right away. The (potentially slow) panel
+        // phase runs inside the stream behind keepalive comments, then the judge
+        // step's SSE is piped through. This avoids first-byte timeouts in real CLIs
+        // (e.g. codex) while the panel solves the task once. Because the 200 + SSE
+        // headers are already sent, failures surface as a terminal error event.
+        const stepUrl = this.#stepUrl;
+        const stepSignal = withDeadline(signal, this.#stepTimeoutMs);
+        const evictOnFailure = () => this.#evictTurn(session, turn);
+        const encoder = new TextEncoder();
+        const decoder = new TextDecoder();
+        const readable = new ReadableStream({
+            async start(controller) {
+                let alive = true;
+                let sseBuffer = "";
+                const keepalive = setInterval(() => {
+                    if (alive) {
+                        try {
+                            controller.enqueue(encoder.encode(": keepalive\n\n"));
+                        }
+                        catch {
+                            alive = false;
+                        }
+                    }
+                }, 3000);
+                const fail = (message) => {
+                    console.error(`fusion: ${message}`);
+                    evictOnFailure();
+                    controller.enqueue(encoder.encode(errorEvent(`fusion error: ${message}`)));
+                };
+                try {
+                    let candidates;
+                    try {
+                        candidates = await resolveCandidates();
+                    }
+                    catch (error) {
+                        fail(errorText(error));
+                        return;
+                    }
+                    emitJudgeRequest(candidates);
+                    if (process.env.FUSION_DEBUG) {
+                        const toolNames = Array.isArray(chat.tools)
+                            ? chat.tools.map((t) => {
+                                const tool = t;
+                                return tool.function?.name ?? tool.name ?? tool.type ?? "?";
+                            })
+                            : [];
+                        console.error(`[fusion-debug] step: messages=${messages.length} roles=${messages.map((m) => m.role).join(",")} ` +
+                            `candidates=${candidates.length} tools=[${toolNames.join(", ")}]`);
+                    }
+                    const upstream = await fetch(stepUrl, {
+                        method: "POST",
+                        headers,
+                        body: buildStepBody(candidates),
+                        signal: stepSignal
+                    });
+                    if (!upstream.ok || upstream.body === null) {
+                        const detail = upstream.body === null ? "no stream" : (await upstream.text()).slice(0, 800);
+                        emitJudgeFinal({ httpStatus: upstream.status, error: detail });
+                        fail(`trajectory:step ${upstream.status}: ${detail}`);
+                        return;
+                    }
+                    const reader = upstream.body.getReader();
+                    for (;;) {
+                        const { done, value } = await reader.read();
+                        if (done)
+                            break;
+                        if (value !== undefined) {
+                            controller.enqueue(value);
+                            if (traceEnabled)
+                                sseBuffer += decoder.decode(value, { stream: true });
+                        }
+                    }
+                    if (traceEnabled) {
+                        const assembled = assembleSseContent(sseBuffer);
+                        if (isTerminalJudgeStep(assembled.toolCalls, assembled.finishReason)) {
+                            emitJudgeFinal({
+                                httpStatus: upstream.status,
+                                ...(assembled.content.length > 0 ? { content: assembled.content } : {}),
+                                ...(assembled.usage !== undefined ? { usage: assembled.usage } : {})
+                            });
+                        }
+                        else {
+                            emitJudgeStep({
+                                ...(assembled.content.length > 0 ? { content: assembled.content } : {}),
+                                toolCalls: assembled.toolCalls,
+                                ...(assembled.usage !== undefined ? { usage: assembled.usage } : {})
+                            });
+                        }
+                    }
+                }
+                catch (error) {
+                    emitJudgeFinal({ error: errorText(error) });
+                    fail(errorText(error));
+                }
+                finally {
+                    alive = false;
+                    clearInterval(keepalive);
+                    try {
+                        controller.close();
+                    }
+                    catch {
+                        // already closed
+                    }
+                }
+            }
+        });
+        return new Response(readable, {
+            status: 200,
+            headers: { "content-type": "text/event-stream", "cache-control": "no-cache" }
+        });
+    }
+    models() {
+        const model = this.defaultModel ?? "fusion-panel";
+        return Promise.resolve(new Response(JSON.stringify({ object: "list", data: [{ id: model, object: "model", owned_by: "fusion-gateway" }] }), { status: 200, headers: { "content-type": "application/json" } }));
+    }
+    embeddings() {
+        return Promise.resolve(new Response(JSON.stringify({ error: { message: "embeddings are not supported by the fusion gateway" } }), {
+            status: 501,
+            headers: { "content-type": "application/json" }
+        }));
+    }
+    /** A stable key for the conversation: system text + first user message. */
+    #sessionKey(messages) {
+        const system = messages
+            .filter((message) => message.role === "system")
+            .map((message) => textOfContent(message.content))
+            .join("\n");
+        const firstUser = messages.find((message) => message.role === "user");
+        const seed = JSON.stringify([system, firstUser ? textOfContent(firstUser.content) : ""]);
+        return createHash("sha256").update(seed).digest("hex").slice(0, 16);
+    }
+    #task(messages) {
+        // The panel task is the *current* request: the most recent user message.
+        // Real CLIs (codex/claude/cursor) put their large agent harness prompt in
+        // the system message and may prepend an <environment_context> user message,
+        // so take the latest user turn (the active instruction) and fall back to
+        // system text only if there is no user content at all. Using the latest
+        // user message means a follow-up turn's panel solves the follow-up request.
+        const userText = messages
+            .filter((message) => message.role === "user")
+            .map((message) => textOfContent(message.content).trim())
+            .filter((text) => text.length > 0);
+        const latest = userText.at(-1);
+        if (latest !== undefined && latest.length > 0)
+            return latest;
+        return messages
+            .filter((message) => message.role === "system")
+            .map((message) => textOfContent(message.content))
+            .join("\n\n")
+            .trim();
+    }
+    /** Drop a turn's cached candidates so the next call for that turn re-runs the panel. */
+    #evictTurn(session, turn) {
+        session.turns.delete(turn);
+    }
+    /** Remove expired sessions so a long-lived gateway does not grow unbounded. */
+    #sweepExpired(now) {
+        for (const [key, session] of this.#sessions) {
+            if (now - session.createdAt >= this.#ttlMs)
+                this.#sessions.delete(key);
+        }
+    }
+    /** Establish (or reuse) the per-conversation session identity. No panel runs here. */
+    #ensureSession(sessionKey) {
+        const now = Date.now();
+        this.#sweepExpired(now);
+        const existing = this.#sessions.get(sessionKey);
+        if (existing !== undefined && now - existing.createdAt < this.#ttlMs)
+            return existing;
+        const session = {
+            traceId: this.#mintTraceId(),
+            sessionSpan: newSpanId(),
+            turns: new Map(),
+            createdAt: now
+        };
+        this.#sessions.set(sessionKey, session);
+        return session;
+    }
+    /**
+     * Run the panel once per user turn and cache its candidates on the session.
+     * Internal tool-loop continuations keep the same `turn` and reuse the result;
+     * a follow-up user message is a new `turn` and triggers a fresh panel run.
+     * A failed turn is evicted so a retry re-runs it (failures are never cached).
+     */
+    #ensureTurnCandidates(session, sessionKey, turn, messages) {
+        const existing = session.turns.get(turn);
+        if (existing !== undefined)
+            return existing;
+        const candidates = this.#runPanels({
+            task: this.#task(messages),
+            messages,
+            traceId: session.traceId,
+            sessionSpanId: session.sessionSpan,
+            sessionKey,
+            turn
+        });
+        session.turns.set(turn, candidates);
+        candidates.catch((error) => {
+            console.error(`fusion: panel run failed for session ${sessionKey} turn ${turn}: ${errorText(error)}`);
+            if (session.turns.get(turn) === candidates)
+                session.turns.delete(turn);
+        });
+        return candidates;
+    }
+}

package/dist/fusion-gateway.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Fusion Harness Gateway — the provider-facing front door that lets a coding
+ * tool (Codex, Claude Code, Cursor via Cursorkit) be the entrypoint. A prompt
+ * sent from the tool hits this gateway, which translates the request into a
+ * dialect-agnostic prompt, runs the unified HandoffKit/FusionKit harness
+ * ensemble through an injected runner, then translates the synthesized final
+ * answer back into the tool's native wire format.
+ *
+ * The runner is injected (not imported) so this package stays free of a
+ * dependency on `@fusionkit/ensemble`, which already depends on this package.
+ */
+import type { AnthropicRequest } from "./adapters/anthropic.js";
+import type { ResponsesRequest } from "./adapters/responses.js";
+export type FrontDoorDialect = "openai-responses" | "anthropic-messages" | "openai-chat";
+export type FrontDoorRunnerInput = {
+    dialect: FrontDoorDialect;
+    prompt: string;
+    requestedModel: string | undefined;
+    requestId: string;
+    /** Correlates this front-door request with all downstream trace events. */
+    traceId: string;
+};
+export type FrontDoorRunnerResult = {
+    finalOutput: string;
+    runId: string;
+    status: "succeeded" | "failed" | "skipped";
+    evidence: string[];
+    reportPath?: string;
+};
+export type FrontDoorRunner = (input: FrontDoorRunnerInput) => Promise<FrontDoorRunnerResult>;
+export type FusionGatewayOptions = {
+    /** Runs the unified harness ensemble for a single front-door prompt. */
+    runner: FrontDoorRunner;
+    /** Bind host; defaults to loopback. */
+    host?: string;
+    /** Bind port; defaults to an ephemeral free port. */
+    port?: number;
+    /** When set, require this bearer token (or matching `x-api-key`). */
+    authToken?: string;
+    /** Model id echoed back in responses and `/v1/models`. */
+    defaultModel?: string;
+};
+export type FusionGateway = {
+    /** Base URL clients should target (without the `/v1` suffix). */
+    url(): string;
+    port(): number;
+    close(): Promise<void>;
+};
+export declare const FUSION_RUN_ID_HEADER = "x-fusion-run-id";
+export declare const FUSION_STATUS_HEADER = "x-fusion-status";
+export declare const FUSION_EVIDENCE_HEADER = "x-fusion-evidence";
+export declare const FUSION_REPORT_HEADER = "x-fusion-report";
+export declare function promptFromResponses(body: ResponsesRequest): string;
+export declare function promptFromAnthropic(body: AnthropicRequest): string;
+type ChatMessage = {
+    role?: string;
+    content?: unknown;
+};
+export type ChatRequest = {
+    model?: string;
+    messages?: ChatMessage[];
+    stream?: boolean;
+};
+export declare function promptFromChat(body: ChatRequest): string;
+export declare function formatResponses(finalOutput: string, model: string): Record<string, unknown>;
+export declare function formatAnthropic(finalOutput: string, model: string): Record<string, unknown>;
+export declare function formatChat(finalOutput: string, model: string): Record<string, unknown>;
+export declare function startFusionGateway(options: FusionGatewayOptions): Promise<FusionGateway>;
+export {};