harness-dispatch 0.4.0

package/dist/dispatchers/openai-compatible.js

@@ -0,0 +1,662 @@
+/**
+ * HTTP endpoint dispatcher for harness-dispatch — handles both wire protocols
+ * a `type: openai_compatible` route can speak, selected by `wire_protocol:`
+ * in config (default `openai_chat_completions`):
+ *
+ *   openai_chat_completions  POST /chat/completions — Ollama, LM Studio,
+ *                            OpenRouter, OpenAI API, NVIDIA NIM, and any
+ *                            other OpenAI-compatible endpoint.
+ *   anthropic_messages       POST /messages — Anthropic's API directly, or
+ *                            any third-party host that mirrors its Messages
+ *                            API shape (different auth headers, request/
+ *                            response body, and SSE event framing from
+ *                            OpenAI's — see the wire-protocol-specific
+ *                            helpers below).
+ *
+ * Transport: global `fetch` (Node 24+). No subprocess, no extra deps.
+ * Quota:     reactive — parses x-ratelimit- and anthropic-ratelimit- headers
+ *            on every response. Local endpoints (Ollama, LM Studio) have no
+ *            rate limits.
+ *
+ * R3: `dispatch()` retains the buffered POST for simplicity + compatibility
+ * with tests that mock `fetch`. `stream()` switches to SSE streaming by
+ * setting `stream: true` in the request body and parsing wire-protocol-
+ * specific SSE frames as they arrive. The `completion` event is built from
+ * the summed delta content across all events.
+ */
+import { BaseDispatcher } from "./base.js";
+import { parseRetryAfter } from "./shared/rate-limit-headers.js";
+const CHAT_PATH = "/chat/completions";
+const MESSAGES_PATH = "/messages";
+const ANTHROPIC_VERSION = "2023-06-01";
+const DEFAULT_TIMEOUT_MS = 120_000;
+/** SSE frames are separated by a blank line — spec-legal as \n\n or \r\n\r\n. */
+const SSE_FRAME_BOUNDARY = /\r?\n\r?\n/;
+/**
+ * Append `path` onto `baseUrl`, inserting `/v1` unless it's already there.
+ * Exported so callers other than this dispatcher (e.g. the `usage` tool's
+ * listModels, which hits GET {baseUrl}/models on the same endpoint) build
+ * URLs the same way — a baseUrl configured without a /v1 suffix must not
+ * 404 on one code path while working on the other.
+ */
+export function endpointUrl(baseUrl, path) {
+    const trimmed = baseUrl.replace(/\/+$/, "");
+    return trimmed.endsWith("/v1") ? `${trimmed}${path}` : `${trimmed}/v1${path}`;
+}
+const DEFAULT_MAX_TOKENS = 8192;
+const DEFAULT_SYSTEM_PROMPT = "You are an expert software engineer. " +
+    "Respond with clear, working code and concise explanations.";
+const _MAX_FILE_BYTES = 512 * 1024; // 512 KB per file
+export class OpenAICompatibleDispatcher extends BaseDispatcher {
+    id;
+    baseUrl;
+    model;
+    apiKey;
+    thinkingLevel;
+    wireProtocol;
+    maxTokens;
+    constructor(svc) {
+        super();
+        this.id = svc.name;
+        const base = svc.baseUrl ?? "";
+        this.baseUrl = base.replace(/\/+$/, "");
+        this.model = svc.model ?? "";
+        this.apiKey = svc.apiKey ?? "";
+        if (svc.thinkingLevel)
+            this.thinkingLevel = svc.thinkingLevel;
+        this.wireProtocol = svc.wireProtocol ?? "openai_chat_completions";
+        this.maxTokens = svc.maxOutputTokens ?? DEFAULT_MAX_TOKENS;
+    }
+    isAvailable() {
+        return this.baseUrl.length > 0 && this.model.length > 0;
+    }
+    async checkQuota() {
+        return { service: this.id, source: "unknown" };
+    }
+    // ---------------------------------------------------------------------
+    // Wire-protocol-specific request building
+    // ---------------------------------------------------------------------
+    #url() {
+        const path = this.wireProtocol === "anthropic_messages" ? MESSAGES_PATH : CHAT_PATH;
+        return endpointUrl(this.baseUrl, path);
+    }
+    #headers(accept) {
+        const headers = {
+            "Content-Type": "application/json",
+            Accept: accept,
+        };
+        if (!this.apiKey)
+            return headers;
+        if (this.wireProtocol === "anthropic_messages") {
+            headers["x-api-key"] = this.apiKey;
+            headers["anthropic-version"] = ANTHROPIC_VERSION;
+        }
+        else {
+            headers["Authorization"] = `Bearer ${this.apiKey}`;
+        }
+        return headers;
+    }
+    #body(model, fullPrompt, stream) {
+        if (this.wireProtocol === "anthropic_messages") {
+            // Anthropic requires max_tokens and puts the system prompt at the
+            // top level, not as a messages[] entry. thinking_level isn't
+            // translated to Anthropic's extended-thinking `thinking:` param yet
+            // (different shape: {type, budget_tokens} with its own max_tokens
+            // interplay) — omitted rather than sent wrong.
+            return {
+                model,
+                max_tokens: this.maxTokens,
+                system: DEFAULT_SYSTEM_PROMPT,
+                messages: [{ role: "user", content: fullPrompt }],
+                stream,
+            };
+        }
+        const body = {
+            model,
+            messages: [
+                { role: "system", content: DEFAULT_SYSTEM_PROMPT },
+                { role: "user", content: fullPrompt },
+            ],
+            stream,
+        };
+        if (this.thinkingLevel)
+            body["reasoning_effort"] = this.thinkingLevel.toLowerCase();
+        // Without this, most OpenAI-compatible servers omit the usage frame
+        // during streaming — tokensUsed would silently be missing for every
+        // `job` and progress-token `code` call (both always use stream()),
+        // while the buffered dispatch() path got it for free.
+        if (stream)
+            body["stream_options"] = { include_usage: true };
+        return body;
+    }
+    // ---------------------------------------------------------------------
+    // Wire-protocol-specific response parsing
+    // ---------------------------------------------------------------------
+    #extractContent(body) {
+        if (this.wireProtocol === "anthropic_messages") {
+            const blocks = body.content;
+            if (!Array.isArray(blocks))
+                return null;
+            const text = blocks
+                .filter((b) => b !== null && typeof b === "object" && b.type === "text" && typeof b.text === "string")
+                .map((b) => b.text)
+                .join("");
+            return text.length > 0 ? text : null;
+        }
+        const choices = body.choices;
+        if (!Array.isArray(choices) || choices.length === 0)
+            return null;
+        const first = choices[0];
+        if (!first || typeof first !== "object")
+            return null;
+        const msg = first.message;
+        if (!msg || typeof msg !== "object")
+            return null;
+        const content = msg.content;
+        return typeof content === "string" ? content : null;
+    }
+    #extractUsage(body) {
+        if (this.wireProtocol === "anthropic_messages") {
+            const usage = body.usage;
+            if (usage && typeof usage.input_tokens === "number" && typeof usage.output_tokens === "number") {
+                return { input: usage.input_tokens, output: usage.output_tokens };
+            }
+            return undefined;
+        }
+        const usage = body.usage;
+        if (usage && typeof usage.prompt_tokens === "number" && typeof usage.completion_tokens === "number") {
+            return { input: usage.prompt_tokens, output: usage.completion_tokens };
+        }
+        return undefined;
+    }
+    #extractErrorMessage(body, rawBody) {
+        if (this.wireProtocol === "anthropic_messages") {
+            const message = body?.error?.message;
+            if (typeof message === "string")
+                return message;
+        }
+        else {
+            const message = body?.error?.message;
+            if (typeof message === "string")
+                return message;
+        }
+        return rawBody.slice(0, 200) || "(empty body)";
+    }
+    #parseBody(rawBody) {
+        if (!rawBody)
+            return null;
+        try {
+            return JSON.parse(rawBody);
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Parse one SSE frame (`event:`/`data:` lines separated by a blank line)
+     * into DispatcherEvents. Anthropic frames carry a named `event:` line;
+     * OpenAI frames don't (only `data:`, with a `[DONE]` sentinel) — both use
+     * the same blank-line frame boundary, so the caller's chunking is shared.
+     */
+    /**
+     * `usage` is PARTIAL per frame on purpose — Anthropic splits input_tokens
+     * (message_start) and output_tokens (message_delta) across two different
+     * frames, unlike OpenAI which sends both together in its one usage-
+     * bearing frame. The caller merges partial updates across the whole
+     * stream rather than overwriting on each frame.
+     *
+     * `error`, when set, means the upstream sent a mid-stream error event
+     * AFTER already returning 200 and streaming some content — a case the
+     * HTTP-status checks earlier in #runStream never see. The caller must
+     * treat this as a failed completion (using whatever partial output
+     * accumulated so far), not silently report success.
+     */
+    #parseSseFrame(frame) {
+        return this.wireProtocol === "anthropic_messages"
+            ? this.#parseAnthropicSseFrame(frame)
+            : this.#parseOpenAiSseFrame(frame);
+    }
+    #parseOpenAiSseFrame(frame) {
+        const out = [];
+        let usage = null;
+        let error;
+        for (const rawLine of frame.split(/\r?\n/)) {
+            const line = rawLine.trim();
+            if (!line || !line.startsWith("data:"))
+                continue;
+            const payload = line.slice("data:".length).trim();
+            if (payload === "[DONE]")
+                continue;
+            let obj;
+            try {
+                obj = JSON.parse(payload);
+            }
+            catch {
+                continue;
+            }
+            if (obj.error) {
+                error = typeof obj.error.message === "string" ? obj.error.message : "upstream error mid-stream";
+                continue;
+            }
+            const choices = obj.choices;
+            if (Array.isArray(choices)) {
+                for (const c of choices) {
+                    const delta = c.delta ?? c.message;
+                    const content = delta?.content;
+                    if (typeof content === "string" && content.length > 0) {
+                        out.push({ type: "stdout", chunk: content });
+                    }
+                }
+            }
+            if (obj.usage) {
+                const input = obj.usage.prompt_tokens;
+                const output = obj.usage.completion_tokens;
+                if (typeof input === "number" && typeof output === "number") {
+                    usage = { input, output };
+                }
+            }
+        }
+        return error !== undefined ? { events: out, usage, error } : { events: out, usage };
+    }
+    /**
+     * Anthropic streams message_start (initial input_tokens),
+     * content_block_delta (text_delta chunks), message_delta (final
+     * output_tokens), and (on failure) a named error event — accumulate
+     * across every `data:` line in the frame rather than keeping only the
+     * last one. That matters beyond just "a frame with two events": if frame
+     * boundaries ever get miscounted upstream (e.g. a proxy that coalesces
+     * writes), multiple real SSE events can land in what we treat as one
+     * frame — dropping all but the last would silently lose content or usage.
+     */
+    #parseAnthropicSseFrame(frame) {
+        const out = [];
+        let inputTokens;
+        let outputTokens;
+        let error;
+        const dataLines = [];
+        for (const rawLine of frame.split(/\r?\n/)) {
+            const line = rawLine.trim();
+            if (line.startsWith("data:"))
+                dataLines.push(line.slice("data:".length).trim());
+        }
+        for (const dataLine of dataLines) {
+            let obj;
+            try {
+                obj = JSON.parse(dataLine);
+            }
+            catch {
+                continue;
+            }
+            if (obj.type === "message_start") {
+                const v = obj.message?.usage?.input_tokens;
+                if (typeof v === "number")
+                    inputTokens = v;
+            }
+            else if (obj.type === "content_block_delta") {
+                const text = obj.delta?.type === "text_delta" ? obj.delta.text : undefined;
+                if (typeof text === "string" && text.length > 0)
+                    out.push({ type: "stdout", chunk: text });
+            }
+            else if (obj.type === "message_delta") {
+                const v = obj.usage?.output_tokens;
+                if (typeof v === "number")
+                    outputTokens = v;
+            }
+            else if (obj.type === "error") {
+                error = typeof obj.error?.message === "string" ? obj.error.message : "upstream error mid-stream";
+            }
+        }
+        const usage = inputTokens !== undefined || outputTokens !== undefined
+            ? {
+                ...(inputTokens !== undefined ? { input: inputTokens } : {}),
+                ...(outputTokens !== undefined ? { output: outputTokens } : {}),
+            }
+            : null;
+        return error !== undefined ? { events: out, usage, error } : { events: out, usage };
+    }
+    // ---------------------------------------------------------------------
+    // Dispatch
+    // ---------------------------------------------------------------------
+    /**
+     * Buffered one-shot: POST with stream=false, parse a single JSON body.
+     * Kept as a fast-path (no incremental parsing overhead) and to preserve
+     * existing mocked-fetch tests.
+     */
+    async dispatch(prompt, files, _workingDir, opts = {}) {
+        const start = Date.now();
+        const fullPrompt = await buildPromptWithFiles(prompt, files);
+        const url = this.#url();
+        const model = opts.modelOverride ?? this.model;
+        const body = this.#body(model, fullPrompt, false);
+        const headers = this.#headers("application/json");
+        const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        timer.unref?.();
+        let res;
+        try {
+            res = await fetch(url, {
+                method: "POST",
+                headers,
+                body: JSON.stringify(body),
+                signal: controller.signal,
+            });
+        }
+        catch (err) {
+            clearTimeout(timer);
+            const errMsg = err instanceof Error ? err.message : String(err);
+            const aborted = err?.name === "AbortError";
+            return {
+                output: "",
+                service: this.id,
+                success: false,
+                error: aborted ? `Timed out after ${timeoutMs}ms` : errMsg,
+                durationMs: Date.now() - start,
+            };
+        }
+        clearTimeout(timer);
+        const responseHeaders = headersToObject(res.headers);
+        const durationMs = Date.now() - start;
+        let rawBody = "";
+        try {
+            rawBody = await res.text();
+        }
+        catch {
+            // Body read failed — treat as empty.
+        }
+        const parsedBody = this.#parseBody(rawBody);
+        if (res.status === 429) {
+            const retryAfter = parseRetryAfter(responseHeaders);
+            const result = {
+                output: "",
+                service: this.id,
+                success: false,
+                error: `Rate limited by ${this.id}`,
+                rateLimited: true,
+                rateLimitHeaders: responseHeaders,
+                durationMs,
+            };
+            if (retryAfter !== null)
+                result.retryAfter = retryAfter;
+            return result;
+        }
+        if (res.status >= 400) {
+            const errMessage = this.#extractErrorMessage(parsedBody, rawBody);
+            return {
+                output: "",
+                service: this.id,
+                success: false,
+                error: `HTTP ${res.status}: ${errMessage}`,
+                durationMs,
+                rateLimitHeaders: responseHeaders,
+            };
+        }
+        const content = parsedBody ? this.#extractContent(parsedBody) : null;
+        if (content === null) {
+            return {
+                output: "",
+                service: this.id,
+                success: false,
+                error: `Unexpected response shape: ${rawBody.slice(0, 300)}`,
+                durationMs,
+                rateLimitHeaders: responseHeaders,
+            };
+        }
+        const result = {
+            output: content,
+            service: this.id,
+            success: true,
+            durationMs,
+            rateLimitHeaders: responseHeaders,
+        };
+        if (parsedBody) {
+            const tokensUsed = this.#extractUsage(parsedBody);
+            if (tokensUsed)
+                result.tokensUsed = tokensUsed;
+        }
+        return result;
+    }
+    stream(prompt, files, workingDir, opts = {}) {
+        return this.#runStream(prompt, files, workingDir, opts);
+    }
+    async *#runStream(prompt, files, _workingDir, opts) {
+        const start = Date.now();
+        const fullPrompt = await buildPromptWithFiles(prompt, files);
+        const url = this.#url();
+        const model = opts.modelOverride ?? this.model;
+        const body = this.#body(model, fullPrompt, true);
+        const headers = this.#headers("text/event-stream");
+        const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        timer.unref?.();
+        let res;
+        try {
+            res = await fetch(url, {
+                method: "POST",
+                headers,
+                body: JSON.stringify(body),
+                signal: controller.signal,
+            });
+        }
+        catch (err) {
+            clearTimeout(timer);
+            const errMsg = err instanceof Error ? err.message : String(err);
+            const aborted = err?.name === "AbortError";
+            yield {
+                type: "completion",
+                result: {
+                    output: "",
+                    service: this.id,
+                    success: false,
+                    error: aborted ? `Timed out after ${timeoutMs}ms` : errMsg,
+                    durationMs: Date.now() - start,
+                },
+            };
+            return;
+        }
+        const responseHeaders = headersToObject(res.headers);
+        if (res.status === 429) {
+            clearTimeout(timer);
+            const retryAfter = parseRetryAfter(responseHeaders);
+            const result = {
+                output: "",
+                service: this.id,
+                success: false,
+                error: `Rate limited by ${this.id}`,
+                rateLimited: true,
+                rateLimitHeaders: responseHeaders,
+                durationMs: Date.now() - start,
+            };
+            if (retryAfter !== null)
+                result.retryAfter = retryAfter;
+            yield { type: "completion", result };
+            return;
+        }
+        if (res.status >= 400) {
+            clearTimeout(timer);
+            const rawBody = await res.text().catch(() => "");
+            const parsedBody = this.#parseBody(rawBody);
+            const errMessage = this.#extractErrorMessage(parsedBody, rawBody);
+            yield {
+                type: "completion",
+                result: {
+                    output: "",
+                    service: this.id,
+                    success: false,
+                    error: `HTTP ${res.status}: ${errMessage}`,
+                    durationMs: Date.now() - start,
+                    rateLimitHeaders: responseHeaders,
+                },
+            };
+            return;
+        }
+        // Stream body — SSE frames are separated by a blank line, which the
+        // spec allows as \n\n OR \r\n\r\n. A literal indexOf("\n\n") never
+        // matches CRLF-framed streams, so the whole body silently piles up in
+        // the trailing flush as one "frame" — use a regex boundary instead.
+        const chunks = [];
+        let buffer = "";
+        // Merged across frames, not overwritten — Anthropic's input/output
+        // token counts arrive on two DIFFERENT frames (see #parseSseFrame).
+        let inputTokens;
+        let outputTokens;
+        let streamError;
+        const mergeUsage = (u) => {
+            if (!u)
+                return;
+            if (u.input !== undefined)
+                inputTokens = u.input;
+            if (u.output !== undefined)
+                outputTokens = u.output;
+        };
+        if (!res.body) {
+            clearTimeout(timer);
+            yield {
+                type: "completion",
+                result: {
+                    output: "",
+                    service: this.id,
+                    success: false,
+                    error: "No response body",
+                    durationMs: Date.now() - start,
+                    rateLimitHeaders: responseHeaders,
+                },
+            };
+            return;
+        }
+        const reader = res.body.getReader();
+        const decoder = new TextDecoder();
+        try {
+            outer: while (true) {
+                const { value, done } = await reader.read();
+                if (done)
+                    break;
+                buffer += decoder.decode(value, { stream: true });
+                let boundary = SSE_FRAME_BOUNDARY.exec(buffer);
+                while (boundary !== null) {
+                    const frame = buffer.slice(0, boundary.index);
+                    buffer = buffer.slice(boundary.index + boundary[0].length);
+                    const evts = this.#parseSseFrame(frame);
+                    for (const e of evts.events) {
+                        yield e;
+                        if (e.type === "stdout")
+                            chunks.push(e.chunk);
+                    }
+                    mergeUsage(evts.usage);
+                    if (evts.error !== undefined) {
+                        streamError = evts.error;
+                        break outer; // Upstream sent a mid-stream error — stop reading.
+                    }
+                    boundary = SSE_FRAME_BOUNDARY.exec(buffer);
+                }
+            }
+        }
+        catch (err) {
+            clearTimeout(timer);
+            const errMsg = err instanceof Error ? err.message : String(err);
+            yield {
+                type: "completion",
+                result: {
+                    output: chunks.join(""),
+                    service: this.id,
+                    success: false,
+                    error: errMsg,
+                    durationMs: Date.now() - start,
+                    rateLimitHeaders: responseHeaders,
+                },
+            };
+            return;
+        }
+        clearTimeout(timer);
+        // A mid-stream error broke out of the read loop with the response body
+        // still open — cancel it so the connection is released now instead of
+        // whenever the server or GC gets around to it.
+        if (streamError !== undefined) {
+            try {
+                await reader.cancel();
+            }
+            catch {
+                // Releasing a broken stream is best-effort.
+            }
+        }
+        // Flush trailing frame if any (skipped if a mid-stream error already
+        // ended the read loop early — nothing meaningful left to parse).
+        if (streamError === undefined && buffer.trim()) {
+            const evts = this.#parseSseFrame(buffer);
+            for (const e of evts.events) {
+                yield e;
+                if (e.type === "stdout")
+                    chunks.push(e.chunk);
+            }
+            mergeUsage(evts.usage);
+            if (evts.error !== undefined)
+                streamError = evts.error;
+        }
+        const output = chunks.join("");
+        const result = streamError !== undefined
+            ? {
+                output,
+                service: this.id,
+                success: false,
+                error: streamError,
+                durationMs: Date.now() - start,
+                rateLimitHeaders: responseHeaders,
+            }
+            : {
+                output,
+                service: this.id,
+                success: true,
+                durationMs: Date.now() - start,
+                rateLimitHeaders: responseHeaders,
+            };
+        if (inputTokens !== undefined && outputTokens !== undefined) {
+            result.tokensUsed = { input: inputTokens, output: outputTokens };
+        }
+        yield { type: "completion", result };
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function headersToObject(h) {
+    const out = {};
+    h.forEach((value, key) => {
+        out[key] = value;
+    });
+    return out;
+}
+async function buildPromptWithFiles(prompt, files) {
+    if (files.length === 0)
+        return prompt;
+    const parts = [prompt];
+    const { stat, readFile } = await import("node:fs/promises");
+    const { extname } = await import("node:path");
+    for (const filePath of files) {
+        try {
+            const info = await stat(filePath);
+            if (!info.isFile()) {
+                parts.push(`\n# Not a file: ${filePath}`);
+                continue;
+            }
+            if (info.size > _MAX_FILE_BYTES) {
+                parts.push(`\n# Skipped ${filePath}: file too large (${Math.floor(info.size / 1024)} KB > ${_MAX_FILE_BYTES / 1024} KB limit)`);
+                continue;
+            }
+            const content = await readFile(filePath, "utf8");
+            const ext = extname(filePath).replace(/^\./, "");
+            parts.push(`\n\n\`\`\`${ext}\n# ${filePath}\n${content}\n\`\`\``);
+        }
+        catch (err) {
+            if (err?.code === "ENOENT") {
+                parts.push(`\n# File not found: ${filePath}`);
+            }
+            else {
+                const msg = err instanceof Error ? err.message : String(err);
+                parts.push(`\n# Could not read ${filePath}: ${msg}`);
+            }
+        }
+    }
+    return parts.join("\n");
+}
+//# sourceMappingURL=openai-compatible.js.map