@zenithfoundry/slm-gate 1.2.1

package/dist/llm-gate/formats/contract.js

@@ -0,0 +1,29 @@
+/**
+ * @fileoverview The one contract every wire-format module implements for the model gate.
+ *
+ * Four real formats (Anthropic Messages, Chat Completions, Responses, Gemini) justify this small shared
+ * shape; everything else stays local to each format module. Request bodies are plain parsed JSON.
+ */
+// Coding tools inject context as blocks wrapped in one tag (<system-reminder>…</system-reminder>,
+// <ide_opened_file>…</ide_opened_file>, <environment_context>…</environment_context>).
+const INJECTED_BLOCK = /^\s*<([A-Za-z][\w-]*)[^>]*>[\s\S]*<\/\1>\s*$/;
+// A slash command runs the tool's own workflow, which only the tool's model can do. Claude Code marks the
+// expanded command with these tags; a client that sends the command literally starts with a bare token
+// (`/review add x`, `/plugin:plan`, `/help`) — never a path (`/Users/me/a.ts`) or a number (`/2`).
+const COMMAND_MARKER = /<command-(?:name|message)>/;
+const LITERAL_COMMAND = /^\/[A-Za-z][\w-]*(?::[\w-]+)?(?:\s|$)/;
+/**
+ * What the person typed, from the text parts of the latest user entry: the last part that is not one
+ * injected tag-wrapped block. Null for a slash command, which is never answered locally. Shared by the
+ * format modules.
+ */
+export function typedText(texts) {
+    if (texts.some(text => COMMAND_MARKER.test(text)))
+        return null;
+    for (let i = texts.length - 1; i >= 0; i--) {
+        const text = texts[i].trim();
+        if (text && !INJECTED_BLOCK.test(text))
+            return LITERAL_COMMAND.test(text) ? null : text;
+    }
+    return null;
+}

package/dist/llm-gate/formats/gemini.js

@@ -0,0 +1,84 @@
+/**
+ * @fileoverview Gemini generateContent, for the model gate contract (formats/contract.ts).
+ *
+ * Tool results are `functionResponse` parts in user contents; Gemini CLI puts the text in
+ * `response.output` (other response shapes are left alone). The call is the `functionCall` part with
+ * the same id (or, without ids, the latest earlier call of that name). Thought signatures are sibling
+ * keys on model parts: parts are never merged, split, reordered or rebuilt, only the one string changes.
+ */
+import crypto from 'node:crypto';
+import { typedText } from './contract.js';
+export function listToolResults(body) {
+    const contents = Array.isArray(body?.contents) ? body.contents : [];
+    const callsById = new Map();
+    const lastCallByName = new Map();
+    let lastModel = -1;
+    const results = [];
+    // One pass in order, so "latest earlier call of that name" means earlier than the response.
+    contents.forEach((content, i) => {
+        const parts = Array.isArray(content?.parts) ? content.parts : [];
+        if (content?.role === 'model') {
+            lastModel = i;
+            for (const part of parts) {
+                const call = part?.functionCall;
+                if (!call)
+                    continue;
+                if (typeof call.id === 'string')
+                    callsById.set(call.id, call.args);
+                if (typeof call.name === 'string')
+                    lastCallByName.set(call.name, call.args);
+            }
+            return;
+        }
+        parts.forEach((part, j) => {
+            const response = part?.functionResponse;
+            if (typeof response?.response?.output !== 'string')
+                return;
+            const name = String(response.name ?? '');
+            const callArgs = typeof response.id === 'string' && callsById.has(response.id) ? callsById.get(response.id) : lastCallByName.get(name);
+            results.push({ location: { content: i, part: j }, text: response.response.output, toolName: name, callArgs, newTurn: false });
+        });
+    });
+    // Only now is the last model turn known.
+    return results.map(result => ({ ...result, newTurn: result.location.content > lastModel }));
+}
+export function replaceToolResultText(params) {
+    const { body, location: { content, part }, text } = params;
+    const contents = [...body.contents];
+    const parts = [...contents[content].parts];
+    const target = parts[part];
+    parts[part] = {
+        ...target,
+        functionResponse: { ...target.functionResponse, response: { ...target.functionResponse.response, output: text } },
+    };
+    contents[content] = { ...contents[content], parts };
+    return { ...body, contents };
+}
+export function firstRequestPrompt(body) {
+    const contents = Array.isArray(body?.contents) ? body.contents : [];
+    if (contents.some(content => content?.role === 'model'))
+        return null;
+    const config = body.generationConfig ?? {};
+    if (config.responseSchema || config.responseJsonSchema || config.responseMimeType === 'application/json')
+        return null;
+    if (body.toolConfig?.functionCallingConfig?.mode === 'ANY')
+        return null;
+    const lastUser = [...contents].reverse().find(content => content?.role === 'user' || content?.role === undefined);
+    const parts = Array.isArray(lastUser?.parts) ? lastUser.parts : [];
+    const text = typedText(parts.filter(part => typeof part?.text === 'string' && !part.thought).map(part => part.text));
+    const toolsListed = Array.isArray(body.tools) && body.tools.some((tool) => !Array.isArray(tool?.functionDeclarations) || tool.functionDeclarations.length > 0);
+    return text ? { text, toolsListed } : null;
+}
+export function buildLocalReply(params) {
+    const { text, stream, model, usage } = params;
+    const reply = {
+        candidates: [{ content: { role: 'model', parts: [{ text }] }, finishReason: 'STOP', index: 0 }],
+        usageMetadata: { promptTokenCount: usage.inputTokens, candidatesTokenCount: usage.outputTokens, totalTokenCount: usage.inputTokens + usage.outputTokens },
+        modelVersion: model,
+        responseId: `slmgate-${crypto.randomUUID()}`,
+    };
+    // Streamed replies are server-sent events (`?alt=sse`), which is what Gemini CLI asks for.
+    return stream
+        ? { contentType: 'text/event-stream', body: `data: ${JSON.stringify(reply)}\r\n\r\n` }
+        : { contentType: 'application/json; charset=UTF-8', body: JSON.stringify(reply) };
+}

package/dist/llm-gate/formats/internal.js

@@ -0,0 +1 @@
+export {};

package/dist/llm-gate/formats/openai.js

@@ -0,0 +1,77 @@
+/**
+ * Parses an incoming OpenAI-formatted chat completion request and translates it
+ * into the agnostic `InternalRequest` structure.
+ * It also hoists the first "system" message out of the messages array for alignment.
+ *
+ * @param body The raw JSON body of an OpenAI API request
+ * @param modelFallback A default model to use if the request omits it
+ */
+export function parseOpenAIRequest(body, modelFallback) {
+    const messages = [];
+    let system;
+    for (const m of (body.messages || [])) {
+        if (m.role === 'system') {
+            system = m.content; // Grab the first system message
+            // We still include it in messages so that when we serialize to OpenAI it remains
+            messages.push({ role: 'system', content: m.content });
+        }
+        else if (m.role === 'tool' || m.role === 'function') {
+            messages.push({ role: 'tool', content: m.content });
+        }
+        else if (m.role === 'user' || m.role === 'assistant') {
+            messages.push({ role: m.role, content: typeof m.content === 'string' ? m.content : JSON.stringify(m.content) });
+        }
+    }
+    return {
+        system,
+        messages,
+        maxTokens: body.max_tokens,
+        stream: !!body.stream,
+        tools: body.tools,
+        model: body.model || modelFallback
+    };
+}
+export function buildOpenAIRequest(internal) {
+    // Strip any existing system messages to avoid duplication
+    const cleanMessages = internal.messages.filter(m => m.role !== 'system');
+    const finalMessages = cleanMessages.map(m => ({ role: m.role, content: m.content }));
+    if (internal.system) {
+        finalMessages.unshift({ role: 'system', content: internal.system });
+    }
+    const req = {
+        model: internal.model,
+        messages: finalMessages,
+    };
+    if (internal.maxTokens !== undefined) {
+        req.max_tokens = internal.maxTokens;
+    }
+    if (internal.tools && internal.tools.length > 0) {
+        req.tools = internal.tools;
+    }
+    if (internal.stream) {
+        req.stream = true;
+        req.stream_options = { include_usage: true };
+    }
+    return req;
+}
+export function formatOpenAIStreamChunk(id, model, content, finishReason = null, usage = null) {
+    const chunk = {
+        id,
+        object: 'chat.completion.chunk',
+        created: Math.floor(Date.now() / 1000),
+        model,
+        choices: []
+    };
+    if (content !== '' || finishReason !== null) {
+        chunk.choices.push({
+            index: 0,
+            delta: content ? { content } : {},
+            finish_reason: finishReason
+        });
+    }
+    if (usage) {
+        chunk.usage = usage;
+    }
+    return `data: ${JSON.stringify(chunk)}\n\n`;
+}
+export const OPENAI_STREAM_DONE = 'data: [DONE]\n\n';

package/dist/llm-gate/formats/responses.js

@@ -0,0 +1,146 @@
+/**
+ * @fileoverview OpenAI Responses (Codex), for the model gate contract (formats/contract.ts).
+ *
+ * Tool results are `function_call_output`, `custom_tool_call_output` or `local_shell_call_output` items
+ * in `input`; their `output` is a string or an array whose `input_text` parts are the text (images are
+ * left alone). The call is the `function_call` / `custom_tool_call` / `local_shell_call` item with the
+ * same `call_id`. Reasoning items (including encrypted reasoning) are never touched.
+ */
+import crypto from 'node:crypto';
+import { typedText } from './contract.js';
+const OUTPUT_TYPES = new Set(['function_call_output', 'custom_tool_call_output', 'local_shell_call_output']);
+function parseArguments(raw) {
+    if (typeof raw !== 'string')
+        return raw;
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        return raw;
+    }
+}
+/** An item the model produced: an assistant message, a reasoning item, or any call it made. */
+function isModelItem(item) {
+    if (item?.role === 'assistant')
+        return true;
+    const type = item?.type;
+    return typeof type === 'string' && type !== 'message' && type !== 'item_reference' && !type.endsWith('_output');
+}
+function callOf(item) {
+    switch (item?.type) {
+        case 'function_call':
+            return { name: String(item.name ?? ''), args: parseArguments(item.arguments) };
+        case 'custom_tool_call':
+            return { name: String(item.name ?? ''), args: parseArguments(item.input) };
+        case 'local_shell_call':
+            return { name: 'local_shell', args: { command: item.action?.command } };
+        default:
+            return null;
+    }
+}
+export function listToolResults(body) {
+    const items = Array.isArray(body?.input) ? body.input : [];
+    const calls = new Map();
+    let lastModelItem = -1;
+    items.forEach((item, i) => {
+        if (isModelItem(item))
+            lastModelItem = i;
+        const call = callOf(item);
+        if (call && typeof item.call_id === 'string')
+            calls.set(item.call_id, call);
+    });
+    const results = [];
+    items.forEach((item, i) => {
+        if (!OUTPUT_TYPES.has(item?.type))
+            return;
+        const call = calls.get(item.call_id);
+        const found = { toolName: call?.name ?? '', callArgs: call?.args, newTurn: i > lastModelItem };
+        if (typeof item.output === 'string') {
+            results.push({ ...found, location: { item: i }, text: item.output });
+        }
+        else if (Array.isArray(item.output)) {
+            item.output.forEach((part, k) => {
+                if (part?.type === 'input_text' && typeof part.text === 'string')
+                    results.push({ ...found, location: { item: i, part: k }, text: part.text });
+            });
+        }
+    });
+    return results;
+}
+export function replaceToolResultText(params) {
+    const { body, location: { item, part }, text } = params;
+    const input = [...body.input];
+    const target = input[item];
+    input[item] = part === undefined
+        ? { ...target, output: text }
+        : { ...target, output: target.output.map((p, k) => (k === part ? { ...p, text } : p)) };
+    return { ...body, input };
+}
+export function firstRequestPrompt(body) {
+    if (body.previous_response_id)
+        return null;
+    if (['json_schema', 'json_object'].includes(body.text?.format?.type))
+        return null;
+    if (body.tool_choice === 'required' || (body.tool_choice && typeof body.tool_choice === 'object'))
+        return null;
+    const toolsListed = Array.isArray(body.tools) && body.tools.length > 0;
+    if (typeof body.input === 'string') {
+        const text = typedText([body.input]);
+        return text ? { text, toolsListed } : null;
+    }
+    const items = Array.isArray(body.input) ? body.input : [];
+    if (items.some(isModelItem))
+        return null;
+    const lastUser = [...items].reverse().find(item => item?.role === 'user');
+    const content = lastUser?.content;
+    const texts = typeof content === 'string'
+        ? [content]
+        : Array.isArray(content)
+            ? content.filter((part) => part?.type === 'input_text' && typeof part.text === 'string').map((part) => part.text)
+            : [];
+    const text = typedText(texts);
+    return text ? { text, toolsListed } : null;
+}
+export function buildLocalReply(params) {
+    const { text, stream, model, usage } = params;
+    const suffix = crypto.randomUUID().replace(/-/g, '');
+    const itemId = `msg_slmgate_${suffix}`;
+    const part = { type: 'output_text', text, annotations: [] };
+    const item = { type: 'message', id: itemId, status: 'completed', role: 'assistant', content: [part] };
+    const response = {
+        id: `resp_slmgate_${suffix}`,
+        object: 'response',
+        created_at: Math.floor(Date.now() / 1000),
+        status: 'completed',
+        model,
+        output: [item],
+        error: null,
+        incomplete_details: null,
+        usage: {
+            input_tokens: usage.inputTokens,
+            input_tokens_details: { cached_tokens: 0 },
+            output_tokens: usage.outputTokens,
+            output_tokens_details: { reasoning_tokens: 0 },
+            total_tokens: usage.inputTokens + usage.outputTokens,
+        },
+    };
+    if (!stream)
+        return { contentType: 'application/json', body: JSON.stringify(response) };
+    const inProgress = { ...response, status: 'in_progress', output: [], usage: null };
+    const at = { item_id: itemId, output_index: 0, content_index: 0 };
+    const events = [
+        ['response.created', { response: inProgress }],
+        ['response.in_progress', { response: inProgress }],
+        ['response.output_item.added', { output_index: 0, item: { ...item, status: 'in_progress', content: [] } }],
+        ['response.content_part.added', { ...at, part: { ...part, text: '' } }],
+        ['response.output_text.delta', { ...at, delta: text }],
+        ['response.output_text.done', { ...at, text }],
+        ['response.content_part.done', { ...at, part }],
+        ['response.output_item.done', { output_index: 0, item }],
+        ['response.completed', { response }],
+    ];
+    return {
+        contentType: 'text/event-stream; charset=utf-8',
+        body: events.map(([type, data], sequence) => `event: ${type}\ndata: ${JSON.stringify({ type, sequence_number: sequence, ...data })}\n\n`).join(''),
+    };
+}

package/dist/llm-gate/forward.js

@@ -0,0 +1,150 @@
+/**
+ * @fileoverview Forwards a model request, unchanged, to the provider its wire format belongs to.
+ *
+ * The gate never translates between formats and never swaps credentials: every client header except
+ * hop-by-hop ones goes upstream, so the tool keeps its own login (API key, claude.ai login, ChatGPT
+ * login). Bytes pass through raw in both directions. That rules out `fetch`, which decompresses a
+ * response but keeps its `content-encoding` header — Claude Code's streams arrive gzip-encoded.
+ * There is no timeout and no retry: streams can run for many minutes, and every tool retries itself.
+ */
+import http from 'node:http';
+import https from 'node:https';
+import { CONFIG } from '../config.js';
+/** Listed in the 404 a tool gets for any other path. */
+export const SUPPORTED_PATHS = [
+    'POST /v1/messages',
+    'POST /v1/messages/count_tokens',
+    'HEAD /api/hello',
+    'GET /v1/models (with an anthropic-version header)',
+    'POST /v1/chat/completions',
+    'POST /v1/responses',
+    'POST /v1beta/models/{model}:generateContent',
+    'POST /v1beta/models/{model}:streamGenerateContent',
+    'POST /v1beta/models/{model}:countTokens',
+];
+// Describe this hop, not the request, so they are never forwarded in either direction.
+const HOP_BY_HOP = new Set(['host', 'connection', 'content-length', 'transfer-encoding', 'keep-alive', 'proxy-connection', 'upgrade', 'te', 'trailer']);
+// countTokens is included because Gemini CLI sends it for prompts that carry media.
+const GEMINI_PATH = /^\/v1beta\/models\/([^/:]+):(generateContent|streamGenerateContent|countTokens)$/;
+function upstreamUrl(params) {
+    return new URL(params.base.replace(/\/+$/, '') + params.path + params.search);
+}
+/**
+ * PROVISIONAL (Slice 0 finding F24, not yet confirmed against a real Codex ChatGPT login): a ChatGPT
+ * login sends a ChatGPT access token, which is a JWT, plus a `chatgpt-account-id` header; an API-key
+ * login sends an `sk-` key. Only the former may go to the ChatGPT backend.
+ */
+function isChatgptLogin(headers) {
+    if (headers['chatgpt-account-id'] !== undefined)
+        return true;
+    const token = /^Bearer\s+(.+)$/i.exec(String(headers.authorization ?? ''))?.[1] ?? '';
+    return /^eyJ[\w-]*\.[\w-]+\.[\w-]*$/.test(token);
+}
+/**
+ * Picks the upstream for a request from its path (and, where two formats share a path, its headers).
+ *
+ * @param params.pathAndQuery The request target as received, e.g. `/v1/messages?beta=true`
+ * @param params.headers The client's request headers
+ * @returns The route, or null when the gate does not handle that path
+ */
+export function resolveUpstream(params) {
+    const { pathAndQuery, headers } = params;
+    const { pathname, search } = new URL(pathAndQuery, 'http://gate.local');
+    const anthropic = (generation) => ({ format: 'anthropic', url: upstreamUrl({ base: CONFIG.UPSTREAM_ANTHROPIC_URL, path: pathname, search }), generation });
+    switch (pathname) {
+        case '/v1/messages':
+            return anthropic(true);
+        case '/v1/messages/count_tokens':
+        case '/api/hello':
+            return anthropic(false);
+        case '/v1/models':
+            // OpenAI-format tools list models on the same path; only Anthropic clients send this header.
+            return headers['anthropic-version'] === undefined ? null : anthropic(false);
+        case '/v1/chat/completions':
+            return { format: 'chat-completions', url: upstreamUrl({ base: CONFIG.UPSTREAM_OPENAI_URL, path: '/chat/completions', search }), generation: true };
+        case '/v1/responses': {
+            const base = isChatgptLogin(headers) ? CONFIG.UPSTREAM_CHATGPT_URL : CONFIG.UPSTREAM_OPENAI_URL;
+            return { format: 'responses', url: upstreamUrl({ base, path: '/responses', search }), generation: true };
+        }
+    }
+    const gemini = GEMINI_PATH.exec(pathname);
+    if (!gemini)
+        return null;
+    return {
+        format: 'gemini',
+        url: upstreamUrl({ base: CONFIG.UPSTREAM_GEMINI_URL, path: pathname, search }),
+        generation: gemini[2] !== 'countTokens',
+        pathModel: gemini[1],
+        ...(gemini[2] === 'streamGenerateContent'
+            ? { geminiStream: new URLSearchParams(search).get('alt') === 'sse' ? 'sse' : 'json-array' }
+            : {}),
+    };
+}
+function withoutHopByHop(headers, keep) {
+    return Object.fromEntries(Object.entries(headers).filter(([name]) => !HOP_BY_HOP.has(name) || name === keep));
+}
+/**
+ * Sends the request upstream and pipes the response back as it arrives, without changing either.
+ *
+ * @param params.req The client request (its method and headers are forwarded)
+ * @param params.res The client response; upstream status, headers and body are written to it
+ * @param params.body The request body exactly as received
+ * @param params.route Where to send it
+ * @returns Resolves once the response has ended, failed, or the client has gone away
+ */
+export function forwardRequest(params) {
+    const { req, res, body, route } = params;
+    const started = Date.now();
+    const headers = withoutHopByHop(req.headers);
+    if (body.length > 0)
+        headers['content-length'] = body.length;
+    const transport = route.url.protocol === 'https:' ? https : http;
+    return new Promise(resolve => {
+        let settled = false;
+        let status = 0;
+        let bytesOut = 0;
+        const settle = (outcome = {}) => {
+            if (settled)
+                return;
+            settled = true;
+            resolve({ status, durationMs: Date.now() - started, bytesOut, clientAborted: false, ...outcome });
+        };
+        const upstreamReq = transport.request(route.url, { method: req.method, headers }, upstreamRes => {
+            status = upstreamRes.statusCode ?? 502;
+            // content-length stays: the bytes pass through unchanged, so it is still correct.
+            res.writeHead(status, withoutHopByHop(upstreamRes.headers, 'content-length'));
+            upstreamRes.on('data', (chunk) => { bytesOut += chunk.length; });
+            upstreamRes.pipe(res);
+            upstreamRes.on('end', () => settle());
+            // The upstream dropped mid-stream, or was destroyed after the client left. pipe() does not
+            // pass source errors on, and an unhandled one would crash the gate.
+            upstreamRes.on('error', err => {
+                settle({ error: err.message });
+                res.destroy(err);
+            });
+        });
+        upstreamReq.on('error', err => {
+            // A dual-stack host refusing every address raises an AggregateError with an empty message.
+            const detail = err instanceof AggregateError ? err.errors.map(String).join('; ') : err.message;
+            settle({ status: 502, error: detail });
+            if (res.destroyed)
+                return;
+            if (res.headersSent) {
+                res.destroy(err);
+                return;
+            }
+            res.writeHead(502, { 'content-type': 'application/json' });
+            res.end(JSON.stringify({
+                error: { type: 'slm_gate_upstream_unreachable', message: `slm-gate could not reach ${route.url.origin}: ${detail}` },
+            }));
+        });
+        // The tool gave up (Esc, its own timeout): stop the upstream generation too.
+        res.on('close', () => {
+            if (res.writableFinished)
+                return;
+            settle({ clientAborted: true });
+            upstreamReq.destroy();
+        });
+        upstreamReq.end(body);
+    });
+}

package/dist/llm-gate/index.js

@@ -0,0 +1,40 @@
+import { requestListener, server } from './server.js';
+import { listenOnThisComputer } from '../utils/local-only.js';
+import { CONFIG } from '../config.js';
+import { installLangfuseFlushLifecycle } from '../ledger/flush-lifecycle.js';
+import { getDb, logLedgerInfo } from '../ledger/index.js';
+import { isEntryPoint } from '../utils/entry-point.js';
+import { warmUpLocalModel } from './distill.js';
+import { warmUpAnsweringModel } from './local-first.js';
+// Ledger writes are synchronous. With better-sqlite3's default 5 s busy wait, another process
+// holding the ledger's write lock would freeze every stream through the gate. Wait briefly instead;
+// a row that still cannot be written is logged and dropped (see recordRequest in server.ts).
+const LEDGER_BUSY_TIMEOUT_MS = 100;
+/**
+ * Entry point for the `llm-gate` layer.
+ *
+ * Can be imported as a module (`export { server }`) for integration testing,
+ * or executed directly via CLI to boot the standalone HTTP proxy server.
+ */
+if (isEntryPoint(import.meta.url)) {
+    const sinks = ['sqlite'];
+    if (CONFIG.LANGFUSE_PUBLIC_KEY && CONFIG.LANGFUSE_SECRET_KEY && CONFIG.LANGFUSE_HOST) {
+        sinks.push('langfuse');
+    }
+    logLedgerInfo('llm-gate');
+    getDb().pragma(`busy_timeout = ${LEDGER_BUSY_TIMEOUT_MS}`);
+    if (CONFIG.LLM_GATE_DISTILL)
+        warmUpLocalModel();
+    if (CONFIG.LLM_GATE_LOCAL_FIRST)
+        warmUpAnsweringModel();
+    listenOnThisComputer({
+        handler: requestListener,
+        port: CONFIG.LLM_GATE_PORT,
+        onListening: () => console.error(`LLM Gate running on port ${CONFIG.LLM_GATE_PORT}, for programs on this computer only (pass-through: anthropic, chat-completions, responses, gemini). sinks: [${sinks.join(', ')}]`),
+    }).catch(err => {
+        console.error(`LLM Gate could not listen on port ${CONFIG.LLM_GATE_PORT}: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(1);
+    });
+    installLangfuseFlushLifecycle('llm-gate');
+}
+export { server };