@juliangruber/harness 1.0.0

package/dist/llm.js

@@ -0,0 +1,52 @@
+import { randomUUID } from 'node:crypto';
+// Speaks the OpenAI compatible /v1/chat/completions API
+// (Ollama, llama.cpp, vLLM, LM Studio, OpenRouter, ...)
+export function createClient({ baseUrl, model, apiKey, retries = 2, onRetry }) {
+    return {
+        async chat(messages, tools) {
+            for (let attempt = 0;; attempt++) {
+                const res = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {})
+                    },
+                    body: JSON.stringify({
+                        model,
+                        messages,
+                        tools: tools.length ? tools : undefined
+                    })
+                });
+                if (res.ok) {
+                    const body = await res.json();
+                    return normalize(body.choices[0].message);
+                }
+                const error = new Error(`LLM request failed: ${res.status} ${res.statusText} ${await res.text()}`);
+                // Server errors are often the server failing to parse sampled model
+                // output, like a malformed tool call, so a retry usually works
+                if (res.status < 500 || attempt >= retries)
+                    throw error;
+                onRetry?.(error);
+            }
+        }
+    };
+}
+// Servers differ slightly: some omit tool call ids or send arguments as
+// objects. Also drops extra fields (like `reasoning`) so they are not sent back.
+function normalize(message) {
+    const toolCalls = (message.tool_calls ?? []).map((call) => ({
+        id: call.id || `call_${randomUUID()}`,
+        type: 'function',
+        function: {
+            name: call.function.name,
+            arguments: typeof call.function.arguments === 'string'
+                ? call.function.arguments
+                : JSON.stringify(call.function.arguments ?? {})
+        }
+    }));
+    return {
+        role: 'assistant',
+        content: message.content ?? null,
+        tool_calls: toolCalls.length ? toolCalls : undefined
+    };
+}

package/dist/markdown.js

@@ -0,0 +1,67 @@
+import { lexer } from 'marked';
+import { stripVTControlCharacters, styleText } from 'node:util';
+// Renders markdown for the terminal: marked parses, util.styleText styles
+export function renderMarkdown(markdown, { color, stream, highlight } = {}) {
+    const style = (format, text) => color === false ? text : styleText(format, text, { validateStream: color === undefined, stream });
+    const inline = (tokens = []) => tokens.map(token => {
+        switch (token.type) {
+            case 'strong': return style('bold', inline(token.tokens));
+            case 'em': return style('italic', inline(token.tokens));
+            case 'del': return style('strikethrough', inline(token.tokens));
+            case 'codespan': return style('cyan', token.text);
+            case 'link': {
+                const text = inline(token.tokens);
+                return text === token.href
+                    ? style('underline', text)
+                    : `${style('underline', text)} ${style('dim', `(${token.href})`)}`;
+            }
+            case 'image': return style('dim', `[image: ${token.text}]`);
+            case 'br': return '\n';
+            case 'text': return token.tokens ? inline(token.tokens) : token.text;
+            default: return 'text' in token ? token.text : token.raw;
+        }
+    }).join('');
+    const block = (tokens = [], separator = '\n\n') => tokens.map(render).filter(Boolean).join(separator);
+    const highlighted = (text) => {
+        const plain = stripVTControlCharacters(text);
+        return highlight?.test(plain) ? style(['bold', 'cyan'], plain) : undefined;
+    };
+    const render = (token) => {
+        switch (token.type) {
+            case 'heading': {
+                const text = inline(token.tokens);
+                return highlighted(text) ?? style(token.depth === 1 ? ['bold', 'underline'] : 'bold', text);
+            }
+            case 'paragraph': {
+                const text = inline(token.tokens);
+                return highlighted(text) ?? text;
+            }
+            case 'text': return token.tokens ? inline(token.tokens) : token.text;
+            case 'code': return token.text.split('\n').map((line) => `  ${style('yellow', line)}`).join('\n');
+            case 'blockquote': return block(token.tokens).split('\n').map(line => `${style('dim', '│')} ${line}`).join('\n');
+            case 'list': return renderList(token);
+            case 'table': return renderTable(token);
+            case 'hr': return style('dim', '─'.repeat(40));
+            case 'space':
+            case 'def': return '';
+            default: return 'text' in token ? token.text : token.raw;
+        }
+    };
+    const renderList = (list) => list.items.map((item, i) => {
+        const bullet = list.ordered ? `${Number(list.start || 1) + i}.` : '-';
+        const checkbox = item.task ? `[${item.checked ? 'x' : ' '}] ` : '';
+        const body = block(item.tokens.filter(t => t.type !== 'checkbox'), list.loose ? '\n\n' : '\n');
+        return `${bullet} ${checkbox}${indent(body, bullet.length + 1).slice(bullet.length + 1)}`;
+    }).join(list.loose ? '\n\n' : '\n');
+    const renderTable = (table) => {
+        const header = table.header.map(cell => style('bold', inline(cell.tokens)));
+        const rows = table.rows.map(row => row.map(cell => inline(cell.tokens)));
+        const widths = header.map((_, i) => Math.max(...[header, ...rows].map(row => width(row[i]))));
+        const line = (row) => row.map((cell, i) => cell + ' '.repeat(widths[i] - width(cell))).join(style('dim', ' │ ')).trimEnd();
+        const divider = style('dim', widths.map(w => '─'.repeat(w)).join('─┼─'));
+        return [line(header), divider, ...rows.map(line)].join('\n');
+    };
+    return block(lexer(markdown));
+}
+const width = (text = '') => stripVTControlCharacters(text).length;
+const indent = (text, n) => text.split('\n').map(line => line && ' '.repeat(n) + line).join('\n');

package/dist/mcp-servers.js

@@ -0,0 +1,34 @@
+// Remote MCP servers the harness uses. Each one is reviewed for fairness: the
+// operator's income doesn't depend on ads or human visits to the content.
+// Reviewed and left out:
+// - Manifold (api.manifold.markets/v0/mcp): decided against it
+// - Exa (mcp.exa.ai): web search, returns content from ad funded websites
+// - GitMCP's fetch_generic_url_content: reads any URL, like the unavailable "fetch web page" tool
+export const researchServers = [
+    {
+        name: 'wolfram',
+        url: 'https://agenttools.wolfram.com/mcp',
+        fairness: 'Wolfram|Alpha is free and funded by Pro subscriptions, apps, paid APIs and enterprise versions, not ads. The cloud MCP service is free for limited personal use, so it is only contacted for maths questions. Code execution (WolframLanguageEvaluator) is left out, since it costs Wolfram compute.',
+        tools: ['WolframAlpha']
+    }
+];
+export const docsServers = [
+    {
+        name: 'deepwiki',
+        url: 'https://mcp.deepwiki.com/mcp',
+        fairness: 'Run by Cognition (Devin) as a free service for agents. Documentation generated from public GitHub repositories, no ads.',
+        tools: ['read_wiki_structure', 'read_wiki_contents', 'ask_question']
+    },
+    {
+        name: 'context7',
+        url: 'https://mcp.context7.com/mcp',
+        fairness: 'Run by Upstash, funded by paid plans. Library documentation made for agents, no ads.',
+        tools: ['resolve-library-id', 'query-docs']
+    },
+    {
+        name: 'microsoft',
+        url: 'https://learn.microsoft.com/api/mcp',
+        fairness: 'Microsoft\'s own product documentation, offered to agents by Microsoft. Documentation supports its products, not ads.',
+        tools: ['microsoft_docs_search', 'microsoft_code_sample_search', 'microsoft_docs_fetch']
+    }
+];

package/dist/mcp.js

@@ -0,0 +1,197 @@
+// A small Model Context Protocol client for remote servers, over the
+// Streamable HTTP transport: JSON-RPC over POST, answered with JSON or an
+// event stream. https://modelcontextprotocol.io/specification/2025-06-18
+import { request } from './http.js';
+const PROTOCOL_VERSION = '2025-06-18';
+const HINT = 'Try another tool instead.';
+const MAX_OUTPUT = 30_000;
+const MAX_TOOL_PAGES = 10;
+export async function connectMcp(url, { timeout = 60_000 } = {}) {
+    let sessionId;
+    let protocolVersion;
+    let nextId = 1;
+    const post = async (message) => {
+        const headers = {
+            'content-type': 'application/json',
+            accept: 'application/json, text/event-stream'
+        };
+        if (sessionId)
+            headers['mcp-session-id'] = sessionId;
+        if (protocolVersion)
+            headers['mcp-protocol-version'] = protocolVersion;
+        const res = await request(url, { method: 'POST', body: JSON.stringify(message), headers, timeout, hint: HINT });
+        sessionId = res.headers.get('mcp-session-id') ?? sessionId;
+        return res;
+    };
+    const call = async (method, params = {}) => {
+        const id = nextId++;
+        const res = await post({ jsonrpc: '2.0', id, method, params });
+        const message = await readResponse(res, id);
+        if (message.error)
+            throw new Error(`MCP ${method} failed: ${message.error.message}`);
+        return message.result;
+    };
+    const notify = async (method) => {
+        const res = await post({ jsonrpc: '2.0', method });
+        await res.body?.cancel();
+    };
+    const initialized = await call('initialize', {
+        protocolVersion: PROTOCOL_VERSION,
+        capabilities: {},
+        clientInfo: { name: 'harness', version: '0.0.0' }
+    });
+    protocolVersion = initialized.protocolVersion ?? PROTOCOL_VERSION;
+    await notify('notifications/initialized');
+    return {
+        serverInfo: initialized.serverInfo ?? { name: new URL(url).hostname },
+        async listTools() {
+            const tools = [];
+            let cursor;
+            for (let page = 0; page < MAX_TOOL_PAGES; page++) {
+                const result = await call('tools/list', cursor ? { cursor } : {});
+                tools.push(...(result.tools ?? []));
+                cursor = result.nextCursor;
+                if (!cursor)
+                    break;
+            }
+            return tools;
+        },
+        async callTool(name, args = {}) {
+            const result = await call('tools/call', { name, arguments: args });
+            const text = formatContent(result);
+            if (result.isError)
+                throw new Error(text || `MCP tool ${name} failed`);
+            return text;
+        },
+        async close() {
+            if (!sessionId)
+                return;
+            try {
+                const headers = { 'mcp-session-id': sessionId, 'mcp-protocol-version': protocolVersion ?? PROTOCOL_VERSION };
+                const res = await request(url, { method: 'DELETE', headers, allowErrors: true, hint: HINT });
+                await res.body?.cancel();
+            }
+            catch { }
+        }
+    };
+}
+// Servers answer a request with JSON, or with an event stream that can carry
+// other messages before the response
+async function readResponse(res, id) {
+    if ((res.headers.get('content-type') ?? '').includes('text/event-stream')) {
+        for await (const message of eventStream(res)) {
+            if (message.id === id && ('result' in message || 'error' in message))
+                return message;
+        }
+        throw new Error('MCP server closed the stream without a response');
+    }
+    const body = await res.json();
+    const message = (Array.isArray(body) ? body : [body]).find((m) => m.id === id);
+    if (!message)
+        throw new Error('MCP server sent no response');
+    return message;
+}
+async function* eventStream(res) {
+    if (!res.body)
+        return;
+    const reader = res.body.pipeThrough(new TextDecoderStream()).getReader();
+    let buffer = '';
+    const parse = (event) => {
+        const data = event.split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).replace(/^ /, '')).join('\n');
+        if (!data)
+            return;
+        try {
+            return JSON.parse(data);
+        }
+        catch { }
+    };
+    try {
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            buffer += value;
+            let match;
+            while ((match = buffer.match(/\r?\n\r?\n/))) {
+                const message = parse(buffer.slice(0, match.index));
+                buffer = buffer.slice(match.index + match[0].length);
+                if (message)
+                    yield message;
+            }
+        }
+        const last = parse(buffer);
+        if (last)
+            yield last;
+    }
+    finally {
+        reader.cancel().catch(() => { });
+    }
+}
+function formatContent(result) {
+    const parts = (result.content ?? []).map((item) => {
+        switch (item.type) {
+            case 'text': return item.text;
+            case 'resource_link': return `${item.name ? `${item.name}: ` : ''}${item.uri}`;
+            case 'resource': return item.resource?.text ?? item.resource?.uri ?? '';
+            default: return `[${item.type}${item.mimeType ? ` ${item.mimeType}` : ''}]`;
+        }
+    });
+    if (!parts.length && result.structuredContent)
+        parts.push(JSON.stringify(result.structuredContent, null, 2));
+    const text = parts.join('\n\n');
+    return text.length > MAX_OUTPUT ? `${text.slice(0, MAX_OUTPUT)}\n[truncated ${text.length - MAX_OUTPUT} chars]` : text;
+}
+// Harness tools for an MCP server's tools, like deepwiki_ask_question. Names
+// that already start with the prefix, like microsoft_docs_search, stay as is.
+export function mcpTools(client, prefix, tools, rename = {}) {
+    return tools.map(tool => ({
+        name: (rename[tool.name] ?? (tool.name.startsWith(`${prefix}_`) ? tool.name : `${prefix}_${tool.name}`)).replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 64),
+        description: tool.description ?? '',
+        parameters: tool.inputSchema ?? { type: 'object' },
+        run: (args) => client.callTool(tool.name, args)
+    }));
+}
+// An MCP tool that only contacts its server when the model calls it
+export function createLazyMcpTool({ server, tool, name, description, parameters, connect = connectMcp }) {
+    if (!server.tools.includes(tool))
+        throw new Error(`${tool} isn't a reviewed tool of MCP server ${server.name}`);
+    let client;
+    return {
+        name,
+        description,
+        parameters,
+        async run(args) {
+            // A failed connection is tried again on the next call
+            client ??= connect(server.url).catch(err => {
+                client = undefined;
+                throw err;
+            });
+            return (await client).callTool(tool, args);
+        }
+    };
+}
+// Connects to servers on first use and keeps the connection. Only reviewed
+// tools are exposed. Failed servers are tried again on the next load.
+export function createMcpToolLoader(servers, { connect = connectMcp, onError } = {}) {
+    const loaded = new Map();
+    const load = async (server) => {
+        const client = await connect(server.url);
+        const tools = (await client.listTools()).filter(tool => server.tools.includes(tool.name));
+        return mcpTools(client, server.name, tools, server.rename);
+    };
+    return async () => {
+        const results = await Promise.all(servers.map(server => {
+            let tools = loaded.get(server.url);
+            if (!tools) {
+                tools = load(server);
+                loaded.set(server.url, tools);
+            }
+            return tools.catch(err => {
+                loaded.delete(server.url);
+                onError?.(server, err instanceof Error ? err : new Error(String(err)));
+                return [];
+            });
+        }));
+        return results.flat();
+    };
+}

package/dist/ratelimit.js

@@ -0,0 +1,98 @@
+// Per host rate limiting that follows what APIs tell us: Retry-After on 429
+// and 503, budget headers (x-ratelimit-*) and rate headers (x-rate-limit-*)
+import { setTimeout } from 'node:timers/promises';
+const DEFAULT_COOLDOWN = 60_000;
+// Warn when this share of a budget is left
+const LOW_REMAINING = 0.1;
+export function createRateLimiter({ intervals = {}, now = Date.now, sleep = setTimeout } = {}) {
+    const hosts = new Map();
+    const state = (host) => {
+        let hostState = hosts.get(host);
+        if (!hostState) {
+            hostState = { interval: intervals[host] ?? 0, nextRequest: 0, blockedUntil: 0, warned: false };
+            hosts.set(host, hostState);
+        }
+        return hostState;
+    };
+    const limiter = {
+        async schedule(host, minInterval = 0) {
+            const hostState = state(host);
+            hostState.interval = Math.max(hostState.interval, minInterval);
+            const blocked = hostState.blockedUntil - now();
+            if (blocked > 0)
+                throw new Error(`${host} is rate limiting, skipping it for ${formatDuration(blocked)}`);
+            const wait = hostState.nextRequest - now();
+            hostState.nextRequest = Math.max(now(), hostState.nextRequest) + hostState.interval;
+            if (wait > 0)
+                await sleep(wait);
+        },
+        failed(host) {
+            const hostState = state(host);
+            hostState.blockedUntil = Math.max(hostState.blockedUntil, now() + DEFAULT_COOLDOWN);
+        },
+        update(host, res) {
+            const hostState = state(host);
+            const header = (name) => res.headers.get(name);
+            const block = (until) => { hostState.blockedUntil = Math.max(hostState.blockedUntil, until); };
+            // Overloaded: too many requests, or the server (or a gateway in front of it) gave up
+            if ([429, 502, 503, 504].includes(res.status)) {
+                block(now() + (parseRetryAfter(header('retry-after'), now()) ?? DEFAULT_COOLDOWN));
+            }
+            // Budgets, like OpenAlex's daily credits
+            const limit = Number(header('x-ratelimit-limit'));
+            const remaining = header('x-ratelimit-remaining') === null ? NaN : Number(header('x-ratelimit-remaining'));
+            const reset = parseReset(header('x-ratelimit-reset'), now());
+            if (remaining <= 0 && reset !== undefined)
+                block(reset);
+            if (limit > 0 && remaining <= limit * LOW_REMAINING) {
+                if (!hostState.warned) {
+                    hostState.warned = true;
+                    limiter.onWarning?.(`${host}: rate limit almost used up, ${remaining} of ${limit} left${reset === undefined ? '' : `, resets in ${formatDuration(reset - now())}`}`);
+                }
+            }
+            else if (remaining > limit * LOW_REMAINING) {
+                hostState.warned = false;
+            }
+            // Rates, like Crossref's "1 request per 1s"
+            const rateLimit = Number(header('x-rate-limit-limit'));
+            const interval = parseDuration(header('x-rate-limit-interval'));
+            if (rateLimit > 0 && interval !== undefined) {
+                hostState.interval = Math.max(intervals[host] ?? 0, interval / rateLimit);
+            }
+        }
+    };
+    return limiter;
+}
+// Retry-After is either seconds or an HTTP date
+function parseRetryAfter(value, now) {
+    if (value === null)
+        return;
+    if (/^\d+$/.test(value.trim()))
+        return Number(value) * 1000;
+    const date = Date.parse(value);
+    return Number.isNaN(date) ? undefined : Math.max(0, date - now);
+}
+// Resets are either seconds from now or a unix timestamp
+function parseReset(value, now) {
+    if (value === null)
+        return;
+    const seconds = Number(value);
+    if (!Number.isFinite(seconds))
+        return;
+    return seconds > 1e9 ? seconds * 1000 : now + seconds * 1000;
+}
+function parseDuration(value) {
+    const match = value?.trim().match(/^(\d+(?:\.\d+)?)\s*(ms|s|m|h)?$/);
+    if (!match)
+        return;
+    const units = { ms: 1, s: 1000, m: 60_000, h: 3_600_000 };
+    return Number(match[1]) * units[match[2] ?? 's'];
+}
+function formatDuration(ms) {
+    const seconds = Math.ceil(ms / 1000);
+    if (seconds < 600)
+        return `${seconds}s`;
+    if (seconds < 7200)
+        return `${Math.ceil(seconds / 60)}m`;
+    return `${Math.ceil(seconds / 3600)}h`;
+}

package/dist/research.js

@@ -0,0 +1,33 @@
+import { connectMcp, createLazyMcpTool } from './mcp.js';
+import { researchServers } from './mcp-servers.js';
+import { sourceTools } from './sources.js';
+import { createSubagentTool } from './subagent.js';
+export const researchPrompt = (date = new Date()) => `You are a research agent. Today's date is ${date.toLocaleDateString('en-CA')}.
+Answer the question using the tools: Wikipedia, Wikidata and scholarly paper databases. Search first, then read the most relevant results. Don't rely on memory for facts you can look up. Paper tools only return abstracts, not full papers, so say when a claim is based on an abstract. Sources can be unavailable or rate limited: if a tool fails, move on to another source instead of retrying it.
+Only use wolfram_alpha when the question is about mathematics, like calculations, equations or statistics. Never use it for other facts.
+Reply with a concise answer, followed by a "Sources" list with the URL of every source you used. If the sources don't answer the question, say so.
+If a regular web search would help the user learn more, for example about news, prices, opinions or anything your sources don't cover, end with a "Suggested web searches:" list of search queries. Leave it out if you have none.`;
+export function createResearchTool({ connect = connectMcp, ...options }) {
+    const wolfram = researchServers.find(server => server.name === 'wolfram');
+    // Wolfram is only contacted when the model calls this tool, for maths
+    const wolframAlpha = createLazyMcpTool({
+        server: wolfram,
+        tool: 'WolframAlpha',
+        name: 'wolfram_alpha',
+        description: 'Compute answers to mathematics questions with Wolfram|Alpha: calculations, equations, statistics. Only use it for maths.',
+        parameters: {
+            type: 'object',
+            properties: { query: { type: 'string', description: 'Wolfram Alpha query, like "integrate x^2 sin(x)"' } },
+            required: ['query']
+        },
+        connect
+    });
+    return createSubagentTool({
+        name: 'research',
+        description: 'Ask a research agent that searches Wikipedia, Wikidata and scholarly papers, and computes maths with Wolfram|Alpha. Returns a concise answer with source URLs, and sometimes suggested web searches. Use it for facts you are unsure about or that may have changed. Include the source URLs and suggested web searches in your answer.',
+        parameter: { name: 'question', description: 'A self-contained question, including all needed context' },
+        prompt: () => researchPrompt(),
+        tools: [...sourceTools, wolframAlpha],
+        maxTurns: 15
+    }, options);
+}