@juliangruber/harness 1.0.0

package/dist/sources.js

@@ -0,0 +1,359 @@
+// Research sources whose income doesn't depend on human visitors. Each uses
+// the source's official keyless API.
+import { cleanText } from './html.js';
+import { CONTACT, getJson, request } from './http.js';
+const MAX_ARTICLE = 15_000;
+const MAX_PROPERTIES = 40;
+const MAX_VALUES = 5;
+const shorten = (text, max) => text.length > max ? `${text.slice(0, max)}...` : text;
+const clampLimit = (limit) => Math.min(Math.max(Math.round(limit) || 5, 1), 20);
+// The language ends up in a hostname, so it must not be arbitrary
+function checkLanguage(language) {
+    if (!/^[a-z]{2,3}(-[a-z]+)?$/.test(language))
+        throw new Error(`Invalid language code ${language}`);
+}
+const languageParam = {
+    type: 'string',
+    description: 'Language code, like "en" or "de". Defaults to "en"'
+};
+// Wikipedia
+const wikipediaApi = (language, params) => getJson(`https://${language}.wikipedia.org/w/api.php?${new URLSearchParams({ format: 'json', formatversion: '2', ...params })}`);
+const articleUrl = (language, title) => `https://${language}.wikipedia.org/wiki/${encodeURIComponent(title.replaceAll(' ', '_'))}`;
+export const wikipediaSearch = {
+    name: 'wikipedia_search',
+    description: 'Search Wikipedia articles. Returns titles, URLs and snippets.',
+    parameters: {
+        type: 'object',
+        properties: {
+            query: { type: 'string' },
+            language: languageParam,
+            limit: { type: 'number', description: 'Maximum results, defaults to 5' }
+        },
+        required: ['query']
+    },
+    async run({ query, language = 'en', limit = 5 }) {
+        checkLanguage(language);
+        const body = await wikipediaApi(language, { action: 'query', list: 'search', srsearch: query, srlimit: String(clampLimit(limit)) });
+        const results = body.query?.search ?? [];
+        if (!results.length)
+            return 'No articles found';
+        return results
+            .map((result, i) => `${i + 1}. ${result.title}\n   ${articleUrl(language, result.title)}\n   ${cleanText(result.snippet)}`)
+            .join('\n');
+    }
+};
+export const wikipediaArticle = {
+    name: 'wikipedia_article',
+    description: 'Read the plain text of a Wikipedia article by its exact title.',
+    parameters: {
+        type: 'object',
+        properties: { title: { type: 'string' }, language: languageParam },
+        required: ['title']
+    },
+    async run({ title, language = 'en' }) {
+        checkLanguage(language);
+        const body = await wikipediaApi(language, { action: 'query', prop: 'extracts', explaintext: '1', redirects: '1', titles: title });
+        const page = body.query?.pages?.[0];
+        if (!page || page.missing)
+            throw new Error(`No Wikipedia article titled ${title}`);
+        const text = page.extract.length > MAX_ARTICLE
+            ? `${page.extract.slice(0, MAX_ARTICLE)}\n[truncated ${page.extract.length - MAX_ARTICLE} chars]`
+            : page.extract;
+        return `${page.title}\n${articleUrl(language, page.title)}\n\n${text}`;
+    }
+};
+// Wikidata
+const wikidataApi = (params) => getJson(`https://www.wikidata.org/w/api.php?${new URLSearchParams({ format: 'json', formatversion: '2', ...params })}`);
+const wikidataUrl = (id) => `https://www.wikidata.org/wiki/${id}`;
+export const wikidataSearch = {
+    name: 'wikidata_search',
+    description: 'Search Wikidata entities (people, places, organizations, concepts). Returns ids like Q42 for wikidata_entity.',
+    parameters: {
+        type: 'object',
+        properties: {
+            query: { type: 'string' },
+            language: languageParam,
+            limit: { type: 'number', description: 'Maximum results, defaults to 5' }
+        },
+        required: ['query']
+    },
+    async run({ query, language = 'en', limit = 5 }) {
+        checkLanguage(language);
+        const body = await wikidataApi({ action: 'wbsearchentities', search: query, language, uselang: language, type: 'item', limit: String(clampLimit(limit)) });
+        const results = body.search ?? [];
+        if (!results.length)
+            return 'No entities found';
+        return results
+            .map((result, i) => `${i + 1}. ${result.id} ${result.label ?? ''}${result.description ? `: ${result.description}` : ''}\n   ${wikidataUrl(result.id)}`)
+            .join('\n');
+    }
+};
+async function wikidataLabels(ids, language) {
+    const labels = new Map();
+    // The API accepts at most 50 ids per request
+    for (let i = 0; i < ids.length; i += 50) {
+        const body = await wikidataApi({ action: 'wbgetentities', ids: ids.slice(i, i + 50).join('|'), props: 'labels', languages: language, languagefallback: '1' });
+        for (const [id, entity] of Object.entries(body.entities ?? {})) {
+            const label = entity.labels?.[language]?.value;
+            if (label)
+                labels.set(id, label);
+        }
+    }
+    return labels;
+}
+function formatSnak(snak, labels) {
+    if (snak.snaktype === 'somevalue')
+        return 'unknown value';
+    if (snak.snaktype === 'novalue')
+        return 'no value';
+    const { type, value } = snak.datavalue ?? {};
+    switch (type) {
+        case 'wikibase-entityid': return labels.get(value.id) ?? value.id;
+        case 'string': return value;
+        case 'monolingualtext': return value.text;
+        case 'time': return value.time.replace(/^\+/, '').replace(/T.*$/, '').replace(/-00/g, '');
+        case 'quantity': return value.amount.replace(/^\+/, '');
+        case 'globecoordinate': return `${value.latitude}, ${value.longitude}`;
+        default: return JSON.stringify(value);
+    }
+}
+export const wikidataEntity = {
+    name: 'wikidata_entity',
+    description: 'Get the facts Wikidata has about an entity, by id like Q42.',
+    parameters: {
+        type: 'object',
+        properties: { id: { type: 'string' }, language: languageParam },
+        required: ['id']
+    },
+    async run({ id, language = 'en' }) {
+        checkLanguage(language);
+        if (!/^[QPL]\d+$/.test(id))
+            throw new Error(`Invalid Wikidata id ${id}, expected something like Q42`);
+        const body = await wikidataApi({ action: 'wbgetentities', ids: id, props: 'labels|descriptions|aliases|claims', languages: language, languagefallback: '1' });
+        const entity = body.entities?.[id];
+        if (!entity || 'missing' in entity)
+            throw new Error(`No Wikidata entity ${id}`);
+        const properties = Object.entries(entity.claims ?? {});
+        const shown = properties.slice(0, MAX_PROPERTIES).map(([property, claims]) => [property, claims.filter(claim => claim.rank !== 'deprecated').slice(0, MAX_VALUES).map(claim => claim.mainsnak)]);
+        const ids = new Set();
+        for (const [property, snaks] of shown) {
+            ids.add(property);
+            for (const snak of snaks) {
+                if (snak.datavalue?.type === 'wikibase-entityid')
+                    ids.add(snak.datavalue.value.id);
+            }
+        }
+        const labels = await wikidataLabels([...ids], language);
+        const lines = [`${entity.labels?.[language]?.value ?? id} (${id})`];
+        const description = entity.descriptions?.[language]?.value;
+        if (description)
+            lines.push(description);
+        const aliases = entity.aliases?.[language] ?? [];
+        if (aliases.length)
+            lines.push(`Also known as: ${aliases.map(alias => alias.value).join(', ')}`);
+        lines.push(wikidataUrl(id), '');
+        for (const [property, snaks] of shown) {
+            if (snaks.length)
+                lines.push(`${labels.get(property) ?? property}: ${snaks.map(snak => formatSnak(snak, labels)).join(', ')}`);
+        }
+        if (properties.length > MAX_PROPERTIES)
+            lines.push(`[${properties.length - MAX_PROPERTIES} more properties]`);
+        return lines.join('\n');
+    }
+};
+const withMailto = (params) => {
+    if (CONTACT)
+        params.set('mailto', CONTACT);
+    return params;
+};
+const openAlex = async (query, limit) => {
+    const params = withMailto(new URLSearchParams({
+        search: query,
+        per_page: String(limit),
+        select: 'id,doi,title,publication_year,authorships,primary_location,abstract_inverted_index'
+    }));
+    const body = await getJson(`https://api.openalex.org/works?${params}`);
+    return (body.results ?? []).map((work) => ({
+        title: work.title ?? 'Untitled',
+        authors: (work.authorships ?? []).map((a) => a.author?.display_name).filter(Boolean),
+        year: work.publication_year ?? undefined,
+        venue: work.primary_location?.source?.display_name ?? undefined,
+        doi: work.doi?.replace(/^https?:\/\/(dx\.)?doi\.org\//i, '') ?? undefined,
+        url: work.primary_location?.landing_page_url ?? work.id,
+        abstract: work.abstract_inverted_index ? uninvert(work.abstract_inverted_index) : undefined,
+        sources: ['openalex']
+    }));
+};
+// OpenAlex stores abstracts as { word: [positions] }
+function uninvert(index) {
+    const words = [];
+    for (const [word, positions] of Object.entries(index)) {
+        for (const position of positions)
+            words[position] = word;
+    }
+    return words.filter(Boolean).join(' ');
+}
+const semanticScholar = async (query, limit) => {
+    const params = new URLSearchParams({ query, limit: String(limit), fields: 'title,year,authors,venue,abstract,externalIds,url' });
+    const body = await getJson(`https://api.semanticscholar.org/graph/v1/paper/search?${params}`);
+    return (body.data ?? []).map((paper) => ({
+        title: paper.title ?? 'Untitled',
+        authors: (paper.authors ?? []).map((a) => a.name),
+        year: paper.year ?? undefined,
+        venue: paper.venue || undefined,
+        doi: paper.externalIds?.DOI,
+        url: paper.url,
+        abstract: paper.abstract ?? undefined,
+        sources: ['semantic_scholar']
+    }));
+};
+const crossref = async (query, limit) => {
+    const params = withMailto(new URLSearchParams({ query, rows: String(limit), select: 'DOI,title,author,issued,container-title,abstract,URL' }));
+    const body = await getJson(`https://api.crossref.org/works?${params}`);
+    return (body.message?.items ?? []).map((item) => ({
+        title: cleanText(item.title?.[0] ?? 'Untitled'),
+        authors: (item.author ?? []).map((a) => [a.given, a.family].filter(Boolean).join(' ') || a.name).filter(Boolean),
+        year: item.issued?.['date-parts']?.[0]?.[0] ?? undefined,
+        venue: item['container-title']?.[0],
+        doi: item.DOI,
+        url: item.URL,
+        // Abstracts are JATS XML, usually starting with an "Abstract" heading
+        abstract: item.abstract ? cleanText(item.abstract).replace(/^abstract\s+/i, '') : undefined,
+        sources: ['crossref']
+    }));
+};
+const europePmc = async (query, limit) => {
+    const params = new URLSearchParams({ query, format: 'json', pageSize: String(limit), resultType: 'core' });
+    const body = await getJson(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?${params}`);
+    return (body.resultList?.result ?? []).map((result) => ({
+        title: cleanText(result.title ?? 'Untitled'),
+        authors: result.authorString ? result.authorString.replace(/\.$/, '').split(', ') : [],
+        year: Number(result.pubYear) || undefined,
+        venue: result.journalInfo?.journal?.title,
+        doi: result.doi,
+        url: `https://europepmc.org/article/${result.source}/${result.id}`,
+        abstract: result.abstractText ? cleanText(result.abstractText) : undefined,
+        sources: ['europe_pmc']
+    }));
+};
+// arXiv often rate limits or times out, so failure is expected: it should be
+// fast, and point the model to sources that also index arXiv
+const ARXIV_HINT = 'Use search_openalex or search_semantic_scholar instead, they also index arXiv papers.';
+const arxiv = async (query, limit) => {
+    const terms = query.split(/\s+/).map(term => term.replace(/[^\p{L}\p{N}-]/gu, '')).filter(Boolean);
+    const params = new URLSearchParams({ search_query: terms.map(term => `all:${term}`).join(' AND '), max_results: String(limit) });
+    const res = await request(`https://export.arxiv.org/api/query?${params}`, { timeout: 10_000, hint: ARXIV_HINT });
+    const xml = await res.text();
+    if (!xml.includes('<feed'))
+        throw new Error(`export.arxiv.org returned an unexpected response. ${ARXIV_HINT}`);
+    return [...xml.matchAll(/<entry>([\s\S]*?)<\/entry>/g)].map(([, entry]) => {
+        const tag = (name) => entry.match(new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`))?.[1];
+        const summary = tag('summary');
+        return {
+            title: cleanText(tag('title') ?? 'Untitled'),
+            authors: [...entry.matchAll(/<name>([\s\S]*?)<\/name>/g)].map(match => cleanText(match[1])),
+            year: Number(tag('published')?.slice(0, 4)) || undefined,
+            venue: 'arXiv',
+            doi: tag('arxiv:doi'),
+            url: tag('id')?.trim(),
+            abstract: summary ? cleanText(summary) : undefined,
+            sources: ['arxiv']
+        };
+    });
+};
+function formatPapers(papers) {
+    return papers.map((paper, i) => {
+        const authors = paper.authors.length > 3 ? `${paper.authors.slice(0, 3).join(', ')} et al.` : paper.authors.join(', ');
+        const lines = [`${i + 1}. ${paper.title}${paper.year ? ` (${paper.year})` : ''}`];
+        if (authors)
+            lines.push(`   Authors: ${authors}`);
+        if (paper.venue)
+            lines.push(`   Venue: ${paper.venue}`);
+        if (paper.doi)
+            lines.push(`   DOI: ${paper.doi}`);
+        if (paper.url)
+            lines.push(`   URL: ${paper.url}`);
+        lines.push(`   Found in: ${paper.sources.join(', ')}`);
+        if (paper.abstract)
+            lines.push(`   Abstract: ${shorten(paper.abstract, 600)}`);
+        return lines.join('\n');
+    }).join('\n\n');
+}
+const paperParameters = {
+    type: 'object',
+    properties: {
+        query: { type: 'string' },
+        limit: { type: 'number', description: 'Maximum results, defaults to 5' }
+    },
+    required: ['query']
+};
+const paperTool = (name, description, search) => ({
+    name,
+    description,
+    parameters: paperParameters,
+    async run({ query, limit = 5 }) {
+        const papers = await search(query, clampLimit(limit));
+        return papers.length ? formatPapers(papers) : 'No papers found';
+    }
+});
+const PAPER_SOURCES = [
+    ['openalex', openAlex],
+    ['semantic_scholar', semanticScholar],
+    ['crossref', crossref],
+    ['europe_pmc', europePmc],
+    ['arxiv', arxiv]
+];
+const normalizeTitle = (title) => title.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '');
+export const papers = {
+    name: 'search_papers',
+    description: 'Search all paper sources at once (OpenAlex, Semantic Scholar, Crossref, Europe PMC, arXiv) and merge duplicates. Use a single source tool when you know the field.',
+    parameters: paperParameters,
+    async run({ query, limit = 5 }) {
+        const results = await Promise.allSettled(PAPER_SOURCES.map(([, search]) => search(query, clampLimit(limit))));
+        const errors = [];
+        const lists = [];
+        results.forEach((result, i) => {
+            if (result.status === 'fulfilled')
+                lists.push(result.value);
+            else
+                errors.push(`${PAPER_SOURCES[i][0]}: ${result.reason instanceof Error ? result.reason.message : result.reason}`);
+        });
+        // Interleave sources so each one's best results come first
+        const merged = new Map();
+        for (let rank = 0; rank < Math.max(0, ...lists.map(list => list.length)); rank++) {
+            for (const paper of lists.map(list => list[rank]).filter(Boolean)) {
+                const key = normalizeTitle(paper.title);
+                const existing = merged.get(key);
+                if (!existing) {
+                    merged.set(key, { ...paper, sources: [...paper.sources] });
+                    continue;
+                }
+                // A source can return the same paper twice, like a preprint and its publication
+                existing.sources = [...new Set([...existing.sources, ...paper.sources])];
+                existing.year ??= paper.year;
+                existing.venue ??= paper.venue;
+                existing.doi ??= paper.doi;
+                existing.url ??= paper.url;
+                existing.abstract ??= paper.abstract;
+                if (!existing.authors.length)
+                    existing.authors = paper.authors;
+            }
+        }
+        let output = merged.size ? formatPapers([...merged.values()]) : 'No papers found';
+        if (errors.length)
+            output += `\n\nUnavailable sources:\n${errors.join('\n')}`;
+        return output;
+    }
+};
+export const sourceTools = [
+    wikipediaSearch,
+    wikipediaArticle,
+    wikidataSearch,
+    wikidataEntity,
+    papers,
+    paperTool('search_openalex', 'Search OpenAlex, a broad index of scholarly works across all fields.', openAlex),
+    paperTool('search_semantic_scholar', 'Search Semantic Scholar, strong in computer science and biomedicine. Shared rate limit, may fail.', semanticScholar),
+    paperTool('search_crossref', 'Search Crossref publication metadata (journals, books, DOIs). Abstracts are often missing.', crossref),
+    paperTool('search_europe_pmc', 'Search Europe PMC for biomedical and life science literature.', europePmc),
+    paperTool('search_arxiv', 'Search arXiv preprints in physics, mathematics, computer science and related fields. Often rate limited, if it fails use search_openalex or search_semantic_scholar.', arxiv)
+];

package/dist/subagent.js

@@ -0,0 +1,66 @@
+import { runAgent } from './agent.js';
+// A tool that delegates to a separate agent, so search results and pages
+// don't fill up the main conversation
+export function createSubagentTool(definition, { maxTurns = definition.maxTurns, extraTools = [], ...options }) {
+    const { parameter } = definition;
+    return {
+        name: definition.name,
+        description: definition.description,
+        parameters: {
+            type: 'object',
+            properties: { [parameter.name]: { type: 'string', description: parameter.description } },
+            required: [parameter.name]
+        },
+        async run(args) {
+            const messages = [
+                { role: 'system', content: definition.prompt() },
+                { role: 'user', content: args[parameter.name] }
+            ];
+            const tools = typeof definition.tools === 'function' ? await definition.tools() : definition.tools;
+            const agentOptions = { ...options, tools: [...tools, ...extraTools], maxTurns };
+            const answer = await runAgent(messages, agentOptions);
+            if (extractUrls(answer).length)
+                return answer;
+            messages.push({ role: 'user', content: 'List the URL of every source you used.' });
+            return `${answer}\n\n${await runAgent(messages, agentOptions)}`;
+        }
+    };
+}
+export function extractUrls(text) {
+    const urls = (text.match(/https?:\/\/[^\s<>"'`\]]+/g) ?? [])
+        // Trailing punctuation and unbalanced closing parens belong to the sentence
+        .map(url => url.replace(/[.,;:!?]+$/, ''))
+        .map(url => url.endsWith(')') && !url.includes('(') ? url.replace(/\)+$/, '') : url);
+    return [...new Set(urls)];
+}
+const appendList = (answer, items, heading, moreHeading, isIncluded) => {
+    const missing = items.filter(item => !isIncluded(item));
+    if (!missing.length)
+        return answer;
+    return `${answer}\n\n${missing.length < items.length ? moreHeading : heading}\n${missing.map(item => `- ${item}`).join('\n')}`;
+};
+// Makes sure every source ends up in the answer
+export const appendSources = (answer, urls) => appendList(answer, urls, 'Sources:', 'More sources:', url => answer.includes(url));
+// Headings of the sources and suggested web searches sections, in plain text
+export const SECTION_HEADING = /^(more )?(sources|suggested web searches)\b[^:]*:?$/i;
+// Reads the queries listed under a "Suggested web searches:" heading. Models
+// like to extend the heading, as in "Suggested web searches for current news:".
+export function extractSearches(text) {
+    const lines = text.split('\n');
+    const start = lines.findIndex(line => /^[#*_\s]*suggested web searches\b[^:]*:?[*_\s]*$/i.test(line));
+    if (start === -1)
+        return [];
+    const searches = [];
+    for (const line of lines.slice(start + 1)) {
+        const item = line.match(/^\s*(?:[-*+]|\d+[.)])\s+(.+)$/);
+        if (item) {
+            searches.push(item[1].trim().replace(/^["'`“]+|["'`”]+$/g, ''));
+        }
+        else if (line.trim() || searches.length) {
+            break;
+        }
+    }
+    return [...new Set(searches)];
+}
+// Makes sure every suggested web search ends up in the answer
+export const appendSearches = (answer, searches) => appendList(answer, searches, 'Suggested web searches:', 'More suggested web searches:', search => answer.toLowerCase().includes(search.toLowerCase()));

package/dist/tools.js

@@ -0,0 +1,251 @@
+import { execFile } from 'node:child_process';
+import { glob as fsGlob, mkdir, readFile, realpath, stat, writeFile } from 'node:fs/promises';
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
+const MAX_OUTPUT = 30_000;
+const MAX_RESULTS = 200;
+const truncate = (text) => text.length > MAX_OUTPUT
+    ? `${text.slice(0, MAX_OUTPUT)}\n[truncated ${text.length - MAX_OUTPUT} chars]`
+    : text;
+const limitResults = (results) => truncate(results.length > MAX_RESULTS
+    ? [...results.slice(0, MAX_RESULTS), `[${results.length - MAX_RESULTS} more results]`].join('\n')
+    : results.join('\n'));
+const IGNORED = /(^|\/)(node_modules|\.git)$/;
+const walk = async (pattern, cwd) => (await Array.fromAsync(fsGlob(pattern, { cwd, exclude: path => IGNORED.test(path) }))).sort();
+// File tools only access files inside this directory. Undefined lifts the limit.
+let fileRoot;
+export function restrictFileTools(directory) {
+    fileRoot = directory === undefined ? undefined : resolve(directory);
+}
+// Follows symlinks. A path that doesn't exist yet resolves through its nearest
+// existing parent, so a new file can't be written through a link either.
+async function realPathOf(path) {
+    try {
+        return await realpath(path);
+    }
+    catch (err) {
+        const code = err.code;
+        if (code !== 'ENOENT' && code !== 'ENOTDIR')
+            throw err;
+        const parent = dirname(path);
+        return parent === path ? path : join(await realPathOf(parent), basename(path));
+    }
+}
+async function isAllowed(path) {
+    if (fileRoot === undefined)
+        return true;
+    const rel = relative(await realPathOf(fileRoot), await realPathOf(resolve(path)));
+    return rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
+}
+async function checkPath(path) {
+    if (!await isAllowed(path)) {
+        throw new Error(`${path} is outside the working directory ${fileRoot}, file tools can only access files inside it`);
+    }
+}
+export const read = {
+    name: 'read',
+    description: 'Read a text file. For large files, use offset and limit to read a range of lines.',
+    parameters: {
+        type: 'object',
+        properties: {
+            path: { type: 'string' },
+            offset: { type: 'number', description: 'First line to read, starting at 1' },
+            limit: { type: 'number', description: 'Maximum number of lines to read' }
+        },
+        required: ['path']
+    },
+    async run({ path, offset = 1, limit }) {
+        await checkPath(path);
+        const lines = (await readFile(path, 'utf8')).split('\n');
+        const end = limit === undefined ? lines.length : offset - 1 + limit;
+        let output = lines.slice(offset - 1, end).join('\n');
+        if (end < lines.length)
+            output += `\n[${lines.length - end} more lines, continue with offset ${end + 1}]`;
+        return truncate(output);
+    }
+};
+export const write = {
+    name: 'write',
+    description: 'Write content to a file, creating parent directories and overwriting existing content.',
+    parameters: {
+        type: 'object',
+        properties: { path: { type: 'string' }, content: { type: 'string' } },
+        required: ['path', 'content']
+    },
+    async run({ path, content }) {
+        await checkPath(path);
+        await mkdir(dirname(path), { recursive: true });
+        await writeFile(path, content);
+        return `Wrote ${content.length} chars to ${path}`;
+    }
+};
+export const edit = {
+    name: 'edit',
+    description: 'Replace an exact string in a file. old_string must match exactly once, unless replace_all is set.',
+    parameters: {
+        type: 'object',
+        properties: {
+            path: { type: 'string' },
+            old_string: { type: 'string', description: 'Exact text to replace, including whitespace' },
+            new_string: { type: 'string' },
+            replace_all: { type: 'boolean', description: 'Replace every occurrence' }
+        },
+        required: ['path', 'old_string', 'new_string']
+    },
+    async run({ path, old_string: oldString, new_string: newString, replace_all: replaceAll = false }) {
+        if (!oldString)
+            throw new Error('old_string must not be empty');
+        await checkPath(path);
+        const parts = (await readFile(path, 'utf8')).split(oldString);
+        const count = parts.length - 1;
+        if (count === 0)
+            throw new Error(`old_string not found in ${path}`);
+        if (count > 1 && !replaceAll) {
+            throw new Error(`old_string found ${count} times in ${path}. Include more surrounding text to make it unique, or set replace_all`);
+        }
+        // split/join instead of String.replace, so `$&` and friends stay literal
+        await writeFile(path, parts.join(newString));
+        return `Replaced ${count} occurrence${count === 1 ? '' : 's'} in ${path}`;
+    }
+};
+const IGNORED_NAMES = new Set(['node_modules', '.git']);
+const formatSize = (bytes) => bytes < 1024 ? `${bytes} B` : bytes < 1024 * 1024 ? `${(bytes / 1024).toFixed(1)} KB` : `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+export const glob = {
+    name: 'glob',
+    description: 'List files and directories matching a glob pattern, like "*" for the directory, "src/**/*.ts" or "{*.md,*/*.md}". Hidden files are included. Directories end with "/", files show their size. Doesn\'t look inside node_modules and .git. Use it instead of find, and ls to list a single directory.',
+    parameters: {
+        type: 'object',
+        properties: {
+            pattern: { type: 'string', description: 'Glob pattern, defaults to "*": everything in the directory' },
+            path: { type: 'string', description: 'Directory to search in, defaults to the working directory' }
+        }
+    },
+    async run({ pattern = '*', path = '.' }) {
+        await checkPath(path);
+        // Globs skip hidden files, so also match the last part with a leading dot:
+        // "*" becomes ".*", "**/*.md" becomes "**/.*.md"
+        const segments = pattern.split('/');
+        const last = segments.pop();
+        const patterns = last.startsWith('.') ? [pattern] : [pattern, [...segments, `.${last}`].join('/')];
+        const entries = new Map();
+        for (const variant of patterns) {
+            // node_modules and .git are listed, but never searched inside
+            for await (const entry of fsGlob(variant, { cwd: path, withFileTypes: true, exclude: dirent => IGNORED_NAMES.has(dirent.name) })) {
+                const fullPath = join(entry.parentPath, entry.name);
+                const relativePath = relative(path, fullPath);
+                // Patterns like "../*" and symlinks can reach outside the working directory
+                if (relativePath && await isAllowed(fullPath))
+                    entries.set(fullPath, { path: join(path, relativePath), fullPath, directory: entry.isDirectory() });
+            }
+        }
+        if (!entries.size)
+            return 'No files found';
+        const sorted = [...entries.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
+        const lines = await Promise.all(sorted.slice(0, MAX_RESULTS).map(async (entry) => {
+            if (entry.directory)
+                return `${entry.path}/`;
+            try {
+                return `${entry.path} (${formatSize((await stat(entry.fullPath)).size)})`;
+            }
+            catch {
+                return entry.path;
+            }
+        }));
+        if (sorted.length > MAX_RESULTS)
+            lines.push(`[${sorted.length - MAX_RESULTS} more results]`);
+        return truncate(lines.join('\n'));
+    }
+};
+// Models reach for ls out of habit, so listing a directory gets its own name
+export const ls = {
+    name: 'ls',
+    description: 'List a directory, like ls -la: hidden files included, directories end with "/", files show their size. Use it instead of running ls in bash.',
+    parameters: {
+        type: 'object',
+        properties: {
+            path: { type: 'string', description: 'Directory to list, defaults to the working directory' }
+        }
+    },
+    run: ({ path = '.' }) => glob.run({ pattern: '*', path })
+};
+export const grep = {
+    name: 'grep',
+    description: 'Search file contents with a JavaScript regular expression. Returns file:line: text. Skips node_modules, .git and binary files.',
+    parameters: {
+        type: 'object',
+        properties: {
+            pattern: { type: 'string', description: 'Regular expression' },
+            path: { type: 'string', description: 'File or directory to search in, defaults to the working directory' },
+            include: { type: 'string', description: 'Glob of files to search, like "**/*.ts"' }
+        },
+        required: ['pattern']
+    },
+    async run({ pattern, path = '.', include = '**/*' }) {
+        const regex = new RegExp(pattern);
+        await checkPath(path);
+        const files = (await stat(path)).isFile()
+            ? [path]
+            : (await walk(include, path)).map(file => join(path, file));
+        const matches = [];
+        for (const file of files) {
+            if (!await isAllowed(file))
+                continue;
+            let content;
+            try {
+                content = await readFile(file, 'utf8');
+            }
+            catch {
+                continue; // directories, unreadable files
+            }
+            if (content.includes('\0'))
+                continue;
+            content.split('\n').forEach((line, i) => {
+                if (regex.test(line))
+                    matches.push(`${file}:${i + 1}: ${line.slice(0, 500)}`);
+            });
+            if (matches.length > MAX_RESULTS)
+                break;
+        }
+        return matches.length ? limitResults(matches) : 'No matches found';
+    }
+};
+export const bash = {
+    name: 'bash',
+    description: 'Run a bash command in the working directory. Returns stdout and stderr. Prefer the dedicated tools when one fits.',
+    parameters: {
+        type: 'object',
+        properties: { command: { type: 'string', description: 'The command to run' } },
+        required: ['command']
+    },
+    run({ command }) {
+        return new Promise(resolve => {
+            execFile('bash', ['-c', command], { timeout: 120_000, maxBuffer: 10 * 1024 * 1024 }, (err, stdout, stderr) => {
+                let output = stdout + stderr;
+                if (err)
+                    output += `\n[exit ${err.code ?? err.signal}]`;
+                resolve(truncate(output));
+            });
+        });
+    }
+};
+// Tools that are missing on purpose, so the model doesn't keep asking for them.
+// Fetching arbitrary pages and web search would send agent traffic to ad funded sites.
+export const UNAVAILABLE_TOOLS = ['fetch web page', 'search the web'];
+// Finds nothing, on purpose: requests show which tools are worth adding
+export function createToolSearch(onSearch) {
+    return {
+        name: 'tool_search',
+        description: `Search for more tools by describing what you need a tool to do. Use it when none of your tools fit. Known to be unavailable, don't search for: ${UNAVAILABLE_TOOLS.join(', ')}.`,
+        parameters: {
+            type: 'object',
+            properties: {
+                description: { type: 'string', description: 'What the tool should do, like "fetch a web page as text"' }
+            },
+            required: ['description']
+        },
+        async run({ description }) {
+            onSearch(description);
+            return 'No matching tools found. Continue with the tools you have.';
+        }
+    };
+}
+export const tools = [read, write, edit, ls, glob, grep, bash];