@xingular/harness 0.1.7 → 1.0.0

package/dist/sdk/variant/variant.js

@@ -0,0 +1,331 @@
+// Reading, writing, validating, repairing and comparing a variant. The edit contract is
+// omitted-is-inherited: an edit names only the files it changes.
+//
+// Ported from Harness² (Apache-2.0, Google LLC; `harness2/core/variant.py`).
+import { spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, relative, sep } from 'node:path';
+import { componentOf, patterns } from "./surface.js";
+/** Scripts the harness itself ships; a reference to one needs no file in the variant. */
+export const RESERVED_SCRIPTS = new Set();
+/** Normalises a relative path and refuses one that escapes the variant. */
+export function normalizeRel(rel) {
+    let r = String(rel).trim().replace(/\\/g, '/');
+    if (r.startsWith('/') || r.split('/').includes('..'))
+        throw new Error(`path escapes the harness: ${JSON.stringify(rel)}`);
+    while (r.startsWith('./'))
+        r = r.slice(2);
+    return r.replace(/^\/+|\/+$/g, '');
+}
+/** Whether a path is on the editable surface. Never throws. */
+export function isAllowed(surface, rel) {
+    let clean;
+    try {
+        clean = normalizeRel(rel);
+    }
+    catch {
+        return false;
+    }
+    return surface.alwaysOn.includes(clean) || componentOf(surface, clean) !== null;
+}
+export function skillNames(surface, variant) {
+    return names(variant, patterns(surface).skill);
+}
+export function subagentNames(surface, variant) {
+    return names(variant, patterns(surface).subagent);
+}
+export function pluginNames(surface, variant) {
+    return names(variant, patterns(surface).plugin);
+}
+function names(variant, rx) {
+    const found = new Set();
+    for (const rel of Object.keys(variant)) {
+        const m = rx.exec(rel);
+        if (m?.[1])
+            found.add(m[1]);
+    }
+    return [...found].sort();
+}
+function stubPaths(surface) {
+    return new Set(surface.stubs.map(([rel]) => rel));
+}
+function walk(dir) {
+    const out = [];
+    let entries;
+    try {
+        entries = readdirSync(dir);
+    }
+    catch {
+        return out;
+    }
+    for (const name of entries.sort()) {
+        if (name === '__pycache__' || name === '.DS_Store' || name === 'node_modules')
+            continue;
+        const path = join(dir, name);
+        let isDir = false;
+        try {
+            isDir = statSync(path).isDirectory();
+        }
+        catch {
+            continue;
+        }
+        if (isDir)
+            out.push(...walk(path));
+        else
+            out.push(path);
+    }
+    return out;
+}
+/**
+ * Every file on the editable surface under `dir`, README stubs excluded. Pass `rejected` to
+ * collect paths that exist but are off the surface.
+ */
+export function readVariant(surface, dir, rejected) {
+    const out = {};
+    const stubs = stubPaths(surface);
+    for (const path of walk(dir)) {
+        const rel = relative(dir, path).split(sep).join('/');
+        if (stubs.has(rel))
+            continue;
+        if (!isAllowed(surface, rel)) {
+            rejected?.push(rel);
+            continue;
+        }
+        out[rel] = readFileSync(path, 'utf8');
+    }
+    return out;
+}
+/** Whether `dir` holds a variant: at least one always-on file. */
+export function isVariant(surface, dir) {
+    return surface.alwaysOn.some((rel) => {
+        try {
+            return statSync(join(dir, rel)).isFile();
+        }
+        catch {
+            return false;
+        }
+    });
+}
+/**
+ * Writes exactly `variant` to `dest`. Always-on files are present even when empty; files off the
+ * surface are kept, editable files already there are replaced.
+ */
+export function writeVariant(surface, variant, dest, options = {}) {
+    mkdirSync(dest, { recursive: true });
+    const stubs = stubPaths(surface);
+    for (const path of walk(dest).reverse()) {
+        const rel = relative(dest, path).split(sep).join('/');
+        if (isAllowed(surface, rel) || stubs.has(rel))
+            unlinkSync(path);
+    }
+    for (const rel of surface.alwaysOn) {
+        if (!(rel in variant)) {
+            const path = join(dest, rel);
+            mkdirSync(dirname(path), { recursive: true });
+            writeFileSync(path, '');
+        }
+    }
+    for (const [rel, text] of Object.entries(variant)) {
+        const path = join(dest, normalizeRel(rel));
+        mkdirSync(dirname(path), { recursive: true });
+        writeFileSync(path, text);
+    }
+    if (options.stubs !== false) {
+        for (const [rel, body] of surface.stubs) {
+            const path = join(dest, rel);
+            try {
+                statSync(path);
+            }
+            catch {
+                mkdirSync(dirname(path), { recursive: true });
+                writeFileSync(path, body);
+            }
+        }
+    }
+    return dest;
+}
+/** The always-on text, as the model sees it, joined in surface order. */
+export function alwaysOnText(surface, variant) {
+    return surface.alwaysOn.map((rel) => variant[rel] ?? '').join('\n\n');
+}
+function frontmatterField(body, field) {
+    const m = new RegExp(`^${field}:\\s*(.+)$`, 'm').exec(body.slice(0, 600));
+    return m?.[1]?.trim() ?? null;
+}
+/**
+ * Gives every skill a trigger line in the always-on text, mutating `variant`. Returns the names
+ * repaired. A line that would push the text over the cap is skipped.
+ */
+export function autotrigger(surface, variant, alwaysOnMax = surface.alwaysOnMax) {
+    let body = alwaysOnText(surface, variant);
+    const fixed = [];
+    for (const name of skillNames(surface, variant)) {
+        if (body.includes(name))
+            continue;
+        const desc = frontmatterField(variant[`${surface.skillDir}/${name}/SKILL.md`] ?? '', 'description')?.replace(/\.$/, '') ?? `the ${name} workflow applies`;
+        const line = surface.triggerLine.replace('{name}', name).replace('{desc}', desc);
+        if (body.length + line.length + 2 > alwaysOnMax)
+            continue;
+        const current = variant[surface.autotriggerFile] ?? '';
+        variant[surface.autotriggerFile] = current.trim() ? `${current.replace(/\s+$/, '')}\n${line}\n` : `${line}\n`;
+        body = alwaysOnText(surface, variant);
+        fixed.push(name);
+    }
+    return fixed;
+}
+/** A variant that does not validate. `problems` is written to be fed back to an editor as-is. */
+export class VariantError extends Error {
+    problems;
+    constructor(problems) {
+        super(problems.join(' | '));
+        this.problems = problems;
+        this.name = 'VariantError';
+    }
+}
+function checkCompiles(rel, src, errs) {
+    const dir = mkdtempSync(join(tmpdir(), 'harness-script-'));
+    try {
+        const file = join(dir, rel.split('/').pop() ?? 'script');
+        writeFileSync(file, src);
+        const python = rel.endsWith('.py');
+        const result = python
+            ? spawnSync('python3', ['-c', 'import ast,sys; ast.parse(open(sys.argv[1]).read(), sys.argv[1])', file], { encoding: 'utf8', timeout: 15_000 })
+            : spawnSync(process.execPath, ['--check', file], { encoding: 'utf8', timeout: 15_000 });
+        // A missing interpreter is not the variant's fault.
+        if (result.error)
+            return;
+        if (result.status !== 0) {
+            const detail = (result.stderr || result.stdout || '').trim().split('\n').filter(Boolean).slice(-2).join(' ');
+            errs.push(`${rel} does not compile: ${detail.slice(0, 200)}`);
+        }
+    }
+    finally {
+        rmSync(dir, { recursive: true, force: true });
+    }
+}
+/** Throws a `VariantError` listing every problem; returns a census of a valid variant. */
+export function validate(surface, variant, options = {}) {
+    const errs = [];
+    const r = patterns(surface);
+    const cap = options.alwaysOnMax ?? surface.alwaysOnMax;
+    const body = alwaysOnText(surface, variant);
+    if (body.length > cap)
+        errs.push(`always-on content is ${body.length} chars, over the ${cap} cap; ${surface.alwaysOnCapReason}`);
+    for (const rel of Object.keys(variant)) {
+        if (!isAllowed(surface, rel))
+            errs.push(`${rel} is not on the editable surface`);
+    }
+    const skills = skillNames(surface, variant);
+    for (const name of skills) {
+        const fm = (variant[`${surface.skillDir}/${name}/SKILL.md`] ?? '').slice(0, 600);
+        if (!/^description:\s*\S/m.test(fm))
+            errs.push(`${surface.skillDir}/${name}/SKILL.md needs a \`description:\` in its frontmatter, or the skill is dropped SILENTLY at load`);
+        const declared = /^name:\s*(\S+)\s*$/m.exec(fm)?.[1];
+        if (surface.skillNameRule === 'must_match') {
+            if (declared !== name)
+                errs.push(`${surface.skillDir}/${name}/SKILL.md needs frontmatter \`name: ${name}\` matching its folder (a mismatch is dropped SILENTLY at load)`);
+        }
+        else if (declared && declared !== name) {
+            errs.push(`${surface.skillDir}/${name}/SKILL.md frontmatter \`name: ${declared}\` overrides the folder name at load, so trigger lines naming \`${name}\` stop matching`);
+        }
+        if (!body.includes(name))
+            errs.push(`skill '${name}' is never named in the always-on text, so the agent will not know to invoke it — an untriggered skill is dead weight. ${surface.triggerHint.replaceAll('{name}', name)}`);
+    }
+    const subagents = subagentNames(surface, variant);
+    if (subagents.length && options.allowSubagents === false)
+        errs.push(`subagents are not available here: ${subagents.join(', ')}`);
+    for (const name of subagents) {
+        const fm = (variant[`${surface.subagentDir}/${name}/AGENT.md`] ?? '').slice(0, 600);
+        if (!/^description:\s*\S/m.test(fm))
+            errs.push(`${surface.subagentDir}/${name}/AGENT.md needs a \`description:\``);
+    }
+    for (const [rel, src] of Object.entries(variant)) {
+        if (options.compileScripts !== false && (r.script.test(rel) || (r.skillScript.test(rel) && /\.(py|mjs)$/.test(rel))))
+            checkCompiles(rel, src, errs);
+        if (r.plugin.test(rel) && !/export\s+default\b/.test(src))
+            errs.push(`${rel} needs a default export: a Pi extension is a function \`export default function (pi) { ... }\``);
+    }
+    const reserved = options.reservedScripts ?? RESERVED_SCRIPTS;
+    const have = new Set(Object.keys(variant).filter((rel) => r.script.test(rel) || r.skillScript.test(rel)));
+    for (const [rel, src] of Object.entries(variant)) {
+        if (!(r.skill.test(rel) || surface.alwaysOn.includes(rel)))
+            continue;
+        for (const m of src.matchAll(r.scriptRef)) {
+            const ref = m[1];
+            if (reserved.has(ref.split('/').pop()))
+                continue;
+            if (!have.has(ref))
+                errs.push(`${rel} invokes \`${ref}\` but that file is not in the harness`);
+        }
+    }
+    if (errs.length)
+        throw new VariantError(errs);
+    return {
+        alwaysOnChars: body.length,
+        files: Object.keys(variant).length,
+        skills: skills.length,
+        subagents: subagents.length,
+        scripts: Object.keys(variant).filter((rel) => r.script.test(rel)).length,
+        plugins: pluginNames(surface, variant).length
+    };
+}
+export function diffSummary(surface, before, after) {
+    const a = new Set(Object.keys(before));
+    const b = new Set(Object.keys(after));
+    return {
+        added: [...b].filter((rel) => !a.has(rel)).sort(),
+        removed: [...a].filter((rel) => !b.has(rel)).sort(),
+        modified: [...a].filter((rel) => b.has(rel) && before[rel] !== after[rel]).sort(),
+        alwaysOnCharDelta: alwaysOnText(surface, after).length - alwaysOnText(surface, before).length
+    };
+}
+export function isUnchanged(surface, before, after) {
+    const d = diffSummary(surface, before, after);
+    return !d.added.length && !d.removed.length && !d.modified.length;
+}
+/** Which components a diff touches. */
+export function touchedComponents(surface, diff) {
+    const out = new Set();
+    for (const rel of [...diff.added, ...diff.removed, ...diff.modified]) {
+        const id = componentOf(surface, rel);
+        if (id)
+            out.add(id);
+    }
+    return [...out];
+}
+/** Pairwise line-Jaccard over variants, so a collapse to one body is visible. */
+export function diversity(variants) {
+    const labels = Object.keys(variants).sort();
+    if (labels.length < 2)
+        return { n: labels.length, distinct: labels.length, meanLineJaccard: 1 };
+    const lines = (variant) => new Set(Object.entries(variant).flatMap(([rel, text]) => text.split('\n').filter((line) => line.trim()).map((line) => `${rel}:${line}`)));
+    const sets = Object.fromEntries(labels.map((label) => [label, lines(variants[label])]));
+    const sims = [];
+    for (let i = 0; i < labels.length; i++) {
+        for (const b of labels.slice(i + 1)) {
+            const x = sets[labels[i]];
+            const y = sets[b];
+            const union = new Set([...x, ...y]);
+            let shared = 0;
+            for (const line of x)
+                if (y.has(line))
+                    shared++;
+            sims.push(union.size ? shared / union.size : 1);
+        }
+    }
+    const distinct = new Set(labels.map((label) => JSON.stringify(Object.entries(variants[label]).sort()))).size;
+    return { n: labels.length, distinct, meanLineJaccard: Math.round((sims.reduce((sum, value) => sum + value, 0) / sims.length) * 10_000) / 10_000 };
+}
+/** Applies an omitted-is-inherited edit: `null` removes a file. */
+export function applyEdit(base, edit) {
+    const next = { ...base };
+    for (const [rel, text] of Object.entries(edit)) {
+        const clean = normalizeRel(rel);
+        if (text === null)
+            delete next[clean];
+        else
+            next[clean] = text;
+    }
+    return next;
+}

package/dist/sdk/web/markdown.d.ts

@@ -0,0 +1,10 @@
+/** Deeper than this, a page is not converted. */
+export declare const MAX_DEPTH = 512;
+export interface PageText {
+    title: string;
+    text: string;
+}
+/** The page's title and its visible content as Markdown. Throws when the page cannot be converted safely. */
+export declare function htmlToMarkdown(html: string): PageText;
+/** Plain text of Markdown: emphasis and link targets dropped, for `format: "text"`. */
+export declare function markdownToText(markdown: string): string;

package/dist/sdk/web/markdown.js

@@ -0,0 +1,66 @@
+// A fetched page as Markdown an agent can read. Content a person would not see is dropped first:
+// hidden text is a known way to slip instructions to a model. A page nested deeper than any real
+// page is refused rather than converted, so a hostile page cannot stall the conversion.
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+/** Deeper than this, a page is not converted. */
+export const MAX_DEPTH = 512;
+const INVISIBLE = 'script, style, noscript, template, iframe, object, embed, svg, canvas, head, link, meta, input[type="hidden"], [hidden], [aria-hidden="true"]';
+const HIDING_STYLE = /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0(?:\.0+)?\s*(?:;|$)|font-size\s*:\s*0(?:px)?\s*(?:;|$))/i;
+function depth(node, limit) {
+    let deepest = 0;
+    const stack = [[node, 0]];
+    while (stack.length) {
+        const [current, level] = stack.pop();
+        if (level > deepest)
+            deepest = level;
+        if (deepest > limit)
+            return deepest;
+        for (let index = 0; index < current.childNodes.length; index += 1) {
+            const child = current.childNodes[index];
+            if (child.nodeType === 1)
+                stack.push([child, level + 1]);
+        }
+    }
+    return deepest;
+}
+function converter() {
+    const service = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced', bulletListMarker: '-', emDelimiter: '*' });
+    service.use(gfm);
+    // An image is a word, not a link the model may follow without reading the page.
+    service.addRule('image', { filter: 'img', replacement: (_content, node) => {
+            const alt = node.getAttribute('alt')?.trim();
+            return alt ? `[image: ${alt}]` : '';
+        } });
+    return service;
+}
+/** The page's title and its visible content as Markdown. Throws when the page cannot be converted safely. */
+export function htmlToMarkdown(html) {
+    const { document } = parseHTML(html.includes('<html') ? html : `<!doctype html><html><body>${html}</body></html>`);
+    const title = (document.querySelector('title')?.textContent ?? '').replace(/\s+/g, ' ').trim();
+    const body = document.body ?? document.documentElement;
+    if (!body)
+        return { title, text: '' };
+    if (depth(body, MAX_DEPTH) > MAX_DEPTH)
+        throw new Error('The page is nested too deeply to convert safely.');
+    for (const element of [...body.querySelectorAll(INVISIBLE)])
+        element.remove();
+    for (const element of [...body.querySelectorAll('[style]')]) {
+        if (HIDING_STYLE.test(element.getAttribute('style') ?? ''))
+            element.remove();
+    }
+    const text = converter().turndown(body.innerHTML)
+        .replace(/\n{3,}/g, '\n\n')
+        .replace(/[ \t]+\n/g, '\n')
+        .trim();
+    return { title, text };
+}
+/** Plain text of Markdown: emphasis and link targets dropped, for `format: "text"`. */
+export function markdownToText(markdown) {
+    return markdown
+        .replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1')
+        .replace(/^#{1,6}\s+/gm, '')
+        .replace(/(\*\*|__|\*|_|`)/g, '')
+        .trim();
+}

package/dist/sdk/web/safeRequest.d.ts

@@ -0,0 +1,34 @@
+import { type LookupFunction } from 'node:net';
+export interface WebResponse {
+    status: number;
+    contentType: string;
+    body: Buffer;
+    /** The body reached the byte limit and was cut short. */
+    truncated: boolean;
+    /** The address finally read, after any redirects. */
+    url: URL;
+    /** Set when the address redirected to another site and `sameOrigin` stopped there. */
+    redirectedTo?: URL;
+}
+export interface WebRequestOptions {
+    headers?: Record<string, string>;
+    /** Longest a single read may stay silent, and the most time the whole request may take. */
+    timeoutMs: number;
+    maxBytes: number;
+    maxRedirects?: number;
+    /** A JSON body sends a POST (search services); without one the request is a GET. */
+    json?: unknown;
+    /** Stop at a redirect to another origin and report it, instead of following it. */
+    sameOrigin?: boolean;
+}
+/** Makes one read. Tests replace it; the default is `safeRequest`. */
+export type WebRequest = (url: URL, options: WebRequestOptions) => Promise<WebResponse>;
+/** Only plain web addresses: no other scheme, and no username or password in the address. */
+export declare function webAddress(input: string): URL;
+/** Whether an address belongs to this machine, a private network, or is otherwise not public. */
+export declare function isPublicAddress(address: string): boolean;
+export declare const publicLookup: LookupFunction;
+/** Reads a web address, following redirects one at a time, each checked the same way. */
+export declare const safeRequest: WebRequest;
+/** A body as text, in the character set its content type declares (UTF-8 when it declares none). */
+export declare function decodeBody(body: Buffer, contentType: string): string;

package/dist/sdk/web/safeRequest.js

@@ -0,0 +1,175 @@
+// Reads a web address on behalf of an agent without reaching the machine's own network. The
+// address is checked before a request and again when the connection is made, so a name that
+// resolves to a private address, or changes between the check and the connection, is refused.
+import { lookup as dnsLookup } from 'node:dns';
+import { request as httpRequest } from 'node:http';
+import { request as httpsRequest } from 'node:https';
+import { isIP } from 'node:net';
+const USER_AGENT = 'Mozilla/5.0 (compatible; XingularHarness/0.1; +web tools)';
+const REDIRECT_CODES = new Set([301, 302, 303, 307, 308]);
+/** Only plain web addresses: no other scheme, and no username or password in the address. */
+export function webAddress(input) {
+    let url;
+    try {
+        url = new URL(input.trim());
+    }
+    catch {
+        throw new Error('That is not a web address.');
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:')
+        throw new Error('Only http and https addresses can be read.');
+    if (url.username || url.password)
+        throw new Error('Addresses with a username or password cannot be read.');
+    if (!url.hostname)
+        throw new Error('That address has no host name.');
+    return url;
+}
+/** Whether an address belongs to this machine, a private network, or is otherwise not public. */
+export function isPublicAddress(address) {
+    const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(address);
+    if (mapped)
+        return isPublicIpv4(mapped[1]);
+    if (address.includes(':'))
+        return isPublicIpv6(address.toLowerCase());
+    return isPublicIpv4(address);
+}
+function isPublicIpv4(address) {
+    const parts = address.split('.').map(Number);
+    if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
+        return false;
+    const [a, b, c] = parts;
+    if (a === 0 || a === 10 || a === 127 || a >= 224)
+        return false;
+    if (a === 100 && b >= 64 && b <= 127)
+        return false;
+    if (a === 169 && b === 254)
+        return false;
+    if (a === 172 && b >= 16 && b <= 31)
+        return false;
+    if (a === 192 && b === 168)
+        return false;
+    if (a === 192 && b === 0 && c === 0)
+        return false;
+    if (a === 198 && (b === 18 || b === 19))
+        return false;
+    return true;
+}
+function isPublicIpv6(address) {
+    // Global unicast only (2000::/3), without documentation (2001:db8::/32), Teredo (2001::/32) or 6to4 (2002::/16), which carry IPv4 addresses.
+    const first = parseInt(address.split(':')[0] || '0', 16);
+    if (Number.isNaN(first) || first < 0x2000 || first > 0x3fff)
+        return false;
+    if (/^2001:db8(:|$)/.test(address) || /^2001:0{0,4}:/.test(address) || /^2002(:|$)/.test(address))
+        return false;
+    return true;
+}
+/** A DNS lookup that refuses any address that is not public. It runs when the connection is made. */
+function publicLookupImpl(hostname, options, callback) {
+    dnsLookup(hostname, { all: true, verbatim: true }, (error, addresses) => {
+        if (error)
+            return callback(error, '', 4);
+        if (addresses.some((entry) => !isPublicAddress(entry.address))) {
+            return callback(Object.assign(new Error(`${hostname} is not a public address.`), { code: 'EPRIVATE' }), '', 4);
+        }
+        if (options.all)
+            return callback(null, addresses);
+        callback(null, addresses[0].address, addresses[0].family);
+    });
+}
+export const publicLookup = publicLookupImpl;
+/**
+ * A literal address is never looked up, so it is checked here. A name is checked by the lookup
+ * at connection time.
+ */
+function assertReachable(url) {
+    const host = url.hostname.replace(/^\[|\]$/g, '');
+    if (isIP(host) && !isPublicAddress(host))
+        throw Object.assign(new Error(`${host} is not a public address.`), { code: 'EPRIVATE' });
+}
+/** Reads a web address, following redirects one at a time, each checked the same way. */
+export const safeRequest = async (start, options) => {
+    let url = start;
+    for (let hop = 0;; hop += 1) {
+        assertReachable(url);
+        const response = await readOnce(url, options);
+        if (REDIRECT_CODES.has(response.status) && response.location) {
+            if (hop >= (options.maxRedirects ?? 5))
+                throw new Error('The address redirected too many times.');
+            const next = webAddress(new URL(response.location, url).toString());
+            // Another site is a new decision: the caller reads it with its own call.
+            if (options.sameOrigin && next.origin !== url.origin)
+                return { status: response.status, contentType: '', body: Buffer.alloc(0), truncated: false, url, redirectedTo: next };
+            url = next;
+            continue;
+        }
+        return { status: response.status, contentType: response.contentType, body: response.body, truncated: response.truncated, url };
+    }
+};
+function readOnce(url, options) {
+    return new Promise((resolve, reject) => {
+        const start = (url.protocol === 'https:' ? httpsRequest : httpRequest);
+        let settled = false;
+        const finish = (value) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(total);
+            resolve(value);
+        };
+        const fail = (error) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(total);
+            reject(error);
+        };
+        const payload = options.json === undefined ? null : Buffer.from(JSON.stringify(options.json));
+        const request = start(url, {
+            method: payload ? 'POST' : 'GET',
+            headers: { 'user-agent': USER_AGENT, accept: 'text/html,text/plain,application/json;q=0.9,*/*;q=0.5', ...(payload ? { 'content-type': 'application/json', 'content-length': String(payload.length) } : {}), ...options.headers },
+            lookup: publicLookup,
+            timeout: options.timeoutMs
+        }, (response) => {
+            const status = response.statusCode ?? 0;
+            const location = response.headers.location;
+            if (REDIRECT_CODES.has(status) && location) {
+                response.resume();
+                finish({ status, contentType: '', body: Buffer.alloc(0), truncated: false, url, location });
+                return;
+            }
+            const chunks = [];
+            let size = 0;
+            let truncated = false;
+            const done = () => finish({ status, contentType: String(response.headers['content-type'] ?? ''), body: Buffer.concat(chunks), truncated, url });
+            response.on('data', (chunk) => {
+                const room = options.maxBytes - size;
+                const kept = chunk.length > room ? chunk.subarray(0, Math.max(room, 0)) : chunk;
+                chunks.push(kept);
+                size += kept.length;
+                if (kept.length < chunk.length) {
+                    truncated = true;
+                    response.destroy();
+                    done();
+                }
+            });
+            response.on('error', (error) => (truncated ? done() : fail(error)));
+            response.on('end', done);
+            response.on('close', done);
+        });
+        // The whole request is capped as well as each silent stretch, so a slow trickle cannot hold a turn open.
+        const total = setTimeout(() => request.destroy(new Error('The address took too long to answer.')), options.timeoutMs * 3);
+        request.on('timeout', () => request.destroy(new Error('The address took too long to answer.')));
+        request.on('error', fail);
+        request.end(payload ?? undefined);
+    });
+}
+/** A body as text, in the character set its content type declares (UTF-8 when it declares none). */
+export function decodeBody(body, contentType) {
+    const label = /charset\s*=\s*"?([^";\s]+)/i.exec(contentType)?.[1]?.toLowerCase() ?? 'utf-8';
+    try {
+        return new TextDecoder(label).decode(body);
+    }
+    catch {
+        throw new Error(`The page uses a character set that cannot be read: ${label}.`);
+    }
+}

package/dist/sdk/web/search.d.ts

@@ -0,0 +1,53 @@
+import { type SecretStore } from "../accounts/secrets.js";
+import { type WebRequest } from "./safeRequest.js";
+export interface SearchHit {
+    title: string;
+    url: string;
+    snippet: string;
+}
+export interface SearchProvider {
+    id: string;
+    title: string;
+    /** The secret holding its key, or null for one that needs none. */
+    key: string | null;
+    search(query: string, count: number, key: string | null, request: WebRequest): Promise<SearchHit[]>;
+}
+/** Raised when a provider asks to be left alone; `retryAfterMs` comes from its answer. */
+export declare class RateLimited extends Error {
+    readonly retryAfterMs: number;
+    constructor(retryAfterMs: number);
+}
+export declare const BRAVE: SearchProvider;
+export declare const TAVILY: SearchProvider;
+export declare const EXA: SearchProvider;
+/** DuckDuckGo's plain results page. No key; off unless a person switched it on. */
+export declare const DUCKDUCKGO: SearchProvider;
+export declare const SEARCH_PROVIDERS: readonly SearchProvider[];
+export interface SearchOptions {
+    secrets: SecretStore;
+    request: WebRequest;
+    /** Whether the keyless provider may be used. */
+    keyless: () => boolean;
+    providers?: readonly SearchProvider[];
+    now?: () => number;
+}
+export interface SearchAnswer {
+    provider: string | null;
+    results: SearchHit[];
+    /** Why each provider tried did not answer, for the model and the log. */
+    failures: string[];
+}
+/** Tries each usable provider in turn. Remembers which ones are resting. */
+export declare class Searcher {
+    private readonly options;
+    private readonly resting;
+    constructor(options: SearchOptions);
+    /** Providers that could answer now: a key saved (or none needed and allowed), not resting. */
+    usable(): Promise<{
+        provider: SearchProvider;
+        key: string | null;
+    }[]>;
+    search(query: string, count: number): Promise<SearchAnswer>;
+}
+/** Several queries' results interleaved, each address once, at most `limit`. */
+export declare function mergeResults(lists: readonly SearchHit[][], limit: number): SearchHit[];