@turri-dev/cli 0.0.0-stage → 0.1.1

package/dist/regenerate/unit-review.js

@@ -0,0 +1,347 @@
+import { basename } from "node:path";
+import { defineTool } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { ask, createExecutor } from "../pi.js";
+import { reportActivity } from "./activity.js";
+import { diffFiles, digestFiles } from "./codebase.js";
+import { confinedTools } from "./confine.js";
+import { REVIEWER_CALLS, reviewKind } from "./conformance-agents.js";
+import { readNotes, recordNote, reflectOnReview, REVIEW_LESSON } from "./notes.js";
+import { diffSinceSnapshot, workDiff, workSnapshot } from "./lineage.js";
+import { REVIEWER_COMPACTION } from "./compaction.js";
+import { describeAccounts, SUBMIT_AFTER_REVIEW, unanswered, unansweredPrompt } from "./submit-work.js";
+import { ARBITER_AFTER, arbitrateReview, dismissedNote, problemLine, REVIEW_LIMIT } from "./arbiter.js";
+import { codeReviewCasesBlock } from "./cases.js";
+import { TEST_FILE_PATTERN } from "./testing.js";
+import { createCanItFailTool } from "./self-check.js";
+/** How many times the reviewer runs can_it_fail in a review session: each is a run of the unit testing script. */
+export const REVIEWER_CAN_IT_FAIL_RUNS = 3;
+/**
+ * How many reviews may send the unit tests back before the arbiter weighs them: from this one on, every review that
+ * sends them back goes to the arbiter first, and only what it upholds reaches the coding agent.
+ */
+export const UNIT_REVIEW_ROUNDS = ARBITER_AFTER;
+/**
+ * How many reviews at most. The review optimises the tests; it does not decide whether the work is done - the
+ * unit testing script does. Past this many, whatever it still finds, blocking or not, is let go, kept in the notes,
+ * and the work goes on.
+ */
+export const UNIT_REVIEW_LIMIT = REVIEW_LIMIT;
+/** The unit test files that differ from what they were when the work started: added or changed. */
+export function changedUnitTests(buildDir, atStart) {
+    const { added, changed } = diffFiles(atStart, digestFiles(buildDir));
+    return [...added, ...changed].filter((file) => TEST_FILE_PATTERN.test(basename(file)));
+}
+/**
+ * The reviewer's task, for its instructions: what it reviews, against which specification, and how. It stays word for
+ * word through every review of the loop; what each review is shown comes as a message (`unitReviewRequest`).
+ */
+export function unitReviewTask(options) {
+    const mechanical = options.cases?.mechanical === true;
+    return [
+        `You review the work on ${options.what}, each time its unit tests pass: passing tests prove only what they`,
+        "check - a test that checks next to nothing passes whatever the code does, and code no test reaches is not checked",
+        "at all. Review both: that the code does what the specification says, and that its unit tests test it.",
+        "",
+        "You are the exacting reviewer: opinionated, thorough, and hard to satisfy. Raise every problem you find with",
+        "the code and its tests, however small - a senior engineer weighs your findings after you, and lets go what is",
+        "not worth the time. Missing one is worse than raising one too many.",
+        "",
+        `The code and its unit tests are in ${options.buildDir}.`,
+        "",
+        "How to read it:",
+        "- <specification> is the specification exactly as the coding agent had it. Review the work against <new in this",
+        "  work>; the rest is there so you know what the words mean and what was there before.",
+        "- The diff you are shown is exactly what changed in the code folder - code and tests - new files whole. It is",
+        "  taken by the harness, not by the agent: start from it, and read a file whole where the diff shows too little of",
+        "  it, or to see how the new code fits what was there. Do not guess what the code does - it is all there.",
+        "- <the coding agent's account> is what the agent that did the work says it did. It is a claim: check every part of",
+        "  it against the diff and the files. A problem it says is fixed that the code does not fix is still a problem; one",
+        "  it declined, weigh its reason - it may be right.",
+        "",
+        "<specification>",
+        options.spec.trim(),
+        "</specification>",
+        "",
+        ...(options.cases !== undefined ? [...codeReviewCasesBlock(options.cases), ""] : []),
+        // A job that touches no functional spec is held only to that: the checks of behaviour are not its.
+        ...(mechanical ? [] : ["Check that the code does what the specification states for this work:"]),
+        ...(mechanical
+            ? []
+            : [
+                "- Every behaviour, edge case, validation rule and error it states is implemented, and implemented as it says -",
+                "  the values, the messages, the order, the limits. Nothing is left as a stub, a placeholder, a TODO or a",
+                "  hard-coded answer that passes a test and nothing else.",
+                "- It keeps to the implementation reqs, and does not break what the code already implements.",
+                "- When the work wrote no code at all, or none for a behaviour, that behaviour is not implemented: raise it.",
+                "",
+                "Check that every behaviour the specification states for this work is tested, and tested well:",
+                "- Each behaviour, edge case, validation rule and error the specification states has a test, and the test",
+                "  asserts the outcome the specification says - the exact value returned, the state stored, the error raised",
+                "  and its message or code - not only that something came back.",
+                "- A test fails when the behaviour it is named for breaks. Weak: it only checks that a function, a route, a",
+                "  file or a field exists; that a result is not null, is truthy, has a type or a length; that a status is 200",
+                "  and nothing about the body; `assert True`, an empty or skipped test, an exception caught and swallowed, an",
+                "  assertion so loose any answer passes, an expected value computed the way the code computes it.",
+                ...(options.canItFail === true
+                    ? [
+                        "- A test you doubt can fail, check: can_it_fail puts a bug in a copy of the code - a condition turned round, a",
+                        "  value changed - and runs the unit tests on it. A test that still passes cannot fail: raise it as bad, with the",
+                        `  bug it missed. ${REVIEWER_CAN_IT_FAIL_RUNS} runs a session: spend them on the tests the work rests on.`,
+                    ]
+                    : []),
+                "- It tests the code, not a stand-in: what is outside the unit - a network service, a clock, a database the",
+                "  unit does not own - may be replaced; the unit under test itself never is, and a test that asserts only",
+                "  what a mock was told to return tests the mock.",
+                "- Deterministic: no real network, no sleeping, no dependence on the time of day or the order tests run in.",
+                ...(options.forChange === true
+                    ? [
+                        "This work is the <spec change> in <new in this work>, section by section: what it adds or changes is what needs",
+                        "testing, and the tests it changed still have to test the rest as they did. A change that adds no behaviour - a",
+                        "description, a renamed concept, a reworded spec - needs no new test.",
+                        "Behaviour is what a functional spec, an acceptance test or an implementation req states. A definition says what",
+                        "a word means: a page, a component or a record that only <definitions> name, and no functional spec or",
+                        "implementation req of this codebase asks for, has no code yet, and a test for it is not missing.",
+                    ]
+                    : []),
+            ]),
+        "Only what this work's specification asks for: a test of something it does not state, or of how the code",
+        "works inside rather than what it does, is not a problem to raise; nor is code written another way than you",
+        "would have, as long as it does what the specification says.",
+        "",
+        "You review the same work more than once. The first time you are shown everything it wrote, and find every problem",
+        "in that one review: the work goes back once with all of them. Each time after, you are shown what it changed since",
+        "and what the coding agent says it did about each problem: check each one you raised first - fixed, or still there -",
+        "and raise a new one only when it would be blocking (below). It is not done again from the start.",
+        "",
+        "Each review ends with submit_review, once: pass when the code does what the specification says and nothing is",
+        'missing or weak; otherwise every problem - the file, the test or the function, its kind - "wrong" for code that does',
+        'not do what the specification says, "missing" for a test that is not there, "bad" for one that is weak or wrong -',
+        "and exactly what, with what the specification says instead. Be specific: the agent that fixes them gets your list",
+        "and nothing else. Mark a problem blocking when because of it a behaviour the specification states is not",
+        "implemented, implemented wrong, or not tested - no test for it, or one that cannot fail. Those are worked on first;",
+        "the rest - ways to make code or tests that already do it better - are let go after a few reviews.",
+    ].join("\n");
+}
+/** What one review is shown: the whole work the first time; after that, what changed since the last, and the problems it raised. */
+export function unitReviewRequest(options) {
+    const diff = options.diff.trim();
+    const again = options.earlier !== undefined;
+    return [
+        again
+            ? "The unit tests pass again, after the coding agent worked on the problems your last review raised. Review it again."
+            : "The unit tests pass. Review the work.",
+        "",
+        again ? "<what changed since your last review - a diff of the code folder>" : "<the code this work wrote - a diff of the code folder from where the work started>",
+        diff !== "" ? diff : again ? "(nothing changed since your last review)" : "(nothing - the code folder is as the work found it)",
+        again ? "</what changed since your last review>" : "</the code this work wrote>",
+        "",
+        "<unit tests added or changed by the work>",
+        options.changed.length > 0 ? options.changed.join("\n") : "(none - no unit test was added or changed)",
+        "</unit tests added or changed by the work>",
+        "",
+        "<the coding agent's account>",
+        options.account,
+        "</the coding agent's account>",
+        // The problems it raised last are its own submit_review, in its session: named, not sent again.
+        ...(again && options.earlier.length > 0
+            ? ["", options.earlier.length === 1 ? "Check first the problem your last review raised." : `Check first each of the ${options.earlier.length} problems your last review raised.`]
+            : []),
+        ...(options.letGo !== undefined && options.letGo.length > 0
+            ? ["", "The arbiter weighed your last review and let these go - they are settled, do not raise them again:", ...options.letGo.map((problem) => `- ${problemLine(problem)}`)]
+            : []),
+        "",
+        "Then call submit_review.",
+    ].join("\n");
+}
+/**
+ * The reviewer: a read-only agent, on the judge model, that says pass or every problem - one session for the whole
+ * loop, so a review after the first starts from what it already read and raised, not from nothing. Its task is its
+ * instructions; its work is compacted as it gets longer.
+ */
+export async function createUnitReviewer(options) {
+    let verdict;
+    const submit = defineTool({
+        name: "submit_review",
+        label: "Submit the review",
+        description: "Say whether the code and its unit tests pass the review, and if not, every problem with them. Call this once, when you are done.",
+        parameters: Type.Object({
+            pass: Type.Boolean({ description: "True only when the code does what the specification says and no test is missing or weak" }),
+            problems: Type.Array(Type.Object({
+                file: Type.String({ description: "The file, relative to the code folder: the code that is wrong, the test file, or the file a missing test belongs in" }),
+                test: Type.Optional(Type.String({ description: "The test or the function in it, when it is one" })),
+                kind: Type.String({
+                    description: '"wrong" when the code does not do what the specification says - or does not do it at all; "missing" when a test the specification' +
+                        ' calls for is not there; "bad" when a test is there but weak or wrong. Any other word is read as one of these - one starting with' +
+                        ' "wrong" as wrong, with "missing" as missing, the rest as bad',
+                }),
+                what: Type.String({ description: "Exactly what is wrong, missing or weak, and what the specification says instead" }),
+                blocking: Type.Boolean({
+                    description: "True when because of it a behaviour the specification states is not implemented, implemented wrong, or not tested - no test, or one that cannot fail." +
+                        " False for a way to make code or a test that already does it better",
+                }),
+            })),
+        }),
+        async execute(_id, params) {
+            // "missing_test", "weak": the verdict is what counts, not the exact word - one bad word never throws a review away.
+            const given = { ...params };
+            given.problems = given.problems.map((problem) => ({ ...problem, kind: /^\s*wrong/i.test(problem.kind) ? "wrong" : reviewKind(problem.kind) }));
+            // A pass with problems, or a fail with none, is not a verdict.
+            if (given.pass === given.problems.length > 0) {
+                return {
+                    content: [{ type: "text", text: given.pass ? "A pass lists no problems - either drop them or fail it." : "A fail has to say what is wrong: list every problem." }],
+                    details: {},
+                };
+            }
+            verdict = given;
+            return { content: [{ type: "text", text: "Recorded." }], details: {}, terminate: true };
+        },
+    });
+    const fence = confinedTools({ workdir: options.buildDir, read: [options.buildDir], write: [], noBash: true });
+    // A test it doubts, it checks: whether the tests catch a bug put in a copy of the code - the code itself untouched.
+    const canItFail = options.unitScript !== undefined ? [createCanItFailTool({ buildDir: options.buildDir, script: options.unitScript, runs: REVIEWER_CAN_IT_FAIL_RUNS })] : [];
+    const session = await createExecutor({
+        cwd: options.buildDir,
+        customTools: [submit, ...canItFail, ...fence.tools],
+        tools: [...fence.names, ...canItFail.map((tool) => tool.name), "submit_review"],
+        model: options.model,
+        label: `${options.label} reviewer`, role: "reviewer",
+        system: fence.note,
+        task: unitReviewTask({ ...options, canItFail: options.unitScript !== undefined }),
+        compaction: { instructions: REVIEWER_COMPACTION },
+        budget: { calls: REVIEWER_CALLS, finish: ["submit_review"] },
+    });
+    return {
+        async review(request) {
+            verdict = undefined;
+            await ask(session, request, options.onDelta);
+            if (verdict === undefined)
+                await ask(session, "You have not called submit_review. Call it now: pass, or every problem.", options.onDelta);
+            return (verdict ?? {
+                pass: false,
+                problems: [{ file: "(all)", kind: "bad", what: "The review gave no verdict. Check the code and every unit test you added or changed against the specification yourself." }],
+            });
+        },
+        dispose: () => session.dispose(),
+    };
+}
+/** The review's problems, for the coding agent. */
+export function unitReviewFeedbackPrompt(problems) {
+    return [
+        "The unit tests pass. A review of the work found where the code does not yet do what the specification says, or its",
+        "tests do not yet test it - each, exactly:",
+        "",
+        ...problems.map((problem, index) => `${index + 1}. ${problemLine(problem)}`),
+        "",
+        "Fix every one: make the code WRONG names do what the specification says, write the tests that are missing, and",
+        "make the weak ones assert what the specification says - the exact outcome, so they fail when it breaks. Then run",
+        "the unit tests. One you are sure the review is wrong about, decline, and say why.",
+        ...SUBMIT_AFTER_REVIEW,
+        ...REVIEW_LESSON,
+    ].join("\n");
+}
+/**
+ * Review the code and its unit tests until they pass it: each time it finds problems, they go to the coding agent
+ * and the unit tests must pass again before the next review. Past UNIT_REVIEW_ROUNDS reviews with
+ * nothing blocking, what it still suggests is let go and kept in the notes; past UNIT_REVIEW_LIMIT,
+ * whatever is left is let go the same way - the review optimises the tests, it never stops the work.
+ * Only unit tests that no longer pass after a round of fixes do.
+ */
+export async function untilUnitTestsReviewed(options) {
+    const reviewer = await createUnitReviewer(options);
+    try {
+        return await reviewLoop(options, reviewer);
+    }
+    finally {
+        reviewer.dispose();
+    }
+}
+async function reviewLoop(options, reviewer) {
+    let earlier;
+    const history = [];
+    const waived = [];
+    /** What the reviewer saw last: the next review is shown only what changed since. */
+    let seen;
+    /** What the arbiter dismissed of the last review: the reviewer is told, so it does not raise them again. */
+    let letGo = [];
+    const said = (problems) => problems.map((problem) => `${problem.file}${problem.test ? ` ${problem.test}` : ""}: ${problem.what.trim()}`).join("; ").slice(0, 600);
+    for (let reviews = 0; reviews < UNIT_REVIEW_LIMIT; reviews += 1) {
+        const changed = changedUnitTests(options.buildDir, options.atStart);
+        // What the coding agent has written by now: it changes between reviews. The arbiter, fresh each time, reads it whole.
+        const code = workDiff(options.buildDir);
+        const account = describeAccounts(options.work?.take() ?? [], earlier);
+        const request = seen === undefined || earlier === undefined
+            ? unitReviewRequest({ diff: code, changed, account })
+            : unitReviewRequest({ diff: diffSinceSnapshot(options.buildDir, seen), changed, account, earlier, letGo });
+        seen = workSnapshot(options.buildDir);
+        letGo = [];
+        options.stage(`reviewing the code and its unit tests (review ${reviews + 1}): does the code do what the spec says, and does each test fail when that breaks`);
+        const review = await reviewer.review(request);
+        if (review.pass) {
+            options.stage("the review has nothing left to improve in the code or its unit tests");
+            return waived.length > 0 ? { passed: true, waived, reviews: reviews + 1 } : { passed: true, reviews: reviews + 1 };
+        }
+        let problems = review.problems;
+        // After the first UNIT_REVIEW_ROUNDS reviews that send them back, the arbiter weighs each first: only what it upholds goes to the coding agent.
+        if (reviews >= UNIT_REVIEW_ROUNDS) {
+            options.stage(`the review sent the work back ${reviews + 1 === 1 ? "once" : `${reviews + 1} times`} - the arbiter weighs what it found`);
+            const ruling = await arbitrateReview({
+                subject: `the code and unit tests of ${options.what}`,
+                workdir: options.buildDir,
+                read: [options.buildDir],
+                uphold: [
+                    "- uphold it when because of it a behaviour this work's specification states - in a functional spec, an",
+                    "  acceptance test or an implementation req - is not implemented, is implemented other than it says, is not tested, or a",
+                    "  test cannot fail when that behaviour breaks. Then the reviewer is right, and the coding agent fixes it.",
+                ],
+                changed,
+                spec: `<specification>\n${options.spec.trim()}\n</specification>`,
+                code,
+                problems: review.problems,
+                history,
+                label: `${options.label} unit test`,
+                ...(options.model !== undefined ? { model: options.model } : {}),
+                ...(options.onDelta !== undefined ? { onDelta: options.onDelta } : {}),
+            });
+            // No ruling: the review stands, as it did before there was an arbiter.
+            if (ruling !== undefined) {
+                waived.push(...ruling.dismissed.map((entry) => entry.problem));
+                if (ruling.dismissed.length > 0)
+                    recordNote(options.buildDir, "pitfalls", dismissedNote(ruling, "unit test"), `${options.label} unit tests`);
+                if (ruling.upheld.length === 0) {
+                    options.stage("the arbiter approved the code and its unit tests as they are - what the review still raised is let go, and in the notes");
+                    return { passed: true, waived, reviews: reviews + 1 };
+                }
+                options.stage(`the arbiter upheld ${ruling.upheld.length} of the review's ${review.problems.length} problem${review.problems.length === 1 ? "" : "s"} - the coding agent fixes ${ruling.upheld.length === 1 ? "it" : "those"}`);
+                problems = ruling.upheld.map((entry) => entry.problem);
+                letGo = ruling.dismissed.map((entry) => entry.problem);
+            }
+        }
+        history.push(review.problems);
+        const blocking = problems.filter((problem) => problem.blocking === true);
+        const wrong = problems.filter((problem) => problem.kind === "wrong");
+        reportActivity(`  the review found ${problems.length} problem${problems.length === 1 ? "" : "s"}: ${problems[0].what.slice(0, 140)}`);
+        options.stage(wrong.length > 0
+            ? `the review found ${wrong.length} place${wrong.length === 1 ? "" : "s"} where the code does not do what the spec says - the coding agent fixes the code${problems.length > wrong.length ? " and its tests" : ""}`
+            : blocking.length > 0
+                ? `the review found ${blocking.length} behaviour${blocking.length === 1 ? "" : "s"} of the spec not yet tested - the coding agent completes the unit tests`
+                : `the review found ${problems.length} way${problems.length === 1 ? "" : "s"} to make the unit tests test more - the coding agent improves them`);
+        earlier = problems;
+        const pitfalls = readNotes(options.buildDir).pitfalls;
+        await options.send(unitReviewFeedbackPrompt(problems));
+        // Every problem answered - fixed, or declined and why - so the reviewer knows what became of each: sent back once.
+        if (options.work !== undefined && options.askForAnswers !== false) {
+            const missing = unanswered(options.work.peek(), problems.length);
+            if (missing.length > 0)
+                await options.send(unansweredPrompt(missing));
+        }
+        // The lesson, left for the agents after it: the next functionality's tests start from it.
+        await reflectOnReview(options.buildDir, pitfalls, options.send);
+        if (!(await options.untilPassing()))
+            return { passed: false, problems, reviews: reviews + 1 };
+    }
+    // An optimiser, not a gate: what the last review sent back was worked on and the unit tests pass - it is not reviewed again.
+    options.stage(`the last of ${UNIT_REVIEW_LIMIT} reviews sent the work back once more; its unit tests pass - going on without another review`);
+    recordNote(options.buildDir, "pitfalls", `the unit tests went on after ${UNIT_REVIEW_LIMIT} reviews, the last one's problems worked on but not reviewed again: ${said(earlier ?? [])}`, `${options.label} unit tests`);
+    return { passed: true, waived: [...waived, ...(earlier ?? [])], reviews: UNIT_REVIEW_LIMIT };
+}

package/dist/regenerate/web.js

@@ -0,0 +1,256 @@
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+import { defineTool } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+/**
+ * The web, for the agents that build: search it, and read a page.
+ *
+ * An agent building on a library it half-remembers, or stuck on an error it has not seen, does
+ * better reading the documentation or the answer than guessing again. So every rendering agent
+ * has two tools: `web_search`, which asks DuckDuckGo and returns the results, and `web_fetch`,
+ * which reads one page and returns it as text.
+ *
+ * Both run here, in turri itself. What reaches the web through them is a request the agent made
+ * through a tool, and what comes back is text it reads.
+ *
+ * Only the public web: `web_fetch` refuses an address on this machine or its network - localhost,
+ * the private ranges, the cloud metadata address - so it cannot be used to reach the cloud's
+ * credentials or a service inside the person's network by way of turri.
+ */
+export const WEB_TOOLS = ["web_search", "web_fetch"];
+const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0";
+const TIMEOUT_MS = 20_000;
+/** How much of a page one call returns; the rest is had with `offset`. */
+export const PAGE_CHARS = 20_000;
+/** How much of a response is read at all. */
+const MAX_BYTES = 5_000_000;
+const MAX_REDIRECTS = 5;
+const ENTITIES = {
+    amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", times: "×", hellip: "…", mdash: "—", ndash: "–",
+    lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", laquo: "«", raquo: "»", copy: "©", reg: "®", trade: "™",
+    bull: "•", middot: "·", rarr: "→", larr: "←", deg: "°", euro: "€",
+};
+export function decodeEntities(text) {
+    return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+\d*);/gi, (whole, name) => {
+        if (name[0] === "#") {
+            const code = name[1] === "x" || name[1] === "X" ? parseInt(name.slice(2), 16) : parseInt(name.slice(1), 10);
+            return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole;
+        }
+        return ENTITIES[name.toLowerCase()] ?? whole;
+    });
+}
+function stripTags(html) {
+    return decodeEntities(html.replace(/<[^>]*>/g, "")).replace(/\s+/g, " ").trim();
+}
+/** A DuckDuckGo link, which may point at its own redirect, as the address it leads to. */
+function resultUrl(href) {
+    const raw = decodeEntities(href);
+    try {
+        const url = new URL(raw, "https://duckduckgo.com");
+        if (url.hostname.endsWith("duckduckgo.com")) {
+            const target = url.searchParams.get("uddg");
+            return target !== null && /^https?:\/\//.test(target) ? target : undefined;
+        }
+        return /^https?:$/.test(url.protocol) ? url.href : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** The results on a page of DuckDuckGo Lite: each a link, then its snippet. Ads are left out. */
+export function parseSearchResults(html) {
+    const results = [];
+    const link = /<a\b[^>]*\bhref=(["'])(.*?)\1[^>]*\bclass=(["'])result-link\3[^>]*>([\s\S]*?)<\/a>|<a\b[^>]*\bclass=(["'])result-link\5[^>]*\bhref=(["'])(.*?)\6[^>]*>([\s\S]*?)<\/a>/gi;
+    const matches = [...html.matchAll(link)];
+    matches.forEach((match, i) => {
+        const url = resultUrl(match[2] ?? match[7] ?? "");
+        if (url === undefined)
+            return;
+        const end = matches[i + 1]?.index ?? html.length;
+        const between = html.slice(match.index + match[0].length, end);
+        const snippet = /<td[^>]*class=(["'])result-snippet\1[^>]*>([\s\S]*?)<\/td>/i.exec(between);
+        results.push({ title: stripTags(match[4] ?? match[8] ?? ""), url, snippet: snippet ? stripTags(snippet[2]) : "" });
+    });
+    return results;
+}
+export async function webSearch(query, limit = 8) {
+    const response = await fetch("https://lite.duckduckgo.com/lite/", {
+        method: "POST",
+        headers: { "user-agent": USER_AGENT, "content-type": "application/x-www-form-urlencoded" },
+        body: new URLSearchParams({ q: query }).toString(),
+        signal: AbortSignal.timeout(TIMEOUT_MS),
+    });
+    const html = await response.text();
+    if (response.status !== 200 || (/anomaly|challenge-form/i.test(html) && !/result-link/.test(html))) {
+        throw new Error(`The search did not answer (DuckDuckGo said ${response.status}${response.status === 202 ? ", which is its rate limit" : ""}). ` +
+            "Wait a little and search again, or fetch a documentation page you know with web_fetch.");
+    }
+    return parseSearchResults(html).slice(0, limit);
+}
+/** Whether an IP address is on this machine or its network, rather than the public internet. */
+export function isPrivateAddress(address) {
+    let ip = address.toLowerCase().replace(/^\[|\]$/g, "");
+    const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(ip);
+    if (mapped)
+        ip = mapped[1];
+    if (isIP(ip) === 4) {
+        const [a, b] = ip.split(".").map(Number);
+        return (a === 0 ||
+            a === 10 ||
+            a === 127 ||
+            (a === 100 && b >= 64 && b <= 127) ||
+            (a === 169 && b === 254) ||
+            (a === 172 && b >= 16 && b <= 31) ||
+            (a === 192 && b === 168) ||
+            (a === 198 && (b === 18 || b === 19)) ||
+            a >= 224);
+    }
+    if (isIP(ip) === 6) {
+        return ip === "::" || ip === "::1" || /^f[cd]/.test(ip) || /^fe[89ab]/.test(ip) || /^ff/.test(ip) || /^::ffff:/.test(ip);
+    }
+    return true;
+}
+const lookupAll = async (host) => (await lookup(host, { all: true })).map((entry) => entry.address);
+/** Refuse a URL that is not http(s) on the public internet; its parsed form when it is. */
+export async function checkPublicUrl(raw, resolve = lookupAll) {
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        throw new Error(`"${raw}" is not a URL. Give the whole address, starting with https://.`);
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:")
+        throw new Error(`Only http and https pages can be fetched, not ${url.protocol}`);
+    const host = url.hostname.replace(/^\[|\]$/g, "");
+    if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local") || host.endsWith(".internal")) {
+        throw new Error(`${url.hostname} is not on the public web, and only the public web can be fetched.`);
+    }
+    const addresses = isIP(host) ? [host] : await resolve(host).catch(() => {
+        throw new Error(`${url.hostname} could not be found.`);
+    });
+    if (addresses.length === 0 || addresses.some(isPrivateAddress)) {
+        throw new Error(`${url.hostname} is not on the public web, and only the public web can be fetched.`);
+    }
+    return url;
+}
+/** A page as readable text: what a person would read, without the markup, scripts and styles. */
+export function htmlToText(html) {
+    const title = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
+    let body = html
+        .replace(/<!--[\s\S]*?-->/g, "")
+        .replace(/<(script|style|noscript|svg|head|template|iframe)\b[\s\S]*?<\/\1>/gi, "")
+        .replace(/<(nav|footer)\b[\s\S]*?<\/\1>/gi, "");
+    // Code keeps its lines; everything else is reflowed.
+    const blocks = [];
+    body = body.replace(/<pre\b[^>]*>([\s\S]*?)<\/pre>/gi, (_whole, code) => {
+        blocks.push(decodeEntities(code.replace(/<[^>]*>/g, "")).replace(/\n+$/, ""));
+        return `\n\u0000${blocks.length - 1}\u0000\n`;
+    });
+    body = body
+        .replace(/<h([1-6])\b[^>]*>/gi, (_whole, level) => `\n\n${"#".repeat(Number(level))} `)
+        .replace(/<li\b[^>]*>/gi, "\n- ")
+        .replace(/<br\s*\/?>/gi, "\n")
+        .replace(/<\/(p|div|section|article|h[1-6]|ul|ol|table|tr|blockquote|dl|dt|dd)>/gi, "\n")
+        .replace(/<(td|th)\b[^>]*>/gi, " | ")
+        .replace(/<code\b[^>]*>([\s\S]*?)<\/code>/gi, "`$1`")
+        .replace(/<[^>]*>/g, "");
+    body = decodeEntities(body)
+        .split("\n")
+        .map((line) => line.replace(/[ \t ]+/g, " ").trim())
+        .join("\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .replace(/\u0000(\d+)\u0000/g, (_whole, i) => `\`\`\`\n${blocks[Number(i)]}\n\`\`\``)
+        .trim();
+    const heading = title ? stripTags(title[1]) : "";
+    return heading !== "" && !body.startsWith(`# ${heading}`) ? `# ${heading}\n\n${body}` : body;
+}
+const READABLE = /^(text\/|application\/(json|xml|javascript|x-yaml|yaml|toml|x-sh|[a-z.+-]*\+(json|xml)))/i;
+async function readLimited(response) {
+    const reader = response.body?.getReader();
+    if (reader === undefined)
+        return "";
+    const chunks = [];
+    let size = 0;
+    while (size < MAX_BYTES) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        chunks.push(value);
+        size += value.byteLength;
+    }
+    await reader.cancel().catch(() => { });
+    return new TextDecoder().decode(Buffer.concat(chunks).subarray(0, MAX_BYTES));
+}
+/** Fetch a public page, following redirects - each checked - and return it as text. */
+export async function webFetch(raw, opts = {}) {
+    const get = opts.fetchImpl ?? fetch;
+    let url = await checkPublicUrl(raw, opts.resolve);
+    for (let hop = 0;; hop++) {
+        const response = await get(url.href, {
+            redirect: "manual",
+            headers: { "user-agent": USER_AGENT, accept: "text/html,text/plain,application/json;q=0.9,*/*;q=0.5" },
+            signal: AbortSignal.timeout(TIMEOUT_MS),
+        });
+        const location = response.headers.get("location");
+        if (response.status >= 300 && response.status < 400 && location !== null) {
+            if (hop >= MAX_REDIRECTS)
+                throw new Error(`${raw} redirected more than ${MAX_REDIRECTS} times.`);
+            url = await checkPublicUrl(new URL(location, url).href, opts.resolve);
+            continue;
+        }
+        const contentType = response.headers.get("content-type") ?? "";
+        if (contentType !== "" && !READABLE.test(contentType)) {
+            await response.body?.cancel().catch(() => { });
+            throw new Error(`${url.href} is ${contentType.split(";")[0]}, not a page that can be read as text.`);
+        }
+        const body = await readLimited(response);
+        const html = /html/i.test(contentType) || (contentType === "" && /^\s*<(!doctype|html)/i.test(body));
+        return { url: url.href, status: response.status, contentType, text: html ? htmlToText(body) : body };
+    }
+}
+/** One window of a long page, and how to read on. */
+export function pageWindow(text, offset) {
+    const start = Math.max(0, Math.min(offset, text.length));
+    const window = text.slice(start, start + PAGE_CHARS);
+    const rest = text.length - (start + window.length);
+    return rest > 0 ? `${window}\n\n[${rest} more characters: call web_fetch again with offset ${start + window.length}]` : window;
+}
+export function createWebTools() {
+    const search = defineTool({
+        name: "web_search",
+        label: "Web search",
+        description: "Search the web (DuckDuckGo) and get back the results: title, address and a snippet each. Use it for a library's " +
+            "documentation, an API you are unsure of, or an error you have not seen. Read a result with web_fetch.",
+        promptSnippet: "search the web for documentation, APIs and errors",
+        parameters: Type.Object({
+            query: Type.String({ description: "What to search for, as you would type it into a search engine" }),
+            limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 20, description: "How many results, 8 by default" })),
+        }),
+        async execute(_id, params) {
+            const results = await webSearch(params.query, params.limit ?? 8);
+            const text = results.length === 0
+                ? `No results for "${params.query}". Try other words.`
+                : results.map((r, i) => `${i + 1}. ${r.title}\n   ${r.url}${r.snippet ? `\n   ${r.snippet}` : ""}`).join("\n\n");
+            return { content: [{ type: "text", text }], details: { results: results.length } };
+        },
+    });
+    const read = defineTool({
+        name: "web_fetch",
+        label: "Fetch a web page",
+        description: "Fetch a page from the public web and get it back as readable text - an HTML page without its markup, or a " +
+            "text, JSON or XML file as it is. Long pages come in parts: call again with the offset it gives to read on.",
+        promptSnippet: "read a web page, such as documentation, as text",
+        parameters: Type.Object({
+            url: Type.String({ description: "The page's full address, starting with https://" }),
+            offset: Type.Optional(Type.Integer({ minimum: 0, description: "Where to start reading, for the next part of a long page" })),
+        }),
+        async execute(_id, params) {
+            const page = await webFetch(params.url);
+            const head = `${page.url} (${page.status}${page.contentType ? `, ${page.contentType.split(";")[0]}` : ""})`;
+            const text = page.text.trim() === "" ? `${head}\n\nThe page has no text.` : `${head}\n\n${pageWindow(page.text, params.offset ?? 0)}`;
+            return { content: [{ type: "text", text }], details: { characters: page.text.length, status: page.status } };
+        },
+    });
+    return [search, read];
+}