@turri-dev/cli 0.1.27-next.763d091 → 0.1.28-next.d5f2680

package/dist/regenerate/test-advisor.js

@@ -0,0 +1,379 @@
+import { defineTool } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { ask, BEGIN, createExecutor } from "../pi.js";
+/** How many times a call that did not answer, or answered what does not hold, is asked again. */
+const RETRIES = 2;
+/** The fewest and most probes a functional spec is given. */
+export const MIN_PROBES = 1;
+export const MAX_PROBES = 10;
+/** What a reader answers for an input the functional spec does not settle: a scenario on it would pin a guess. */
+export const UNSPECIFIED = "unspecified";
+/** The readers: the same task, decorrelated by their system prompts alone. */
+export const READERS = [
+    {
+        name: "literal reader",
+        system: "You read a functional spec literally: you predict exactly what its words say, and nothing they do not. Where the words are silent, you take the narrowest reading they allow.",
+    },
+    {
+        name: "edge-case hunter",
+        system: "You read a functional spec as a tester hunting for edge cases: empty and huge values, limits, duplicates, order, rounding, time zones, missing data. You predict what a careful implementation would do at each edge.",
+    },
+    {
+        name: "another domain's reader",
+        system: "You read a functional spec as someone from another domain would - a different field, a different country, a different habit of mind - bringing the conventions that domain takes for granted. You predict what you would expect the output to be.",
+    },
+];
+/** Ask one call, and ask again with what was wrong, until it holds or the retries are spent. */
+async function askAdvisor(options) {
+    let answer;
+    const tool = options.tool((given) => (answer = given));
+    const session = await createExecutor({
+        cwd: options.cwd,
+        customTools: [tool],
+        tools: [tool.name],
+        model: options.model,
+        label: options.label,
+        role: "advisor",
+        task: options.task,
+        ...(options.system !== undefined ? { system: options.system } : {}),
+        budget: { calls: 4, finish: [tool.name] },
+    });
+    try {
+        await ask(session, BEGIN);
+        for (let again = 0; again < RETRIES; again += 1) {
+            if (answer !== undefined && options.check(answer).length === 0)
+                return answer;
+            const problems = answer === undefined ? [`You did not call ${tool.name}.`] : options.check(answer);
+            answer = undefined;
+            await ask(session, `That does not hold:\n- ${problems.join("\n- ")}\n\nCall ${tool.name} again, with it fixed.`);
+        }
+        return answer !== undefined && options.check(answer).length === 0 ? answer : undefined;
+    }
+    finally {
+        session.dispose();
+    }
+}
+/** Text compared as the readers' outputs are: whitespace collapsed, case left aside. */
+export function normalised(text) {
+    return text.trim().replace(/\s+/g, " ").toLowerCase();
+}
+/** What every call is given: the functional spec, its module, and the project's own words on what matters. */
+function subjectBlock(subject) {
+    return [
+        `<functional_spec module="${subject.module}" func_id="${subject.funcId}">`,
+        subject.text.trim(),
+        "</functional_spec>",
+        "",
+        `<module name="${subject.module}">`,
+        subject.moduleText.trim(),
+        "</module>",
+        ...(subject.testedIn !== undefined
+            ? [
+                "",
+                `Its scenarios are tested against the code of ${subject.testedIn.module}, which is built on ${subject.module}: what a command`,
+                `or a listing shows there includes ${subject.testedIn.module}'s own functional specs.`,
+                `<module name="${subject.testedIn.module}">`,
+                subject.testedIn.text.trim(),
+                "</module>",
+            ]
+            : []),
+        ...(subject.context.length > 0 ? ["", "<the project's own words on what matters>", ...subject.context.map((text) => text.trim()), "</the project's own words on what matters>"] : []),
+    ];
+}
+const stringOf = (value) => (typeof value === "string" ? value.trim() : "");
+/**
+ * The classification, from the call's answer: each yes kept only with its evidence - a yes without it, or with it
+ * malformed (two readings that are the same), is a no. The verdict is yes on silent failure, divergence or cost.
+ */
+export function classificationOf(answer) {
+    const questions = [];
+    const add = (question, yes, evidence) => {
+        if (yes && evidence !== "")
+            questions.push({ question, evidence });
+    };
+    add("silent-failure", answer.silent_failure?.answer === true, stringOf(answer.silent_failure?.wrong_output));
+    const input = stringOf(answer.divergence?.input);
+    const a = stringOf(answer.divergence?.reading_a);
+    const b = stringOf(answer.divergence?.reading_b);
+    const diverges = answer.divergence?.answer === true && input !== "" && a !== "" && b !== "" && normalised(a) !== normalised(b);
+    add("divergence", diverges, diverges ? `${input} - one reading: ${a}; the other: ${b}` : "");
+    add("revisited", answer.revisited?.answer === true, stringOf(answer.revisited?.would_change));
+    add("cost", answer.cost?.answer === true, stringOf(answer.cost?.consequence));
+    const decisive = questions.some((answered) => answered.question !== "revisited");
+    return { verdict: decisive ? "yes" : "no", questions, ...(diverges ? { divergence: { input, readings: [a, b] } } : {}) };
+}
+function classifyPrompt(subject) {
+    return [
+        "You decide whether one functional spec of a ***plain module needs explicit testing scenarios: black-box tests written",
+        "from scenarios, beside the unit tests its coding agent writes anyway. Not everything should be tested this way - only",
+        "functionality where a wrong implementation would go unnoticed, where readers would disagree, or where a bug is costly.",
+        "",
+        ...subjectBlock(subject),
+        ...(subject.reason !== undefined && subject.reason.trim() !== ""
+            ? ["", "<why it was asked for>", subject.reason.trim(), "</why it was asked for>", "This is context, not evidence: answer the questions from the functional spec, its module and the project."]
+            : []),
+        "",
+        "Answer four questions about the functional spec. Each yes needs concrete evidence; without it, answer no.",
+        "1. silent_failure: could an incorrect implementation give output that looks plausible but is wrong? Evidence: one such",
+        "   wrong output, concretely - the input and the wrong output it would give.",
+        "2. divergence: is there an input for which two reasonable readings of the text give different outputs? Evidence: the",
+        "   input, and the output under each reading.",
+        "3. revisited: is this functionality likely to be changed or built on later? Evidence: what would change.",
+        "4. cost: would a bug in it cause financial loss, security exposure, data loss, or harm to users' trust? Evidence: the",
+        "   consequence.",
+        "",
+        "Judge only this functional spec, from what the module and the project say. Do not answer yes to be safe: a yes",
+        "starts work, and a functional spec with nothing at stake is better left to its unit tests.",
+        "Call submit_classification once.",
+    ].join("\n");
+}
+/** Classify a functional spec: one call, four evidence-backed questions. Throws when the call gives no answer. */
+export async function classify(subject, options) {
+    const answer = await askAdvisor({
+        cwd: options.cwd,
+        model: options.model,
+        label: `${subject.module}:${subject.funcId} test advisor classifying`,
+        task: classifyPrompt(subject),
+        check: () => [],
+        tool: (take) => defineTool({
+            name: "submit_classification",
+            label: "Submit the classification",
+            description: "The four answers, each yes with its evidence.",
+            parameters: Type.Object({
+                silent_failure: Type.Object({ answer: Type.Boolean(), wrong_output: Type.Optional(Type.String({ description: "An input, and the plausible but wrong output an incorrect implementation would give." })) }),
+                divergence: Type.Object({
+                    answer: Type.Boolean(),
+                    input: Type.Optional(Type.String({ description: "The input two reasonable readings disagree on." })),
+                    reading_a: Type.Optional(Type.String({ description: "The output under one reading." })),
+                    reading_b: Type.Optional(Type.String({ description: "The output under the other." })),
+                }),
+                revisited: Type.Object({ answer: Type.Boolean(), would_change: Type.Optional(Type.String({ description: "What would change, or be built on it." })) }),
+                cost: Type.Object({ answer: Type.Boolean(), consequence: Type.Optional(Type.String({ description: "What a bug would cost, and whom." })) }),
+            }),
+            async execute(_id, params) {
+                take(params);
+                return { content: [{ type: "text", text: "Recorded." }], details: {}, terminate: true };
+            },
+        }),
+    });
+    if (answer === undefined)
+        throw new Error("the classification call gave no answer");
+    return classificationOf(answer);
+}
+function probesPrompt(subject, classification) {
+    return [
+        "This functional spec needs explicit testing scenarios. Pick the inputs they should cover.",
+        "",
+        ...subjectBlock(subject),
+        "",
+        "<why it needs them>",
+        ...classification.questions.map((answered) => `- ${answered.question}: ${answered.evidence}`),
+        "</why it needs them>",
+        "",
+        `Write the probes - as many as the functional spec has real boundaries, ${MIN_PROBES} to ${MAX_PROBES}, one when it has a single input:`,
+        "concrete inputs at the boundaries of what the functional spec says - limits, empty and missing values, duplicates,",
+        "order, the cases its wording leaves open - each one an input a black-box test can give through the interface the",
+        "functional spec names. Concrete values, never descriptions (\"a long name\").",
+        "Only inputs whose output the functional spec bears on: never a command, flag, alias or option it does not name, and",
+        "never one that exercises another functional spec of the module instead - a test of what it does not say would pin a guess.",
+        "Never pad to reach a number: an input that tests nothing new is noise in the scenarios. Do not say what the output",
+        "should be. Call submit_probes once.",
+    ].join("\n");
+}
+/** The probes: 1-10 boundary inputs, and the divergence's input when the classification found one. */
+async function probes(subject, classification, options) {
+    const answer = await askAdvisor({
+        cwd: options.cwd,
+        model: options.model,
+        label: `${subject.module}:${subject.funcId} test advisor probes`,
+        task: probesPrompt(subject, classification),
+        check: (given) => {
+            const inputs = given.probes.map((probe) => stringOf(probe.input)).filter((input) => input !== "");
+            return inputs.length < MIN_PROBES || inputs.length > MAX_PROBES ? [`Give ${MIN_PROBES} to ${MAX_PROBES} probes, each a concrete input; you gave ${inputs.length}.`] : [];
+        },
+        tool: (take) => defineTool({
+            name: "submit_probes",
+            label: "Submit the probes",
+            description: `${MIN_PROBES} to ${MAX_PROBES} concrete boundary inputs.`,
+            parameters: Type.Object({ probes: Type.Array(Type.Object({ input: Type.String() })) }),
+            async execute(_id, params) {
+                take(params);
+                return { content: [{ type: "text", text: "Recorded." }], details: {}, terminate: true };
+            },
+        }),
+    });
+    if (answer === undefined)
+        throw new Error("the probes call gave no probes");
+    const inputs = [];
+    const add = (input) => {
+        if (input !== "" && !inputs.some((known) => normalised(known) === normalised(input)))
+            inputs.push(input);
+    };
+    for (const probe of answer.probes)
+        add(stringOf(probe.input));
+    if (classification.divergence !== undefined)
+        add(classification.divergence.input);
+    return inputs;
+}
+function readerPrompt(subject, inputs) {
+    return [
+        "Predict what an implementation of this functional spec outputs for each input below.",
+        "",
+        ...subjectBlock(subject),
+        "",
+        "<inputs>",
+        ...inputs.map((input, index) => `${index + 1}. ${input}`),
+        "</inputs>",
+        "",
+        "For every input, by its number, give the output you expect - concrete, as a test would check it, in a line. Where",
+        "the functional spec leaves it open, give the output your reading of it leads to. Where it says nothing that bears on",
+        `the input at all - a command or an option it does not name - answer exactly ${UNSPECIFIED}. Call submit_predictions once.`,
+    ].join("\n");
+}
+/** One reader's predictions, by probe - or nothing when it gave none that hold. */
+async function predictions(subject, inputs, reader, options) {
+    const answer = await askAdvisor({
+        cwd: options.cwd,
+        model: options.model,
+        label: `${subject.module}:${subject.funcId} test advisor ${reader.name}`,
+        task: readerPrompt(subject, inputs),
+        system: reader.system,
+        check: (given) => {
+            const missing = inputs.flatMap((_input, index) => (given.predictions.filter((prediction) => prediction.input === index + 1 && stringOf(prediction.output) !== "").length === 1 ? [] : [index + 1]));
+            return missing.length > 0 ? [`Give exactly one output for every input; not for: ${missing.join(", ")}.`] : [];
+        },
+        tool: (take) => defineTool({
+            name: "submit_predictions",
+            label: "Submit the predictions",
+            description: "The output expected for every input, by its number.",
+            parameters: Type.Object({ predictions: Type.Array(Type.Object({ input: Type.Number(), output: Type.String() })) }),
+            async execute(_id, params) {
+                take(params);
+                return { content: [{ type: "text", text: "Recorded." }], details: {}, terminate: true };
+            },
+        }),
+    });
+    if (answer === undefined)
+        return undefined;
+    return inputs.map((_input, index) => stringOf(answer.predictions.find((prediction) => prediction.input === index + 1)?.output));
+}
+/**
+ * The inputs the readers split on - every probe where any two predictions differ once normalised: the union, never a
+ * majority - each with its distinct readings, in the order the readers gave them.
+ */
+export function splitInputs(readings) {
+    const split = new Map();
+    const count = readings[0]?.length ?? 0;
+    for (let index = 0; index < count; index += 1) {
+        const distinct = [];
+        for (const reader of readings) {
+            const output = reader[index] ?? "";
+            if (!distinct.some((known) => normalised(known) === normalised(output)))
+                distinct.push(output);
+        }
+        if (distinct.length > 1)
+            split.set(index, distinct);
+    }
+    return split;
+}
+function verifyPrompt(subject, candidates) {
+    return [
+        "Readers of this functional spec disagreed on the outputs of some inputs. For each, say which readings a reasonable",
+        "person could hold, given the functional spec, its module and the project.",
+        "",
+        ...subjectBlock(subject),
+        "",
+        "<disagreements>",
+        ...candidates.flatMap((candidate, index) => [`${index + 1}. input: ${candidate.input}`, ...candidate.readings.map((reading, at) => `   reading ${at + 1}: ${reading}`)]),
+        "</disagreements>",
+        "",
+        "For every disagreement, by its number, list the numbers of the readings a reasonable person could hold. Leave out a",
+        `reading only a misreading or a mistake leads to. A reading of ${UNSPECIFIED} holds when the functional spec says nothing`,
+        "that settles the input. Call submit_verification once.",
+    ].join("\n");
+}
+/** Of each candidate, the readings a reasonable person could hold - or nothing when the call gave no answer that holds. */
+async function verify(subject, candidates, options) {
+    const answer = await askAdvisor({
+        cwd: options.cwd,
+        model: options.model,
+        label: `${subject.module}:${subject.funcId} test advisor verifier`,
+        task: verifyPrompt(subject, candidates),
+        check: (given) => {
+            const problems = [];
+            candidates.forEach((candidate, index) => {
+                const said = given.inputs.filter((item) => item.input === index + 1);
+                if (said.length !== 1)
+                    problems.push(`Say once which readings of disagreement ${index + 1} hold.`);
+                else if (said[0].reasonable.some((number) => !Number.isInteger(number) || number < 1 || number > candidate.readings.length))
+                    problems.push(`Disagreement ${index + 1} has readings 1 to ${candidate.readings.length} only.`);
+            });
+            return problems;
+        },
+        tool: (take) => defineTool({
+            name: "submit_verification",
+            label: "Submit the verification",
+            description: "For every disagreement, by its number, the numbers of the readings a reasonable person could hold.",
+            parameters: Type.Object({ inputs: Type.Array(Type.Object({ input: Type.Number(), reasonable: Type.Array(Type.Number()) })) }),
+            async execute(_id, params) {
+                take(params);
+                return { content: [{ type: "text", text: "Recorded." }], details: {}, terminate: true };
+            },
+        }),
+    });
+    if (answer === undefined)
+        return undefined;
+    return candidates.map((candidate, index) => {
+        const reasonable = answer.inputs.find((item) => item.input === index + 1)?.reasonable ?? [];
+        return candidate.readings.filter((_reading, at) => reasonable.includes(at + 1));
+    });
+}
+/**
+ * Pick the inputs a functional spec's scenarios should cover: the probes, the readers' predictions of them, the
+ * disagreements in code, and the verifier's word on each. An input the readers agree on comes with their output; one
+ * they split on, with every reading the verifier kept - a single reading left is an agreed one, and none left drops it.
+ */
+export async function pickInputs(subject, classification, options) {
+    const inputs = await probes(subject, classification, options);
+    // Two readers first; the third only when they agree on every probe - a disagreement already found is enough.
+    const first = await Promise.all(READERS.slice(0, 2).map((reader) => predictions(subject, inputs, reader, options)));
+    const readings = first.filter((reading) => reading !== undefined);
+    if (readings.length === 0 || splitInputs(readings).size === 0) {
+        for (const reader of READERS.slice(2)) {
+            const more = await predictions(subject, inputs, reader, options);
+            if (more !== undefined)
+                readings.push(more);
+        }
+    }
+    if (readings.length === 0)
+        throw new Error("no reader predicted the probes' outputs");
+    const split = splitInputs(readings);
+    const candidates = [...split].map(([index, distinct]) => ({ index, input: inputs[index], readings: distinct }));
+    // Without the verifier's word, every reading stands: a split left unsettled is the chat's to settle as it writes.
+    const kept = candidates.length > 0 ? ((await verify(subject, candidates, options)) ?? candidates.map((candidate) => candidate.readings)) : [];
+    const out = [];
+    inputs.forEach((input, index) => {
+        const at = candidates.findIndex((candidate) => candidate.index === index);
+        // An input the functional spec does not settle is left out: a scenario on it would pin a guess.
+        if (at === -1) {
+            if (normalised(readings[0][index]) !== UNSPECIFIED)
+                out.push({ input, agreed: readings[0][index] });
+            return;
+        }
+        const held = kept[at] ?? [];
+        if (held.some((reading) => normalised(reading) === UNSPECIFIED))
+            return;
+        if (held.length > 1)
+            out.push({ input, readings: held });
+        else if (held.length === 1)
+            out.push({ input, agreed: held[0] });
+    });
+    return out;
+}
+/** The whole judgement: classify, and on a yes pick the inputs. Throws when a call that cannot be done without fails. */
+export async function advise(subject, options) {
+    const classification = await classify(subject, options);
+    if (classification.verdict === "no")
+        return { verdict: "no", questions: classification.questions, inputs: [] };
+    return { verdict: "yes", questions: classification.questions, inputs: await pickInputs(subject, classification, options) };
+}

package/dist/regenerate/test-kinds.d.ts

@@ -1,46 +1,84 @@
+import { type MarshalledPlainSource } from "../plain/spec.ts";
 /**
- * The two kinds of test the conformance step writes, and how each is told apart in the code.
+ * How the conformance tests are labelled, and how the labels are checked.
  *
- * Conformance tests are written from the functional spec itself: what the functionality does,
- * its edge cases, its errors - as if no acceptance tests had been given. Acceptance tests are the
- * ones the spec's author wrote, each implemented as a test of its own, doing what it says. The
- * two are kept apart even when the acceptance tests would seem to cover the functionality: tests
- * derived only from the acceptance tests check what the author thought to check, and nothing
- * the spec says beyond it.
+ * A functional spec's conformance tests are its scenarios, and nothing else: each conformance scenario and each
+ * acceptance scenario of its test plain file is carried out by at least one test that does what it says. No test is
+ * invented beside them. A functional spec with no scenarios has no conformance tests.
  *
- * Each kind is labelled where anyone reading the tests can see it - in its file names, and in a
- * comment the harness looks for - so which test is which, and which acceptance test a test
- * implements, never has to be guessed.
+ * Each is labelled where anyone reading the tests can see it - a comment the harness looks for - so which functional
+ * spec a file tests, and which scenario a test carries out, never has to be guessed:
+ * - a file: `conformance tests of <module>:<func-id>`, one line for each functional spec it tests;
+ * - a test: `conformance scenario <scenario-id> of <module>:<func-id>` or `acceptance scenario <scenario-id> of
+ *   <module>:<func-id>`, right above it.
  */
-export declare function conformanceLabel(frid: string): string;
-export declare function acceptanceLabel(frid: string, n: number): string;
-/** An acceptance test as the spec gives it, without its bullet. */
-export declare function acceptanceText(item: string): string;
+/** A scenario as the conformance agent is given it: its id, and its text. */
+export interface Scenario {
+    id: string;
+    text: string;
+}
 /**
- * What is missing from the labelling of a functionality's tests: the conformance label, when no
- * test file carries it, and each acceptance test no test is labelled as implementing.
+ * A functional spec's scenarios: those its conformance tests carry out. `module`: a module below's - the tip's suite holds
+ * the tests of the whole chain - else the codebase's own.
  */
-export declare function missingTestLabels(testsDir: string, frid: string, acceptanceTests: string[]): string[];
+export interface TestedSpec {
+    funcId: string;
+    module?: string;
+    conformance: Scenario[];
+    acceptance: Scenario[];
+}
+export type ScenarioKind = "conformance" | "acceptance";
+/** The label of a file of a functional spec's conformance tests. */
+export declare function conformanceLabel(module: string, funcId: string): string;
+/** The label right above a test that carries out a scenario. */
+export declare function scenarioLabel(kind: ScenarioKind, scenarioId: string, module: string, funcId: string): string;
+/** Every scenario of a functional spec, each with its kind, in its file's order: the conformance ones, then the acceptance ones. */
+export declare function scenariosOf(spec: TestedSpec): Array<Scenario & {
+    kind: ScenarioKind;
+}>;
+/** A scenario as the agents are given it: its label, and its text under it. */
+export declare function scenarioText(module: string, funcId: string, scenario: Scenario & {
+    kind: ScenarioKind;
+}): string;
 /**
- * The test files that carry a functionality's labels - its conformance tests' or any of its acceptance tests'. The
- * harness's own files beside them - the notes, the record - are not tests, whatever they mention.
+ * What is missing from the labelling of a functional spec's tests: the file label, when no test file carries it, and each
+ * scenario no test is labelled as carrying out.
  */
-export declare function labelledFiles(testsDir: string, frid: string): string[];
-/** Sent back when the labels are missing: which, and how to label them. */
-export declare function labelsPrompt(frid: string, missing: string[], empty?: boolean, hasAcceptance?: boolean): string;
+export declare function missingTestLabels(testsDir: string, codebase: string, spec: TestedSpec): string[];
 /**
- * Sent back when the labels of several functionalities are missing - a spec change that touches them: every one that is
- * missing, in one request, so none is asked for in place of another.
+ * Sent back when the labels of functional specs are missing - one, or every one a spec change touches: every one that
+ * is missing, in one request, so none is asked for in place of another.
  */
-export declare function labelsPromptFor(items: Array<{
-    frid: string;
+export declare function labelsPromptFor(codebase: string, items: Array<{
+    funcId: string;
+    module?: string;
     missing: string[];
-}>, empty?: boolean, hasAcceptance?: boolean): string;
+}>, empty?: boolean): string;
 /**
- * Conformance and acceptance tests in the code folder, where they do not belong: they live in the
- * module's tests folder, which the harness runs as the conformance suite. Found by their label, or
- * by a test file under a name or folder that says conformance or acceptance - one written without
- * a label is misplaced all the same.
+ * The functional specs of a tree of what conformance tests are written for, with their scenarios - those of the modules
+ * below with their module, the codebase's own after them.
+ */
+export declare function testedSpecsOf(tree: MarshalledPlainSource | null | undefined): TestedSpec[];
+/**
+ * The labels in the tests folder that name nothing the tests are written for: a functional spec, or a scenario of one,
+ * that `specs` does not have - left over from a scenario or a functional spec taken out or renamed, mistyped, or
+ * invented. Each with the file it is in.
+ */
+export declare function orphanedLabels(testsDir: string, codebase: string, specs: TestedSpec[]): Array<{
+    file: string;
+    label: string;
+}>;
+/** Sent back when labels name nothing the tests are written for. */
+export declare function orphansPrompt(orphans: Array<{
+    file: string;
+    label: string;
+}>): string;
+/** Sent back when the labels of one functional spec are missing. */
+export declare function labelsPrompt(module: string, funcId: string, missing: string[], empty?: boolean): string;
+/**
+ * Conformance tests in the code folder, where they do not belong: they live in the module's tests folder, which the
+ * harness runs as the conformance suite. Found by their label, or by a test file under a name or folder that says
+ * conformance or acceptance - one written without a label is misplaced all the same.
  */
 export declare function misplacedConformanceTests(codeDir: string): string[];
 /** Sent back when there are: to the agent writing the code, or to the one writing the tests. */