decisium 0.0.0-stage → 0.1.2

package/dist/decision.js

@@ -0,0 +1,143 @@
+/** The page itself as a target (task 0065): scrolling it, history, tabs, keys, waiting. */
+export const PAGE = "page";
+/** Everything a target can be used for (task 0065, ADR 0005); capabilities() picks a target's own. */
+export const ACTIONS = {
+    click: "Click it.",
+    type: "Type a value from `step` into it (an empty value clears it).",
+    select: "Select an option in it.",
+    upload: "Click it to choose a file to upload, one `step` names.",
+    paste: "Paste the copied text into it.",
+    hover: "Only hover over it with the mouse pointer (e.g. to open a menu or show a tooltip).",
+    "double click": "Double-click it (e.g. to edit it inline).",
+    "right click": "Right-click it (e.g. to open its context menu).",
+    drag: "Drag it with the mouse and drop it onto another element (e.g. to reorder it or move it to another list).",
+    "scroll down": "Scroll it down: show or load more of its content further down (the page: the needed element may load further down).",
+    "scroll up": "Scroll it up: show or load more of its content further up.",
+    wait: "Wait: the page is still loading what is needed.",
+    back: "Go back to the previous page in the browser history.",
+    forward: "Go forward in the browser history.",
+    reload: "Reload the page.",
+    "close tab": "Close this browser tab and return to the previous one.",
+    "switch tab": "Switch to the previous browser tab, keeping this one open.",
+    enter: "Press Enter in it, to submit what was typed (e.g. run a search).",
+};
+/**
+ * What `target` can be used for. Elements: their role's verb (a typeable field also takes paste, a clickable one an
+ * upload), click-like gestures on every target (no DOM evidence for them, task 0064), drag/scroll only with the fact.
+ * The page: scroll, wait, history; tabs while more than one is open; keys the Step names (`keys`, task 0039). Enter:
+ * the focused text field's.
+ */
+export function capabilities(target, obs, keys = []) {
+    if (target === PAGE)
+        return ["scroll down", "scroll up", "wait", "back", "forward", "reload",
+            ...obs.readable.some(e => e.ref === "tabs") ? ["close tab", "switch tab"] : [],
+            ...keys.filter(k => k !== "Enter").map(k => `press ${k}`)];
+    // Enter goes to the focused field: offered there, so the field is the target (as the page, Jev split it with the
+    // Search button: 0.47 / 0.29, AMBIGUOUS on Wikipedia).
+    const own = target.verb === "TYPE" ? ["type", "paste", ...(/\[active\]/.test(target.state) ? ["enter"] : [])]
+        : target.verb === "SELECT" ? ["select"] : ["click", "upload"];
+    return [...own, "hover", "double click", "right click",
+        ...(/\[draggable\]/.test(target.state) ? ["drag"] : []), ...(/\[scrollable\]/.test(target.state) ? ["scroll down", "scroll up"] : [])];
+}
+/** Choice text of an action: ACTIONS, or a key the Step names. */
+export const describeAction = (a) => ACTIONS[a] ?? `Press the ${a.slice(6)} key, as \`step\` says.`;
+/**
+ * Equivalent targets sum their confidence before the gate (task 0065, 0057): links with the same name and URL lead to
+ * the same page (Wikipedia's two `Deutsch` links, 42% / 38%). The first keeps the sum. Buttons never: their handlers
+ * may differ.
+ */
+export function grouped(p, obs) {
+    const first = new Map();
+    const probabilities = {};
+    for (const [k, v] of Object.entries(p.probabilities)) {
+        const e = obs.elements.find(x => x.ref === k);
+        const id = e?.role === "link" && e.url ? `${e.name}|${e.url}` : undefined;
+        const into = id ? (first.get(id) ?? (first.set(id, k), k)) : k;
+        probabilities[into] = (probabilities[into] ?? 0) + v;
+    }
+    const choice = Object.entries(probabilities).sort((a, b) => b[1] - a[1])[0]?.[0] ?? p.choice;
+    return { choice, probabilities };
+}
+export const ASSERTION_KINDS = {
+    value: "Checks that some element shows a specific text or value.",
+    presence: "Checks that something is shown or exists on the page.",
+    absence: "Checks that something is not shown or does not exist on the page.",
+    state: "Checks whether a control is disabled, enabled, checked, expanded, selected or pressed.",
+    fuzzy: "A subjective or holistic check that none of the other kinds can express.",
+};
+/** State labels Jev can pick; code reads the matching snapshot flag. */
+export const STATES = {
+    disabled: "disabled", enabled: "enabled (not disabled)",
+    checked: "checked", unchecked: "not checked",
+    expanded: "expanded / open", collapsed: "collapsed / closed",
+    selected: "selected", unselected: "not selected",
+    pressed: "pressed / toggled on", unpressed: "not pressed / toggled off",
+};
+export const AREA_PAGE = "page";
+export const AREA_MISSING = "missing";
+const SPECIAL = {
+    none: "nothing", [AREA_PAGE]: "anywhere on the page", [AREA_MISSING]: "an area not on this screen",
+};
+/** A Jev answer for people: element refs replaced by the element, top 3 as percentages. */
+export const named = (p, obs) => Object.entries(p.probabilities).sort((a, b) => b[1] - a[1]).slice(0, 3).map(([k, v]) => {
+    const el = obs && [...obs.readable, ...obs.areas, ...obs.elements].find(e => e.ref === k);
+    return `${el ? el.line : SPECIAL[k] ?? k} ${Math.round(v * 100)}%`;
+}).join(", ");
+export const fmt = (p) => Object.entries(p.probabilities).sort((a, b) => b[1] - a[1]).slice(0, 3)
+    .map(([k, v]) => `${k}=${v.toFixed(2)}`).join(" ");
+/**
+ * Thrown by a Decider that needs a later Observation (a replayed control not on a still-loading page): the runner
+ * waits for the page to change and asks again, and throws `final` once ASSERT_TIMEOUT_MS has passed.
+ */
+export class NotYet extends Error {
+    final;
+    constructor(final) { super(final.message); this.final = final; }
+}
+/** Confidence gate: act only on a clear, confident top answer. */
+export const GATE_TOP = 0.8;
+export const GATE_MARGIN = 0.3;
+/**
+ * Which element to act on is gated lower than a check (task 0002): a wrong action fails a later Step loudly,
+ * while a wrong check can pass falsely. Fitted on 26 labelled AMBIGUOUS Turns from the real-site corpus, where
+ * every top answer naming an element was the right one down to 0.47; below 0.1 margin the two answers are a coin flip.
+ */
+export const GATE_TARGET_TOP = 0.5;
+export const GATE_TARGET_MARGIN = 0.1;
+export function gate(pick, minTop = GATE_TOP, minMargin = GATE_MARGIN) {
+    const [top = 0, second = 0] = Object.values(pick.probabilities).sort((a, b) => b - a);
+    return top >= minTop && top - second >= minMargin;
+}
+/** Most likely answers first, until together they cover the gate's confidence (cassette.ts stores exactly these). */
+export function plausible(pick) {
+    const out = [];
+    let covered = 0;
+    for (const [k, p] of Object.entries(pick.probabilities).sort((x, y) => y[1] - x[1])) {
+        if (covered >= GATE_TOP)
+            break;
+        out.push(k);
+        covered += p;
+    }
+    return out;
+}
+/**
+ * `d` with each answer written as a trace event named after its question (turn, value, assess...): the Step (every
+ * question's first argument) and the answer. A question answering plainly needs no code here; a value chosen for a
+ * Secret field is masked, and a dialog answer carries the dialog. Only Decider questions are reached through it.
+ */
+export function traced(d, event) {
+    return new Proxy(d, {
+        get(target, question) {
+            const ask = Reflect.get(target, question);
+            if (typeof ask !== "function")
+                return ask;
+            return async (...args) => {
+                const answer = await ask.apply(target, args);
+                const el = args[2]; // value(step, obs, el, candidates)
+                const extra = question === "value" ? { element: el.line, answer: el.secret ? "(secret)" : answer }
+                    : question === "dialog" ? { dialog: args[1].dialog, answer } : { answer };
+                event(question, { step: args[0], ...extra });
+                return answer;
+            };
+        },
+    });
+}

package/dist/doctor.d.ts

@@ -0,0 +1,3 @@
+/** Node with type stripping on by default: the CLI runs .ts files directly. */
+export declare const nodeOk: (version?: string) => boolean;
+export declare function doctor(files: string[]): Promise<number>;

package/dist/doctor.js

@@ -0,0 +1,46 @@
+// `decisium doctor`: the setup checks a CI job wants before the first run, one [OK]/[FAIL] line each.
+import { noul, TypeSafeClient } from "@typesafe-ai/sdk";
+import { existsSync, readFileSync } from "node:fs";
+import { chromium } from "playwright";
+import { Browser } from "./browser.js";
+import { parseFeature, toUrl } from "./feature.js";
+import { MODEL } from "./jev.js";
+/** Node with type stripping on by default: the CLI runs .ts files directly. */
+export const nodeOk = (version = process.versions.node) => {
+    const [major, minor] = version.split(".").map(Number);
+    return major > 23 || (major === 23 && minor >= 6) || (major === 22 && minor >= 18);
+};
+export async function doctor(files) {
+    let failed = false;
+    const line = (ok, what) => { failed ||= !ok; console.log(`[${ok ? "OK" : "FAIL"}] ${what}`); };
+    const attempt = async (what, fn) => {
+        try {
+            await fn();
+            line(true, what);
+        }
+        catch (err) {
+            line(false, `${what}: ${err.message.split("\n")[0]}`);
+        }
+    };
+    line(nodeOk(), `Node ${process.versions.node} (needs 22.18+ or 23.6+)`);
+    line(existsSync(chromium.executablePath()), `Chromium at ${chromium.executablePath()} (else: npx playwright install chromium)`);
+    if (!process.env.TYPESAFE_API_KEY)
+        line(false, "TYPESAFE_API_KEY is not set");
+    else
+        await attempt(`TYPESAFE_API_KEY accepted by Jev (${MODEL})`, () => new TypeSafeClient({ defaultModel: MODEL })
+            .systemOne({ state: { word: "yes" }, questions: { q: noul("Is `word` yes?") } }));
+    const scenarios = [];
+    for (const f of files) {
+        try {
+            scenarios.push(...parseFeature(readFileSync(f, "utf8"), f));
+            line(true, `${f} parses`);
+        }
+        catch (err) {
+            line(false, `${f}: ${err.message}`);
+        }
+    }
+    for (const url of new Set(scenarios.map(s => s.url).filter(u => u !== undefined))) {
+        await attempt(`opens ${url}`, async () => (await Browser.open(toUrl(url))).close());
+    }
+    return failed ? 2 : 0;
+}

package/dist/execute.d.ts

@@ -0,0 +1,34 @@
+import { type Cassette } from "./cassette.ts";
+import type { Decider } from "./decision.ts";
+import { type Runtime, type StepResult } from "./runner.ts";
+import { type StepArgument, type StepKind } from "./step.ts";
+export interface StepContext {
+    rt: Runtime;
+    live: Decider;
+    cassette?: Cassette;
+    /** The Step's cassette key (stepKey); a table row adds ` (row n)`. */
+    key: string;
+    kind: StepKind;
+    arg: StepArgument;
+    /** Scenario memory: quoted value -> the Step that named it. This Step's values are added once it ends. */
+    earlier: Record<string, string>;
+    log: (line: string) => void;
+    /** Wraps each Decider the Step uses (the run's trace). */
+    trace?: (d: Decider) => Decider;
+    judged?: boolean;
+    /** A Marginal pass fails as AMBIGUOUS. */
+    strict?: boolean;
+}
+export interface StepReport extends StepResult {
+    /** How the Cassette served the Step; undefined without one, or when nothing was kept. */
+    cassette?: "replayed" | "recorded" | "healed";
+    seconds: number;
+    jevCalls: number;
+    inputTokens: number;
+    outputTokens: number;
+}
+/**
+ * Runs one Step: an Action step's Data table as one Step per row (the first that is not DONE stops it), each with its
+ * own Cassette entry. A page too large for Jev and a broken replay are Step outcomes; any other error is thrown (setup).
+ */
+export declare function executeStep(text: string, ctx: StepContext): Promise<StepReport>;

package/dist/execute.js

@@ -0,0 +1,61 @@
+// Step execution (task 0051): one Gherkin Step from text to Step outcome, without cucumber. steps.ts adapts it.
+import { UiChanged } from "./cassette.js";
+import { jevUsage } from "./jev.js";
+import { runAssertion, runStep } from "./runner.js";
+import { readStep } from "./step.js";
+/**
+ * Runs one Step: an Action step's Data table as one Step per row (the first that is not DONE stops it), each with its
+ * own Cassette entry. A page too large for Jev and a broken replay are Step outcomes; any other error is thrown (setup).
+ */
+export async function executeStep(text, ctx) {
+    const { rt, live, cassette, key, kind, arg, earlier, log, trace = d => d } = ctx;
+    const start = Date.now(), before = { ...jevUsage };
+    const { table, docString } = arg;
+    const perRow = !!table && kind !== "assertion";
+    const { rows } = readStep({ text, kind, ...arg });
+    // A header row alone (an action table with 3+ columns) leaves no row to act on; otherwise the rows decide.
+    let r = rows.length ? { outcome: "DONE" }
+        : { outcome: "AMBIGUOUS", reason: "the table has only its header row, so there is no row to act on" };
+    const served = [];
+    for (const [i, row] of rows.entries()) {
+        const entry = cassette?.step(perRow ? `${key} (row ${i + 1})` : key, live, log);
+        const decider = trace(entry ?? live);
+        try {
+            r = kind === "assertion"
+                ? await runAssertion(text, rt, decider, log, !!ctx.judged, { ...earlier }, arg)
+                : await runStep(row, rt, decider, log, kind === "attempt", docString);
+        }
+        catch (err) {
+            if (/max_tokens_exceeded/.test(String(err?.message))) {
+                // The page, not the setup: fail this Step and let the rest of the feature run (task 0021).
+                log(`  => PAGE_TOO_LARGE`);
+                r = { outcome: "PAGE_TOO_LARGE", reason: "the page is too large for one Jev request, even reduced to the sections that matter" };
+            }
+            else if (err instanceof UiChanged) {
+                log(`  => UI_CHANGED`);
+                r = { outcome: "UI_CHANGED", reason: err.reason };
+            }
+            else
+                throw err;
+        }
+        served.push(entry?.done(!["UI_CHANGED", "PAGE_TOO_LARGE"].includes(r.outcome)));
+        if (r.outcome !== "DONE" && perRow)
+            r = { ...r, reason: `row ${i + 1} (${row})${r.reason ? `: ${r.reason}` : ""}` };
+        if (r.outcome !== "DONE")
+            break;
+    }
+    if (r.marginal && ctx.strict)
+        r = { outcome: "AMBIGUOUS", reason: `passed, but Jev was barely confident (--strict): ${r.marginal}` };
+    for (const row of rows)
+        for (const v of readStep({ text: row, kind }).quoted)
+            earlier[v] = row;
+    // Cost and time per Step: live Jev calls and tokens (cassette replays cost 0), wall-clock seconds.
+    return {
+        ...r,
+        cassette: served.find(c => c !== "replayed") ?? served[0],
+        seconds: (Date.now() - start) / 1000,
+        jevCalls: jevUsage.calls - before.calls,
+        inputTokens: jevUsage.inputTokens - before.inputTokens,
+        outputTokens: jevUsage.outputTokens - before.outputTokens,
+    };
+}

package/dist/feature.d.ts

@@ -0,0 +1,37 @@
+import { type PickleStepArgument, type PickleStepType } from "@cucumber/messages";
+import { type BrowserContextOptions } from "playwright";
+import { type ParsedStep } from "./step.ts";
+export interface Scenario {
+    name: string;
+    tags: string[];
+    url?: string;
+    steps: ParsedStep[];
+}
+/** Start URL from the last `@url:<url>` tag (pickle tags list feature tags first). */
+export declare const urlFromTags: (tags: readonly {
+    name: string;
+}[]) => string | undefined;
+/** The value of the last `@<name>:<value>` tag. */
+export declare const tagValue: (tags: readonly {
+    name: string;
+}[], name: string) => string | undefined;
+/**
+ * Browser context options from `@viewport:<w>x<h>` (or a Playwright device name, dashes for spaces: `@viewport:iPhone-13`),
+ * `@locale:<bcp47>`, `@timezone:<IANA id>`, `@geolocation:<lat>,<lon>` (task 0048). A malformed value throws.
+ */
+export declare function contextFromTags(tags: readonly {
+    name: string;
+}[]): BrowserContextOptions;
+/** A pickle step as a Step: its text, kind and Data table or Doc string. */
+export declare const fromPickle: (step: {
+    text: string;
+    type?: PickleStepType;
+    argument?: PickleStepArgument;
+}) => ParsedStep;
+/** Parses a `.feature` source. Throws on Gherkin syntax errors. */
+export declare function parseFeature(source: string, uri?: string): Scenario[];
+/** One Step given on the command line, e.g. `Then the status should be "Draft"`; no keyword means `When`. */
+export declare function parseStep(line: string): ParsedStep;
+/** A one-Step `.feature` source, so `decisium step` runs through the same runner as files. */
+export declare function stepFeature(line: string): string;
+export declare const toUrl: (u: string) => string;

package/dist/feature.js

@@ -0,0 +1,92 @@
+// Gherkin `.feature` files -> runnable Scenarios. The official parser expands Background steps and Scenario
+// Outline rows ("pickles"), so the runner only sees flat Step lists.
+import { AstBuilder, compile, GherkinClassicTokenMatcher, Parser } from "@cucumber/gherkin";
+import { IdGenerator } from "@cucumber/messages";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { devices } from "playwright";
+import { readStep, stepKind } from "./step.js";
+/** Start URL from the last `@url:<url>` tag (pickle tags list feature tags first). */
+export const urlFromTags = (tags) => tagValue(tags, "url");
+/** The value of the last `@<name>:<value>` tag. */
+export const tagValue = (tags, name) => tags.map(t => t.name.startsWith(`@${name}:`) ? t.name.slice(name.length + 2) : undefined).filter(Boolean).at(-1);
+/**
+ * Browser context options from `@viewport:<w>x<h>` (or a Playwright device name, dashes for spaces: `@viewport:iPhone-13`),
+ * `@locale:<bcp47>`, `@timezone:<IANA id>`, `@geolocation:<lat>,<lon>` (task 0048). A malformed value throws.
+ */
+export function contextFromTags(tags) {
+    const o = {};
+    const viewport = tagValue(tags, "viewport"), geo = tagValue(tags, "geolocation");
+    if (viewport) {
+        const size = /^(\d+)x(\d+)$/.exec(viewport);
+        const device = devices[viewport.replaceAll("-", " ")];
+        if (size)
+            o.viewport = { width: Number(size[1]), height: Number(size[2]) };
+        else if (device) {
+            const { defaultBrowserType: _, ...d } = device;
+            Object.assign(o, d);
+        }
+        else
+            throw new Error(`@viewport:${viewport}: expected <width>x<height> or a Playwright device name such as iPhone-13`);
+    }
+    if (geo) {
+        const m = /^(-?[\d.]+),(-?[\d.]+)$/.exec(geo);
+        if (!m)
+            throw new Error(`@geolocation:${geo}: expected <latitude>,<longitude>`);
+        o.geolocation = { latitude: Number(m[1]), longitude: Number(m[2]) };
+        o.permissions = ["geolocation"];
+    }
+    const locale = tagValue(tags, "locale"), timezoneId = tagValue(tags, "timezone");
+    if (locale)
+        o.locale = locale;
+    if (timezoneId)
+        o.timezoneId = timezoneId;
+    return o;
+}
+/** A pickle step's data table or doc string, only the one it has. */
+const stepArgument = ({ argument }) => argument?.dataTable ? { table: argument.dataTable.rows.map(r => r.cells.map(c => c.value)) }
+    : argument?.docString ? { docString: argument.docString.content } : {};
+/** A pickle step as a Step: its text, kind and Data table or Doc string. */
+export const fromPickle = (step) => ({ text: step.text, kind: stepKind(step), ...stepArgument(step) });
+/** Parses a `.feature` source. Throws on Gherkin syntax errors. */
+export function parseFeature(source, uri = "test.feature") {
+    const newId = IdGenerator.uuid();
+    const doc = new Parser(new AstBuilder(newId), new GherkinClassicTokenMatcher()).parse(source);
+    const pickles = compile(doc, uri, newId);
+    // Gherkin reads keyword-less lines before the first step as a description, so a typo can leave no steps at all.
+    const empty = pickles.find(p => !p.steps.length);
+    if (empty)
+        throw new Error(`Scenario "${empty.name}" has no steps (lines without Given/When/Then/And/But are read as description)`);
+    if (!pickles.length)
+        throw new Error("No scenarios found");
+    const lines = new Map((doc.feature?.children ?? []).flatMap(c => c.rule ? c.rule.children : [c])
+        .flatMap(c => (c.background ?? c.scenario)?.steps ?? []).map(s => [s.id, s.location.line]));
+    for (const p of pickles)
+        for (const s of p.steps) {
+            const cell = readStep(fromPickle(s)).badCell;
+            if (cell !== undefined)
+                throw new Error(`Line ${lines.get(s.astNodeIds[0])}: an action's table cannot have ${cell.trim() ? `a cell with a double quote (${cell})` : "an empty cell"}; it becomes a quoted value`);
+        }
+    return pickles.map(pickle => ({
+        name: pickle.name,
+        tags: pickle.tags.map(t => t.name),
+        url: urlFromTags(pickle.tags),
+        steps: pickle.steps.map(step => ({ ...fromPickle(step), line: lines.get(step.astNodeIds[0]) })),
+    }));
+}
+/** One Step given on the command line, e.g. `Then the status should be "Draft"`; no keyword means `When`. */
+export function parseStep(line) {
+    return parseFeature(stepFeature(line))[0].steps[0];
+}
+/** A one-Step `.feature` source, so `decisium step` runs through the same runner as files. */
+export function stepFeature(line) {
+    const text = /^(Given|When|Then|And|But|\*)\s/i.test(line) ? line : `When ${line}`;
+    return `Feature: step\n  Scenario: step\n    ${text}\n`;
+}
+// Local paths keep their ?query (e.g. index.html?decoy=1) instead of having it encoded into the file name.
+export const toUrl = (u) => {
+    if (/^[a-z]+:\/\//i.test(u))
+        return u;
+    const [path, query] = u.split(/(?=\?)/);
+    return pathToFileURL(resolve(path)).href + (query ?? "");
+};

package/dist/formatter.d.ts

@@ -0,0 +1,5 @@
+import { Formatter, type IFormatterOptions } from "@cucumber/cucumber";
+export default class DecisiumFormatter extends Formatter {
+    static readonly documentation = "Decisium console output";
+    constructor(options: IFormatterOptions);
+}

package/dist/formatter.js

@@ -0,0 +1,119 @@
+// cucumber-js console formatter: one ✓/✗ line per Step, the reason under a failed one, and an end-of-run Failures block.
+import { Formatter, formatterHelpers, Status } from "@cucumber/cucumber";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, relative } from "node:path";
+import { fileURLToPath } from "node:url";
+import { fromPickle, urlFromTags } from "./feature.js";
+import { readStep } from "./step.js";
+import { agentReport, failureReport, KIND, MARGINAL, MARGINAL_HINT, NO_TRACE, parseOutcome, stepEvents } from "./report.js";
+const env = process.env;
+/** Shell-quoted for a copy-paste command line. */
+const q = (s) => /^[\w./:@%+=,-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`;
+/** The command that reruns one scenario (or the one command-line Step) with this run's options. */
+function rerun(target) {
+    const cli = relative(process.cwd(), fileURLToPath(import.meta.resolve(`./cli${import.meta.url.slice(-3)}`)));
+    const flags = [env.DECISIUM_URL && `--url ${q(env.DECISIUM_URL)}`, env.DECISIUM_JUDGED === "1" && "--judged", env.DECISIUM_STRICT === "1" && "--strict"];
+    return ["node", cli, ...target.map(q), ...flags].filter(Boolean).join(" ");
+}
+export default class DecisiumFormatter extends Formatter {
+    static documentation = "Decisium console output";
+    constructor(options) {
+        super(options);
+        const scenarios = [];
+        const failures = []; // built at the end: the Playwright trace is saved after the step
+        const warnings = [];
+        const hookIds = new Map(); // testRunHookStarted id -> hook definition id
+        const hookFailures = []; // a failed BeforeAll/AfterAll (e.g. no TYPESAFE_API_KEY): cucumber runs no scenario
+        options.eventBroadcaster.on("envelope", (e) => {
+            if (e.testCaseStarted) {
+                const { pickle } = this.eventDataCollector.getTestCaseAttempt(e.testCaseStarted.id);
+                this.log(`\n${pickle.name}  (${process.env.DECISIUM_URL || urlFromTags(pickle.tags) || "no URL"})\n`);
+            }
+            else if (e.testStepFinished) {
+                const { testCaseStartedId, testStepId, testStepResult: result } = e.testStepFinished;
+                const { pickle, gherkinDocument, testCase, stepAttachments } = this.eventDataCollector.getTestCaseAttempt(testCaseStartedId);
+                const step = pickle.steps.find(s => s.id === testCase.testSteps.find(t => t.id === testStepId)?.pickleStepId);
+                const failed = result.status === Status.FAILED;
+                if (!step) { // hook: only infrastructure errors (browser could not open, ...)
+                    if (failed)
+                        this.log(`  Error: ${result.message ?? "hook failed"}\n`);
+                    return;
+                }
+                if (result.status === Status.UNDEFINED) { // no Decisium step definitions loaded (a wrong "import" path)
+                    this.log(`  ? Step: ${step.text}\n  no step definition: is Decisium's steps file in "import"?\n`);
+                    return;
+                }
+                if (![Status.PASSED, Status.FAILED].includes(result.status))
+                    return; // skipped after a failure
+                const parsed = fromPickle(step);
+                const { kind } = parsed;
+                const label = readStep(parsed).label; // with its table or doc string
+                const attachments = stepAttachments[testStepId] ?? [];
+                const judged = attachments.some(a => a.body.startsWith("JUDGED_PASSED"));
+                this.log(`  ${failed ? "✗" : "✓"} ${KIND[kind]}: ${label}${judged ? "  (judged by Jev)" : ""}\n`);
+                const marginal = attachments.find(a => a.body.startsWith(MARGINAL))?.body.slice(MARGINAL.length);
+                if (marginal) {
+                    this.log(`      ⚠ marginal pass: ${marginal}\n`);
+                    warnings.push(`⚠ ${pickle.name}\n  ${KIND[kind]}: ${label}\n  Jev was barely confident: ${marginal}`);
+                }
+                if (!failed)
+                    return;
+                const r = parseOutcome(result.message ?? "");
+                this.log(`      ${r ? `${r.outcome}${r.reason ? `: ${r.reason}` : ""}` : `Error: ${result.message}`}`.replace(/\n/g, "\n      ") + "\n");
+                if (!r)
+                    return;
+                const line = formatterHelpers.GherkinDocumentParser.getGherkinStepMap(gherkinDocument)[step.astNodeIds[0]]?.location.line;
+                const screenshot = attachments.findLast(a => a.mediaType === "image/png")?.fileName;
+                const cmdLine = env.DECISIUM_STEP === "1";
+                const where = cmdLine ? "command line" : `${pickle.uri}:${line}`;
+                // The scenario's own line; for an Outline, its Examples row (the last AST node).
+                const scenarioLine = formatterHelpers.GherkinDocumentParser.getGherkinScenarioLocationMap(gherkinDocument)[pickle.astNodeIds.at(-1)]?.line;
+                failures.push(() => {
+                    const trace = screenshot && join(dirname(screenshot), "playwright-trace.zip");
+                    const ui = screenshot && join(dirname(screenshot), "ui.txt");
+                    const noTrace = screenshot && join(dirname(screenshot), NO_TRACE);
+                    return {
+                        where, scenario: pickle.name, kind, step: label, ...r, screenshot,
+                        trace: trace && existsSync(trace) ? trace : noTrace && existsSync(noTrace) ? `not kept: ${readFileSync(noTrace, "utf8").trim()}` : undefined, ui: ui && existsSync(ui) ? ui : undefined,
+                        rerun: rerun(cmdLine ? ["step", step.text] : ["run", `${pickle.uri}:${scenarioLine}`]),
+                    };
+                });
+            }
+            else if (e.testCaseFinished && !e.testCaseFinished.willBeRetried) {
+                const { pickle, worstTestStepResult } = this.eventDataCollector.getTestCaseAttempt(e.testCaseFinished.testCaseStartedId);
+                scenarios.push({ name: pickle.name, passed: worstTestStepResult.status === Status.PASSED });
+            }
+            else if (e.testRunHookStarted) {
+                hookIds.set(e.testRunHookStarted.id, e.testRunHookStarted.hookId);
+            }
+            else if (e.testRunHookFinished?.result.status === Status.FAILED) {
+                const before = this.supportCodeLibrary.beforeTestRunHookDefinitions.some(h => h.id === hookIds.get(e.testRunHookFinished.testRunHookStartedId));
+                const { message, exception } = e.testRunHookFinished.result;
+                hookFailures.push(`✗ ${before ? "BeforeAll" : "AfterAll"} hook failed\n  ${(exception?.message ?? message ?? "").split("\n")[0]}`);
+            }
+            else if (e.testRunFinished) {
+                if (hookFailures.length)
+                    this.log(`\n${hookFailures.join("\n")}\n`);
+                const failed = failures.map(f => f());
+                if (failed.length)
+                    this.log(`\n${"─".repeat(60)}\nFailures\n\n${failed.map(f => failureReport(f.where, f.scenario, f.kind, f.step, f, f.screenshot, f.trace)).join("\n\n")}\n`);
+                if (warnings.length)
+                    this.log(`\n${"─".repeat(60)}\nMarginal passes\n\n${warnings.join("\n\n")}\n\nWhat to do: ${MARGINAL_HINT}\n`);
+                const failedNames = scenarios.filter(s => !s.passed).map(s => s.name);
+                if (!scenarios.length && hookFailures.length)
+                    return;
+                this.log(`\n${scenarios.length - failedNames.length}/${scenarios.length} scenarios passed${failedNames.length ? `; failed: ${failedNames.join(", ")}` : ""}\n`);
+                // The run directory: the CLI's, or (raw cucumber-js) the one the screenshots are in.
+                const shot = failed.find(f => f.screenshot)?.screenshot;
+                const runDir = env.DECISIUM_RUN_DIR || (shot && dirname(dirname(shot)));
+                if (failed.length && runDir && env.DECISIUM_NO_AGENT_REPORT !== "1") {
+                    const tracePath = join(runDir, "trace.jsonl");
+                    const events = existsSync(tracePath) ? readFileSync(tracePath, "utf8").trim().split("\n").filter(Boolean).map(l => JSON.parse(l)) : [];
+                    const path = join(runDir, "report.md");
+                    writeFileSync(path, agentReport(failed.map(f => ({ ...f, events: stepEvents(events, f.scenario, f.step) }))));
+                    this.log(`Agent report: ${path}\n`);
+                }
+            }
+        });
+    }
+}

package/dist/jev.d.ts

@@ -0,0 +1,36 @@
+import { type Assessment, type Decider, type Pick, type TurnAnswer } from "./decision.ts";
+import type { Element, Observation } from "./observation.ts";
+export declare const MODEL = "jev-1.14.0";
+/**
+ * Pages can offer more candidates than one question accepts (GitHub: 336 links and buttons). Ask in batches, keep each
+ * batch's most likely candidates, repeat until they fit; the caller asks the real question over what is left, so the
+ * confidence gate still reads one comparable distribution. `fixed` answers (none, scroll, ...) are in every batch.
+ */
+export declare function narrow(candidates: Record<string, string>, fixed: Record<string, string>, ask: (criteria: Record<string, string>) => Promise<Pick>, max?: number): Promise<Record<string, string>>;
+/**
+ * Jev rejects requests over its token limit (`max_tokens_exceeded`): measured between 115k and 117k characters of
+ * Wikipedia snapshot in one request. `ui` stays under this budget so 255 choices still fit next to it (task 0021).
+ */
+export declare const UI_BUDGET = 80000;
+/**
+ * A `ui` over budget, reduced to the sections that matter for `step`. Sections are cut at lines and carry their
+ * ancestor lines (so a list item keeps its list); narrow() keeps the likeliest ones per batch until they fit, so every
+ * question of the Turn reads one reduced `ui` and their probabilities stay comparable.
+ */
+export declare function reduceUi(ui: string, ask: (sections: Record<string, string>) => Promise<Pick>, budget?: number, section?: number): Promise<string>;
+/** Live Jev requests of this process and their tokens; cassette replays cost nothing and are not counted. */
+export declare const jevUsage: {
+    calls: number;
+    inputTokens: number;
+    outputTokens: number;
+};
+/** Every question sends the Step text as-is, so a value quoted in a Step (a password included) reaches the API. */
+export declare class JevDecider implements Decider {
+    #private;
+    turn(step: string, obs: Observation, recent: string[], attempt?: boolean, keys?: string[]): Promise<TurnAnswer>;
+    drop(step: string, obs: Observation, source: Element): Promise<Pick>;
+    recall(step: string, earlier: Record<string, string>): Promise<Pick>;
+    dialog(step: string, obs: Observation, answers: string[]): Promise<Pick>;
+    value(step: string, obs: Observation, el: Element, candidates: string[]): Promise<Pick>;
+    assess(step: string, obs: Observation, judged: boolean): Promise<Assessment>;
+}