decisium 0.0.0-stage → 0.1.2

package/dist/steps.js

@@ -0,0 +1,205 @@
+// cucumber-js support code (ADR 0003): one catch-all step definition runs every Step through Jev; hooks give each
+// scenario a fresh browser, its run-directory folder, the feature's cassette and an empty memory.
+// Options come from worldParameters.decisium and DECISIUM_* env (src/options.ts), so raw `npx cucumber-js` works too.
+import { After, AfterAll, AfterStep, Before, BeforeAll, BeforeStep, defineStep, formatterHelpers, setDefaultTimeout, setParallelCanAssign, setWorldConstructor, Status, supportCodeLibraryBuilder, World } from "@cucumber/cucumber";
+import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { Browser } from "./browser.js";
+import { Cassette, RECORD_MODES, stepKey } from "./cassette.js";
+import { executeStep } from "./execute.js";
+import { contextFromTags, fromPickle, tagValue, toUrl, urlFromTags } from "./feature.js";
+import { JevDecider, MODEL } from "./jev.js";
+import { MARGINAL, NO_TRACE, outcomeMessage } from "./report.js";
+import { placeholder, readStep } from "./step.js";
+import { INFRA_MARKER, RunDir } from "./trace.js";
+import { applyOptions } from "./options.js";
+const env = process.env;
+const opt = (name) => env[`DECISIUM_${name}`] === "1";
+let record; // set in BeforeAll, once worldParameters are known
+// Jev steps take seconds per Turn; the runner has its own caps (8 Turns, 5 s waits).
+setDefaultTimeout(10 * 60_000);
+// Parallel workers (`--parallel`) are separate processes: the CLI creates the run directory (DECISIUM_RUN_DIR) and an
+// infrastructure error is signalled through a marker file in it, read by the CLI and by the other workers.
+let run;
+const cliOwnsRun = !!env.DECISIUM_RUN_DIR; // the CLI prints the run directory once, not each worker
+const broken = () => !!run && existsSync(join(run.path, INFRA_MARKER)); // after one, the remaining scenarios are skipped
+const deciders = new Map(); // per feature file
+class Scenario extends World {
+    browser;
+    rt;
+    cassette;
+    live;
+    key = ""; // the running Step in its cassette: scenario, position, text
+    dir = "";
+    kind = "action";
+    arg = {}; // the running Step's data table or doc string
+    at = ""; // file:line of the running Step, in its trace event (scripts/cookbook.ts)
+    earlier = {}; // memory: quoted value -> the Step that named it
+    /** Public API for client step definitions (task 0048): the current tab (a popup becomes it) and its Playwright context. */
+    get page() { return this.browser.page; }
+    get context() { return this.browser.context; }
+}
+setWorldConstructor(Scenario);
+// A feature's scenarios run one after another (they share its cassette); different features run side by side.
+setParallelCanAssign((pickle, running) => !running.some(p => p.uri === pickle.uri));
+BeforeAll(function () {
+    if (existsSync(".env"))
+        process.loadEnvFile(".env"); // TYPESAFE_API_KEY; never logged
+    applyOptions(this.parameters.decisium);
+    record = (env.DECISIUM_RECORD || "new");
+    if (!RECORD_MODES.includes(record))
+        throw new Error(`DECISIUM_RECORD must be one of ${RECORD_MODES.join(", ")}`);
+    if (!env.TYPESAFE_API_KEY && (record !== "none" || opt("STEP") || opt("JUDGED")))
+        throw new Error("TYPESAFE_API_KEY is not set.");
+    run = new RunDir("runs", env.DECISIUM_RUN_DIR || undefined);
+    if (!cliOwnsRun)
+        console.log(`Model: ${MODEL}\nRun directory: ${run.path}`);
+});
+Before(async function ({ pickle }) {
+    if (broken())
+        return "skipped";
+    const url = env.DECISIUM_URL || urlFromTags(pickle.tags);
+    if (!url)
+        throw infra(new Error(`No start URL for "${pickle.name}": pass --url or add an @url:<url> tag.`));
+    try {
+        if (!deciders.has(pickle.uri)) { // a corrupt cassette is a setup error, not a failed scenario
+            const cassette = opt("STEP") ? undefined : new Cassette(pickle.uri.replace(/\.feature$/, "") + ".cassette.json", MODEL, record, run.path);
+            deciders.set(pickle.uri, { cassette, live: new JevDecider() });
+        }
+    }
+    catch (err) {
+        throw infra(err);
+    }
+    ({ cassette: this.cassette, live: this.live } = deciders.get(pickle.uri));
+    // Secret placeholders ("$NAME"): a missing variable is a setup error, not a failed Step (task 0030).
+    const secrets = {};
+    for (const token of pickle.steps.flatMap(s => readStep(fromPickle(s)).placeholders)) {
+        const name = placeholder(token);
+        if (!env[name])
+            throw infra(new Error(`"${token}" in "${pickle.name}": environment variable ${name} is not set (or empty).`));
+        secrets[token] = env[name];
+    }
+    run.log(`\n=== Scenario: ${pickle.name}  (${url})`);
+    this.dir = run.scenario(pickle.name);
+    const glob = tagValue(pickle.tags, "replay-network");
+    // One recording per scenario: replay matches the n-th call to a URL. Outline rows share a name, so the last row's wins.
+    const har = `${pickle.uri.replace(/\.feature$/, "")}.network/${pickle.name.replace(/[^\w-]+/g, "_")}.har`;
+    // Follows --record: all re-records, none replays, new replays an existing recording and records a missing one.
+    const network = glob ? { glob, har, mode: record === "all" || (record === "new" && !existsSync(har)) ? "record" : "replay" } : undefined;
+    try {
+        this.browser = await Browser.open(toUrl(url), opt("HEADED"), true, secrets, dirname(pickle.uri), this.dir, // fresh browser per scenario: no shared state
+        contextFromTags(pickle.tags), network);
+    }
+    catch (err) {
+        throw infra(err);
+    }
+    this.rt = run.runtime(this.browser);
+    networkMissed(this.browser);
+});
+/** Fails the Step (or the page load) that made a `@replay-network` request the recording does not have. */
+function networkMissed(browser) {
+    const missed = browser?.networkMisses.splice(0);
+    if (missed?.length) {
+        const err = new Error(`network not in recording: ${missed.join(", ")} (re-record with --record all)`);
+        err.stack = err.message;
+        throw err;
+    }
+}
+AfterStep(function () { networkMissed(this.browser); });
+BeforeStep(function ({ pickle, pickleStep, gherkinDocument }) {
+    const line = formatterHelpers.GherkinDocumentParser.getGherkinStepMap(gherkinDocument)[pickleStep.astNodeIds[0]]?.location.line;
+    this.at = `${pickle.uri}:${line}`;
+    const { kind, text: _, ...arg } = fromPickle(pickleStep);
+    this.kind = kind;
+    this.arg = arg;
+    const labels = pickle.steps.map(s => readStep(fromPickle(s)).label);
+    this.key = stepKey(pickle.name, labels, pickle.steps.findIndex(s => s.id === pickleStep.id));
+    Object.defineProperty(step, "length", { value: pickleStep.argument ? 2 : 1 });
+});
+// cucumber-js passes a data table or doc string as one more argument and checks the function's arity against it;
+// the argument is read from the pickle step in BeforeStep instead, so the arity follows the Step (set there).
+async function step(text) {
+    const log = (line) => { run.log(line); if (opt("VERBOSE"))
+        console.log(line); };
+    let r;
+    try {
+        r = await executeStep(text, { rt: this.rt, live: this.live, cassette: this.cassette, key: this.key, kind: this.kind, arg: this.arg,
+            earlier: this.earlier, log, trace: d => run.decider(d), judged: opt("JUDGED"), strict: opt("STRICT") });
+    }
+    catch (err) {
+        throw infra(err);
+    }
+    run.event("step", { kind: this.kind, step: readStep({ text, kind: this.kind, ...this.arg }).label, at: this.at, ...r });
+    if (r.outcome === "JUDGED_PASSED")
+        this.log(`JUDGED_PASSED: judged by Jev (${r.reason})`);
+    // A file attachment any reporter keeps (text/plain opens as .txt); the console formatter reads it by its prefix.
+    if (r.marginal)
+        this.attach(`${MARGINAL}${r.marginal}`, { mediaType: "text/plain", fileName: "marginal-pass.txt" });
+    if (!["DONE", "PASSED", "JUDGED_PASSED"].includes(r.outcome)) {
+        // Failed Step only; fileName carries the path to the console formatter.
+        const shot = run.lastScreenshot;
+        if (shot && existsSync(shot))
+            this.attach(readFileSync(shot), { mediaType: "image/png", fileName: shot });
+        const err = new Error(outcomeMessage(r));
+        err.stack = err.message; // no stack frames: reporters such as Qase's read keywords in them
+        throw err;
+    }
+}
+const CATCH_ALL = /^(.*)$/;
+defineStep(CATCH_ALL, step);
+// Client step definitions (task 0048): a line another definition matches runs that one, not Jev. Without this cucumber
+// reports it Ambiguous. Relies on cucumber internals (finalize, expression.match); "client steps" in test/decisium.test.ts
+// fails when they change.
+const finalize = supportCodeLibraryBuilder.finalize.bind(supportCodeLibraryBuilder);
+supportCodeLibraryBuilder.finalize = canonicalIds => {
+    const lib = finalize(canonicalIds);
+    const ours = lib.stepDefinitions.find(d => d.pattern === CATCH_ALL);
+    const others = lib.stepDefinitions.filter(d => d !== ours);
+    const match = ours.expression.match.bind(ours.expression);
+    ours.expression.match = text => (others.some(d => d.matchesStepName(text)) ? null : match(text));
+    return lib;
+};
+After(async function ({ result }) {
+    run.flush(); // the scenario's whole log in one write
+    if (!this.browser)
+        return;
+    const ok = result?.status === Status.PASSED;
+    if (!ok)
+        run.failure(this.browser.consoleErrors);
+    // A passed scenario keeps its trace events but not its screenshots: a screenshot per Turn filled runs/ (2 GB).
+    else
+        for (const f of readdirSync(this.dir))
+            if (f.endsWith(".png"))
+                rmSync(join(this.dir, f));
+    if (!ok && this.browser.typedSecret)
+        writeFileSync(join(this.dir, NO_TRACE), "The scenario typed a secret value; Playwright traces store typed text unmasked, so none is kept.\n");
+    await this.browser.close(ok ? undefined : `${this.dir}/playwright-trace.zip`);
+    run.event("result", { passed: ok });
+    for (const { cassette } of deciders.values())
+        cassette?.save(); // per scenario, so a crash keeps what was recorded
+    run.flush();
+});
+AfterAll(function () {
+    if (!run)
+        return;
+    run.flush(); // a scenario skipped or broken before its browser opened
+    if (!cliOwnsRun)
+        run.done();
+    if (!cliOwnsRun)
+        console.log(`\nRun directory: ${run.path} (log.txt has the full decision log)`);
+    for (const { cassette: c } of deciders.values())
+        if (c)
+            console.log(`Cassette ${c.path}: ${c.replayed} replayed, ${c.recorded} recorded, ${c.healed} healed`);
+});
+/** Infrastructure error (app not running, bad API key, network): one clear line, `--debug` for the stack. Exit code 2. */
+function infra(err) {
+    process.exitCode = 2;
+    if (run)
+        writeFileSync(join(run.path, INFRA_MARKER), String(err?.message ?? err)); // read by `decisium run`
+    if (opt("DEBUG"))
+        return err;
+    const e = err;
+    const out = new Error(`${e.message.split("\n")[0]}${e.status === 401 ? " (check TYPESAFE_API_KEY in .env)" : ""}\nRerun with --debug for details.`);
+    out.stack = out.message;
+    return out;
+}

package/dist/trace.d.ts

@@ -0,0 +1,28 @@
+import type { Browser } from "./browser.ts";
+import { type Decider } from "./decision.ts";
+import type { Runtime } from "./runner.ts";
+/** File in the run directory marking an infrastructure error (exit 2), written by any worker, read by the CLI. */
+export declare const INFRA_MARKER = "infrastructure-error";
+/** runs/<timestamp>/: trace.jsonl for the whole run, one folder per scenario with a screenshot per Turn. */
+export declare class RunDir {
+    #private;
+    readonly path: string;
+    /** `existing`: a run directory the CLI created, shared by parallel workers. */
+    constructor(root?: string, existing?: string);
+    /** Full verbose log of the run (what -v prints). Buffered per scenario so parallel workers do not interleave lines. */
+    log(line: string): void;
+    /** Appends the buffered scenario log to log.txt in one write. Call at the end of every scenario. */
+    flush(): void;
+    /** End of the run: drop the folder-number claims. */
+    done(): void;
+    /** Last screenshot of the current scenario, for failure reports. */
+    get lastScreenshot(): string | undefined;
+    event(type: string, data?: Record<string, unknown>): void;
+    scenario(name: string): string;
+    /** Every Observation is a Turn: screenshot it and log the Observation text. */
+    runtime(b: Browser): Runtime;
+    /** Jev's answers with their probabilities, one event per question. Values chosen for secret fields are masked. */
+    decider(d: Decider): Decider;
+    /** On failure: console errors and the last page text next to the screenshots (the Playwright trace is saved by Browser.close). */
+    failure(errors: string[]): void;
+}

package/dist/trace.js

@@ -0,0 +1,112 @@
+import { appendFileSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { traced } from "./decision.js";
+/** File in the run directory marking an infrastructure error (exit 2), written by any worker, read by the CLI. */
+export const INFRA_MARKER = "infrastructure-error";
+/** runs/<timestamp>/: trace.jsonl for the whole run, one folder per scenario with a screenshot per Turn. */
+export class RunDir {
+    path;
+    #scenario = "";
+    #dir = "";
+    #shot = 0;
+    #ui = ""; // the scenario's last Observation, written next to the screenshots on failure
+    /** `existing`: a run directory the CLI created, shared by parallel workers. */
+    constructor(root = "runs", existing) {
+        if (existing) {
+            mkdirSync(existing, { recursive: true });
+            this.path = existing;
+            return;
+        }
+        mkdirSync(root, { recursive: true });
+        const base = join(root, new Date().toISOString().replace(/[:.]/g, "-"));
+        // Parallel runs can start in the same millisecond: mkdir without `recursive` fails if the directory exists.
+        for (let i = 1;; i++) {
+            const path = i === 1 ? base : `${base}-${i}`;
+            try {
+                mkdirSync(path);
+                this.path = path;
+                return;
+            }
+            catch (err) {
+                if (err.code !== "EEXIST")
+                    throw err;
+            }
+        }
+    }
+    #lines = [];
+    /** Full verbose log of the run (what -v prints). Buffered per scenario so parallel workers do not interleave lines. */
+    log(line) {
+        this.#lines.push(line);
+    }
+    /** Appends the buffered scenario log to log.txt in one write. Call at the end of every scenario. */
+    flush() {
+        if (this.#lines.length)
+            appendFileSync(join(this.path, "log.txt"), this.#lines.join("\n") + "\n");
+        this.#lines = [];
+    }
+    /** End of the run: drop the folder-number claims. */
+    done() {
+        rmSync(join(this.path, ".claimed"), { recursive: true, force: true });
+    }
+    /** Last screenshot of the current scenario, for failure reports. */
+    get lastScreenshot() {
+        return this.#shot ? join(this.#dir, `${String(this.#shot).padStart(3, "0")}.png`) : undefined;
+    }
+    event(type, data = {}) {
+        appendFileSync(join(this.path, "trace.jsonl"), JSON.stringify({ t: new Date().toISOString(), scenario: this.#scenario, type, ...data }) + "\n");
+    }
+    scenario(name) {
+        this.#scenario = name;
+        this.#shot = 0;
+        this.#ui = "";
+        const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60);
+        // Numbered in start order across parallel workers: claim a number with an exclusive mkdir, then name the folder.
+        mkdirSync(join(this.path, ".claimed"), { recursive: true });
+        let i = 1;
+        for (;; i++) {
+            try {
+                mkdirSync(join(this.path, ".claimed", String(i)));
+                break;
+            }
+            catch (err) {
+                if (err.code !== "EEXIST")
+                    throw err;
+            }
+        }
+        this.#dir = join(this.path, `${String(i).padStart(2, "0")}-${slug}`);
+        mkdirSync(this.#dir, { recursive: true });
+        this.event("scenario");
+        return this.#dir;
+    }
+    /** Every Observation is a Turn: screenshot it and log the Observation text. */
+    runtime(b) {
+        const rt = {
+            observe: async () => {
+                const obs = await b.observe();
+                const screenshot = `${String(++this.#shot).padStart(3, "0")}.png`;
+                await b.screenshot(join(this.#dir, screenshot));
+                this.event("observe", { screenshot, ui: obs.text });
+                this.#ui = obs.text;
+                return obs;
+            },
+            execute: async (a) => {
+                this.event("execute", { ...a, value: a.value === undefined ? undefined : "(see log; may be secret)" });
+                await b.execute(a);
+            },
+            waitForChange: (since, ms) => b.waitForChange(since, ms),
+            stale: () => b.stale(),
+            actionStep: (run, log) => b.actionStep(run, log, rt.execute),
+        };
+        return rt;
+    }
+    /** Jev's answers with their probabilities, one event per question. Values chosen for secret fields are masked. */
+    decider(d) {
+        return traced(d, (type, data) => this.event(type, data));
+    }
+    /** On failure: console errors and the last page text next to the screenshots (the Playwright trace is saved by Browser.close). */
+    failure(errors) {
+        writeFileSync(join(this.#dir, "console-errors.txt"), errors.length ? errors.join("\n") + "\n" : "(none)\n");
+        if (this.#ui)
+            writeFileSync(join(this.#dir, "ui.txt"), this.#ui + "\n");
+    }
+}

package/docs/configuration.md

@@ -0,0 +1,53 @@
+# Configuration and reporters
+
+`decisium run` loads configuration exactly like cucumber-js:
+your `cucumber.{js,mjs,cjs,json,yaml}`, `-p <profile>`, and cucumber's `--format name[:path]` and `--import <file>`
+flags, added to the file's values. Decisium adds its own step definitions to `import`.
+
+## Decisium options
+
+In `worldParameters.decisium`, so raw `npx cucumber-js` reads them too. Flags (`--url`, `--record`, `--strict`, ...)
+and `DECISIUM_*` env (`DECISIUM_URL`, `DECISIUM_RECORD`, `DECISIUM_STRICT=1`, ...) override them.
+
+```js
+// cucumber.mjs
+export default {
+  worldParameters: { decisium: { record: "none", strict: true } },
+  // url, record, headed, judged, strict, verbose, debug, agentReport
+};
+```
+
+## Reporters
+
+Any cucumber formatter: cucumber's own `junit`, `json`, `html`, `message`, or a package such as the official Qase
+reporter. Only one formatter may write to stdout: one configured without a path takes it; otherwise Decisium's console
+output does.
+
+```sh
+decisium run features/login.feature --format junit:reports/junit.xml
+```
+
+### Qase
+
+Decisium ships nothing Qase-specific; wire the official reporter as in any cucumber project.
+
+```sh
+npm install --save-dev cucumberjs-qase-reporter
+echo '{"mode":"report"}' > qase.config.json   # or mode "testops" with QASE_TESTOPS_API_TOKEN, QASE_TESTOPS_PROJECT
+decisium run features/login.feature --format cucumberjs-qase-reporter:/dev/null
+```
+
+Or in your configuration, which raw `npx cucumber-js` uses too:
+
+```js
+// cucumber.mjs
+export default {
+  // Raw cucumber-js only: Decisium's steps and formatter (decisium run adds both itself).
+  import: ["node_modules/decisium/dist/steps.js"],
+  format: ["decisium/formatter", "cucumberjs-qase-reporter:/dev/null"],
+};
+```
+
+Tag scenarios `@QaseID=<id>`. A failing Step's message starts with its outcome; app failures (`ASSERTION_FAILED`,
+`JUDGED_FAILED`, `BLOCKED`, `ACTION_FAILED`) report as `failed`, every other outcome as `invalid`. A marginal pass
+reaches every reporter as a `text/plain` attachment `marginal-pass.txt` on its Step (Qase stores it as `file.txt`).