as-test 1.0.11 → 1.0.13

package/bin/commands/run-core.js

@@ -327,6 +327,9 @@ function formatReadableLog(file, suites, modeName, buildCommand, runCommand, sna
         lines.push("", "Failures:");
         for (const failure of failures) {
             lines.push(`FAIL  ${failure.title}${failure.where.length ? ` (${failure.where})` : ""}`);
+            if (failure.suitePath.length) {
+                lines.push(`Repro: ${buildSuiteReproCommand(file, failure.suitePath, modeName)}`);
+            }
             if (failure.message.length)
                 lines.push(`Message: ${failure.message}`);
             if (failure.left.length)
@@ -352,6 +355,139 @@ function formatInvocation(invocation) {
         .map((part) => (/[\s"'\\]/.test(part) ? JSON.stringify(part) : part))
         .join(" ");
 }
+function filterSelectedSuites(suites, selectors, file, modeName) {
+    const annotated = annotateSuitePaths(suites, []);
+    const matches = resolveSuiteSelectionMatches(annotated, selectors, file);
+    const selected = new Set(matches.map((match) => match.resolvedPath));
+    return cloneSelectedSuites(annotated, selected, file, modeName);
+}
+function annotateSuitePaths(suites, pathParts) {
+    return suites.map((suite) => annotateSuiteNode(suite, pathParts));
+}
+function annotateSuiteNode(suite, pathParts) {
+    const description = String(suite?.description ?? "unknown");
+    const slug = slugifySelectorSegment(description);
+    const nextParts = [...pathParts, slug];
+    const nextSuites = Array.isArray(suite?.suites)
+        ? suite.suites
+        : [];
+    const annotatedSuites = annotateSuitePaths(nextSuites, nextParts);
+    return {
+        ...suite,
+        path: nextParts.join("/"),
+        suites: annotatedSuites,
+    };
+}
+function resolveSuiteSelectionMatches(suites, selectors, file) {
+    const matches = [];
+    for (const selector of selectors) {
+        const normalized = selector.trim();
+        if (!normalized.length)
+            continue;
+        if (normalized.includes("/")) {
+            const resolved = resolveExplicitSuitePath(suites, normalized);
+            if (!resolved) {
+                throw new Error(`No suites matched "${selector}" in ${path.basename(file)}.`);
+            }
+            matches.push({
+                kind: "path",
+                raw: selector,
+                resolvedPath: resolved.path,
+                depth: resolved.depth,
+            });
+            continue;
+        }
+        const resolved = resolveBareSuiteSelector(suites, normalized);
+        if (!resolved) {
+            throw new Error(`No suites matched "${selector}" in ${path.basename(file)}.`);
+        }
+        matches.push({
+            kind: "bare",
+            raw: selector,
+            resolvedPath: resolved.path,
+            depth: resolved.depth,
+        });
+    }
+    return matches;
+}
+function resolveExplicitSuitePath(suites, selector) {
+    const normalized = selector
+        .split("/")
+        .map((part) => slugifySelectorSegment(part))
+        .filter((part) => part.length)
+        .join("/");
+    if (!normalized.length)
+        return null;
+    let match = null;
+    walkSuites(suites, (suite, depth) => {
+        if (suite.path == normalized) {
+            match = { path: suite.path, depth };
+            return true;
+        }
+        return false;
+    });
+    return match;
+}
+function resolveBareSuiteSelector(suites, selector) {
+    const slug = slugifySelectorSegment(selector);
+    if (!slug.length)
+        return null;
+    const matches = [];
+    walkSuites(suites, (suite, depth) => {
+        const leaf = String(suite.path ?? "").split("/").pop() ?? "";
+        if (leaf == slug) {
+            matches.push({ path: String(suite.path), depth });
+        }
+        return false;
+    });
+    if (!matches.length)
+        return null;
+    matches.sort((a, b) => a.depth - b.depth || a.path.localeCompare(b.path));
+    const shallowest = matches[0];
+    const ambiguous = matches.filter((match) => match.depth == shallowest.depth);
+    if (ambiguous.length > 1) {
+        throw new Error(`Suite selector "${selector}" is ambiguous. Matches: ${ambiguous.map((match) => match.path).join(", ")}`);
+    }
+    return shallowest;
+}
+function walkSuites(suites, visitor, depth = 0) {
+    for (const suite of suites) {
+        if (visitor(suite, depth))
+            return true;
+        const childSuites = Array.isArray(suite?.suites) ? suite.suites : [];
+        if (walkSuites(childSuites, visitor, depth + 1))
+            return true;
+    }
+    return false;
+}
+function cloneSelectedSuites(suites, selected, file, modeName) {
+    const out = [];
+    for (const suite of suites) {
+        const childSuites = Array.isArray(suite.suites) ? suite.suites : [];
+        const selectedChildren = cloneSelectedSuites(childSuites, selected, file, modeName);
+        const keep = selected.has(String(suite.path ?? "")) || selectedChildren.length > 0;
+        if (!keep)
+            continue;
+        out.push({
+            ...suite,
+            file,
+            modeName,
+            suites: selectedChildren,
+        });
+    }
+    return out;
+}
+function slugifySelectorSegment(value) {
+    return value
+        .trim()
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "");
+}
+function buildSuiteReproCommand(file, suitePath, modeName) {
+    const modeArg = modeName && modeName != "default" ? ` --mode ${modeName}` : "";
+    return `ast run ${file}${modeArg} --suite ${suitePath}`;
+}
 function collectReadableFailures(suites, file, pathParts) {
     const out = [];
     for (const suite of suites) {
@@ -372,6 +508,7 @@ function collectReadableFailures(suites, file, pathParts) {
                 message: String(test.message ?? ""),
                 left: JSON.stringify(test.left ?? ""),
                 right: JSON.stringify(test.right ?? ""),
+                suitePath: String(suiteAny.path ?? ""),
             });
         }
         const childSuites = Array.isArray(suiteAny.suites)
@@ -492,6 +629,9 @@ export async function run(flags = {}, configPath = DEFAULT_CONFIG_PATH, selector
             throw new Error(`Failed to run ${path.basename(file)} in mode ${modeLabel} with ${details}`);
         }
         const normalized = normalizeReport(report);
+        const selectedSuites = options.suiteSelectors?.length
+            ? filterSelectedSuites(normalized.suites, options.suiteSelectors, file, options.modeName ?? "default")
+            : normalized.suites;
         snapshotStore.flush();
         snapshotSummary.matched += snapshotStore.matched;
         snapshotSummary.created += snapshotStore.created;
@@ -500,7 +640,7 @@ export async function run(flags = {}, configPath = DEFAULT_CONFIG_PATH, selector
         reports.push({
             file,
             modeName: options.modeName ?? "default",
-            suites: normalized.suites,
+            suites: selectedSuites,
             coverage: normalized.coverage,
             runCommand: runCommandForLog,
             snapshotSummary: {
@@ -1483,6 +1623,31 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
     let stderrPendingLine = "";
     let stdoutBuffer = "";
     let spawnError = null;
+    let sawChannelClose = false;
+    const runtimeEvents = {
+        sawFileStart: false,
+        sawFileEnd: false,
+        fileName: path.basename(specFile),
+        fileVerdict: "none",
+        fileTime: "",
+        suiteStarts: 0,
+        suiteEnds: 0,
+        assertionFails: 0,
+        warnings: 0,
+        logs: 0,
+    };
+    const reportStream = {
+        dataFrames: 0,
+        dataBytes: 0,
+        sawChunkStart: false,
+        sawChunkEnd: false,
+        chunkCountExpected: 0,
+        chunkBytesExpected: 0,
+        chunkTotalBytesExpected: 0,
+        chunkFramesReceived: 0,
+        chunkBytesReceived: 0,
+        chunks: [],
+    };
     child.on("error", (error) => {
         spawnError = error;
     });
@@ -1512,6 +1677,7 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
             const event = msg;
             const kind = String(event.kind ?? "");
             if (kind === "event:assert-fail") {
+                runtimeEvents.assertionFails++;
                 reporter.onAssertionFail?.({
                     key: String(event.key ?? ""),
                     instr: String(event.instr ?? ""),
@@ -1522,6 +1688,8 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
                 return;
             }
             if (kind === "event:file-start") {
+                runtimeEvents.sawFileStart = true;
+                runtimeEvents.fileName = String(event.file ?? runtimeEvents.fileName);
                 reporter.onFileStart?.({
                     file: String(event.file ?? "unknown"),
                     depth: 0,
@@ -1531,6 +1699,10 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
                 return;
             }
             if (kind === "event:file-end") {
+                runtimeEvents.sawFileEnd = true;
+                runtimeEvents.fileName = String(event.file ?? runtimeEvents.fileName);
+                runtimeEvents.fileVerdict = String(event.verdict ?? "none");
+                runtimeEvents.fileTime = String(event.time ?? "");
                 reporter.onFileEnd?.({
                     file: String(event.file ?? "unknown"),
                     depth: 0,
@@ -1542,6 +1714,7 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
                 return;
             }
             if (kind === "event:suite-start") {
+                runtimeEvents.suiteStarts++;
                 reporter.onSuiteStart?.({
                     file: String(event.file ?? "unknown"),
                     depth: Number(event.depth ?? 0),
@@ -1551,6 +1724,7 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
                 return;
             }
             if (kind === "event:suite-end") {
+                runtimeEvents.suiteEnds++;
                 reporter.onSuiteEnd?.({
                     file: String(event.file ?? "unknown"),
                     depth: Number(event.depth ?? 0),
@@ -1561,12 +1735,14 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
                 return;
             }
             if (kind === "event:warn") {
+                runtimeEvents.warnings++;
                 reporter.onWarning?.({
                     message: String(event.message ?? ""),
                 });
                 return;
             }
             if (kind === "event:log") {
+                runtimeEvents.logs++;
                 reporter.onLog?.({
                     file: String(event.file ?? "unknown"),
                     depth: Number(event.depth ?? 0),
@@ -1584,16 +1760,43 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
                 this.send(MessageType.CALL, Buffer.from(`${result.ok ? "1" : "0"}\n${result.expected}`, "utf8"));
                 return;
             }
+            if (kind === "report:start") {
+                reportStream.sawChunkStart = true;
+                reportStream.sawChunkEnd = false;
+                reportStream.chunkCountExpected = Number(event.chunkCount ?? 0);
+                reportStream.chunkBytesExpected = Number(event.chunkBytes ?? 0);
+                reportStream.chunkTotalBytesExpected = Number(event.totalBytes ?? 0);
+                reportStream.chunkFramesReceived = 0;
+                reportStream.chunkBytesReceived = 0;
+                reportStream.chunks = [];
+                return;
+            }
+            if (kind === "report:end") {
+                reportStream.sawChunkEnd = true;
+                return;
+            }
             this.sendJSON(MessageType.CALL, { ok: true, expected: "" });
         }
         onDataMessage(data) {
+            reportStream.dataFrames++;
+            reportStream.dataBytes += data.length;
+            if (reportStream.sawChunkStart && !reportStream.sawChunkEnd) {
+                reportStream.chunkFramesReceived++;
+                reportStream.chunkBytesReceived += data.length;
+                reportStream.chunks.push(data.toString("utf8"));
+                return;
+            }
             try {
                 report = JSON.parse(data.toString("utf8"));
+                parseError = null;
             }
             catch (error) {
                 parseError = String(error);
             }
         }
+        onClose() {
+            sawChannelClose = true;
+        }
     }
     const _channel = new TestChannel(child.stdout, child.stdin);
     const code = await new Promise((resolve) => {
@@ -1602,8 +1805,9 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
     if (stderrPendingLine.length && !shouldSuppressWasiWarningLine(stderrPendingLine)) {
         stderrBuffer += stderrPendingLine;
     }
-    if (spawnError) {
-        const errorText = spawnError.stack ?? spawnError.message;
+    const processSpawnError = spawnError;
+    if (processSpawnError) {
+        const errorText = processSpawnError.stack ?? processSpawnError.message;
         persistCrashRecord(crashDir, {
             kind: "test",
             file: specFile,
@@ -1614,29 +1818,93 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
         });
         return createRuntimeFailureReport(specFile, modeName, "failed to start test runtime", errorText, stdoutBuffer, stderrBuffer);
     }
+    if (reportStream.sawChunkStart) {
+        if (!reportStream.sawChunkEnd) {
+            parseError =
+                parseError ??
+                    "missing report:end marker for chunked report payload";
+        }
+        else {
+            const chunkedPayload = reportStream.chunks.join("");
+            try {
+                report = JSON.parse(chunkedPayload);
+                parseError = null;
+            }
+            catch (error) {
+                parseError = `could not parse chunked report payload: ${String(error)}`;
+            }
+            if (reportStream.chunkCountExpected > 0 &&
+                reportStream.chunkFramesReceived !== reportStream.chunkCountExpected) {
+                parseError =
+                    parseError ??
+                        `chunk count mismatch: expected ${reportStream.chunkCountExpected}, received ${reportStream.chunkFramesReceived}`;
+            }
+            if (reportStream.chunkTotalBytesExpected > 0 &&
+                reportStream.chunkBytesReceived !== reportStream.chunkTotalBytesExpected) {
+                parseError =
+                    parseError ??
+                        `chunk size mismatch: expected ${reportStream.chunkTotalBytesExpected} bytes, received ${reportStream.chunkBytesReceived}`;
+            }
+        }
+    }
     if (parseError) {
         const errorText = `could not parse report payload: ${parseError}`;
+        const diagnostics = buildRuntimeReportDiagnostics(code, sawChannelClose, reportStream, runtimeEvents);
+        const fullError = `${errorText}\n${diagnostics}`;
         persistCrashRecord(crashDir, {
             kind: "test",
             file: specFile,
             mode: modeName ?? "default",
-            error: errorText,
+            error: fullError,
             stdout: stdoutBuffer,
             stderr: stderrBuffer,
         });
-        return createRuntimeFailureReport(specFile, modeName, "runtime returned an invalid report payload", errorText, stdoutBuffer, stderrBuffer);
+        return createRuntimeFailureReport(specFile, modeName, "runtime returned an invalid report payload", fullError, stdoutBuffer, stderrBuffer);
     }
     if (!report) {
+        const synthesized = synthesizeReportFromRuntimeEvents(specFile, runtimeEvents);
+        if (synthesized) {
+            reporter.onWarning?.({
+                message: "runtime report payload missing; reconstructed result from streamed lifecycle events",
+            });
+            if (code !== 0 || hasMeaningfulRuntimeOutput(stderrBuffer)) {
+                const errorParts = [];
+                if (code !== 0) {
+                    errorParts.push(`child process exited with code ${code}`);
+                }
+                const stderrText = normalizeRuntimeOutput(stderrBuffer);
+                if (stderrText.length) {
+                    errorParts.push(stderrText);
+                }
+                const diagnostics = buildRuntimeReportDiagnostics(code, sawChannelClose, reportStream, runtimeEvents);
+                errorParts.push(diagnostics);
+                const errorText = errorParts.join("\n\n");
+                persistCrashRecord(crashDir, {
+                    kind: "test",
+                    file: specFile,
+                    mode: modeName ?? "default",
+                    error: errorText || "runtime reported an unknown error",
+                    stdout: stdoutBuffer,
+                    stderr: stderrBuffer,
+                });
+                return appendRuntimeFailureReport(synthesized, specFile, modeName, code !== 0
+                    ? `test runtime failed with exit code ${code}`
+                    : "test runtime wrote to stderr", errorText, stdoutBuffer, stderrBuffer);
+            }
+            return synthesized;
+        }
         const errorText = "missing report payload from test runtime";
+        const diagnostics = buildRuntimeReportDiagnostics(code, sawChannelClose, reportStream, runtimeEvents);
+        const fullError = `${errorText}\n${diagnostics}`;
         persistCrashRecord(crashDir, {
             kind: "test",
             file: specFile,
             mode: modeName ?? "default",
-            error: errorText,
+            error: fullError,
             stdout: stdoutBuffer,
             stderr: stderrBuffer,
         });
-        return createRuntimeFailureReport(specFile, modeName, "test runtime exited without sending a report", errorText, stdoutBuffer, stderrBuffer);
+        return createRuntimeFailureReport(specFile, modeName, "test runtime exited without sending a report", fullError, stdoutBuffer, stderrBuffer);
     }
     if (code !== 0 || hasMeaningfulRuntimeOutput(stderrBuffer)) {
         const errorParts = [];
@@ -1662,6 +1930,52 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
     }
     return report;
 }
+function synthesizeReportFromRuntimeEvents(specFile, runtimeEvents) {
+    if (!runtimeEvents.sawFileEnd &&
+        runtimeEvents.suiteStarts <= 0 &&
+        runtimeEvents.suiteEnds <= 0) {
+        return null;
+    }
+    let verdict = runtimeEvents.fileVerdict;
+    if (verdict == "none" && runtimeEvents.assertionFails > 0) {
+        verdict = "fail";
+    }
+    else if (verdict == "none" && runtimeEvents.sawFileEnd) {
+        verdict = "ok";
+    }
+    return {
+        suites: [
+            {
+                file: specFile,
+                description: runtimeEvents.fileName || path.basename(specFile),
+                depth: 0,
+                kind: "file",
+                verdict,
+                time: {
+                    start: 0,
+                    end: 0,
+                },
+                suites: [],
+                logs: [],
+                tests: [],
+            },
+        ],
+        coverage: {
+            total: 0,
+            covered: 0,
+            uncovered: 0,
+            percent: 100,
+            points: [],
+        },
+    };
+}
+function buildRuntimeReportDiagnostics(code, sawChannelClose, reportStream, runtimeEvents) {
+    return [
+        `runtime diagnostics: exitCode=${code}, channelClose=${sawChannelClose ? "yes" : "no"}`,
+        `report stream: dataFrames=${reportStream.dataFrames}, dataBytes=${reportStream.dataBytes}, chunked=${reportStream.sawChunkStart ? "yes" : "no"}, chunkStart=${reportStream.sawChunkStart ? "yes" : "no"}, chunkEnd=${reportStream.sawChunkEnd ? "yes" : "no"}, chunkFrames=${reportStream.chunkFramesReceived}, expectedChunkFrames=${reportStream.chunkCountExpected}, chunkBytes=${reportStream.chunkBytesReceived}, expectedChunkBytes=${reportStream.chunkTotalBytesExpected}`,
+        `runtime events: fileStart=${runtimeEvents.sawFileStart ? "yes" : "no"}, fileEnd=${runtimeEvents.sawFileEnd ? "yes" : "no"}, fileVerdict=${runtimeEvents.fileVerdict}, suiteStarts=${runtimeEvents.suiteStarts}, suiteEnds=${runtimeEvents.suiteEnds}, assertionFails=${runtimeEvents.assertionFails}, warnings=${runtimeEvents.warnings}, logs=${runtimeEvents.logs}`,
+    ].join("\n");
+}
 function createRuntimeFailureReport(specFile, modeName, title, details, stdout, stderr) {
     return appendRuntimeFailureReport({
         suites: [],

package/bin/commands/run.js

@@ -1,6 +1,7 @@
 export { createRunReporter, run } from "./run-core.js";
 export async function executeRunCommand(rawArgs, flags, configPath, selectedModes, deps) {
     const commandArgs = deps.resolveCommandArgs(rawArgs, "run");
+    const suiteSelectors = deps.resolveSuiteSelectors(rawArgs, "run");
     const listFlags = deps.resolveListFlags(rawArgs, "run");
     const featureToggles = deps.resolveFeatureToggles(rawArgs, "run");
     const runFlags = {
@@ -20,5 +21,5 @@ export async function executeRunCommand(rawArgs, flags, configPath, selectedMode
         await deps.listExecutionPlan("run", configPath, commandArgs, modeTargets, listFlags);
         return;
     }
-    await deps.runRuntimeModes(runFlags, configPath, commandArgs, modeTargets);
+    await deps.runRuntimeModes(runFlags, configPath, commandArgs, suiteSelectors, modeTargets);
 }

package/bin/commands/test.js

@@ -1,5 +1,7 @@
 export async function executeTestCommand(rawArgs, flags, configPath, selectedModes, deps) {
     const commandArgs = deps.resolveCommandArgs(rawArgs, "test");
+    const suiteSelectors = deps.resolveSuiteSelectors(rawArgs, "test");
+    const fuzzerSelectors = deps.resolveFuzzerSelectors(rawArgs, "test");
     const listFlags = deps.resolveListFlags(rawArgs, "test");
     const featureToggles = deps.resolveFeatureToggles(rawArgs, "test");
     const buildFeatureToggles = {
@@ -25,5 +27,5 @@ export async function executeTestCommand(rawArgs, flags, configPath, selectedMod
         await deps.listExecutionPlan("test", configPath, commandArgs, modeTargets, listFlags, fuzzEnabled);
         return;
     }
-    await deps.runTestModes(runFlags, configPath, commandArgs, modeTargets, buildFeatureToggles, fuzzEnabled, fuzzOverrides);
+    await deps.runTestModes(runFlags, configPath, commandArgs, suiteSelectors, fuzzerSelectors, modeTargets, buildFeatureToggles, fuzzEnabled, fuzzOverrides);
 }