as-test 0.5.0 → 0.5.2

package/bin/build.js

@@ -3,31 +3,45 @@ import { glob } from "glob";
 import chalk from "chalk";
 import { execSync } from "child_process";
 import * as path from "path";
-import { getPkgRunner, loadConfig } from "./util.js";
+import { applyMode, getPkgRunner, loadConfig } from "./util.js";
 const DEFAULT_CONFIG_PATH = path.join(process.cwd(), "./as-test.config.json");
-export async function build(configPath = DEFAULT_CONFIG_PATH, selectors = []) {
-    const config = loadConfig(configPath, false);
+export async function build(configPath = DEFAULT_CONFIG_PATH, selectors = [], modeName) {
+    const loadedConfig = loadConfig(configPath, false);
+    const mode = applyMode(loadedConfig, modeName);
+    const config = mode.config;
     ensureDeps(config);
     const pkgRunner = getPkgRunner();
     const inputPatterns = resolveInputPatterns(config.input, selectors);
-    const inputFiles = await glob(inputPatterns);
+    const inputFiles = (await glob(inputPatterns)).sort((a, b) => a.localeCompare(b));
     const buildArgs = getBuildArgs(config);
     for (const file of inputFiles) {
         let cmd = `${pkgRunner} asc ${file}${buildArgs}`;
-        const outFile = `${config.outDir}/${file.slice(file.lastIndexOf("/") + 1).replace(".ts", ".wasm")}`;
+        const outFile = `${config.outDir}/${resolveArtifactFileName(file, config.buildOptions.target, modeName)}`;
         if (config.outDir) {
             cmd += " -o " + outFile;
         }
         try {
-            buildFile(cmd);
+            buildFile(cmd, mode.env);
         }
         catch (error) {
             throw new Error(`Failed to build ${path.basename(file)} with ${getBuildStderr(error)}`);
         }
     }
 }
+function resolveArtifactFileName(file, target, modeName) {
+    const base = path
+        .basename(file)
+        .replace(/\.spec\.ts$/, "")
+        .replace(/\.ts$/, "");
+    if (!modeName) {
+        return `${path.basename(file).replace(".ts", ".wasm")}`;
+    }
+    return `${base}.${modeName}.${target}.wasm`;
+}
 function resolveInputPatterns(configured, selectors) {
-    const configuredInputs = Array.isArray(configured) ? configured : [configured];
+    const configuredInputs = Array.isArray(configured)
+        ? configured
+        : [configured];
     if (!selectors.length)
         return configuredInputs;
     const patterns = new Set();
@@ -60,15 +74,12 @@ function ensureDeps(config) {
             process.exit(1);
         }
     }
-    if (!hasJsonAsTransform()) {
-        console.log(`${chalk.bgRed(" ERROR ")}${chalk.dim(":")} could not find json-as. Install it to compile as-test suites.`);
-        process.exit(1);
-    }
 }
-function buildFile(command) {
+function buildFile(command, env) {
     execSync(command, {
         stdio: ["ignore", "pipe", "pipe"],
         encoding: "utf8",
+        env,
     });
 }
 function getBuildStderr(error) {
@@ -90,7 +101,6 @@ function getBuildStderr(error) {
 function getBuildArgs(config) {
     let buildArgs = "";
     buildArgs += " --transform as-test/transform";
-    buildArgs += " --transform json-as/transform";
     if (hasTryAsRuntime()) {
         buildArgs += " --transform try-as/transform";
     }
@@ -124,8 +134,3 @@ function hasTryAsRuntime() {
     return (existsSync(path.join(process.cwd(), "node_modules/try-as")) ||
         existsSync(path.join(process.cwd(), "node_modules/try-as/package.json")));
 }
-function hasJsonAsTransform() {
-    return (existsSync(path.join(process.cwd(), "node_modules/json-as/transform.js")) ||
-        existsSync(path.join(process.cwd(), "node_modules/json-as/transform.ts")) ||
-        existsSync(path.join(process.cwd(), "node_modules/json-as/transform")));
-}

package/bin/index.js

@@ -3,7 +3,7 @@ import chalk from "chalk";
 import { build } from "./build.js";
 import { createRunReporter, run } from "./run.js";
 import { init } from "./init.js";
-import { getCliVersion, loadConfig } from "./util.js";
+import { getCliVersion, loadConfig, resolveModeNames } from "./util.js";
 import * as path from "path";
 import { glob } from "glob";
 const _args = process.argv.slice(2);
@@ -12,6 +12,7 @@ const args = [];
 const COMMANDS = ["run", "build", "test", "init"];
 const version = getCliVersion();
 const configPath = resolveConfigPath(_args);
+const selectedModes = resolveModeNames(_args);
 for (const arg of _args) {
     if (arg.startsWith("-"))
         flags.push(arg);
@@ -37,16 +38,19 @@ else if (COMMANDS.includes(args[0])) {
         verbose: flags.includes("--verbose"),
     };
     if (command === "build") {
-        build(configPath).catch((error) => {
+        runBuildModes(configPath, commandArgs, selectedModes).catch((error) => {
             printCliError(error);
             process.exit(1);
         });
     }
     else if (command === "run") {
-        run(runFlags, configPath);
+        runRuntimeModes(runFlags, configPath, commandArgs, selectedModes).catch((error) => {
+            printCliError(error);
+            process.exit(1);
+        });
     }
     else if (command === "test") {
-        runTestSequential(runFlags, configPath, commandArgs).catch((error) => {
+        runTestModes(runFlags, configPath, commandArgs, selectedModes).catch((error) => {
             printCliError(error);
             process.exit(1);
         });
@@ -109,6 +113,10 @@ function info() {
         "Initialize an empty testing template");
     console.log("");
     console.log(chalk.bold("Flags:"));
+    console.log("   " +
+        chalk.bold.blue("--mode <name[,name...]>") +
+        "       " +
+        "Run one or multiple named config modes");
     console.log("   " +
         chalk.bold.blue("--config <path>") +
         "               " +
@@ -133,6 +141,10 @@ function info() {
         chalk.bold.blue("--verbose") +
         "                     " +
         "Print each suite start/end line");
+    console.log("   " +
+        chalk.bold.blue("--reporter <name|path>") +
+        "        " +
+        "Use built-in reporter (default|tap) or custom module path");
     console.log("");
     console.log(chalk.dim("If your using this, consider dropping a star, it would help a lot!") + "\n");
     console.log("View the repo:                   " +
@@ -176,9 +188,26 @@ function resolveCommandArgs(rawArgs, command) {
             i++;
             continue;
         }
+        if (arg == "--mode") {
+            i++;
+            continue;
+        }
         if (arg.startsWith("--config=")) {
             continue;
         }
+        if (arg.startsWith("--mode=")) {
+            continue;
+        }
+        if (arg == "--reporter") {
+            i++;
+            continue;
+        }
+        if (arg.startsWith("--reporter=")) {
+            continue;
+        }
+        if (arg == "--tap") {
+            continue;
+        }
         if (arg.startsWith("-")) {
             continue;
         }
@@ -200,7 +229,7 @@ function resolveCommandTokens(rawArgs, command) {
     }
     return values;
 }
-async function runTestSequential(runFlags, configPath, selectors) {
+async function runTestSequential(runFlags, configPath, selectors, modeName) {
     const files = await resolveSelectedFiles(configPath, selectors);
     if (!files.length) {
         const scope = selectors.length > 0
@@ -208,7 +237,7 @@ async function runTestSequential(runFlags, configPath, selectors) {
             : "configured input patterns";
         throw new Error(`No test files matched: ${scope}`);
     }
-    const reporterSession = await createRunReporter(configPath);
+    const reporterSession = await createRunReporter(configPath, undefined, modeName);
     const reporter = reporterSession.reporter;
     const snapshotEnabled = runFlags.snapshot !== false;
     reporter.onRunStart?.({
@@ -221,7 +250,7 @@ async function runTestSequential(runFlags, configPath, selectors) {
     const results = [];
     let failed = false;
     for (const file of files) {
-        await build(configPath, [file]);
+        await build(configPath, [file], modeName);
         const artifactKey = path.basename(file).replace(/[^a-zA-Z0-9._-]/g, "_");
         const result = await run(runFlags, configPath, [file], false, {
             reporter,
@@ -229,6 +258,7 @@ async function runTestSequential(runFlags, configPath, selectors) {
             emitRunComplete: false,
             logFileName: `test.${artifactKey}.log.json`,
             coverageFileName: `coverage.${artifactKey}.log.json`,
+            modeName,
         });
         results.push(result);
         if (result?.failed)
@@ -244,6 +274,34 @@ async function runTestSequential(runFlags, configPath, selectors) {
         stats: summary.stats,
         reports: summary.reports,
     });
+    return failed;
+}
+async function runBuildModes(configPath, selectors, modes) {
+    const targets = modes.length ? modes : [undefined];
+    for (const modeName of targets) {
+        await build(configPath, selectors, modeName);
+    }
+}
+async function runRuntimeModes(runFlags, configPath, selectors, modes) {
+    const targets = modes.length ? modes : [undefined];
+    let failed = false;
+    for (const modeName of targets) {
+        const result = await run(runFlags, configPath, selectors, false, {
+            modeName,
+        });
+        if (result.failed)
+            failed = true;
+    }
+    process.exit(failed ? 1 : 0);
+}
+async function runTestModes(runFlags, configPath, selectors, modes) {
+    const targets = modes.length ? modes : [undefined];
+    let failed = false;
+    for (const modeName of targets) {
+        const modeFailed = await runTestSequential(runFlags, configPath, selectors, modeName);
+        if (modeFailed)
+            failed = true;
+    }
     process.exit(failed ? 1 : 0);
 }
 async function resolveSelectedFiles(configPath, selectors) {
@@ -252,7 +310,7 @@ async function resolveSelectedFiles(configPath, selectors) {
     const patterns = resolveInputPatterns(config.input, selectors);
     const matches = await glob(patterns);
     const specs = matches.filter((file) => file.endsWith(".spec.ts"));
-    return [...new Set(specs)];
+    return [...new Set(specs)].sort((a, b) => a.localeCompare(b));
 }
 function resolveInputPatterns(configured, selectors) {
     const configuredInputs = Array.isArray(configured) ? configured : [configured];

package/bin/init.js

@@ -23,7 +23,7 @@ export async function init(rawArgs) {
         printPlan(root, target, example);
         if (!options.yes) {
             const cont = (await ask("Continue? [Y/n] ", rl)).toLowerCase().trim();
-            if (["y", "yes"].includes(cont)) {
+            if (["n", "no"].includes(cont)) {
                 console.log("Exiting.");
                 return;
             }
@@ -148,11 +148,9 @@ function printPlan(root, target, example) {
     if (example != "none") {
         console.log(chalk.dim("    assembly/__tests__/example.spec.ts"));
     }
-    if (target == "wasi") {
+    if (target == "wasi" || target == "bindings") {
         console.log(chalk.dim("    .as-test/runners/default.wasi.js"));
-    }
-    if (target == "bindings") {
-        console.log(chalk.dim("    .as-test/runners/default.run.js"));
+        console.log(chalk.dim("    .as-test/runners/default.bindings.js"));
     }
     console.log(chalk.dim("    package.json"));
     console.log("");
@@ -171,6 +169,7 @@ function applyInit(root, target, example, force) {
     if (target == "wasi" || target == "bindings") {
         ensureDir(root, ".as-test/runners", summary);
     }
+    ensureGitignoreIncludesAsTestDirs(root, summary);
     const configPath = path.join(root, "as-test.config.json");
     const config = loadConfig(configPath, false);
     config.$schema = "./node_modules/as-test/as-test.config.schema.json";
@@ -179,7 +178,7 @@ function applyInit(root, target, example, force) {
         config.runOptions.runtime.cmd = "node ./.as-test/runners/default.wasi.js <file>";
     }
     else {
-        config.runOptions.runtime.cmd = "node ./.as-test/runners/default.run.js <file>";
+        config.runOptions.runtime.cmd = "node ./.as-test/runners/default.bindings.js <file>";
     }
     writeJson(configPath, config, summary, "as-test.config.json");
     if (example != "none") {
@@ -187,13 +186,13 @@ function applyInit(root, target, example, force) {
         const content = example == "minimal" ? buildMinimalExampleSpec() : buildFullExampleSpec();
         writeManagedFile(examplePath, content, force, summary, "assembly/__tests__/example.spec.ts");
     }
-    if (target == "wasi") {
+    if (target == "wasi" || target == "bindings") {
         const runnerPath = path.join(root, ".as-test/runners/default.wasi.js");
         writeManagedFile(runnerPath, buildWasiRunner(), force, summary, ".as-test/runners/default.wasi.js");
     }
-    if (target == "bindings") {
-        const runnerPath = path.join(root, ".as-test/runners/default.run.js");
-        writeManagedFile(runnerPath, buildBindingsRunner(), force, summary, ".as-test/runners/default.run.js");
+    if (target == "wasi" || target == "bindings") {
+        const runnerPath = path.join(root, ".as-test/runners/default.bindings.js");
+        writeManagedFile(runnerPath, buildBindingsRunner(), force, summary, ".as-test/runners/default.bindings.js");
     }
     const pkgPath = path.join(root, "package.json");
     const pkg = existsSync(pkgPath)
@@ -232,6 +231,31 @@ function ensureDir(root, rel, summary) {
     mkdirSync(full, { recursive: true });
     summary.created.push(rel + "/");
 }
+function ensureGitignoreIncludesAsTestDirs(root, summary) {
+    const rel = ".gitignore";
+    const fullPath = path.join(root, rel);
+    const entries = ["!.as-test/runners/", "!.as-test/snapshots/"];
+    const existed = existsSync(fullPath);
+    const source = existed ? readFileSync(fullPath, "utf8") : "";
+    const lines = source.split(/\r?\n/);
+    const missing = entries.filter((entry) => !lines.some((line) => line.trim() == entry));
+    if (!missing.length) {
+        return;
+    }
+    const eol = source.includes("\r\n") ? "\r\n" : "\n";
+    let output = source;
+    if (output.length &&
+        !output.endsWith("\n") &&
+        !output.endsWith("\r\n")) {
+        output += eol;
+    }
+    output += missing.join(eol) + eol;
+    writeFileSync(fullPath, output);
+    if (existed)
+        summary.updated.push(rel);
+    else
+        summary.created.push(rel);
+}
 function writeJson(fullPath, value, summary, displayPath) {
     const rel = displayPath ??
         path.relative(process.cwd(), fullPath) ??
@@ -449,7 +473,7 @@ function withNodeIo(imports = {}) {
 
 const wasmPathArg = process.argv[2];
 if (!wasmPathArg) {
-  process.stderr.write("usage: node ./.as-test/runners/default.run.js <file.wasm>\\n");
+  process.stderr.write("usage: node ./.as-test/runners/default.bindings.js <file.wasm>\\n");
   process.exit(1);
 }
 

package/bin/reporters/tap.js

@@ -0,0 +1,294 @@
+import * as path from "path";
+import { mkdirSync, writeFileSync } from "fs";
+export function createTapReporter(context, config = {}) {
+    return new TapReporter(context, normalizeTapConfig(config));
+}
+export const createReporter = (context) => {
+    return createTapReporter(context);
+};
+class TapReporter {
+    constructor(context, config) {
+        this.context = context;
+        this.config = config;
+    }
+    onRunComplete(event) {
+        const points = collectTapPoints(event.reports);
+        const output = buildTapDocument(points);
+        this.context.stdout.write(output);
+        for (const point of points) {
+            if (point.status != "fail")
+                continue;
+            emitGitHubAnnotation(this.context, point);
+        }
+        this.writeArtifacts(points, output);
+    }
+    writeArtifacts(points, output) {
+        if (this.config.mode == "per-file") {
+            this.writePerFileArtifacts(points);
+            return;
+        }
+        const outFile = path.resolve(process.cwd(), this.config.outFile);
+        mkdirSync(path.dirname(outFile), { recursive: true });
+        writeFileSync(outFile, output);
+    }
+    writePerFileArtifacts(points) {
+        const groups = new Map();
+        for (const point of points) {
+            const key = point.file?.length ? point.file : "unknown";
+            if (!groups.has(key))
+                groups.set(key, []);
+            groups.get(key).push(point);
+        }
+        let unknownIndex = 0;
+        const outDir = path.resolve(process.cwd(), this.config.outDir);
+        mkdirSync(outDir, { recursive: true });
+        for (const [fileKey, filePoints] of groups) {
+            const fileName = fileKey == "unknown"
+                ? `unknown-${++unknownIndex}.tap`
+                : toTapFileName(fileKey);
+            const outFile = path.join(outDir, fileName);
+            writeFileSync(outFile, buildTapDocument(filePoints));
+        }
+    }
+}
+function normalizeTapConfig(config) {
+    const mode = config.mode == "per-file" ? "per-file" : "single-file";
+    const outDir = typeof config.outDir == "string" && config.outDir.trim().length
+        ? config.outDir.trim()
+        : "./.as-test/reports";
+    const outFile = typeof config.outFile == "string" && config.outFile.trim().length
+        ? config.outFile.trim()
+        : path.join(outDir, "report.tap");
+    return {
+        mode,
+        outDir,
+        outFile,
+    };
+}
+function toTapFileName(file) {
+    const normalized = file
+        .replace(/^[.\\/]+/, "")
+        .replace(/[\\/]/g, "__")
+        .replace(/\.[^.]+$/, "");
+    return `${normalized}.tap`;
+}
+function buildTapDocument(points) {
+    const totals = {
+        pass: 0,
+        fail: 0,
+        skip: 0,
+    };
+    const lines = [];
+    lines.push("TAP version 13");
+    lines.push(`1..${points.length}`);
+    for (let i = 0; i < points.length; i++) {
+        const point = points[i];
+        const id = i + 1;
+        const name = sanitizeTap(point.name.length ? point.name : `test ${id}`);
+        if (point.status == "fail") {
+            totals.fail++;
+            lines.push(`not ok ${id} - ${name}`);
+            lines.push(...buildFailDetails(point));
+            continue;
+        }
+        if (point.status == "skip") {
+            totals.skip++;
+            lines.push(`ok ${id} - ${name} # SKIP`);
+            continue;
+        }
+        totals.pass++;
+        lines.push(`ok ${id} - ${name}`);
+    }
+    lines.push(`# tests ${points.length}`);
+    lines.push(`# pass ${totals.pass}`);
+    if (totals.skip) {
+        lines.push(`# skip ${totals.skip}`);
+    }
+    lines.push(`# fail ${totals.fail}`);
+    return lines.join("\n") + "\n";
+}
+function buildFailDetails(point) {
+    const lines = ["  ---", `  message: ${JSON.stringify(point.message ?? "assertion failed")}`];
+    if (point.file) {
+        lines.push(`  file: ${JSON.stringify(point.file)}`);
+    }
+    if (point.line) {
+        lines.push(`  line: ${point.line}`);
+    }
+    if (point.column) {
+        lines.push(`  column: ${point.column}`);
+    }
+    if (point.matcher) {
+        lines.push(`  matcher: ${JSON.stringify(point.matcher)}`);
+    }
+    if (point.expected != null) {
+        lines.push(`  expected: ${JSON.stringify(point.expected)}`);
+    }
+    if (point.actual != null) {
+        lines.push(`  actual: ${JSON.stringify(point.actual)}`);
+    }
+    if (point.durationMs != null) {
+        lines.push(`  duration_ms: ${Math.round(point.durationMs * 1000) / 1000}`);
+    }
+    lines.push("  ...");
+    return lines;
+}
+function collectTapPoints(reports) {
+    const points = [];
+    if (!Array.isArray(reports))
+        return points;
+    for (const report of reports) {
+        const reportAny = report;
+        const file = String(reportAny.file ?? "");
+        const suites = Array.isArray(reportAny.suites)
+            ? reportAny.suites
+            : [];
+        for (const suite of suites) {
+            collectTapPointsFromSuite(suite, file, [], points);
+        }
+    }
+    return points;
+}
+function collectTapPointsFromSuite(suite, file, pathStack, points) {
+    const suiteAny = suite;
+    const description = String(suiteAny.description ?? "suite");
+    const fullPath = [...pathStack, description];
+    const localFile = suiteAny.file ? String(suiteAny.file) : file;
+    const childSuites = Array.isArray(suiteAny.suites)
+        ? suiteAny.suites
+        : [];
+    const tests = Array.isArray(suiteAny.tests)
+        ? suiteAny.tests
+        : [];
+    const suiteKind = String(suiteAny.kind ?? "");
+    const durationMs = suiteDuration(suiteAny.time);
+    if (tests.length > 0) {
+        for (let i = 0; i < tests.length; i++) {
+            const test = tests[i];
+            const location = parseLocation(test.location);
+            const name = tests.length > 1
+                ? `${fullPath.join(" > ")} #${i + 1}`
+                : fullPath.join(" > ");
+            const status = normalizeStatus(test.verdict);
+            const matcher = stringifyValue(test.instr);
+            const expected = stringifyValue(test.right);
+            const actual = stringifyValue(test.left);
+            const message = buildFailureMessage(stringifyValue(test.message), matcher, expected, actual);
+            points.push({
+                name,
+                status,
+                file: localFile,
+                line: location.line,
+                column: location.column,
+                matcher,
+                expected,
+                actual,
+                message,
+                durationMs,
+            });
+        }
+    }
+    else if (childSuites.length == 0 &&
+        (suiteKind == "test" ||
+            suiteKind == "it" ||
+            suiteKind == "xtest" ||
+            suiteKind == "xit")) {
+        points.push({
+            name: fullPath.join(" > "),
+            status: normalizeStatus(suiteAny.verdict),
+            file: localFile,
+            durationMs,
+        });
+    }
+    for (const child of childSuites) {
+        collectTapPointsFromSuite(child, localFile, fullPath, points);
+    }
+}
+function suiteDuration(value) {
+    const time = value;
+    if (!time)
+        return undefined;
+    const start = Number(time.start ?? 0);
+    const end = Number(time.end ?? 0);
+    if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) {
+        return undefined;
+    }
+    return end - start;
+}
+function parseLocation(value) {
+    const text = String(value ?? "").trim();
+    if (!text.length)
+        return {};
+    const match = /^(\d+)(?::(\d+))?$/.exec(text);
+    if (!match)
+        return {};
+    const line = Number(match[1]);
+    const column = match[2] ? Number(match[2]) : undefined;
+    return {
+        line: Number.isFinite(line) && line > 0 ? line : undefined,
+        column: typeof column == "number" && Number.isFinite(column) && column > 0
+            ? column
+            : undefined,
+    };
+}
+function normalizeStatus(verdict) {
+    const value = String(verdict ?? "none");
+    if (value == "fail")
+        return "fail";
+    if (value == "ok")
+        return "ok";
+    return "skip";
+}
+function stringifyValue(value) {
+    if (value == null)
+        return "";
+    if (typeof value == "string")
+        return value;
+    try {
+        return JSON.stringify(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+function buildFailureMessage(message, matcher, expected, actual) {
+    if (message.length)
+        return message;
+    if (matcher.length && expected.length && actual.length) {
+        return `${matcher} expected ${expected} but received ${actual}`;
+    }
+    if (matcher.length)
+        return `${matcher} failed`;
+    return "assertion failed";
+}
+function sanitizeTap(name) {
+    return name.replace(/\s+/g, " ").replace(/#/g, "\\#").trim();
+}
+function emitGitHubAnnotation(context, point) {
+    if (process.env.GITHUB_ACTIONS != "true" || point.status != "fail")
+        return;
+    const properties = [];
+    if (point.file) {
+        properties.push(`file=${escapeGithubValue(point.file, true)}`);
+    }
+    if (point.line) {
+        properties.push(`line=${point.line}`);
+    }
+    if (point.column) {
+        properties.push(`col=${point.column}`);
+    }
+    properties.push(`title=${escapeGithubValue("as-test", true)}`);
+    const message = point.message?.length ? point.message : "assertion failed";
+    const detail = `${message} | test=${point.name}`;
+    context.stdout.write(`::error ${properties.join(",")}::${escapeGithubValue(detail)}\n`);
+}
+function escapeGithubValue(value, property = false) {
+    let output = value
+        .replace(/%/g, "%25")
+        .replace(/\r/g, "%0D")
+        .replace(/\n/g, "%0A");
+    if (property) {
+        output = output.replace(/:/g, "%3A").replace(/,/g, "%2C");
+    }
+    return output;
+}