@stsepelin/checktrail 0.1.0-alpha.1

package/dist/src/mutation.js

@@ -0,0 +1,370 @@
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, realpath, rm, writeFile, } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { z } from "zod";
+import { createPlan, validate } from "./engine.js";
+import { inventory, withinRoot } from "./inventory.js";
+import { VERSION } from "./types.js";
+const count = z.number().int().nonnegative();
+const digest = z.string().regex(/^[a-f0-9]{64}$/);
+const relative = z
+    .string()
+    .min(1)
+    .max(512)
+    .refine((value) => !value.includes("\\") &&
+    !value.includes("\0") &&
+    !path.posix.isAbsolute(value) &&
+    value
+        .split("/")
+        .every((part) => part !== "" && part !== "." && part !== ".."));
+export const mutationRecipeSchema = z.strictObject({
+    schemaVersion: z.literal(1),
+    profile: z.literal("node-flat-tests"),
+    mutations: z
+        .array(z.strictObject({
+        id: z
+            .string()
+            .min(1)
+            .max(128)
+            .regex(/^[a-z0-9][a-z0-9.-]*$/),
+        file: relative,
+        expected: z.string().min(1).max(4096),
+        replacement: z.string().max(4096),
+    }))
+        .min(1)
+        .max(8),
+});
+const testsSchema = z.strictObject({
+    total: count,
+    passed: count,
+    failed: count,
+    skipped: count,
+});
+const observationSchema = z.strictObject({
+    runId: z.string().uuid(),
+    outcome: z.enum(["passed", "failed", "incomplete"]),
+    durationMs: count,
+    sourceFingerprint: digest,
+    sourceChanged: z.boolean(),
+    sourceError: z.boolean(),
+    tests: testsSchema.optional(),
+});
+const trialSchema = z.strictObject({
+    id: z.string(),
+    file: relative,
+    status: z.enum(["killed", "survived", "invalid", "inconclusive", "not-run"]),
+    reason: z.string(),
+    observation: observationSchema.optional(),
+});
+const metadata = {
+    schemaVersion: z.literal(1),
+    engineVersion: z.string(),
+    profile: z.literal("node-flat-tests"),
+    channel: z.literal("advisory"),
+    provenance: z.literal("temporary-copy-mutation-experiment"),
+    recipeDigest: digest,
+    sourceFingerprint: digest,
+    finalSourceFingerprint: digest.nullable(),
+    complete: z.boolean(),
+    reason: z.string(),
+    durationMs: count,
+    runtime: z.strictObject({ name: z.literal("node"), version: z.string() }),
+    counts: z.strictObject({
+        total: count,
+        killed: count,
+        survived: count,
+        invalid: count,
+        inconclusive: count,
+        notRun: count,
+    }),
+};
+export const mutationReportSchema = z.strictObject({
+    ...metadata,
+    excluded: z.array(z.string()),
+    baseline: observationSchema.optional(),
+    trials: z.array(trialSchema).max(8),
+});
+export const mutationSummarySchema = z.strictObject(metadata);
+function observation(report) {
+    return {
+        runId: report.runId,
+        outcome: report.outcome,
+        durationMs: Math.round(report.durationMs),
+        sourceFingerprint: report.sourceFingerprint,
+        sourceChanged: report.sourceChanged,
+        sourceError: report.sourceError,
+        ...(report.checks[0]?.tests ? { tests: report.checks[0].tests } : {}),
+    };
+}
+function testIdentities(report, root) {
+    const check = report.checks[0];
+    if (report.sourceChanged ||
+        report.sourceError ||
+        report.checks.length !== 1 ||
+        check?.id !== "javascript.node-test" ||
+        !check.tests ||
+        check.tests.skipped ||
+        check.processes.length !== 1 ||
+        check.tools?.some((tool) => tool.status !== "identified"))
+        return undefined;
+    const identities = [];
+    let assertionFailures = 0;
+    try {
+        for (const line of check.processes[0].stdout.split("\n").filter(Boolean)) {
+            const event = JSON.parse(line);
+            if (event.type !== "test:pass" && event.type !== "test:fail")
+                continue;
+            const data = event.data;
+            if (!data ||
+                data.nesting !== 0 ||
+                data.details?.type !== "test" ||
+                typeof data.file !== "string" ||
+                typeof data.name !== "string" ||
+                !Number.isSafeInteger(data.line) ||
+                !Number.isSafeInteger(data.column))
+                return undefined;
+            const file = path.relative(root, data.file).split(path.sep).join("/");
+            if (!check.scope.includes(file))
+                return undefined;
+            identities.push(JSON.stringify([file, data.name, data.line, data.column]));
+            if (event.type === "test:fail") {
+                const error = data.details.error;
+                if (error?.failureType !== "testCodeFailure" ||
+                    error?.cause?.name !== "AssertionError" ||
+                    error?.cause?.code !== "ERR_ASSERTION")
+                    return undefined;
+                assertionFailures++;
+            }
+        }
+    }
+    catch {
+        return undefined;
+    }
+    if (!identities.length ||
+        new Set(identities).size !== identities.length ||
+        identities.length !== check.tests.total ||
+        assertionFailures !== check.tests.failed)
+        return undefined;
+    return { identities: identities.sort(), assertionFailures };
+}
+export async function runMutations(root, input, options) {
+    if (!options.trusted)
+        throw new Error("Mutation experiments require operator execution trust");
+    const serialized = JSON.stringify(input);
+    if (serialized === undefined || Buffer.byteLength(serialized) > 128 * 1024)
+        throw new Error("Mutation recipe exceeds input limits");
+    const recipe = mutationRecipeSchema.parse(input);
+    if (new Set(recipe.mutations.map((mutation) => mutation.id)).size !==
+        recipe.mutations.length)
+        throw new Error("Mutation IDs must be unique");
+    const timeoutMs = options.timeoutMs ?? 30_000;
+    if (!Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 120_000)
+        throw new Error("Timeout must be between 1 and 120000 milliseconds");
+    const started = performance.now();
+    const remaining = () => Math.floor(timeoutMs - (performance.now() - started));
+    const { source, plan } = await createPlan(root);
+    if (plan.projects.length !== 1 ||
+        plan.projects[0]?.path !== "." ||
+        plan.checks.length !== 1 ||
+        plan.checks[0]?.id !== "javascript.node-test" ||
+        plan.checks[0]?.unavailableReason ||
+        plan.checks[0]?.environment?.length)
+        throw new Error("Mutation profile requires one root Node test project without additional checks or environment requirements");
+    const manifest = JSON.parse(await readFile(await withinRoot(source.root, "package.json"), "utf8"));
+    for (const key of [
+        "dependencies",
+        "devDependencies",
+        "optionalDependencies",
+        "peerDependencies",
+    ])
+        if (manifest[key] && Object.keys(manifest[key]).length)
+            throw new Error("Mutation profile requires a dependency-free Node project");
+    const snapshot = new Map();
+    let snapshotBytes = 0;
+    for (const file of source.files) {
+        const bytes = await readFile(await withinRoot(source.root, file));
+        snapshotBytes += bytes.length;
+        if (bytes.length > 8 * 1024 * 1024 || snapshotBytes > 64 * 1024 * 1024)
+            throw new Error("Mutation snapshot exceeds inventory limits");
+        snapshot.set(file, bytes);
+    }
+    if ((await inventory(source.root)).fingerprint !== source.fingerprint)
+        throw new Error("Source changed while preparing mutation copies");
+    const trials = [];
+    const edits = new Map();
+    for (const mutation of recipe.mutations) {
+        const bytes = snapshot.get(mutation.file);
+        let reason = "";
+        if (!bytes ||
+            !/\.[cm]?js$/.test(mutation.file) ||
+            plan.checks[0].scope.includes(mutation.file)) {
+            reason =
+                "Target must be inventoried JavaScript source outside the selected test files";
+        }
+        else {
+            try {
+                const original = new TextDecoder("utf-8", {
+                    fatal: true,
+                    ignoreBOM: true,
+                }).decode(bytes);
+                const first = original.indexOf(mutation.expected);
+                if (first < 0 ||
+                    original.indexOf(mutation.expected, first + 1) !== -1 ||
+                    mutation.expected === mutation.replacement)
+                    reason =
+                        "Mutation must replace one unique occurrence with different text";
+                else
+                    edits.set(mutation.id, original.slice(0, first) +
+                        mutation.replacement +
+                        original.slice(first + mutation.expected.length));
+            }
+            catch {
+                reason = "Mutation target must contain valid UTF-8";
+            }
+        }
+        trials.push({
+            id: mutation.id,
+            file: mutation.file,
+            status: reason ? "invalid" : "not-run",
+            reason: reason || "Experiment has not run",
+        });
+    }
+    let baseline;
+    let reason = "Requested experiments finished";
+    const temporary = await mkdtemp(path.join(tmpdir(), "checktrail-mutations-"));
+    try {
+        async function trialCopy(name, edit) {
+            if (remaining() <= 0 || options.signal?.aborted)
+                return undefined;
+            const copy = path.join(temporary, name);
+            await mkdir(copy, { mode: 0o700 });
+            try {
+                for (const [file, bytes] of snapshot) {
+                    if (remaining() <= 0 || options.signal?.aborted)
+                        return undefined;
+                    const destination = path.join(copy, file);
+                    await mkdir(path.dirname(destination), { recursive: true });
+                    await writeFile(destination, edit?.file === file ? edit.contents : bytes);
+                }
+                const time = remaining();
+                if (time <= 0 || options.signal?.aborted)
+                    return undefined;
+                return {
+                    report: await validate(copy, {
+                        trusted: true,
+                        timeoutMs: Math.min(time, 120_000),
+                        ...(options.signal ? { signal: options.signal } : {}),
+                    }),
+                    root: await realpath(copy),
+                };
+            }
+            finally {
+                await rm(copy, { recursive: true, force: true });
+            }
+        }
+        if (!edits.size)
+            reason = "No valid mutation target was supplied";
+        else {
+            const initial = await trialCopy("baseline");
+            baseline = initial?.report;
+            const baselineTests = initial && testIdentities(initial.report, initial.root);
+            if (baseline?.outcome !== "passed" || !baselineTests)
+                reason =
+                    "A passing baseline with complete flat-test evidence is required";
+            else {
+                for (const [index, mutation] of recipe.mutations.entries()) {
+                    const contents = edits.get(mutation.id);
+                    if (contents === undefined)
+                        continue;
+                    const trial = trials[index];
+                    const run = await trialCopy(`trial-${index}`, {
+                        file: mutation.file,
+                        contents,
+                    });
+                    if (!run) {
+                        reason = "Experiment interrupted or total time budget exhausted";
+                        break;
+                    }
+                    trial.observation = observation(run.report);
+                    const evidence = testIdentities(run.report, run.root);
+                    if (!evidence ||
+                        JSON.stringify(evidence.identities) !==
+                            JSON.stringify(baselineTests.identities)) {
+                        trial.status = "inconclusive";
+                        trial.reason =
+                            "Missing or changed test evidence, non-assertion failure, or source mutation during execution";
+                    }
+                    else if (run.report.outcome === "passed") {
+                        trial.status = "survived";
+                        trial.reason = "The same observed tests passed with this mutation";
+                    }
+                    else if (run.report.outcome === "failed" &&
+                        evidence.assertionFailures > 0) {
+                        trial.status = "killed";
+                        trial.reason =
+                            "The same observed tests completed with native assertion failures";
+                    }
+                    else {
+                        trial.status = "inconclusive";
+                        trial.reason =
+                            "Native validation did not provide conclusive mutation evidence";
+                    }
+                }
+            }
+        }
+    }
+    finally {
+        await rm(temporary, { recursive: true, force: true });
+    }
+    let finalSourceFingerprint = null;
+    try {
+        finalSourceFingerprint = (await inventory(source.root)).fingerprint;
+    }
+    catch {
+        /* Retain experiment observations when final source inspection fails. */
+    }
+    const counts = {
+        total: trials.length,
+        killed: 0,
+        survived: 0,
+        invalid: 0,
+        inconclusive: 0,
+        notRun: 0,
+    };
+    for (const trial of trials)
+        counts[trial.status === "not-run" ? "notRun" : trial.status]++;
+    const complete = finalSourceFingerprint === source.fingerprint &&
+        counts.invalid + counts.inconclusive + counts.notRun === 0;
+    if (finalSourceFingerprint !== source.fingerprint)
+        reason =
+            "Original source changed or could not be inspected after the experiment";
+    else if (!complete && reason === "Requested experiments finished")
+        reason = "Some requested experiments lack conclusive evidence";
+    return mutationReportSchema.parse({
+        schemaVersion: 1,
+        engineVersion: VERSION,
+        profile: recipe.profile,
+        channel: "advisory",
+        provenance: "temporary-copy-mutation-experiment",
+        recipeDigest: createHash("sha256")
+            .update(JSON.stringify(recipe))
+            .digest("hex"),
+        sourceFingerprint: source.fingerprint,
+        finalSourceFingerprint,
+        excluded: source.excluded,
+        complete,
+        reason,
+        durationMs: Math.round(performance.now() - started),
+        runtime: { name: "node", version: process.versions.node },
+        counts,
+        ...(baseline ? { baseline: observation(baseline) } : {}),
+        trials,
+    });
+}
+export function projectMutations(report, detailed) {
+    const parsed = mutationReportSchema.parse(report);
+    if (detailed)
+        return parsed;
+    return mutationSummarySchema.parse(Object.fromEntries(Object.entries(parsed).filter(([key]) => key !== "excluded" && key !== "baseline" && key !== "trials")));
+}

package/dist/src/mypy-evidence.d.ts

@@ -0,0 +1,2 @@
+import type { Check, CheckResult, ProcessResult } from "./types.js";
+export declare function mypyEvidence(check: Check, processes: ProcessResult[], root: string | undefined): Pick<CheckResult, "status" | "reason">;

package/dist/src/mypy-evidence.js

@@ -0,0 +1,63 @@
+import path from "node:path";
+import { z } from "zod";
+const schema = z.object({
+    format: z.literal("checktrail-mypy-1"),
+    version: z.literal("2.3.1"),
+    files: z.array(z.string()),
+    suppressedFiles: z.array(z.string()),
+    stdout: z.string(),
+    stderr: z.string(),
+    exitCode: z.number().int().min(0).max(2),
+});
+export function mypyEvidence(check, processes, root) {
+    const incomplete = {
+        status: "inconclusive",
+        reason: "Mypy evidence is incomplete, suppresses module errors, or does not account for every planned source.",
+    };
+    if (!root || processes.length !== 1 || !check.scope.length)
+        return incomplete;
+    const process = processes[0];
+    let value;
+    try {
+        value = JSON.parse(process.stdout);
+    }
+    catch {
+        return incomplete;
+    }
+    const unavailable = z
+        .object({
+        format: z.literal("checktrail-mypy-1"),
+        unavailable: z.string().min(1),
+    })
+        .safeParse(value);
+    if (process.exitCode === 3 && unavailable.success)
+        return { status: "unavailable", reason: unavailable.data.unavailable };
+    const parsed = schema.safeParse(value);
+    if (!parsed.success || parsed.data.exitCode !== process.exitCode)
+        return incomplete;
+    const report = parsed.data;
+    if (report.exitCode === 2)
+        return {
+            status: "error",
+            reason: "Mypy reported a configuration or execution error.",
+        };
+    if (report.exitCode === 1)
+        return {
+            status: "failed",
+            reason: "Mypy reported type-checking diagnostics.",
+        };
+    const expected = new Set(check.scope.map((file) => path.resolve(root, check.project, file)));
+    const summary = /^Success: no issues found in (\d+) source files?\s*$/.exec(report.stdout);
+    if (report.stderr.trim() ||
+        !summary ||
+        Number(summary[1]) !== expected.size ||
+        report.files.length !== expected.size ||
+        new Set(report.files).size !== expected.size ||
+        report.files.some((file) => !expected.has(file)) ||
+        report.suppressedFiles.length)
+        return incomplete;
+    return {
+        status: "passed",
+        reason: "Mypy checked every explicit source and reported no diagnostics or broad module-error suppression.",
+    };
+}

package/dist/src/mypy.d.ts

@@ -0,0 +1,2 @@
+import type { Check, Project } from "./types.js";
+export declare function mypyCheck(project: Project): Check;

package/dist/src/mypy.js

@@ -0,0 +1,59 @@
+const runner = String.raw `
+import contextlib
+import io
+import json
+import os
+import pathlib
+import sys
+
+def unavailable(reason):
+    print(json.dumps({"format": "checktrail-mypy-1", "unavailable": reason}))
+    sys.exit(3)
+
+try:
+    from mypy import api
+    from mypy.version import __version__
+except ModuleNotFoundError as error:
+    if error.name != "mypy":
+        raise
+    unavailable("mypy is not installed")
+if __version__ != "2.3.1":
+    unavailable("mypy version is outside the verified options API contract: " + __version__)
+
+from mypy.main import process_options
+
+files = [str(pathlib.Path(file).resolve()) for file in sys.argv[1:]]
+config = next((name for name in ["mypy.ini", ".mypy.ini", "pyproject.toml", "setup.cfg"] if pathlib.Path(name).is_file()), os.devnull)
+args = ["--config-file", config, "--no-incremental", "--cache-dir", os.devnull, "--no-install-types", "--check-untyped-defs", "--warn-unused-ignores", "--no-pretty", "--no-color-output", "--error-summary", "--", *files]
+capture = io.StringIO()
+with contextlib.redirect_stdout(sys.stderr):
+    sources, options = process_options(args, stdout=capture, stderr=capture)
+    suppressed = [str(pathlib.Path(source.path).resolve()) for source in sources if source.path and options.clone_for_module(source.module).ignore_errors]
+    stdout, stderr, code = api.run(args)
+print(json.dumps({"format": "checktrail-mypy-1", "version": __version__, "files": [str(pathlib.Path(source.path).resolve()) for source in sources if source.path], "suppressedFiles": suppressed, "stdout": stdout, "stderr": capture.getvalue() + stderr, "exitCode": code}))
+sys.exit(code)
+`;
+export function mypyCheck(project) {
+    const files = project.files.filter((file) => /\.pyi?$/.test(file));
+    const check = {
+        id: "python.mypy",
+        adapter: project.adapter,
+        project: project.path,
+        scope: files,
+        kind: "analysis",
+        parser: "mypy-json",
+        commands: [
+            {
+                executable: "python3",
+                args: ["-c", runner, ...files],
+                cwd: project.path,
+                env: { PYTHONDONTWRITEBYTECODE: "1" },
+            },
+        ],
+        reason: "Run installed mypy with explicit source accounting, disabled stub installation/cache writes and broad error-suppression detection.",
+    };
+    if (!files.length)
+        check.unavailableReason =
+            "No Python source or stub files were inventoried.";
+    return check;
+}

package/dist/src/node-reporter.d.ts

@@ -0,0 +1 @@
+export default function reporter(events: AsyncIterable<unknown>): AsyncGenerator<string>;

package/dist/src/node-reporter.js

@@ -0,0 +1,28 @@
+export default async function* reporter(events) {
+    for await (const event of events) {
+        yield `${JSON.stringify(event, (_key, value) => value instanceof Error
+            ? {
+                name: value.name,
+                message: value.message,
+                stack: value.stack,
+                ...("code" in value && typeof value.code === "string"
+                    ? { code: value.code }
+                    : {}),
+                ...("failureType" in value && typeof value.failureType === "string"
+                    ? { failureType: value.failureType }
+                    : {}),
+                ...(value.cause instanceof Error
+                    ? {
+                        cause: {
+                            name: value.cause.name,
+                            ...("code" in value.cause &&
+                                typeof value.cause.code === "string"
+                                ? { code: value.cause.code }
+                                : {}),
+                        },
+                    }
+                    : {}),
+            }
+            : value)}\n`;
+    }
+}

package/dist/src/nuxt-evidence.d.ts

@@ -0,0 +1,2 @@
+import type { Check, CheckResult, ProcessResult } from "./types.js";
+export declare function nuxtEvidence(check: Check, processes: ProcessResult[]): Partial<CheckResult>;