@specific.dev/spectest 0.91.2 → 0.92.0

package/src/daemon.ts

@@ -103,7 +103,9 @@ import {
   sanitizeSegment,
 } from "./harness/volume-paths.js";
 import { pollUntilReady } from "./harness/ready-poll.js";
-import { fsUsageDiagnostics, runWrapperRules } from "./harness/wrapper-rules.js";
+import { fsUsageDiagnostics } from "./harness/wrapper-rules.js";
+import { runTypecheck, type TypecheckReport } from "./harness/typecheck.js";
+import { evaluateChecked } from "./harness/eval-typecheck.js";
 import type { WrapperDiagnostic } from "./harness/wrapper-rules.js";
 import { cpus, tmpdir } from "node:os";
 import { APP_DIR, WORKSPACE, resolveExistingProjectPath, resolveProjectPath, takeProjectReads } from "./project-files.js";
@@ -6246,7 +6248,12 @@ async function evalCode(code: string): Promise<EvalResult> {
     filePath = path.join(EVAL_DIR, `${randomUUID()}.ts`);
     await fs.writeFile(filePath, prepared.code);
     const moduleUrl = pathToFileURL(filePath).href;
-    const mod = (await runInstrumented(fetchScope, () => import(moduleUrl))) as { default?: unknown };
+    const mod = await evaluateChecked(
+      { file: filePath, appDir: APP_DIR, typecheckDir: TYPECHECK_DIR, run: shx,
+        projectEntry: loaded ? resolveEntry() : undefined },
+      (text) => { chunks.push(text); },
+      () => runInstrumented(fetchScope, () => import(moduleUrl)) as Promise<{ default?: unknown }>,
+    );
     outcome = { ok: true, result: safeSerialize(mod.default) };
   } catch (err) {
     const e = err as Error;
@@ -6327,46 +6334,11 @@ async function evalCode(code: string): Promise<EvalResult> {
  *  blocking rules, which always use THIS copy rather than the project's own —
  *  can be driven outside a VM. */
 const TYPECHECK_DIR = process.env.SPECTEST_TYPECHECK_DIR ?? "/opt/spectest/typecheck";
-/** Cap on errors shipped in the report; `totalErrors` carries the true count. */
-const TYPECHECK_ERROR_CAP = 50;
-
-interface TypecheckError {
-  /** Path relative to the project's spectest/ dir (e.g. `tests/checkout.ts`). */
-  file: string;
-  line: number;
-  column: number;
-  /** `TS2345` from the compiler, or a `SPECTEST….` rule code. */
-  code: string;
-  message: string;
-}
-
-interface TypecheckReport {
-  status: "ok" | "errors" | "skipped" | "failed";
-  errors: TypecheckError[];
-  totalErrors: number;
-  durationMs: number;
-  /** Why the check was skipped / how it failed. */
-  detail?: string;
-  /**
-   * Findings that FAIL the run, as opposed to `errors`, which only advise.
-   * A diagnostic qualifies only when it is true of the program that runs
-   * rather than of the declared types alone — see `harness/wrapper-rules.ts`
-   * for why no `tsc` code can be in here and why these can.
-   *
-   * The list is the rollout mechanism as well as the payload: an older SDK
-   * never sends the field, `#[serde(default)]` leaves it empty on the server,
-   * and the gate is inert for every project that has not moved to a version
-   * carrying it. The version a project runs is already its own choice
-   * (`sdk.rs` honours the declared lower bound), so the bump is the consent.
-   */
-  blocking?: TypecheckError[];
-}
-
 let TYPECHECK: Promise<TypecheckReport> | null = null;
 
 /** Fire-and-forget kickoff; the promise is parked for POST /typecheck. */
 function startTypecheck(): void {
-  TYPECHECK = runTypecheck().catch((err) => ({
+  TYPECHECK = runProjectTypecheck().catch((err) => ({
     status: "failed" as const,
     errors: [],
     totalErrors: 0,
@@ -6377,149 +6349,9 @@ function startTypecheck(): void {
   TYPECHECK.catch(() => {});
 }
 
-/** The tsconfig used when the project doesn't ship its own. `lib` is left to
- *  the target default (which includes DOM — browser.evaluate callbacks
- *  reference `document`); `types` pulls Bun's ambient globals from the baked
- *  install via `typeRoots` (the walk-up from APP_DIR never reaches it). */
-function generatedTsconfig(): string {
-  return JSON.stringify(
-    {
-      compilerOptions: {
-        target: "esnext",
-        module: "esnext",
-        moduleResolution: "bundler",
-        strict: true,
-        noEmit: true,
-        skipLibCheck: true,
-        esModuleInterop: true,
-        resolveJsonModule: true,
-        jsx: "react-jsx",
-        types: ["bun"],
-        typeRoots: [
-          path.join(APP_DIR, "node_modules", "@types"),
-          path.join(TYPECHECK_DIR, "node_modules", "@types"),
-        ],
-      },
-      include: ["**/*.ts", "**/*.tsx"],
-      exclude: ["node_modules"],
-    },
-    null,
-    2,
-  );
-}
-
-/** Parse `--pretty false` tsc output: `file(line,col): error TScode: msg`,
- *  with indented elaboration lines folded into the preceding error. */
-function parseTscOutput(output: string): {
-  errors: TypecheckError[];
-  total: number;
-  suppressed: number;
-} {
-  const errors: TypecheckError[] = [];
-  let total = 0;
-  let suppressed = 0;
-  let last: TypecheckError | null = null;
-  for (const line of output.split("\n")) {
-    const m = /^(.+?)\((\d+),(\d+)\): error (TS\d+): (.*)$/.exec(line);
-    if (!m) {
-      // Elaboration lines are indented; fold them into the last kept error.
-      if (last && /^\s+\S/.test(line)) last.message += `\n${line.trimEnd()}`;
-      continue;
-    }
-    const [, rawFile, ln, col, code, message] = m;
-    // Keep only diagnostics in the user's files. tsc runs with cwd=APP_DIR
-    // (the daemon's cwd), so project files come out relative ("tests/x.ts")
-    // and anything outside — the SDK at /opt/spectest/sdk, node_modules —
-    // is `../…` or absolute. Errors in our own SDK are ours to fix, not the
-    // user's to read.
-    const abs = path.resolve(APP_DIR, rawFile!);
-    const inApp = abs.startsWith(APP_DIR + path.sep) && !abs.includes(`${path.sep}node_modules${path.sep}`);
-    if (!inApp) {
-      suppressed += 1;
-      last = null;
-      continue;
-    }
-    total += 1;
-    const err: TypecheckError = {
-      file: path.relative(APP_DIR, abs),
-      line: Number(ln),
-      column: Number(col),
-      code: code!,
-      message: message!,
-    };
-    if (errors.length < TYPECHECK_ERROR_CAP) {
-      errors.push(err);
-      last = err;
-    } else {
-      last = null;
-    }
-  }
-  return { errors, total, suppressed };
-}
-
-async function runTypecheck(): Promise<TypecheckReport> {
-  const started = Date.now();
-  // The project's own typescript wins over the baked copy.
-  const tsc = [
-    path.join(APP_DIR, "node_modules", "typescript", "bin", "tsc"),
-    path.join(TYPECHECK_DIR, "node_modules", "typescript", "bin", "tsc"),
-  ].find((p) => existsSync(p));
-  if (!tsc) {
-    return {
-      status: "skipped",
-      errors: [],
-      totalErrors: 0,
-      durationMs: 0,
-      detail: "typescript is not installed (base image predates the baked typechecker)",
-    };
-  }
-  let config = path.join(APP_DIR, "tsconfig.json");
-  if (!existsSync(config)) {
-    config = path.join(APP_DIR, ".spectest-tsconfig.json");
-    await fs.writeFile(config, generatedTsconfig());
-  }
-  const res = await shx(process.execPath, [tsc, "--noEmit", "--pretty", "false", "-p", config], 120_000);
-  const durationMs = Date.now() - started;
-  if (res.code === 124) {
-    return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: "typecheck timed out" };
-  }
-  const { errors, total, suppressed } = parseTscOutput(res.stdout + res.stderr);
-  const wrapper = [...(await runWrapperRulesReport(config)), ...(await rawFsReport())];
-  const strip = ({ file, line, column, code, message }: WrapperDiagnostic): TypecheckError => ({
-    file,
-    line,
-    column,
-    code,
-    message,
-  });
-  const blocking = wrapper.filter((d) => d.blocking).map(strip);
-  const advisory = wrapper.filter((d) => !d.blocking).map(strip);
-  if (errors.length === 0 && wrapper.length === 0) {
-    // Exit 0 → clean. Non-zero with no *user-file* diagnostics is either
-    // all-suppressed (still ok from the user's perspective) or a compiler
-    // crash (config not found, OOM) — surface the latter.
-    if (res.code !== 0 && suppressed === 0) {
-      const tail = (res.stderr || res.stdout).trim().split("\n").slice(-5).join("\n");
-      return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: tail || `tsc exited ${res.code}` };
-    }
-    if (suppressed > 0) {
-      console.warn(`[typecheck] ${suppressed} diagnostic(s) outside the project suppressed`);
-    }
-    return { status: "ok", errors: [], totalErrors: 0, durationMs };
-  }
-  // The wrapper findings ride the same list the CLI and the dashboard already
-  // render; `blocking` is a second view of the ones that also fail the run, so
-  // nothing has to learn a new shape to show them. Blocking leads, because
-  // `errors` is capped and the entries that stopped the run must never be the
-  // ones the cap drops.
-  const all = [...blocking, ...advisory, ...errors];
-  return {
-    status: "errors",
-    errors: all.slice(0, TYPECHECK_ERROR_CAP),
-    totalErrors: total + wrapper.length,
-    durationMs,
-    ...(blocking.length > 0 ? { blocking } : {}),
-  };
+async function runProjectTypecheck(): Promise<TypecheckReport> {
+  return runTypecheck({ appDir: APP_DIR, typecheckDir: TYPECHECK_DIR, run: shx,
+    projectEntry: loaded ? resolveEntry() : undefined, extraDiagnostics: rawFsReport });
 }
 
 /**
@@ -6545,37 +6377,6 @@ async function rawFsReport(): Promise<WrapperDiagnostic[]> {
   }
 }
 
-/**
- * The blocking rules, run against the BAKED compiler whatever the project
- * pins. Best-effort in every direction: an install that predates the API, a
- * config the rules cannot open, or a throw from `typescript/unstable/*` all
- * yield no findings, so a run proceeds exactly as it does today. Only a
- * definite finding can stop one.
- */
-async function runWrapperRulesReport(config: string): Promise<WrapperDiagnostic[]> {
-  const typescriptDir = path.join(TYPECHECK_DIR, "node_modules", "typescript");
-  if (!existsSync(path.join(typescriptDir, "dist", "api", "async", "api.js"))) return [];
-  try {
-    const run = await runWrapperRules({
-      typescriptDir,
-      configFile: config,
-      appDir: APP_DIR,
-      readFile: (f) => fs.readFile(f, "utf8"),
-      relative: path.relative,
-      join: path.join,
-      dirname: path.dirname,
-    });
-    if (run.status !== "ok") {
-      console.warn(`[typecheck] wrapper rules ${run.status}: ${run.detail ?? "no detail"}`);
-      return [];
-    }
-    return run.diagnostics;
-  } catch (err) {
-    console.warn(`[typecheck] wrapper rules threw: ${(err as Error)?.message ?? err}`);
-    return [];
-  }
-}
-
 // ────────────────────────────────────────────────────────────────────────
 // HTTP server
 // ────────────────────────────────────────────────────────────────────────

package/src/harness/eval-typecheck.test.ts

@@ -0,0 +1,163 @@
+import { afterEach, beforeEach, expect, test } from "bun:test";
+import fs from "node:fs/promises";
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { tmpdir } from "node:os";
+import { execFile } from "node:child_process";
+import { checkEval, evaluateChecked } from "./eval-typecheck.js";
+import { runTypecheck } from "./typecheck.js";
+
+const sdkDir = path.resolve(import.meta.dir, "../..");
+// CI/dev can select the same native TypeScript installation as the VM.
+const typecheckDir = process.env.SPECTEST_TYPECHECK_DIR ?? sdkDir;
+const hasWrapperChecker = existsSync(path.join(typecheckDir, "node_modules/typescript/dist/api/async/api.js"));
+let appDir: string;
+let file: string;
+let projectEntry: string;
+
+beforeEach(async () => {
+  appDir = await fs.mkdtemp(path.join(tmpdir(), "spectest-eval-types-"));
+  await fs.mkdir(path.join(appDir, "node_modules/@specific.dev"), { recursive: true });
+  await fs.symlink(sdkDir, path.join(appDir, "node_modules/@specific.dev/spectest"));
+  await fs.symlink(path.join(sdkDir, "node_modules/@types"), path.join(appDir, "node_modules/@types"));
+  await fs.mkdir(path.join(appDir, "spectest"));
+  await fs.mkdir(path.join(appDir, ".spectest-eval"));
+  file = path.join(appDir, ".spectest-eval/snippet.ts");
+  projectEntry = path.join(appDir, "spectest/index.ts");
+  await fs.writeFile(projectEntry, `
+import { defineEnvironment, defineFake } from "@specific.dev/spectest";
+import { supabase } from "@specific.dev/spectest/components";
+export const env = defineEnvironment({
+  name: "eval-typecheck",
+  services: { supabase: supabase().group },
+  fakes: { sample: defineFake({ hostnames: ["example.test"], state: () => ({}),
+    handler: () => new Response(), helpers: () => ({ count: () => 3 }) }) },
+});
+export default env.project([]);
+`);
+});
+
+afterEach(async () => { await fs.rm(appDir, { recursive: true, force: true }); });
+
+function run(command: string, args: string[], timeout: number): Promise<{ code: number; stdout: string; stderr: string }> {
+  return new Promise((resolve) => {
+    execFile(command, args, { cwd: appDir, timeout, maxBuffer: 16 * 1024 * 1024 }, (error, stdout, stderr) => {
+      resolve({ code: error ? (typeof error.code === "number" ? error.code : 1) : 0, stdout, stderr });
+    });
+  });
+}
+
+const opts = () => ({ appDir, file, projectEntry, typecheckDir, run });
+const query = 'const rows = await ctx.svc.supabase.sql<{ analyzed: boolean }>`SELECT true AS analyzed`;';
+
+test("eval receives the default-exported project's service/fake types and globals", async () => {
+  await fs.writeFile(file, `${query}
+const ready: boolean = rows[0].analyzed.unwrap();
+const count: number = ctx.fakes.sample.count().unwrap();
+expect(rows[0].analyzed).toBe(true);
+expectRaw(count, "count").toBe(3);
+assert(ready);
+state.saved = ready;
+export default ready;
+`);
+  const report = await checkEval(opts());
+  expect(report).toMatchObject({ status: "ok", errors: [], totalErrors: 0 });
+  expect(await fs.readdir(path.dirname(file))).toEqual(["snippet.ts"]);
+}, 30_000);
+
+test("compiler catches the Listen comparison and reports original snippet lines", async () => {
+  await fs.writeFile(file, `${query}\nconst done = rows[0].analyzed === true;\nexport default done;`);
+  const report = await checkEval(opts());
+  expect(report.errors).toContainEqual(expect.objectContaining({ file: "eval.ts", line: 2, code: "TS2367" }));
+}, 30_000);
+
+test("an untyped SQL row and nonexistent service remain diagnostics, not any", async () => {
+  await fs.writeFile(file, 'const rows = await ctx.svc.supabase.sql`SELECT true AS analyzed`;\nexport default rows[0].analyzed;\nctx.svc.missing;');
+  const report = await checkEval(opts());
+  expect(report.errors.filter((e) => e.code === "TS2339").map((e) => e.line)).toEqual([2, 3]);
+}, 30_000);
+
+test.skipIf(!hasWrapperChecker)("blocking wrapper finding prevents all snippet side effects", async () => {
+  await fs.writeFile(file, `${query}\nexport default rows[0].analyzed === true;`);
+  let executed = false;
+  const logs: string[] = [];
+  await expect(evaluateChecked(opts(), (s) => logs.push(s), async () => { executed = true; })).rejects.toThrow("snippet did not run");
+  expect(executed).toBe(false);
+  expect(logs.join("")).toContain("SPECTEST2002");
+  expect(logs.join("")).toContain(".unwrap()");
+}, 30_000);
+
+test("ordinary errors are advisory and unrelated project errors do not enter eval output", async () => {
+  await fs.appendFile(projectEntry, '\nconst unrelated: number = "wrong";');
+  await fs.writeFile(file, 'const x: number = "wrong";\nexport default x;');
+  const logs: string[] = [];
+  expect(await evaluateChecked(opts(), (s) => logs.push(s), async () => "executed")).toBe("executed");
+  expect(logs.join("")).toContain("(advisory)");
+  expect(logs.join("")).toContain("eval.ts:1:");
+  expect(logs.join("")).not.toContain("index.ts");
+}, 30_000);
+
+test.skipIf(!hasWrapperChecker)("unwrapped comparisons execute and nullable presence checks remain allowed", async () => {
+  await fs.writeFile(file, `${query}\nexport default rows[0].analyzed.unwrap() === true;`);
+  expect(await evaluateChecked(opts(), () => {}, async () => "executed")).toBe("executed");
+  await fs.writeFile(file, 'const rows = await ctx.svc.supabase.sql<{ label: string | null }>`SELECT null AS label`;\nexport default rows[0].label === null;');
+  expect((await checkEval(opts())).blocking).toBeUndefined();
+}, 30_000);
+
+test("eval without a loaded project still checks basic globals", async () => {
+  await fs.writeFile(file, 'assert(true); state.value = 1; export default ctx.testName;');
+  expect(await checkEval({ ...opts(), projectEntry: undefined })).toMatchObject({ status: "ok", totalErrors: 0 });
+}, 30_000);
+
+test("project paths are respected even when its include excludes the eval directory", async () => {
+  await fs.writeFile(path.join(appDir, "spectest/tsconfig.json"), JSON.stringify({
+    compilerOptions: { paths: { "@fixture": ["./fixture.ts"] } }, include: ["tests/**/*.ts"],
+  }));
+  await fs.writeFile(path.join(appDir, "spectest/fixture.ts"), 'export const count = 3;');
+  await fs.writeFile(file, 'import { count } from "@fixture";\nconst x: number = count;\nexport default x;');
+  expect(await checkEval(opts())).toMatchObject({ status: "ok", totalErrors: 0 });
+}, 30_000);
+
+test("unavailable compiler is reported, evaluation continues, and scratch files are removed", async () => {
+  await fs.writeFile(file, "export default 1;");
+  const logs: string[] = [];
+  expect(await evaluateChecked({ ...opts(), typecheckDir: path.join(appDir, "missing") }, (s) => logs.push(s), async () => 1)).toBe(1);
+  expect(logs.join("")).toContain("[typecheck] skipped:");
+  expect(await fs.readdir(path.dirname(file))).toEqual(["snippet.ts"]);
+});
+
+test("shared managed-test checker still reports project diagnostics", async () => {
+  await fs.appendFile(projectEntry, '\nconst wrong: number = "bad";');
+  const report = await runTypecheck({ appDir, typecheckDir, run });
+  expect(report.errors).toContainEqual(expect.objectContaining({ file: "spectest/index.ts", code: "TS2322" }));
+}, 30_000);
+
+test.skipIf(!hasWrapperChecker)("managed tests and evals classify the same comparison identically", async () => {
+  const body = `${query}\nreturn rows[0].analyzed === true;`;
+  await fs.writeFile(file, `export default await (async () => {\n${body}\n})();`);
+  const evalReport = await checkEval(opts());
+  await fs.mkdir(path.join(appDir, "spectest/tests"));
+  await fs.writeFile(path.join(appDir, "spectest/tests/comparison.ts"),
+    `import { env } from "../index";\nenv.test("comparison", async (ctx) => {\n${body}\n});`);
+  const suiteReport = await runTypecheck({ appDir, projectEntry, typecheckDir, run });
+  const findings = (report: typeof evalReport) => report.errors
+    .filter((e) => e.code === "TS2367" || e.code === "SPECTEST2002")
+    .map(({ code, message }) => ({ code, message }));
+  expect(findings(evalReport)).toEqual(findings(suiteReport));
+  expect(evalReport.blocking?.map((e) => e.code)).toEqual(["SPECTEST2002"]);
+  expect(suiteReport.blocking?.map((e) => e.code)).toEqual(["SPECTEST2002"]);
+}, 60_000);
+
+test("eval and tests honor the same project strictness", async () => {
+  await fs.writeFile(path.join(appDir, "spectest/tsconfig.json"), JSON.stringify({
+    compilerOptions: { strict: false, skipLibCheck: true, target: "esnext", module: "esnext", moduleResolution: "bundler" },
+  }));
+  await fs.writeFile(file, 'const value: number = null; export default value;');
+  await fs.writeFile(path.join(appDir, "spectest/null.ts"), 'const value: number = null; export default value;');
+  const evalReport = await checkEval(opts());
+  const suiteReport = await runTypecheck({ appDir, projectEntry, typecheckDir, run });
+  expect(evalReport.errors.some((d) => d.code === "TS2322")).toBe(false);
+  expect(suiteReport.errors.some((d) => d.code === "TS2322")).toBe(false);
+  expect(evalReport.blocking).toBeUndefined();
+  expect(suiteReport.blocking).toBeUndefined();
+}, 30_000);

package/src/harness/eval-typecheck.ts

@@ -0,0 +1,100 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { generatedTsconfig, projectTypecheckConfig, runTypecheck, type TypecheckOptions, type TypecheckReport } from "./typecheck.js";
+
+interface EvalTypecheckOptions extends Pick<TypecheckOptions, "appDir" | "typecheckDir" | "run"> {
+  file: string;
+  projectEntry?: string;
+}
+
+/** Describe the globals installed by evalCode, without importing/executing the project. */
+function globals(projectEntry?: string): string {
+  return `import type * as SDK from "@specific.dev/spectest";
+${projectEntry ? `import type project from ${JSON.stringify(projectEntry)};` : ""}
+type EvalContext = ${projectEntry
+    ? "typeof project extends SDK.Project<infer S, infer F> ? SDK.TestContext<undefined, S, F> : SDK.TestContext<undefined>"
+    : "SDK.TestContext<undefined>"};
+declare global {
+  var ctx: EvalContext;
+  var expect: typeof SDK.expect;
+  var expectRaw: typeof SDK.expectRaw;
+  var assert: typeof SDK.assert;
+  var state: Record<string, unknown>;
+}
+`;
+}
+
+export async function checkEval(opts: EvalTypecheckOptions): Promise<TypecheckReport> {
+  const started = Date.now();
+  let scratch: string | undefined;
+  try {
+    scratch = await fs.mkdtemp(path.join(path.dirname(opts.file), "check-"));
+    const declarations = path.join(scratch, "globals.d.ts");
+    await fs.writeFile(declarations, globals(opts.projectEntry));
+    const projectConfig = projectTypecheckConfig(opts);
+    const defaults = JSON.parse(generatedTsconfig(opts.appDir, opts.typecheckDir));
+    const config = {
+      ...(projectConfig ? { extends: projectConfig } : defaults),
+      compilerOptions: {
+        ...(!projectConfig ? defaults.compilerOptions : {}),
+        // Bun runs eval as ESM TypeScript, regardless of the app's build output.
+        noEmit: true,
+        skipLibCheck: true,
+        target: "esnext",
+        module: "esnext",
+        moduleResolution: "bundler",
+        moduleDetection: "force",
+        allowImportingTsExtensions: true,
+        composite: false,
+        incremental: false,
+        rootDir: opts.appDir,
+      },
+      // Project includes/excludes may omit hidden files or include the entire
+      // application. Check this snippet, with imports resolved normally.
+      files: [opts.file, declarations],
+      include: [],
+      exclude: [],
+    };
+    const configFile = path.join(scratch, "tsconfig.json");
+    await fs.writeFile(configFile, JSON.stringify(config));
+    const report = await runTypecheck({ ...opts, configFile, diagnosticFile: opts.file });
+    // Stable, snippet-relative locations; don't expose random temporary paths.
+    const normalize = (d: TypecheckReport["errors"][number]) => ({ ...d, file: "eval.ts" });
+    return {
+      ...report,
+      errors: report.errors.map(normalize),
+      ...(report.blocking ? { blocking: report.blocking.map(normalize) } : {}),
+    };
+  } catch (error) {
+    // Match managed tests: unavailable/broken checkers report their failure,
+    // while only a definite blocking finding prevents execution.
+    return {
+      status: "failed", errors: [], totalErrors: 0, durationMs: Date.now() - started,
+      detail: error instanceof Error ? error.message : String(error),
+    };
+  } finally {
+    if (scratch) await fs.rm(scratch, { recursive: true, force: true });
+  }
+}
+
+/** The same advisory/blocking policy as a managed test, before module import. */
+export async function evaluateChecked<T>(
+  opts: EvalTypecheckOptions,
+  log: (text: string) => void,
+  evaluate: () => Promise<T>,
+): Promise<T> {
+  const report = await checkEval(opts);
+  if (report.status === "failed" || report.status === "skipped") {
+    log(`[typecheck] ${report.status}: ${report.detail ?? "unavailable"}\n`);
+  } else if (report.totalErrors > 0) {
+    log(`[typecheck] ${report.totalErrors} diagnostic(s)${report.blocking?.length ? "" : " (advisory)"}\n`);
+    for (const d of report.errors) log(`${d.file}:${d.line}:${d.column}: ${d.code}: ${d.message}\n`);
+    if (report.totalErrors > report.errors.length) {
+      log(`[typecheck] ${report.totalErrors - report.errors.length} more diagnostic(s)\n`);
+    }
+  }
+  if (report.blocking?.length) {
+    throw new Error(`Typecheck: ${report.blocking.length} blocking error(s) in eval — the snippet did not run. Fix the diagnostics above before retrying.`);
+  }
+  return evaluate();
+}