@specific.dev/spectest 0.91.1 → 0.92.0

package/src/components/k3s-apply.test.ts

@@ -0,0 +1,90 @@
+import { expect, spyOn, test } from "bun:test";
+import { KubernetesObjectApi } from "@kubernetes/client-node";
+import { ApiException } from "@kubernetes/client-node/dist/gen/apis/exception.js";
+import { isTransientApplyError, k3s } from "./k3s";
+
+const missingNamespace = {
+  apiVersion: "v1", kind: "Status", status: "Failure", code: 404, reason: "NotFound",
+  message: 'namespaces "agent-sandbox-system" not found',
+  details: { name: "agent-sandbox-system", kind: "namespaces" },
+};
+const exception = (code: number, body: unknown) => new ApiException(code, "Unknown API Status Code!", body, {});
+
+for (const format of ["object", "JSON string"] as const) {
+  const encode = (body: unknown) => format === "object" ? body : JSON.stringify(body);
+  test(`missing-namespace ApiException with ${format} body is transient`, () => {
+    const error = exception(404, encode(missingNamespace));
+    // This is the production client's escaped representation, not a hand-made Error.
+    expect(/namespaces? "[^"]+" not found/i.test(error.message)).toBe(false);
+    expect(isTransientApplyError(error)).toBe(true);
+    expect(isTransientApplyError(exception(404, encode({ ...missingNamespace, message: "Not Found" })))).toBe(true);
+    // The Status message also works when structured details are absent.
+    expect(isTransientApplyError(exception(404, encode({ ...missingNamespace, details: undefined })))).toBe(true);
+  });
+
+  test(`apply converges after a missing namespace with ${format} response body`, async () => {
+    const error = exception(404, encode(missingNamespace));
+    let attempts = 0;
+    const patch = spyOn(KubernetesObjectApi.prototype, "patch").mockImplementation(async doc => {
+      if (++attempts === 1) throw error;
+      return doc as never;
+    });
+    try {
+      const helpers = await makeHelpers();
+      const applied = await helpers.apply(JSON.stringify(policy), { timeoutMs: 2000 });
+      expect(attempts).toBe(2);
+      expect(applied[0].unwrap()).toEqual(policy);
+    } finally { patch.mockRestore(); }
+  });
+
+  test(`unrelated 404, forbidden and invalid Status bodies remain terminal (${format})`, () => {
+    for (const body of [
+      { ...missingNamespace, message: 'deployments.apps "controller" not found', details: { name: "controller", kind: "deployments", group: "apps" } },
+      { ...missingNamespace, message: 'namespaces.example.dev "widget" not found', details: { name: "widget", kind: "namespaces", group: "example.dev" } },
+      { code: 404, reason: "NotFound", message: "Not Found" },
+      { code: 403, reason: "Forbidden", message: 'namespaces "agent-sandbox-system" is forbidden', details: missingNamespace.details },
+      { code: 422, reason: "Invalid", message: 'NetworkPolicy "policy" is invalid', details: { causes: [{ message: missingNamespace.message }] } },
+    ]) expect(isTransientApplyError(exception(body.code, encode(body)))).toBe(false);
+  });
+}
+
+test("plain client errors and transient HTTP statuses retain their retry behavior", () => {
+  for (const code of [429, 500, 503, 504]) {
+    expect(isTransientApplyError(exception(code, { message: "Try again" }))).toBe(true);
+  }
+  for (const message of [missingNamespace.message, 'no matches for kind "Widget"', "Failed to fetch resource metadata", "failed calling webhook", "HTTP-Code: 503\nMessage: unavailable"]) {
+    expect(isTransientApplyError(new Error(message))).toBe(true);
+  }
+  for (const body of ["not JSON", "{", null, [], 42]) {
+    expect(isTransientApplyError(exception(404, body))).toBe(false);
+  }
+  expect(isTransientApplyError(null)).toBe(false);
+});
+
+const policy = {
+  apiVersion: "networking.k8s.io/v1", kind: "NetworkPolicy",
+  metadata: { name: "policy", namespace: "agent-sandbox-system" },
+  spec: { podSelector: {}, policyTypes: ["Ingress"] },
+};
+
+async function makeHelpers() {
+  return k3s().helpers({ name: "cluster", exec: async () => ({ exitCode: 0, stderr: "", stdout: JSON.stringify({
+    apiVersion: "v1", kind: "Config", clusters: [{ name: "test", cluster: { server: "https://cluster.internal:6443" } }],
+    users: [{ name: "test", user: {} }], contexts: [{ name: "test", context: { cluster: "test", user: "test" } }],
+    "current-context": "test",
+  }) }) } as never);
+}
+
+test("apply fails immediately on an unrelated 404 and still times out on a missing namespace", async () => {
+  const helpers = await makeHelpers();
+  const patch = spyOn(KubernetesObjectApi.prototype, "patch");
+  try {
+    patch.mockRejectedValue(exception(404, { reason: "NotFound", message: 'deployments.apps "controller" not found' }));
+    await expect(helpers.apply(JSON.stringify(policy))).rejects.toThrow("applying NetworkPolicy/policy");
+    expect(patch).toHaveBeenCalledTimes(1);
+    patch.mockClear();
+    patch.mockRejectedValue(exception(404, missingNamespace));
+    await expect(helpers.apply(JSON.stringify(policy), { timeoutMs: 0 })).rejects.toThrow("apply did not converge");
+    expect(patch).toHaveBeenCalledTimes(1);
+  } finally { patch.mockRestore(); }
+});

package/src/components/k3s.ts

@@ -1421,8 +1421,27 @@ function errorText(err: unknown): string {
   return (err as Error)?.message ?? String(err);
 }
 
-function isTransientApplyError(err: unknown): boolean {
-  const msg = errorText(err);
+/** ApiException formats its body with JSON.stringify, escaping Status.message.
+ * Read the actual response instead of matching that presentation string. */
+function applyErrorStatus(err: unknown): V1Status | undefined {
+  let body = (err as { body?: unknown } | null)?.body;
+  if (typeof body === "string") {
+    try { body = JSON.parse(body); } catch { return undefined; }
+  }
+  return body && typeof body === "object" && !Array.isArray(body) ? body as V1Status : undefined;
+}
+
+/** @internal Exported for regression tests; not part of the components API. */
+export function isTransientApplyError(err: unknown): boolean {
+  const status = applyErrorStatus(err);
+  const code = status?.code ?? (err as { code?: unknown } | null)?.code;
+  if ([429, 500, 503, 504].includes(code as number)) return true;
+  // A missing namespace can converge while another apply creates it. An
+  // arbitrary resource 404 cannot: don't classify by HTTP status alone.
+  if (code === 404 && status?.reason === "NotFound" &&
+      status.details?.kind === "namespaces" && !status.details.group &&
+      typeof status.details.name === "string" && status.details.name.length > 0) return true;
+  const msg = typeof status?.message === "string" ? status.message : errorText(err);
   return TRANSIENT_APPLY_PATTERNS.some((re) => re.test(msg));
 }
 

package/src/daemon.ts

@@ -103,7 +103,9 @@ import {
   sanitizeSegment,
 } from "./harness/volume-paths.js";
 import { pollUntilReady } from "./harness/ready-poll.js";
-import { fsUsageDiagnostics, runWrapperRules } from "./harness/wrapper-rules.js";
+import { fsUsageDiagnostics } from "./harness/wrapper-rules.js";
+import { runTypecheck, type TypecheckReport } from "./harness/typecheck.js";
+import { evaluateChecked } from "./harness/eval-typecheck.js";
 import type { WrapperDiagnostic } from "./harness/wrapper-rules.js";
 import { cpus, tmpdir } from "node:os";
 import { APP_DIR, WORKSPACE, resolveExistingProjectPath, resolveProjectPath, takeProjectReads } from "./project-files.js";
@@ -6246,7 +6248,12 @@ async function evalCode(code: string): Promise<EvalResult> {
     filePath = path.join(EVAL_DIR, `${randomUUID()}.ts`);
     await fs.writeFile(filePath, prepared.code);
     const moduleUrl = pathToFileURL(filePath).href;
-    const mod = (await runInstrumented(fetchScope, () => import(moduleUrl))) as { default?: unknown };
+    const mod = await evaluateChecked(
+      { file: filePath, appDir: APP_DIR, typecheckDir: TYPECHECK_DIR, run: shx,
+        projectEntry: loaded ? resolveEntry() : undefined },
+      (text) => { chunks.push(text); },
+      () => runInstrumented(fetchScope, () => import(moduleUrl)) as Promise<{ default?: unknown }>,
+    );
     outcome = { ok: true, result: safeSerialize(mod.default) };
   } catch (err) {
     const e = err as Error;
@@ -6327,46 +6334,11 @@ async function evalCode(code: string): Promise<EvalResult> {
  *  blocking rules, which always use THIS copy rather than the project's own —
  *  can be driven outside a VM. */
 const TYPECHECK_DIR = process.env.SPECTEST_TYPECHECK_DIR ?? "/opt/spectest/typecheck";
-/** Cap on errors shipped in the report; `totalErrors` carries the true count. */
-const TYPECHECK_ERROR_CAP = 50;
-
-interface TypecheckError {
-  /** Path relative to the project's spectest/ dir (e.g. `tests/checkout.ts`). */
-  file: string;
-  line: number;
-  column: number;
-  /** `TS2345` from the compiler, or a `SPECTEST….` rule code. */
-  code: string;
-  message: string;
-}
-
-interface TypecheckReport {
-  status: "ok" | "errors" | "skipped" | "failed";
-  errors: TypecheckError[];
-  totalErrors: number;
-  durationMs: number;
-  /** Why the check was skipped / how it failed. */
-  detail?: string;
-  /**
-   * Findings that FAIL the run, as opposed to `errors`, which only advise.
-   * A diagnostic qualifies only when it is true of the program that runs
-   * rather than of the declared types alone — see `harness/wrapper-rules.ts`
-   * for why no `tsc` code can be in here and why these can.
-   *
-   * The list is the rollout mechanism as well as the payload: an older SDK
-   * never sends the field, `#[serde(default)]` leaves it empty on the server,
-   * and the gate is inert for every project that has not moved to a version
-   * carrying it. The version a project runs is already its own choice
-   * (`sdk.rs` honours the declared lower bound), so the bump is the consent.
-   */
-  blocking?: TypecheckError[];
-}
-
 let TYPECHECK: Promise<TypecheckReport> | null = null;
 
 /** Fire-and-forget kickoff; the promise is parked for POST /typecheck. */
 function startTypecheck(): void {
-  TYPECHECK = runTypecheck().catch((err) => ({
+  TYPECHECK = runProjectTypecheck().catch((err) => ({
     status: "failed" as const,
     errors: [],
     totalErrors: 0,
@@ -6377,149 +6349,9 @@ function startTypecheck(): void {
   TYPECHECK.catch(() => {});
 }
 
-/** The tsconfig used when the project doesn't ship its own. `lib` is left to
- *  the target default (which includes DOM — browser.evaluate callbacks
- *  reference `document`); `types` pulls Bun's ambient globals from the baked
- *  install via `typeRoots` (the walk-up from APP_DIR never reaches it). */
-function generatedTsconfig(): string {
-  return JSON.stringify(
-    {
-      compilerOptions: {
-        target: "esnext",
-        module: "esnext",
-        moduleResolution: "bundler",
-        strict: true,
-        noEmit: true,
-        skipLibCheck: true,
-        esModuleInterop: true,
-        resolveJsonModule: true,
-        jsx: "react-jsx",
-        types: ["bun"],
-        typeRoots: [
-          path.join(APP_DIR, "node_modules", "@types"),
-          path.join(TYPECHECK_DIR, "node_modules", "@types"),
-        ],
-      },
-      include: ["**/*.ts", "**/*.tsx"],
-      exclude: ["node_modules"],
-    },
-    null,
-    2,
-  );
-}
-
-/** Parse `--pretty false` tsc output: `file(line,col): error TScode: msg`,
- *  with indented elaboration lines folded into the preceding error. */
-function parseTscOutput(output: string): {
-  errors: TypecheckError[];
-  total: number;
-  suppressed: number;
-} {
-  const errors: TypecheckError[] = [];
-  let total = 0;
-  let suppressed = 0;
-  let last: TypecheckError | null = null;
-  for (const line of output.split("\n")) {
-    const m = /^(.+?)\((\d+),(\d+)\): error (TS\d+): (.*)$/.exec(line);
-    if (!m) {
-      // Elaboration lines are indented; fold them into the last kept error.
-      if (last && /^\s+\S/.test(line)) last.message += `\n${line.trimEnd()}`;
-      continue;
-    }
-    const [, rawFile, ln, col, code, message] = m;
-    // Keep only diagnostics in the user's files. tsc runs with cwd=APP_DIR
-    // (the daemon's cwd), so project files come out relative ("tests/x.ts")
-    // and anything outside — the SDK at /opt/spectest/sdk, node_modules —
-    // is `../…` or absolute. Errors in our own SDK are ours to fix, not the
-    // user's to read.
-    const abs = path.resolve(APP_DIR, rawFile!);
-    const inApp = abs.startsWith(APP_DIR + path.sep) && !abs.includes(`${path.sep}node_modules${path.sep}`);
-    if (!inApp) {
-      suppressed += 1;
-      last = null;
-      continue;
-    }
-    total += 1;
-    const err: TypecheckError = {
-      file: path.relative(APP_DIR, abs),
-      line: Number(ln),
-      column: Number(col),
-      code: code!,
-      message: message!,
-    };
-    if (errors.length < TYPECHECK_ERROR_CAP) {
-      errors.push(err);
-      last = err;
-    } else {
-      last = null;
-    }
-  }
-  return { errors, total, suppressed };
-}
-
-async function runTypecheck(): Promise<TypecheckReport> {
-  const started = Date.now();
-  // The project's own typescript wins over the baked copy.
-  const tsc = [
-    path.join(APP_DIR, "node_modules", "typescript", "bin", "tsc"),
-    path.join(TYPECHECK_DIR, "node_modules", "typescript", "bin", "tsc"),
-  ].find((p) => existsSync(p));
-  if (!tsc) {
-    return {
-      status: "skipped",
-      errors: [],
-      totalErrors: 0,
-      durationMs: 0,
-      detail: "typescript is not installed (base image predates the baked typechecker)",
-    };
-  }
-  let config = path.join(APP_DIR, "tsconfig.json");
-  if (!existsSync(config)) {
-    config = path.join(APP_DIR, ".spectest-tsconfig.json");
-    await fs.writeFile(config, generatedTsconfig());
-  }
-  const res = await shx(process.execPath, [tsc, "--noEmit", "--pretty", "false", "-p", config], 120_000);
-  const durationMs = Date.now() - started;
-  if (res.code === 124) {
-    return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: "typecheck timed out" };
-  }
-  const { errors, total, suppressed } = parseTscOutput(res.stdout + res.stderr);
-  const wrapper = [...(await runWrapperRulesReport(config)), ...(await rawFsReport())];
-  const strip = ({ file, line, column, code, message }: WrapperDiagnostic): TypecheckError => ({
-    file,
-    line,
-    column,
-    code,
-    message,
-  });
-  const blocking = wrapper.filter((d) => d.blocking).map(strip);
-  const advisory = wrapper.filter((d) => !d.blocking).map(strip);
-  if (errors.length === 0 && wrapper.length === 0) {
-    // Exit 0 → clean. Non-zero with no *user-file* diagnostics is either
-    // all-suppressed (still ok from the user's perspective) or a compiler
-    // crash (config not found, OOM) — surface the latter.
-    if (res.code !== 0 && suppressed === 0) {
-      const tail = (res.stderr || res.stdout).trim().split("\n").slice(-5).join("\n");
-      return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: tail || `tsc exited ${res.code}` };
-    }
-    if (suppressed > 0) {
-      console.warn(`[typecheck] ${suppressed} diagnostic(s) outside the project suppressed`);
-    }
-    return { status: "ok", errors: [], totalErrors: 0, durationMs };
-  }
-  // The wrapper findings ride the same list the CLI and the dashboard already
-  // render; `blocking` is a second view of the ones that also fail the run, so
-  // nothing has to learn a new shape to show them. Blocking leads, because
-  // `errors` is capped and the entries that stopped the run must never be the
-  // ones the cap drops.
-  const all = [...blocking, ...advisory, ...errors];
-  return {
-    status: "errors",
-    errors: all.slice(0, TYPECHECK_ERROR_CAP),
-    totalErrors: total + wrapper.length,
-    durationMs,
-    ...(blocking.length > 0 ? { blocking } : {}),
-  };
+async function runProjectTypecheck(): Promise<TypecheckReport> {
+  return runTypecheck({ appDir: APP_DIR, typecheckDir: TYPECHECK_DIR, run: shx,
+    projectEntry: loaded ? resolveEntry() : undefined, extraDiagnostics: rawFsReport });
 }
 
 /**
@@ -6545,37 +6377,6 @@ async function rawFsReport(): Promise<WrapperDiagnostic[]> {
   }
 }
 
-/**
- * The blocking rules, run against the BAKED compiler whatever the project
- * pins. Best-effort in every direction: an install that predates the API, a
- * config the rules cannot open, or a throw from `typescript/unstable/*` all
- * yield no findings, so a run proceeds exactly as it does today. Only a
- * definite finding can stop one.
- */
-async function runWrapperRulesReport(config: string): Promise<WrapperDiagnostic[]> {
-  const typescriptDir = path.join(TYPECHECK_DIR, "node_modules", "typescript");
-  if (!existsSync(path.join(typescriptDir, "dist", "api", "async", "api.js"))) return [];
-  try {
-    const run = await runWrapperRules({
-      typescriptDir,
-      configFile: config,
-      appDir: APP_DIR,
-      readFile: (f) => fs.readFile(f, "utf8"),
-      relative: path.relative,
-      join: path.join,
-      dirname: path.dirname,
-    });
-    if (run.status !== "ok") {
-      console.warn(`[typecheck] wrapper rules ${run.status}: ${run.detail ?? "no detail"}`);
-      return [];
-    }
-    return run.diagnostics;
-  } catch (err) {
-    console.warn(`[typecheck] wrapper rules threw: ${(err as Error)?.message ?? err}`);
-    return [];
-  }
-}
-
 // ────────────────────────────────────────────────────────────────────────
 // HTTP server
 // ────────────────────────────────────────────────────────────────────────

package/src/harness/eval-typecheck.test.ts

@@ -0,0 +1,163 @@
+import { afterEach, beforeEach, expect, test } from "bun:test";
+import fs from "node:fs/promises";
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { tmpdir } from "node:os";
+import { execFile } from "node:child_process";
+import { checkEval, evaluateChecked } from "./eval-typecheck.js";
+import { runTypecheck } from "./typecheck.js";
+
+const sdkDir = path.resolve(import.meta.dir, "../..");
+// CI/dev can select the same native TypeScript installation as the VM.
+const typecheckDir = process.env.SPECTEST_TYPECHECK_DIR ?? sdkDir;
+const hasWrapperChecker = existsSync(path.join(typecheckDir, "node_modules/typescript/dist/api/async/api.js"));
+let appDir: string;
+let file: string;
+let projectEntry: string;
+
+beforeEach(async () => {
+  appDir = await fs.mkdtemp(path.join(tmpdir(), "spectest-eval-types-"));
+  await fs.mkdir(path.join(appDir, "node_modules/@specific.dev"), { recursive: true });
+  await fs.symlink(sdkDir, path.join(appDir, "node_modules/@specific.dev/spectest"));
+  await fs.symlink(path.join(sdkDir, "node_modules/@types"), path.join(appDir, "node_modules/@types"));
+  await fs.mkdir(path.join(appDir, "spectest"));
+  await fs.mkdir(path.join(appDir, ".spectest-eval"));
+  file = path.join(appDir, ".spectest-eval/snippet.ts");
+  projectEntry = path.join(appDir, "spectest/index.ts");
+  await fs.writeFile(projectEntry, `
+import { defineEnvironment, defineFake } from "@specific.dev/spectest";
+import { supabase } from "@specific.dev/spectest/components";
+export const env = defineEnvironment({
+  name: "eval-typecheck",
+  services: { supabase: supabase().group },
+  fakes: { sample: defineFake({ hostnames: ["example.test"], state: () => ({}),
+    handler: () => new Response(), helpers: () => ({ count: () => 3 }) }) },
+});
+export default env.project([]);
+`);
+});
+
+afterEach(async () => { await fs.rm(appDir, { recursive: true, force: true }); });
+
+function run(command: string, args: string[], timeout: number): Promise<{ code: number; stdout: string; stderr: string }> {
+  return new Promise((resolve) => {
+    execFile(command, args, { cwd: appDir, timeout, maxBuffer: 16 * 1024 * 1024 }, (error, stdout, stderr) => {
+      resolve({ code: error ? (typeof error.code === "number" ? error.code : 1) : 0, stdout, stderr });
+    });
+  });
+}
+
+const opts = () => ({ appDir, file, projectEntry, typecheckDir, run });
+const query = 'const rows = await ctx.svc.supabase.sql<{ analyzed: boolean }>`SELECT true AS analyzed`;';
+
+test("eval receives the default-exported project's service/fake types and globals", async () => {
+  await fs.writeFile(file, `${query}
+const ready: boolean = rows[0].analyzed.unwrap();
+const count: number = ctx.fakes.sample.count().unwrap();
+expect(rows[0].analyzed).toBe(true);
+expectRaw(count, "count").toBe(3);
+assert(ready);
+state.saved = ready;
+export default ready;
+`);
+  const report = await checkEval(opts());
+  expect(report).toMatchObject({ status: "ok", errors: [], totalErrors: 0 });
+  expect(await fs.readdir(path.dirname(file))).toEqual(["snippet.ts"]);
+}, 30_000);
+
+test("compiler catches the Listen comparison and reports original snippet lines", async () => {
+  await fs.writeFile(file, `${query}\nconst done = rows[0].analyzed === true;\nexport default done;`);
+  const report = await checkEval(opts());
+  expect(report.errors).toContainEqual(expect.objectContaining({ file: "eval.ts", line: 2, code: "TS2367" }));
+}, 30_000);
+
+test("an untyped SQL row and nonexistent service remain diagnostics, not any", async () => {
+  await fs.writeFile(file, 'const rows = await ctx.svc.supabase.sql`SELECT true AS analyzed`;\nexport default rows[0].analyzed;\nctx.svc.missing;');
+  const report = await checkEval(opts());
+  expect(report.errors.filter((e) => e.code === "TS2339").map((e) => e.line)).toEqual([2, 3]);
+}, 30_000);
+
+test.skipIf(!hasWrapperChecker)("blocking wrapper finding prevents all snippet side effects", async () => {
+  await fs.writeFile(file, `${query}\nexport default rows[0].analyzed === true;`);
+  let executed = false;
+  const logs: string[] = [];
+  await expect(evaluateChecked(opts(), (s) => logs.push(s), async () => { executed = true; })).rejects.toThrow("snippet did not run");
+  expect(executed).toBe(false);
+  expect(logs.join("")).toContain("SPECTEST2002");
+  expect(logs.join("")).toContain(".unwrap()");
+}, 30_000);
+
+test("ordinary errors are advisory and unrelated project errors do not enter eval output", async () => {
+  await fs.appendFile(projectEntry, '\nconst unrelated: number = "wrong";');
+  await fs.writeFile(file, 'const x: number = "wrong";\nexport default x;');
+  const logs: string[] = [];
+  expect(await evaluateChecked(opts(), (s) => logs.push(s), async () => "executed")).toBe("executed");
+  expect(logs.join("")).toContain("(advisory)");
+  expect(logs.join("")).toContain("eval.ts:1:");
+  expect(logs.join("")).not.toContain("index.ts");
+}, 30_000);
+
+test.skipIf(!hasWrapperChecker)("unwrapped comparisons execute and nullable presence checks remain allowed", async () => {
+  await fs.writeFile(file, `${query}\nexport default rows[0].analyzed.unwrap() === true;`);
+  expect(await evaluateChecked(opts(), () => {}, async () => "executed")).toBe("executed");
+  await fs.writeFile(file, 'const rows = await ctx.svc.supabase.sql<{ label: string | null }>`SELECT null AS label`;\nexport default rows[0].label === null;');
+  expect((await checkEval(opts())).blocking).toBeUndefined();
+}, 30_000);
+
+test("eval without a loaded project still checks basic globals", async () => {
+  await fs.writeFile(file, 'assert(true); state.value = 1; export default ctx.testName;');
+  expect(await checkEval({ ...opts(), projectEntry: undefined })).toMatchObject({ status: "ok", totalErrors: 0 });
+}, 30_000);
+
+test("project paths are respected even when its include excludes the eval directory", async () => {
+  await fs.writeFile(path.join(appDir, "spectest/tsconfig.json"), JSON.stringify({
+    compilerOptions: { paths: { "@fixture": ["./fixture.ts"] } }, include: ["tests/**/*.ts"],
+  }));
+  await fs.writeFile(path.join(appDir, "spectest/fixture.ts"), 'export const count = 3;');
+  await fs.writeFile(file, 'import { count } from "@fixture";\nconst x: number = count;\nexport default x;');
+  expect(await checkEval(opts())).toMatchObject({ status: "ok", totalErrors: 0 });
+}, 30_000);
+
+test("unavailable compiler is reported, evaluation continues, and scratch files are removed", async () => {
+  await fs.writeFile(file, "export default 1;");
+  const logs: string[] = [];
+  expect(await evaluateChecked({ ...opts(), typecheckDir: path.join(appDir, "missing") }, (s) => logs.push(s), async () => 1)).toBe(1);
+  expect(logs.join("")).toContain("[typecheck] skipped:");
+  expect(await fs.readdir(path.dirname(file))).toEqual(["snippet.ts"]);
+});
+
+test("shared managed-test checker still reports project diagnostics", async () => {
+  await fs.appendFile(projectEntry, '\nconst wrong: number = "bad";');
+  const report = await runTypecheck({ appDir, typecheckDir, run });
+  expect(report.errors).toContainEqual(expect.objectContaining({ file: "spectest/index.ts", code: "TS2322" }));
+}, 30_000);
+
+test.skipIf(!hasWrapperChecker)("managed tests and evals classify the same comparison identically", async () => {
+  const body = `${query}\nreturn rows[0].analyzed === true;`;
+  await fs.writeFile(file, `export default await (async () => {\n${body}\n})();`);
+  const evalReport = await checkEval(opts());
+  await fs.mkdir(path.join(appDir, "spectest/tests"));
+  await fs.writeFile(path.join(appDir, "spectest/tests/comparison.ts"),
+    `import { env } from "../index";\nenv.test("comparison", async (ctx) => {\n${body}\n});`);
+  const suiteReport = await runTypecheck({ appDir, projectEntry, typecheckDir, run });
+  const findings = (report: typeof evalReport) => report.errors
+    .filter((e) => e.code === "TS2367" || e.code === "SPECTEST2002")
+    .map(({ code, message }) => ({ code, message }));
+  expect(findings(evalReport)).toEqual(findings(suiteReport));
+  expect(evalReport.blocking?.map((e) => e.code)).toEqual(["SPECTEST2002"]);
+  expect(suiteReport.blocking?.map((e) => e.code)).toEqual(["SPECTEST2002"]);
+}, 60_000);
+
+test("eval and tests honor the same project strictness", async () => {
+  await fs.writeFile(path.join(appDir, "spectest/tsconfig.json"), JSON.stringify({
+    compilerOptions: { strict: false, skipLibCheck: true, target: "esnext", module: "esnext", moduleResolution: "bundler" },
+  }));
+  await fs.writeFile(file, 'const value: number = null; export default value;');
+  await fs.writeFile(path.join(appDir, "spectest/null.ts"), 'const value: number = null; export default value;');
+  const evalReport = await checkEval(opts());
+  const suiteReport = await runTypecheck({ appDir, projectEntry, typecheckDir, run });
+  expect(evalReport.errors.some((d) => d.code === "TS2322")).toBe(false);
+  expect(suiteReport.errors.some((d) => d.code === "TS2322")).toBe(false);
+  expect(evalReport.blocking).toBeUndefined();
+  expect(suiteReport.blocking).toBeUndefined();
+}, 30_000);

package/src/harness/eval-typecheck.ts

@@ -0,0 +1,100 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { generatedTsconfig, projectTypecheckConfig, runTypecheck, type TypecheckOptions, type TypecheckReport } from "./typecheck.js";
+
+interface EvalTypecheckOptions extends Pick<TypecheckOptions, "appDir" | "typecheckDir" | "run"> {
+  file: string;
+  projectEntry?: string;
+}
+
+/** Describe the globals installed by evalCode, without importing/executing the project. */
+function globals(projectEntry?: string): string {
+  return `import type * as SDK from "@specific.dev/spectest";
+${projectEntry ? `import type project from ${JSON.stringify(projectEntry)};` : ""}
+type EvalContext = ${projectEntry
+    ? "typeof project extends SDK.Project<infer S, infer F> ? SDK.TestContext<undefined, S, F> : SDK.TestContext<undefined>"
+    : "SDK.TestContext<undefined>"};
+declare global {
+  var ctx: EvalContext;
+  var expect: typeof SDK.expect;
+  var expectRaw: typeof SDK.expectRaw;
+  var assert: typeof SDK.assert;
+  var state: Record<string, unknown>;
+}
+`;
+}
+
+export async function checkEval(opts: EvalTypecheckOptions): Promise<TypecheckReport> {
+  const started = Date.now();
+  let scratch: string | undefined;
+  try {
+    scratch = await fs.mkdtemp(path.join(path.dirname(opts.file), "check-"));
+    const declarations = path.join(scratch, "globals.d.ts");
+    await fs.writeFile(declarations, globals(opts.projectEntry));
+    const projectConfig = projectTypecheckConfig(opts);
+    const defaults = JSON.parse(generatedTsconfig(opts.appDir, opts.typecheckDir));
+    const config = {
+      ...(projectConfig ? { extends: projectConfig } : defaults),
+      compilerOptions: {
+        ...(!projectConfig ? defaults.compilerOptions : {}),
+        // Bun runs eval as ESM TypeScript, regardless of the app's build output.
+        noEmit: true,
+        skipLibCheck: true,
+        target: "esnext",
+        module: "esnext",
+        moduleResolution: "bundler",
+        moduleDetection: "force",
+        allowImportingTsExtensions: true,
+        composite: false,
+        incremental: false,
+        rootDir: opts.appDir,
+      },
+      // Project includes/excludes may omit hidden files or include the entire
+      // application. Check this snippet, with imports resolved normally.
+      files: [opts.file, declarations],
+      include: [],
+      exclude: [],
+    };
+    const configFile = path.join(scratch, "tsconfig.json");
+    await fs.writeFile(configFile, JSON.stringify(config));
+    const report = await runTypecheck({ ...opts, configFile, diagnosticFile: opts.file });
+    // Stable, snippet-relative locations; don't expose random temporary paths.
+    const normalize = (d: TypecheckReport["errors"][number]) => ({ ...d, file: "eval.ts" });
+    return {
+      ...report,
+      errors: report.errors.map(normalize),
+      ...(report.blocking ? { blocking: report.blocking.map(normalize) } : {}),
+    };
+  } catch (error) {
+    // Match managed tests: unavailable/broken checkers report their failure,
+    // while only a definite blocking finding prevents execution.
+    return {
+      status: "failed", errors: [], totalErrors: 0, durationMs: Date.now() - started,
+      detail: error instanceof Error ? error.message : String(error),
+    };
+  } finally {
+    if (scratch) await fs.rm(scratch, { recursive: true, force: true });
+  }
+}
+
+/** The same advisory/blocking policy as a managed test, before module import. */
+export async function evaluateChecked<T>(
+  opts: EvalTypecheckOptions,
+  log: (text: string) => void,
+  evaluate: () => Promise<T>,
+): Promise<T> {
+  const report = await checkEval(opts);
+  if (report.status === "failed" || report.status === "skipped") {
+    log(`[typecheck] ${report.status}: ${report.detail ?? "unavailable"}\n`);
+  } else if (report.totalErrors > 0) {
+    log(`[typecheck] ${report.totalErrors} diagnostic(s)${report.blocking?.length ? "" : " (advisory)"}\n`);
+    for (const d of report.errors) log(`${d.file}:${d.line}:${d.column}: ${d.code}: ${d.message}\n`);
+    if (report.totalErrors > report.errors.length) {
+      log(`[typecheck] ${report.totalErrors - report.errors.length} more diagnostic(s)\n`);
+    }
+  }
+  if (report.blocking?.length) {
+    throw new Error(`Typecheck: ${report.blocking.length} blocking error(s) in eval — the snippet did not run. Fix the diagnostics above before retrying.`);
+  }
+  return evaluate();
+}