@specific.dev/spectest 0.91.1 → 0.92.0

package/src/harness/typecheck.ts

@@ -0,0 +1,237 @@
+import fs from "node:fs/promises";
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { runWrapperRules, type WrapperDiagnostic } from "./wrapper-rules.js";
+
+export interface TypecheckOptions {
+  appDir: string;
+  typecheckDir: string;
+  projectEntry?: string;
+  run: (file: string, args: string[], timeoutMs: number) => Promise<{ code: number; stdout: string; stderr: string }>;
+  configFile?: string;
+  /** Eval reports only the submitted snippet, not unrelated imported project code. */
+  diagnosticFile?: string;
+  extraDiagnostics?: () => Promise<WrapperDiagnostic[]>;
+}
+
+/** Both entry points honor the same project config before generating defaults. */
+export function projectTypecheckConfig(opts: Pick<TypecheckOptions, "appDir" | "projectEntry">): string | undefined {
+  return [
+    path.join(opts.appDir, "tsconfig.json"),
+    ...(opts.projectEntry ? [path.join(path.dirname(opts.projectEntry), "tsconfig.json")] : []),
+  ].find(existsSync);
+}
+
+/** Cap on errors shipped in the report; `totalErrors` carries the true count. */
+const TYPECHECK_ERROR_CAP = 50;
+
+export interface TypecheckError {
+  /** Path relative to the project's spectest/ dir (e.g. `tests/checkout.ts`). */
+  file: string;
+  line: number;
+  column: number;
+  /** `TS2345` from the compiler, or a `SPECTEST….` rule code. */
+  code: string;
+  message: string;
+}
+
+export interface TypecheckReport {
+  status: "ok" | "errors" | "skipped" | "failed";
+  errors: TypecheckError[];
+  totalErrors: number;
+  durationMs: number;
+  /** Why the check was skipped / how it failed. */
+  detail?: string;
+  /**
+   * Findings that FAIL the run, as opposed to `errors`, which only advise.
+   * A diagnostic qualifies only when it is true of the program that runs
+   * rather than of the declared types alone — see `harness/wrapper-rules.ts`
+   * for why no `tsc` code can be in here and why these can.
+   *
+   * The list is the rollout mechanism as well as the payload: an older SDK
+   * never sends the field, `#[serde(default)]` leaves it empty on the server,
+   * and the gate is inert for every project that has not moved to a version
+   * carrying it. The version a project runs is already its own choice
+   * (`sdk.rs` honours the declared lower bound), so the bump is the consent.
+   */
+  blocking?: TypecheckError[];
+}
+
+/** The tsconfig used when the project doesn't ship its own. `lib` is left to
+ *  the target default (which includes DOM — browser.evaluate callbacks
+ *  reference `document`); `types` pulls Bun's ambient globals from the baked
+ *  install via `typeRoots` (the walk-up from APP_DIR never reaches it). */
+export function generatedTsconfig(appDir: string, typecheckDir: string): string {
+  return JSON.stringify(
+    {
+      compilerOptions: {
+        target: "esnext",
+        module: "esnext",
+        moduleResolution: "bundler",
+        strict: true,
+        noEmit: true,
+        skipLibCheck: true,
+        esModuleInterop: true,
+        resolveJsonModule: true,
+        jsx: "react-jsx",
+        types: ["bun"],
+        typeRoots: [
+          path.join(appDir, "node_modules", "@types"),
+          path.join(typecheckDir, "node_modules", "@types"),
+        ],
+      },
+      include: ["**/*.ts", "**/*.tsx"],
+      exclude: ["node_modules", ".spectest-eval"],
+    },
+    null,
+    2,
+  );
+}
+
+/** Parse `--pretty false` tsc output: `file(line,col): error TScode: msg`,
+ *  with indented elaboration lines folded into the preceding error. */
+function parseTscOutput(output: string, appDir: string, diagnosticFile?: string): {
+  errors: TypecheckError[];
+  total: number;
+  suppressed: number;
+} {
+  const errors: TypecheckError[] = [];
+  let total = 0;
+  let suppressed = 0;
+  let last: TypecheckError | null = null;
+  for (const line of output.split("\n")) {
+    const m = /^(.+?)\((\d+),(\d+)\): error (TS\d+): (.*)$/.exec(line);
+    if (!m) {
+      // Elaboration lines are indented; fold them into the last kept error.
+      if (last && /^\s+\S/.test(line)) last.message += `\n${line.trimEnd()}`;
+      continue;
+    }
+    const [, rawFile, ln, col, code, message] = m;
+    // Keep only diagnostics in the user's files. tsc runs with cwd=APP_DIR
+    // (the daemon's cwd), so project files come out relative ("tests/x.ts")
+    // and anything outside — the SDK at /opt/spectest/sdk, node_modules —
+    // is `../…` or absolute. Errors in our own SDK are ours to fix, not the
+    // user's to read.
+    const abs = path.resolve(appDir, rawFile!);
+    const inApp = abs.startsWith(appDir + path.sep) && !abs.includes(`${path.sep}node_modules${path.sep}`);
+    if (!inApp || (diagnosticFile && abs !== diagnosticFile)) {
+      suppressed += 1;
+      last = null;
+      continue;
+    }
+    total += 1;
+    const err: TypecheckError = {
+      file: path.relative(appDir, abs),
+      line: Number(ln),
+      column: Number(col),
+      code: code!,
+      message: message!,
+    };
+    if (errors.length < TYPECHECK_ERROR_CAP) {
+      errors.push(err);
+      last = err;
+    } else {
+      last = null;
+    }
+  }
+  return { errors, total, suppressed };
+}
+
+export async function runTypecheck(opts: TypecheckOptions): Promise<TypecheckReport> {
+  const { appDir, typecheckDir, run } = opts;
+  const started = Date.now();
+  // The project's own typescript wins over the baked copy.
+  const tsc = [
+    path.join(appDir, "node_modules", "typescript", "bin", "tsc"),
+    path.join(typecheckDir, "node_modules", "typescript", "bin", "tsc"),
+  ].find((p) => existsSync(p));
+  if (!tsc) {
+    return {
+      status: "skipped",
+      errors: [],
+      totalErrors: 0,
+      durationMs: 0,
+      detail: "typescript is not installed (base image predates the baked typechecker)",
+    };
+  }
+  let config = opts.configFile ?? projectTypecheckConfig(opts);
+  if (!config) {
+    config = path.join(appDir, ".spectest-tsconfig.json");
+    await fs.writeFile(config, generatedTsconfig(appDir, typecheckDir));
+  }
+  const res = await run(process.execPath, [tsc, "--noEmit", "--pretty", "false", "-p", config], 120_000);
+  const durationMs = Date.now() - started;
+  if (res.code === 124) {
+    return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: "typecheck timed out" };
+  }
+  const { errors, total, suppressed } = parseTscOutput(res.stdout + res.stderr, appDir, opts.diagnosticFile);
+  const wrapper = [...(await runWrapperRulesReport(config, opts)), ...(await opts.extraDiagnostics?.() ?? [])];
+  const strip = ({ file, line, column, code, message }: WrapperDiagnostic): TypecheckError => ({
+    file,
+    line,
+    column,
+    code,
+    message,
+  });
+  const blocking = wrapper.filter((d) => d.blocking).map(strip);
+  const advisory = wrapper.filter((d) => !d.blocking).map(strip);
+  if (errors.length === 0 && wrapper.length === 0) {
+    // Exit 0 → clean. Non-zero with no *user-file* diagnostics is either
+    // all-suppressed (still ok from the user's perspective) or a compiler
+    // crash (config not found, OOM) — surface the latter.
+    if (res.code !== 0 && suppressed === 0) {
+      const tail = (res.stderr || res.stdout).trim().split("\n").slice(-5).join("\n");
+      return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: tail || `tsc exited ${res.code}` };
+    }
+    if (suppressed > 0) {
+      console.warn(`[typecheck] ${suppressed} diagnostic(s) outside the project suppressed`);
+    }
+    return { status: "ok", errors: [], totalErrors: 0, durationMs };
+  }
+  // The wrapper findings ride the same list the CLI and the dashboard already
+  // render; `blocking` is a second view of the ones that also fail the run, so
+  // nothing has to learn a new shape to show them. Blocking leads, because
+  // `errors` is capped and the entries that stopped the run must never be the
+  // ones the cap drops.
+  const all = [...blocking, ...advisory, ...errors];
+  return {
+    status: "errors",
+    errors: all.slice(0, TYPECHECK_ERROR_CAP),
+    totalErrors: total + wrapper.length,
+    durationMs,
+    ...(blocking.length > 0 ? { blocking } : {}),
+  };
+}
+
+/**
+ * The blocking rules, run against the BAKED compiler whatever the project
+ * pins. Best-effort in every direction: an install that predates the API, a
+ * config the rules cannot open, or a throw from `typescript/unstable/*` all
+ * yield no findings, so a run proceeds exactly as it does today. Only a
+ * definite finding can stop one.
+ */
+async function runWrapperRulesReport(config: string, opts: TypecheckOptions): Promise<WrapperDiagnostic[]> {
+  const { appDir, typecheckDir } = opts;
+  const typescriptDir = path.join(typecheckDir, "node_modules", "typescript");
+  if (!existsSync(path.join(typescriptDir, "dist", "api", "async", "api.js"))) return [];
+  try {
+    const run = await runWrapperRules({
+      typescriptDir,
+      configFile: config,
+      appDir,
+      diagnosticFile: opts.diagnosticFile,
+      readFile: (f) => fs.readFile(f, "utf8"),
+      relative: path.relative,
+      join: path.join,
+      dirname: path.dirname,
+    });
+    if (run.status !== "ok") {
+      console.warn(`[typecheck] wrapper rules ${run.status}: ${run.detail ?? "no detail"}`);
+      return [];
+    }
+    return run.diagnostics;
+  } catch (err) {
+    console.warn(`[typecheck] wrapper rules threw: ${(err as Error)?.message ?? err}`);
+    return [];
+  }
+}

package/src/harness/wrapper-rules.ts

@@ -351,11 +351,6 @@ export interface WrapperRuleRun {
   durationMs: number;
 }
 
-/** `TypeFlags.Object | TypeFlags.Union` — the only shapes a carrier can wear.
- *  Filtering on the flags the batch call already returned keeps the printed
- *  name (one round trip each) off every ordinary `boolean` condition. */
-const OBJECT_OR_UNION = (1 << 19) | (1 << 20);
-
 /** Candidate positions, and what each one means if the operand is a carrier. */
 interface Candidate {
   node: TsNode;
@@ -431,6 +426,8 @@ export async function runWrapperRules(opts: {
   configFile: string;
   /** Only files under here produce diagnostics. */
   appDir: string;
+  /** Restrict eval findings to its submitted module. */
+  diagnosticFile?: string;
   readFile: (p: string) => Promise<string>;
   relative: (from: string, to: string) => string;
   join: (...parts: string[]) => string;
@@ -456,6 +453,13 @@ export async function runWrapperRules(opts: {
     const utils = (await import(opts.join(base, "ast", "utils.js"))) as {
       formatSyntaxKind(k: number): string;
     };
+    // Native TypeScript's flag values differ from the JavaScript compiler's.
+    // Read them from the compiler serving this API; hard-coded TS 5 flags
+    // silently miss wrapped boolean unions in TS 7.
+    const { TypeFlags } = await import(opts.join(base, "enums", "typeFlags.js")) as {
+      TypeFlags: { Object: number; Union: number };
+    };
+    const objectOrUnion = TypeFlags.Object | TypeFlags.Union;
     const visitor = (await import(opts.join(base, "ast", "visitor.js"))) as {
       visitEachChild(n: unknown, cb: (c: unknown) => unknown, ctx: unknown): unknown;
     };
@@ -505,7 +509,8 @@ export async function runWrapperRules(opts: {
     const names = await project.program.getSourceFileNames();
     const files = names.filter(
       (f) =>
-        f.startsWith(`${opts.appDir}/`) && !f.includes("/node_modules/") && !f.endsWith(".d.ts"),
+        f.startsWith(`${opts.appDir}/`) && !f.includes("/node_modules/") && !f.endsWith(".d.ts") &&
+        (!opts.diagnosticFile || f === opts.diagnosticFile),
     );
 
     const diagnostics: WrapperDiagnostic[] = [];
@@ -522,7 +527,7 @@ export async function runWrapperRules(opts: {
       const printed = new Map<number, string>();
       for (let i = 0; i < nodes.length; i += 1) {
         const t = types[i];
-        if (t && (t.flags & OBJECT_OR_UNION) !== 0) {
+        if (t && (t.flags & objectOrUnion) !== 0) {
           printed.set(i, await project.checker.typeToString(t));
         }
       }