@avi2dg/checks 0.9.0 → 0.11.0

package/scripts/test-layout.ts

@@ -3,7 +3,7 @@ import { parse } from "@swc/core";
 import { Console, Effect, FileSystem, Path, Schema } from "effect";
 import { git } from "./git.ts";
 import { runMain } from "./main.ts";
-import { ENTRY_POINT } from "./gates.ts";
+import { ENTRY_POINT, TEST_ENTRY_POINT } from "./gates.ts";
 
 export type Violation = {
   readonly file: string;
@@ -39,13 +39,13 @@ const BANNED_BUN_NAMES: readonly string[] = [
 ];
 const BANNED_GLOBAL_CALLS: readonly string[] = ["fetch"];
 
-export const REQUIRED_TEST_SCRIPT = "bun test --randomize";
 const REQUIRED_TEST_TABLE: Record<string, unknown> = {
   pathIgnorePatterns: ["**/tests/quarantine/**"],
 };
 export const LAYOUT_CHECK_MARK = "scripts/test-layout.ts";
 export const LAYOUT_CHECK_BIN = "checks-test-layout";
 const OWN_ENTRY_POINT = `scripts/${ENTRY_POINT.script}`;
+const TEST_SCRIPTS: readonly string[] = [TEST_ENTRY_POINT.bin, `bun scripts/${TEST_ENTRY_POINT.script}`];
 
 export class LayoutError extends Schema.TaggedError<LayoutError>()("LayoutError", {
   message: Schema.String,
@@ -226,11 +226,11 @@ export function scriptViolations(manifest: unknown): readonly Violation[] {
   const test = isRecord(scripts) ? scripts["test"] : undefined;
   const lint = isRecord(scripts) ? scripts["lint"] : undefined;
   const violations: Violation[] = [];
-  if (test !== REQUIRED_TEST_SCRIPT) {
+  if (typeof test !== "string" || !TEST_SCRIPTS.includes(test)) {
     violations.push({
       file,
       line: undefined,
-      message: `scripts.test must be exactly "${REQUIRED_TEST_SCRIPT}", found ${JSON.stringify(test ?? null)}`,
+      message: `scripts.test must be exactly "${TEST_ENTRY_POINT.bin}", which runs bun test --randomize and judges its skips, found ${JSON.stringify(test ?? null)}`,
     });
   }
   if (typeof lint !== "string" || !runsLayoutCheck(lint)) {

package/scripts/test-report.ts

@@ -0,0 +1,115 @@
+import { Effect, Schema } from "effect";
+
+export type Outcome = "passed" | "failed" | "skipped" | "todo";
+
+export type TestResult = {
+  readonly file: string;
+  readonly name: string;
+  readonly line: number;
+  readonly outcome: Outcome;
+};
+
+export class ReportError extends Schema.TaggedError<ReportError>()("ReportError", {
+  message: Schema.String,
+}) {}
+
+type Element = {
+  readonly tag: string;
+  readonly attributes: Readonly<Record<string, string>>;
+  readonly children: Element[];
+};
+
+// bun's console prints a test inside describe blocks as "outer > inner > name".
+export const NAME_SEPARATOR = " > ";
+
+export function reporterArgs(outfile: string): readonly string[] {
+  return ["--reporter=junit", `--reporter-outfile=${outfile}`];
+}
+
+const MARKUP =
+  /<(?:\?[\s\S]*?\?>|!--[\s\S]*?-->|!\[CDATA\[[\s\S]*?\]\]>|(\/?)([A-Za-z_][\w.:-]*)((?:\s+[A-Za-z_][\w.:-]*\s*=\s*"[^"<]*")*)\s*(\/?)>)/g;
+const ATTRIBUTE = /([A-Za-z_][\w.:-]*)\s*=\s*"([^"<]*)"/g;
+const ENTITY = /&(?:(amp|lt|gt|quot|apos)|#(\d+)|#x([0-9A-Fa-f]+));/g;
+const NAMED_ENTITIES: Readonly<Record<string, string>> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
+
+const decodeSuite = Schema.decodeUnknownEffect(Schema.Struct({ name: Schema.String }));
+const decodeCase = Schema.decodeUnknownEffect(
+  Schema.Struct({ name: Schema.String, file: Schema.String, line: Schema.NumberFromString.check(Schema.isInt()) }),
+);
+
+function unescape(text: string): string {
+  return text.replace(ENTITY, (_, named: string | undefined, decimal: string | undefined, hex: string | undefined) => {
+    if (named !== undefined) return NAMED_ENTITIES[named] ?? "";
+    return String.fromCodePoint(decimal === undefined ? Number.parseInt(hex ?? "", 16) : Number.parseInt(decimal, 10));
+  });
+}
+
+function attributesOf(source: string): Readonly<Record<string, string>> {
+  const attributes: Record<string, string> = {};
+  for (const [, name, value] of source.matchAll(ATTRIBUTE)) {
+    if (name !== undefined && value !== undefined) attributes[name] = unescape(value);
+  }
+  return attributes;
+}
+
+const parseElements = Effect.fnUntraced(function* (xml: string): Effect.fn.Return<Element, ReportError> {
+  const malformed = (reason: string) => new ReportError({ message: `the junit report is malformed: ${reason}` });
+  const document: Element = { tag: "", attributes: {}, children: [] };
+  const open: Element[] = [document];
+  let consumed = 0;
+  for (const match of xml.matchAll(MARKUP)) {
+    if (xml.slice(consumed, match.index).includes("<")) return yield* malformed(`stray < before offset ${match.index}`);
+    consumed = match.index + match[0].length;
+    const [, closing, tag, attributes, selfClosing] = match;
+    if (tag === undefined) continue;
+    const parent = open.at(-1);
+    if (parent === undefined) return yield* malformed(`<${tag}> after the root closed`);
+    if (closing === "/") {
+      if (parent.tag !== tag) return yield* malformed(`</${tag}> closes <${parent.tag}>`);
+      open.pop();
+      continue;
+    }
+    const element: Element = { tag, attributes: attributesOf(attributes ?? ""), children: [] };
+    parent.children.push(element);
+    if (selfClosing !== "/") open.push(element);
+  }
+  if (xml.slice(consumed).includes("<")) return yield* malformed("stray < after the last element");
+  if (open.length !== 1) return yield* malformed(`<${open.at(-1)?.tag}> never closes`);
+  const [root, ...extra] = document.children;
+  if (root?.tag !== "testsuites" || extra.length > 0) return yield* malformed("the root is not one <testsuites>");
+  return root;
+});
+
+function outcomeOf(testcase: Element): Outcome {
+  const skipped = testcase.children.find((child) => child.tag === "skipped");
+  if (skipped !== undefined) return skipped.attributes["message"] === "TODO" ? "todo" : "skipped";
+  return testcase.children.some((child) => child.tag === "failure" || child.tag === "error") ? "failed" : "passed";
+}
+
+const collect = Effect.fnUntraced(function* (
+  suite: Element,
+  describes: readonly string[],
+  results: TestResult[],
+): Effect.fn.Return<void, ReportError> {
+  const unreadable = (tag: string) => (cause: { readonly message: string }) =>
+    new ReportError({ message: `the junit report holds a <${tag}> bun never writes: ${cause.message}` });
+  for (const child of suite.children) {
+    if (child.tag === "testsuite") {
+      const { name } = yield* decodeSuite(child.attributes).pipe(Effect.mapError(unreadable(child.tag)));
+      yield* collect(child, [...describes, name], results);
+    } else if (child.tag === "testcase") {
+      const { name, file, line } = yield* decodeCase(child.attributes).pipe(Effect.mapError(unreadable(child.tag)));
+      results.push({ file, name: [...describes, name].join(NAME_SEPARATOR), line, outcome: outcomeOf(child) });
+    }
+  }
+});
+
+// A top-level <testsuite> is named for its file, so only the describe blocks nested in it join a test's name.
+export const parseReport = Effect.fn("parseReport")(function* (xml: string) {
+  const root = yield* parseElements(xml);
+  const results: TestResult[] = [];
+  for (const fileSuite of root.children) {
+    if (fileSuite.tag === "testsuite") yield* collect(fileSuite, [], results);
+  }
+  return results;
+});

package/scripts/test.ts

@@ -0,0 +1,153 @@
+#!/usr/bin/env bun
+import { Config, Console, Effect, FileSystem, Path, Schema } from "effect";
+import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
+import { TEST_ENTRY_POINT } from "./gates.ts";
+import { runMain, Usage } from "./main.ts";
+import { NAME_SEPARATOR, parseReport, ReportError, reporterArgs, type TestResult } from "./test-report.ts";
+
+const Environment = Schema.Literals(["ci", "local"]);
+
+export type Environment = typeof Environment.Type;
+
+const SkipDeclaration = Schema.Struct({
+  file: Schema.NonEmptyString,
+  test: Schema.NonEmptyString,
+  reason: Schema.NonEmptyString,
+  when: Schema.optionalKey(Environment),
+});
+
+export type SkipDeclaration = typeof SkipDeclaration.Type;
+
+export type Verdict = {
+  readonly environment: Environment;
+  readonly skipped: number;
+  readonly undeclared: readonly TestResult[];
+  readonly stale: readonly SkipDeclaration[];
+  readonly unjudged: number;
+};
+
+export class SkipGateError extends Schema.TaggedError<SkipGateError>()("SkipGateError", {
+  message: Schema.String,
+}) {}
+
+const NAME = TEST_ENTRY_POINT.bin;
+const DECLARATIONS = "package.json testSkips";
+const USAGE = `usage: ${NAME} takes no arguments, since a narrowed run skips every test it leaves out; run bun test --randomize <args> for one`;
+
+const decodeManifest = Schema.decodeUnknownEffect(
+  Schema.fromJsonString(Schema.Struct({ testSkips: Schema.optionalKey(Schema.Array(SkipDeclaration)) })),
+);
+
+function matches(declaration: SkipDeclaration, result: TestResult): boolean {
+  return declaration.file === result.file && declaration.test === result.name;
+}
+
+function byPlace(a: TestResult, b: TestResult): number {
+  return a.file.localeCompare(b.file) || a.line - b.line || a.name.localeCompare(b.name);
+}
+
+export function judgeSkips(
+  results: readonly TestResult[],
+  declarations: readonly SkipDeclaration[],
+  environment: Environment,
+): Verdict {
+  const skipped = results.filter((result) => result.outcome === "skipped" || result.outcome === "todo");
+  const applying = declarations.filter((declaration) => (declaration.when ?? environment) === environment);
+  return {
+    environment,
+    skipped: skipped.length,
+    undeclared: skipped.filter((result) => !applying.some((declaration) => matches(declaration, result))).sort(byPlace),
+    stale: applying.filter((declaration) => !skipped.some((result) => matches(declaration, result))),
+    unjudged: declarations.length - applying.length,
+  };
+}
+
+function refusedStale(verdict: Verdict): readonly SkipDeclaration[] {
+  return verdict.environment === "ci" ? verdict.stale : [];
+}
+
+export function passes(verdict: Verdict): boolean {
+  return verdict.undeclared.length === 0 && refusedStale(verdict).length === 0;
+}
+
+function staleLine(declaration: SkipDeclaration): string {
+  return `  ${declaration.file}${NAME_SEPARATOR}${declaration.test}: declared, but no such test skipped; delete the declaration`;
+}
+
+function verdictLines(verdict: Verdict): readonly string[] {
+  const { environment, skipped, undeclared, unjudged } = verdict;
+  const run = `this ${environment} run`;
+  if (passes(verdict)) {
+    const other = environment === "ci" ? "local" : "ci";
+    const aside = unjudged === 0 ? "" : `; ${unjudged} declaration(s) for ${other} not judged in ${run}`;
+    return [skipped === 0 ? `${NAME}: no test skipped${aside}` : `${NAME}: ${skipped} skipped test(s), each declared in ${DECLARATIONS}${aside}`];
+  }
+  const stale = refusedStale(verdict);
+  const counted = environment === "ci" ? ` and ${stale.length} declaration(s) matching no skipped test` : "";
+  return [
+    `${NAME}: ${undeclared.length} skipped test(s) undeclared${counted} in ${run}:`,
+    ...undeclared.map((result) => {
+      const kind = result.outcome === "todo" ? "a todo" : "skipped";
+      return `  ${result.file}:${result.line} ${result.name}: ${kind} with no declaration; run it, or declare it in ${DECLARATIONS} with its reason`;
+    }),
+    ...stale.map(staleLine),
+  ];
+}
+
+export function report(verdict: Verdict): string {
+  const warned = verdict.environment === "local" ? verdict.stale : [];
+  const warning =
+    warned.length === 0
+      ? []
+      : [
+          `${NAME}: warning: ${warned.length} declaration(s) matching no skipped test in this local run, refused only in a ci run:`,
+          ...warned.map(staleLine),
+        ];
+  return [...verdictLines(verdict), ...warning].join("\n");
+}
+
+const readDeclarations = Effect.fn("readDeclarations")(function* (root: string) {
+  const manifest = (yield* Path.Path).join(root, "package.json");
+  const { testSkips } = yield* (yield* FileSystem.FileSystem).readFileString(manifest).pipe(
+    Effect.flatMap(decodeManifest),
+    Effect.mapError((cause) => new SkipGateError({ message: `cannot read ${DECLARATIONS}: ${cause.message}` })),
+  );
+  return testSkips ?? [];
+});
+
+const environmentOf = Config.Boolean("CI").pipe(
+  Config.withDefault(false),
+  Config.map((ci): Environment => (ci ? "ci" : "local")),
+  Effect.mapError((cause) => new SkipGateError({ message: `cannot tell a ci run from a local one: ${cause.message}` })),
+);
+
+const runSuite = Effect.fn("runSuite")(function* (outfile: string) {
+  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+  const args = ["test", "--randomize", ...reporterArgs(outfile)];
+  return yield* spawner.exitCode(
+    ChildProcess.make(process.execPath, args, { stdin: "ignore", stdout: "inherit", stderr: "inherit" }),
+  );
+});
+
+const testEntry = Effect.gen(function* () {
+  if (process.argv.length > 2) return yield* new Usage({ message: USAGE });
+  const fs = yield* FileSystem.FileSystem;
+  const declarations = yield* readDeclarations(process.cwd());
+  const environment = yield* environmentOf;
+
+  const outfile = (yield* Path.Path).join(yield* fs.makeTempDirectoryScoped({ prefix: "checks-test-" }), "junit.xml");
+  const suitePassed = (yield* runSuite(outfile)) === ChildProcessSpawner.ExitCode(0);
+  if (!(yield* fs.exists(outfile))) {
+    if (!suitePassed) {
+      yield* Console.error(`${NAME}: bun test failed before it wrote a report, so no skip was judged`);
+      return false;
+    }
+    return yield* new ReportError({ message: `bun test passed but wrote no report to ${outfile}` });
+  }
+
+  const verdict = judgeSkips(yield* parseReport(yield* fs.readFileString(outfile)), declarations, environment);
+  yield* passes(verdict) ? Console.log(report(verdict)) : Console.error(report(verdict));
+  return suitePassed && passes(verdict);
+}).pipe(Effect.scoped);
+
+if (import.meta.main) runMain(NAME, testEntry);