@avi2dg/checks 0.20.0 → 0.22.0

package/scripts/test-skips.ts

@@ -0,0 +1,180 @@
+import { Effect, FileSystem, Path, Schema } from "effect";
+import { identifierName, isRecord, lineOf, parseTypeScript, spanStart, stringValue } from "./swc.ts";
+
+export type Environment = "ci" | "local";
+export type TestTier = "live" | "pixel";
+export type SkipDeclaration = {
+  readonly file: string;
+  readonly line: number;
+  readonly name: string;
+  readonly reason: string;
+  readonly when?: Environment;
+};
+
+export class SkipDeclarationError extends Schema.TaggedError<SkipDeclarationError>()("SkipDeclarationError", {
+  message: Schema.String,
+}) {}
+
+const TEST_FILES = ["tests/**/*.test.ts", "tests/**/*.test.tsx"] as const;
+const SKIP_HELPER_MODULE = "test-skips.ts";
+const whenSchema = Schema.Literals(["ci", "local"]);
+
+export function skipReason(_reason: string, label: string, _when?: Environment): string {
+  return label;
+}
+
+function expressionOf(argument: unknown): unknown {
+  return isRecord(argument) ? argument["expression"] : undefined;
+}
+
+function argumentsOf(node: Record<string, unknown>): readonly unknown[] {
+  const args = node["arguments"];
+  return Array.isArray(args) ? args : [];
+}
+
+function skipReasonAliases(statement: unknown): readonly string[] {
+  if (!isRecord(statement) || statement["type"] !== "ImportDeclaration") return [];
+  const source = stringValue(statement["source"]);
+  const specifiers = statement["specifiers"];
+  if (source === undefined || !source.endsWith(SKIP_HELPER_MODULE) || !Array.isArray(specifiers)) return [];
+  return specifiers.flatMap((specifier) => {
+    if (!isRecord(specifier) || specifier["type"] !== "ImportSpecifier") return [];
+    const imported = identifierName(specifier["imported"]) ?? identifierName(specifier["local"]);
+    const local = identifierName(specifier["local"]);
+    return imported === "skipReason" && local !== undefined ? [local] : [];
+  });
+}
+
+function importedSkipReasonNames(module: unknown): ReadonlySet<string> {
+  if (!isRecord(module) || !Array.isArray(module["body"])) return new Set();
+  return new Set(module["body"].flatMap(skipReasonAliases));
+}
+
+type Member = { readonly object: string | undefined; readonly property: string | undefined };
+
+function memberOf(node: unknown): Member | undefined {
+  if (!isRecord(node) || node["type"] !== "MemberExpression") return undefined;
+  return { object: identifierName(node["object"]), property: identifierName(node["property"]) };
+}
+
+type SkipReasonScan =
+  | { readonly kind: "none" }
+  | { readonly kind: "invalid"; readonly message: string }
+  | { readonly kind: "valid"; readonly name: string; readonly reason: string; readonly when?: Environment };
+
+function skipReasonCall(node: unknown, aliases: ReadonlySet<string>): SkipReasonScan {
+  if (!isRecord(node) || node["type"] !== "CallExpression") return { kind: "none" };
+  if (!aliases.has(identifierName(node["callee"]) ?? "")) return { kind: "none" };
+  const [reasonArgument, labelArgument, whenArgument] = argumentsOf(node);
+  const reason = stringValue(expressionOf(reasonArgument));
+  const label = stringValue(expressionOf(labelArgument));
+  if (reason === undefined || reason.trim() === "" || label === undefined || label.trim() === "") {
+    return { kind: "invalid", message: "skipReason needs a non-empty literal reason and a literal test name" };
+  }
+  const when = whenArgument === undefined ? undefined : stringValue(expressionOf(whenArgument));
+  if (whenArgument !== undefined && when === undefined) {
+    return { kind: "invalid", message: "skipReason's when value must be \"ci\" or \"local\"" };
+  }
+  if (when !== undefined && !Schema.is(whenSchema)(when)) {
+    return {
+      kind: "invalid",
+      message: `skipReason's when value must be "ci" or "local", found ${JSON.stringify(when)}`,
+    };
+  }
+  return { kind: "valid", name: label, reason, ...(when === undefined ? {} : { when }) };
+}
+
+type RegisteredSkip = { readonly onDescribe: boolean; readonly first: unknown };
+
+function registeredSkip(node: Record<string, unknown>): RegisteredSkip | undefined {
+  if (node["type"] !== "CallExpression") return undefined;
+  const callee = node["callee"];
+  const curried = isRecord(callee) && callee["type"] === "CallExpression";
+  const member = memberOf(curried ? callee["callee"] : callee);
+  const methods = curried ? ["skipIf", "if"] : ["skip", "todo"];
+  if (member === undefined || !methods.includes(member.property ?? "")) return undefined;
+  return { onDescribe: member.object === "describe", first: expressionOf(argumentsOf(node)[0]) };
+}
+
+function registeredSkips(module: unknown): readonly RegisteredSkip[] {
+  const found: RegisteredSkip[] = [];
+  const visit = (node: unknown): void => {
+    if (Array.isArray(node)) {
+      for (const child of node) visit(child);
+      return;
+    }
+    if (!isRecord(node)) return;
+    const registered = registeredSkip(node);
+    if (registered !== undefined) found.push(registered);
+    for (const value of Object.values(node)) visit(value);
+  };
+  visit(module);
+  return found;
+}
+
+type DeclarationsRead =
+  | { readonly kind: "valid"; readonly declarations: readonly SkipDeclaration[] }
+  | { readonly kind: "invalid"; readonly message: string };
+
+function declarationsIn(source: string, file: string, module: unknown): DeclarationsRead {
+  const aliases = importedSkipReasonNames(module);
+  if (aliases.size === 0) return { kind: "valid", declarations: [] };
+  const declarations: SkipDeclaration[] = [];
+  for (const { onDescribe, first } of registeredSkips(module)) {
+    if (!isRecord(first)) continue;
+    const metadata = skipReasonCall(first, aliases);
+    if (metadata.kind === "none") continue;
+    const line = lineOf(source, spanStart(first));
+    if (onDescribe) {
+      return {
+        kind: "invalid",
+        message: `line ${line}: skipReason cannot declare a describe; declare each test inside it with its own skipReason`,
+      };
+    }
+    if (metadata.kind === "invalid") return metadata;
+    const { name, reason, when } = metadata;
+    declarations.push({ file, line, name, reason, ...(when === undefined ? {} : { when }) });
+  }
+  return { kind: "valid", declarations };
+}
+
+function selectedTierFile(file: string, tier: TestTier | undefined): boolean {
+  if (file.startsWith("tests/quarantine/")) return false;
+  if (tier === undefined) return !file.startsWith("tests/live/") && !file.startsWith("tests/pixel/");
+  return file.startsWith(`tests/${tier}/`);
+}
+
+export const readSkipDeclarations = Effect.fn("readSkipDeclarations")(function* (
+  root: string,
+  tier: TestTier | undefined,
+) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const files = yield* Effect.tryPromise({
+    try: () =>
+      Promise.all(
+        TEST_FILES.map((pattern) =>
+          Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: root, onlyFiles: true })),
+        ),
+      ).then((groups) => groups.flat().filter((file) => selectedTierFile(file, tier)).sort()),
+    catch: (cause) => new SkipDeclarationError({ message: `cannot find test files: ${String(cause)}` }),
+  });
+  const declarations: SkipDeclaration[] = [];
+  for (const file of files) {
+    const source = yield* fs.readFileString(path.join(root, file)).pipe(
+      Effect.mapError((cause) => new SkipDeclarationError({ message: `cannot read ${file}: ${cause.message}` })),
+    );
+    const module = yield* parseTypeScript(file, source).pipe(
+      Effect.mapError((error) => new SkipDeclarationError({ message: error.message })),
+    );
+    const parsed = yield* Effect.try({
+      try: () => declarationsIn(source, file, module),
+      catch: (cause) => new SkipDeclarationError({ message: `cannot read skip declarations in ${file}: ${String(cause)}` }),
+    });
+    if (parsed.kind === "invalid") {
+      return yield* new SkipDeclarationError({ message: `${file}: ${parsed.message}` });
+    }
+    declarations.push(...parsed.declarations);
+  }
+  return declarations;
+});

package/scripts/test.ts

@@ -3,20 +3,10 @@ import { Config, Console, Effect, FileSystem, Path, Schema } from "effect";
 import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
 import { TEST_ENTRY_POINT } from "./gates.ts";
 import { runMain, Usage } from "./main.ts";
+import { readSkipDeclarations, type Environment, type SkipDeclaration, type TestTier } from "./test-skips.ts";
 import { NAME_SEPARATOR, parseReport, ReportError, reporterArgs, type TestResult } from "./test-report.ts";
 
-const Environment = Schema.Literals(["ci", "local"]);
-
-export type Environment = typeof Environment.Type;
-
-const SkipDeclaration = Schema.Struct({
-  file: Schema.NonEmptyString,
-  test: Schema.NonEmptyString,
-  reason: Schema.NonEmptyString,
-  when: Schema.optionalKey(Environment),
-});
-
-export type SkipDeclaration = typeof SkipDeclaration.Type;
+export type { Environment, SkipDeclaration } from "./test-skips.ts";
 
 export type Verdict = {
   readonly environment: Environment;
@@ -31,15 +21,23 @@ export class SkipGateError extends Schema.TaggedError<SkipGateError>()("SkipGate
 }) {}
 
 const NAME = TEST_ENTRY_POINT.bin;
-const DECLARATIONS = "package.json testSkips";
-const USAGE = `usage: ${NAME} takes no arguments, since a narrowed run skips every test it leaves out; run bun test --randomize <args> for one`;
-
-const decodeManifest = Schema.decodeUnknownEffect(
-  Schema.fromJsonString(Schema.Struct({ testSkips: Schema.optionalKey(Schema.Array(SkipDeclaration)) })),
-);
+const USAGE = `usage: ${NAME} takes no arguments except --tier=live or --tier=pixel`;
+const TIERS = new Map<string, TestTier>([
+  ["--tier=live", "live"],
+  ["--tier=pixel", "pixel"],
+]);
 
 function matches(declaration: SkipDeclaration, result: TestResult): boolean {
-  return declaration.file === result.file && declaration.test === result.name;
+  const { name } = declaration;
+  return (
+    declaration.file === result.file &&
+    declaration.line === result.line &&
+    (result.name === name || result.name.endsWith(`${NAME_SEPARATOR}${name}`))
+  );
+}
+
+function skippedOutcome(result: TestResult): boolean {
+  return result.outcome === "skipped" || result.outcome === "todo";
 }
 
 function byPlace(a: TestResult, b: TestResult): number {
@@ -51,7 +49,7 @@ export function judgeSkips(
   declarations: readonly SkipDeclaration[],
   environment: Environment,
 ): Verdict {
-  const skipped = results.filter((result) => result.outcome === "skipped" || result.outcome === "todo");
+  const skipped = results.filter(skippedOutcome);
   const applying = declarations.filter((declaration) => (declaration.when ?? environment) === environment);
   return {
     environment,
@@ -71,7 +69,7 @@ export function passes(verdict: Verdict): boolean {
 }
 
 function staleLine(declaration: SkipDeclaration): string {
-  return `  ${declaration.file}${NAME_SEPARATOR}${declaration.test}: declared, but no such test skipped; delete the declaration`;
+  return `  ${declaration.file}:${declaration.line}: no test skipped with this declaration; reason: ${declaration.reason}`;
 }
 
 function verdictLines(verdict: Verdict): readonly string[] {
@@ -80,15 +78,18 @@ function verdictLines(verdict: Verdict): readonly string[] {
   if (passes(verdict)) {
     const other = environment === "ci" ? "local" : "ci";
     const aside = unjudged === 0 ? "" : `; ${unjudged} declaration(s) for ${other} not judged in ${run}`;
-    return [skipped === 0 ? `${NAME}: no test skipped${aside}` : `${NAME}: ${skipped} skipped test(s), each declared in ${DECLARATIONS}${aside}`];
+    return [skipped === 0 ? `${NAME}: no test skipped${aside}` : `${NAME}: ${skipped} skipped test(s), each declared at its test site${aside}`];
   }
   const stale = refusedStale(verdict);
   const counted = environment === "ci" ? ` and ${stale.length} declaration(s) matching no skipped test` : "";
   return [
     `${NAME}: ${undeclared.length} skipped test(s) undeclared${counted} in ${run}:`,
     ...undeclared.map((result) => {
-      const kind = result.outcome === "todo" ? "a todo" : "skipped";
-      return `  ${result.file}:${result.line} ${result.name}: ${kind} with no declaration; run it, or declare it in ${DECLARATIONS} with its reason`;
+      const [kind, usage] =
+        result.outcome === "todo"
+          ? ["a todo", "test.todo(skipReason(reason, name))"]
+          : ["skipped", "test.skipIf(condition)(skipReason(reason, name), fn)"];
+      return `  ${result.file}:${result.line} ${result.name}: ${kind} with no reason at its test site; use ${usage}`;
     }),
     ...stale.map(staleLine),
   ];
@@ -106,37 +107,31 @@ export function report(verdict: Verdict): string {
   return [...verdictLines(verdict), ...warning].join("\n");
 }
 
-const readDeclarations = Effect.fn("readDeclarations")(function* (root: string) {
-  const manifest = (yield* Path.Path).join(root, "package.json");
-  const { testSkips } = yield* (yield* FileSystem.FileSystem).readFileString(manifest).pipe(
-    Effect.flatMap(decodeManifest),
-    Effect.mapError((cause) => new SkipGateError({ message: `cannot read ${DECLARATIONS}: ${cause.message}` })),
-  );
-  return testSkips ?? [];
-});
-
 const environmentOf = Config.Boolean("CI").pipe(
   Config.withDefault(false),
   Config.map((ci): Environment => (ci ? "ci" : "local")),
   Effect.mapError((cause) => new SkipGateError({ message: `cannot tell a ci run from a local one: ${cause.message}` })),
 );
 
-const runSuite = Effect.fn("runSuite")(function* (outfile: string) {
+const runSuite = Effect.fn("runSuite")(function* (outfile: string, tier: TestTier | undefined) {
   const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
-  const args = ["test", "--randomize", ...reporterArgs(outfile)];
+  const tierArgs = tier === undefined ? [] : ["--path-ignore-patterns", "", `./tests/${tier}`];
+  const args = ["test", "--randomize", ...tierArgs, ...reporterArgs(outfile)];
   return yield* spawner.exitCode(
     ChildProcess.make(process.execPath, args, { stdin: "ignore", stdout: "inherit", stderr: "inherit" }),
   );
 });
 
 const testEntry = Effect.gen(function* () {
-  if (process.argv.length > 2) return yield* new Usage({ message: USAGE });
+  const args = process.argv.slice(2);
+  const tier = args.length === 0 ? undefined : args.length === 1 ? TIERS.get(args[0] ?? "") : undefined;
+  if (args.length > 0 && tier === undefined) return yield* new Usage({ message: USAGE });
   const fs = yield* FileSystem.FileSystem;
-  const declarations = yield* readDeclarations(process.cwd());
+  const declarations = yield* readSkipDeclarations(process.cwd(), tier);
   const environment = yield* environmentOf;
 
   const outfile = (yield* Path.Path).join(yield* fs.makeTempDirectoryScoped({ prefix: "checks-test-" }), "junit.xml");
-  const suitePassed = (yield* runSuite(outfile)) === ChildProcessSpawner.ExitCode(0);
+  const suitePassed = (yield* runSuite(outfile, tier)) === ChildProcessSpawner.ExitCode(0);
   if (!(yield* fs.exists(outfile))) {
     if (!suitePassed) {
       yield* Console.error(`${NAME}: bun test failed before it wrote a report, so no skip was judged`);

package/scripts/vendor.ts

@@ -1,12 +1,10 @@
 #!/usr/bin/env bun
 import { Console, Effect, FileSystem, Option, Path, Schema } from "effect";
-import { QUALITY_FILE } from "./gates.ts";
 import { git } from "./git.ts";
 import { runMain, Usage } from "./main.ts";
-import { readQuality, type Library } from "./quality-file.ts";
 
 const NAME = "checks-vendor";
-const USAGE = `usage: ${NAME} takes no arguments, since quality.json names the libraries`;
+const USAGE = `usage: ${NAME} [--library <name> --package <package> --repository <remote> --tag <template> [--path <manifest>]]...`;
 const CACHE_HOME = ".cache/avi2dg-checks";
 const RECORD_SUFFIX = ".commit";
 const LINKS = "repos";
@@ -23,6 +21,55 @@ export class Unreachable extends Schema.TaggedError<Unreachable>()("Unreachable"
   message: Schema.String,
 }) {}
 
+const Library = Schema.Struct({
+  name: Schema.String.check(Schema.isPattern(/^[a-z0-9]+(?:-[a-z0-9]+)*$/)),
+  package: Schema.NonEmptyString,
+  repository: Schema.NonEmptyString,
+  tag: Schema.String.check(Schema.isPattern(/\{version\}/)),
+  path: Schema.optionalKey(Schema.String.check(Schema.isPattern(/^(?:[\w.@+-]+\/)*[\w.@+-]+\.\w+$/))),
+});
+type Library = typeof Library.Type;
+
+const Libraries = Schema.Array(Library).check(
+  Schema.makeFilter((libraries) => {
+    const names = libraries.map(({ name }) => name);
+    const repeated = names.find((name, index) => names.indexOf(name) !== index);
+    return repeated === undefined || `--library ${repeated} appears more than once`;
+  }),
+);
+
+const OPENER = "--library";
+const FIELDS = new Map<string, keyof Library>([
+  ["--package", "package"],
+  ["--repository", "repository"],
+  ["--tag", "tag"],
+  ["--path", "path"],
+]);
+
+function misuse(message: string): Usage {
+  return new Usage({ message: `${message}\n${USAGE}` });
+}
+
+const librariesFrom = Effect.fn("librariesFrom")(function* (args: readonly string[]) {
+  const groups: Record<string, string>[] = [];
+  for (let index = 0; index < args.length; index += 2) {
+    const flag = args[index] ?? "";
+    const value = args[index + 1];
+    if (value === undefined || value.startsWith("--")) return yield* misuse(`${flag} takes a value`);
+    if (flag === OPENER) {
+      groups.push({ name: value });
+      continue;
+    }
+    const field = FIELDS.get(flag);
+    const current = groups.at(-1);
+    if (field === undefined) return yield* misuse(`${flag} is not an argument`);
+    if (current === undefined) return yield* misuse(`${flag} comes before any ${OPENER}`);
+    if (field in current) return yield* misuse(`${flag} appears twice for ${OPENER} ${current["name"]}`);
+    current[field] = value;
+  }
+  return yield* Schema.decodeUnknownEffect(Libraries)(groups).pipe(Effect.mapError((cause) => misuse(cause.message)));
+});
+
 export function tagFor(template: string, version: string): string {
   return template.replaceAll(VERSION_TOKEN, version);
 }
@@ -296,8 +343,11 @@ const cacheRoot = Effect.fn("cacheRoot")(function* () {
 });
 
 const main = Effect.gen(function* () {
-  const [extra] = process.argv.slice(2);
-  if (extra !== undefined) return yield* new Usage({ message: USAGE });
+  const libraries = yield* librariesFrom(process.argv.slice(2));
+  if (libraries.length === 0) {
+    yield* Console.error(`${NAME}: no ${OPENER} argument names a library, so nothing is pinned`);
+    return true;
+  }
   const toplevel = yield* git(["rev-parse", "--show-toplevel"]).pipe(
     Effect.map((output) => Option.some(output.trim())),
     Effect.catchTag("GitFailure", (failure) =>
@@ -308,12 +358,6 @@ const main = Effect.gen(function* () {
   );
   if (Option.isNone(toplevel)) return true;
   const root = toplevel.value;
-  const { quality } = yield* readQuality(root);
-  const libraries = quality.sources?.libraries ?? [];
-  if (libraries.length === 0) {
-    yield* Console.error(`${NAME}: ${QUALITY_FILE} declares no libraries, so nothing is pinned`);
-    return true;
-  }
   const cache = yield* cacheRoot();
   let passed = true;
   for (const library of libraries) {