@avi2dg/checks 0.15.0 → 0.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@avi2dg/checks",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "Deterministic checks shared across the captain's TypeScript repos",
   "license": "MIT",
   "repository": {
@@ -41,6 +41,8 @@
     "scripts/quality-file.ts",
     "scripts/quality.ts",
     "scripts/size-budget.ts",
+    "scripts/size-rules.ts",
+    "scripts/repetition.ts",
     "scripts/feature-owners.ts",
     "scripts/docs.ts",
     "scripts/doc-outline.ts",
@@ -79,6 +81,7 @@
     "./scripts/quality-file.ts": "./scripts/quality-file.ts",
     "./scripts/quality.ts": "./scripts/quality.ts",
     "./scripts/size-budget.ts": "./scripts/size-budget.ts",
+    "./scripts/repetition.ts": "./scripts/repetition.ts",
     "./scripts/feature-owners.ts": "./scripts/feature-owners.ts",
     "./scripts/docs.ts": "./scripts/docs.ts",
     "./templates/readme.md": "./templates/readme.md",
@@ -113,6 +116,7 @@
     "checks-backtest": "scripts/backtest.ts",
     "checks-quality": "scripts/quality.ts",
     "checks-size-budget": "scripts/size-budget.ts",
+    "checks-repetition": "scripts/repetition.ts",
     "checks-feature-owners": "scripts/feature-owners.ts",
     "checks-docs": "scripts/docs.ts"
   },
@@ -127,6 +131,7 @@
     "dependency-cruiser": "18.4.0",
     "@effect/tsgo": "0.45.0",
     "effect": "4.0.0-rc.115",
+    "jscpd": "5.3.2",
     "oxlint": "1.83.0",
     "oxlint-tsgolint": "7.0.2002"
   },
@@ -138,6 +143,7 @@
     "ajv": "8.20.0",
     "dependency-cruiser": "18.4.0",
     "effect": "4.0.0-rc.115",
+    "jscpd": "5.3.2",
     "oxlint": "1.83.0",
     "oxlint-tsgolint": "7.0.2002",
     "typescript": "7.0.2"

package/quality.schema.json

@@ -47,6 +47,7 @@
               "checks-docs",
               "checks-quality",
               "checks-size-budget",
+              "checks-repetition",
               "checks-feature-owners"
             ]
           },
@@ -148,32 +149,76 @@
     "size": {
       "type": "object",
       "properties": {
-        "fileLines": {
-          "type": "integer",
-          "exclusiveMinimum": 0,
-          "description": "The most lines a file may hold, blank and comment lines counted"
-        },
-        "functionLines": {
-          "type": "integer",
-          "exclusiveMinimum": 0,
-          "description": "The most lines a function may span, blank and comment lines counted"
-        },
         "applies": {
           "type": "string",
           "enum": [
-            "changed",
+            "ratchet",
             "all"
           ],
-          "description": "Which production files the budget holds: changed, the ones a range adds or changes; all, every one. The rest are reported as advisory"
+          "description": "Which production and test files the budget holds: ratchet, the ones a range adds or changes, to no more overrun per rule than at the range's base; all, every one. ratchet when absent. The rest are listed as advisory"
+        },
+        "production": {
+          "type": "object",
+          "properties": {
+            "fileLines": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "description": "The most lines a file may hold, blank and comment lines counted; 400 when absent"
+            },
+            "functionLines": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "description": "The most lines a function may span, blank and comment lines counted; 100 when absent"
+            },
+            "statements": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "description": "The most statements a function may hold; 30 when absent"
+            },
+            "complexity": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "description": "The highest cyclomatic complexity a function may reach, a switch counted once; 15 when absent"
+            },
+            "depth": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "description": "The deepest a block may nest inside a function; 4 when absent"
+            }
+          },
+          "additionalProperties": false,
+          "description": "The budget of the files under sources.production, and of the files listed as advisory outside tests/"
+        },
+        "tests": {
+          "type": "object",
+          "properties": {
+            "fileLines": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "description": "The most lines a file may hold, blank and comment lines counted; 600 when absent"
+            },
+            "statements": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "description": "The most statements a function may hold; 50 when absent"
+            },
+            "complexity": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "description": "The highest cyclomatic complexity a function may reach, a switch counted once; 15 when absent"
+            },
+            "depth": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "description": "The deepest a block may nest inside a function; 4 when absent"
+            }
+          },
+          "additionalProperties": false,
+          "description": "The budget of the files under tests/, which sets no limit on a function's lines"
         }
       },
-      "required": [
-        "fileLines",
-        "functionLines",
-        "applies"
-      ],
       "additionalProperties": false,
-      "description": "The line budget oxlint holds production files to, read by checks-size-budget"
+      "description": "The size budget oxlint holds production and test files to, read by checks-size-budget"
     },
     "features": {
       "type": "array",

package/scripts/gates.ts

@@ -40,6 +40,7 @@ export const KIT_GATES = [
   { bin: "checks-docs", script: "docs.ts", reads: "range", appliesTo: EVERY_REPOSITORY },
   { bin: "checks-quality", script: "quality.ts", reads: "tree", args: ["--check"], appliesTo: QUALITY_DECLARATION },
   { bin: "checks-size-budget", script: "size-budget.ts", reads: "range", appliesTo: TYPESCRIPT_SOURCE },
+  { bin: "checks-repetition", script: "repetition.ts", reads: "range", appliesTo: TYPESCRIPT_SOURCE },
   { bin: "checks-feature-owners", script: "feature-owners.ts", reads: "range", appliesTo: TYPESCRIPT_SOURCE },
 ] as const satisfies readonly KitGate[];
 

package/scripts/git.ts

@@ -1,5 +1,6 @@
-import { Effect, Schema, Stream } from "effect";
+import { Effect, Path, Schema, Stream } from "effect";
 import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
+import { Usage } from "./main.ts";
 
 export class GitFailure extends Schema.TaggedError<GitFailure>()("GitFailure", {
   message: Schema.String,
@@ -159,3 +160,20 @@ export const rangeEnds = Effect.fn("rangeEnds")(function* (first: string, second
   if (second === undefined) return { base: yield* parentOrEmptyTree(first, cwd), head: first };
   return { base: (yield* git(["merge-base", first, second], cwd)).trim(), head: second };
 });
+
+export const rangeFromArgs = Effect.fn("rangeFromArgs")(function* (args: readonly string[], usage: string, cwd?: string) {
+  const [first, second, ...extra] = args;
+  if (first === undefined || extra.length > 0) return yield* new Usage({ message: usage });
+  return yield* rangeEnds(first, second, cwd);
+});
+
+// A scratch index leaves the repository's own index and working tree untouched.
+export const checkoutFiles = Effect.fn("checkoutFiles")(function* (rev: string, files: readonly string[], scratch: string, cwd?: string) {
+  const path = yield* Path.Path;
+  const env = { GIT_INDEX_FILE: path.join(scratch, "index") };
+  const tree = path.join(scratch, "tree");
+  yield* git(["read-tree", rev], cwd, { env });
+  const listed = files.map((file) => `${file}\0`).join("");
+  yield* git(["checkout-index", "-z", "--stdin", `--prefix=${tree}${path.sep}`], cwd, { env, input: listed });
+  return tree;
+});

package/scripts/quality-file.ts

@@ -1,5 +1,6 @@
 import { Console, Effect, FileSystem, JsonSchema, Path, Schema } from "effect";
 import { LintGates, QUALITY_FILE } from "./gates.ts";
+import { Size } from "./size-rules.ts";
 
 const SEGMENT = String.raw`(?!\.\.?(?:/|$))(?:\*\*|(?:[\w.@+-]|\*(?!\*))+)`;
 
@@ -97,19 +98,6 @@ const Sources = Schema.Struct({
   effect: Schema.optionalKey(EffectSources),
 });
 
-const LineBudget = Schema.Int.check(Schema.isGreaterThan(0));
-
-const Size = Schema.Struct({
-  fileLines: LineBudget.annotate({ description: "The most lines a file may hold, blank and comment lines counted" }),
-  functionLines: LineBudget.annotate({
-    description: "The most lines a function may span, blank and comment lines counted",
-  }),
-  applies: Schema.Literals(["changed", "all"]).annotate({
-    description:
-      "Which production files the budget holds: changed, the ones a range adds or changes; all, every one. The rest are reported as advisory",
-  }),
-});
-
 const Feature = Schema.Struct({
   name: Schema.String.check(
     Schema.isPattern(/^[a-z0-9]+(?:-[a-z0-9]+)*$/, { expected: "a feature name in kebab case" }),
@@ -192,9 +180,7 @@ export const Quality = Schema.Struct({
   gates: Schema.optionalKey(Gates),
   commitIdentity: Schema.optionalKey(CommitIdentity),
   sources: Schema.optionalKey(Sources),
-  size: Schema.optionalKey(
-    Size.annotate({ description: "The line budget oxlint holds production files to, read by checks-size-budget" }),
-  ),
+  size: Schema.optionalKey(Size),
   features: Schema.optionalKey(
     Features.annotate({
       description: "The feature owners dependency-cruiser holds to their entries and checks-feature-owners maps a change to",
@@ -215,7 +201,7 @@ export const Quality = Schema.Struct({
   .check(
     Schema.makeFilter(
       ({ size, sources }) =>
-        size === undefined || (sources?.production ?? []).length > 0 || "declares size, which holds nothing without sources.production",
+        size === undefined || (sources?.production ?? []).length > 0 || "declares size, which holds no production file without sources.production",
       {
         toJsonSchema: () => ({
           if: { required: ["size"] },

package/scripts/quality.ts

@@ -120,10 +120,10 @@ const fragmentProblems = Effect.fn("fragmentProblems")(function* (root: string,
 
 const unmatchedPaths = Effect.fn("unmatchedPaths")(function* (root: string, quality: Quality) {
   const declared = [
-    ...(quality.size === undefined ? [] : (quality.sources?.production ?? [])).map((glob) => ({
+    ...(quality.sources?.production ?? []).map((glob) => ({
       glob,
       key: "sources.production",
-      holds: "no source to the size budget",
+      holds: "no source to checks-size-budget or checks-repetition",
     })),
     ...(quality.sources?.effect?.paths ?? []).map((glob) => ({ glob, key: "sources.effect.paths", holds: "nothing to the Effect rules" })),
   ];

package/scripts/repetition.ts

@@ -0,0 +1,161 @@
+#!/usr/bin/env bun
+import { Console, Effect, FileSystem, Path, Schema } from "effect";
+import { changedPaths, checkoutFiles, collect, git, pathsAt, rangeFromArgs } from "./git.ts";
+import { runMain } from "./main.ts";
+import { readQuality } from "./quality-file.ts";
+
+type Fragment = {
+  readonly file: string;
+  readonly start: number;
+  readonly end: number;
+};
+
+type Clone = readonly [Fragment, Fragment];
+
+type Rise = {
+  readonly file: string;
+  readonly before: number;
+  readonly after: number;
+  readonly clones: readonly Clone[];
+};
+
+type Held = {
+  readonly measured: number;
+  readonly rises: readonly Rise[];
+  readonly advisory: ReadonlyMap<string, number>;
+};
+
+class JscpdUnreadable extends Schema.TaggedError<JscpdUnreadable>()("JscpdUnreadable", {
+  message: Schema.String,
+}) {}
+
+const NAME = "repetition";
+const USAGE = "usage: repetition.ts <ref> | <base-ref> <head-ref>";
+const MIN_TOKENS = 50;
+const MIN_LINES = 5;
+const MEASURE = `at ${MIN_TOKENS} tokens and ${MIN_LINES} lines`;
+const DECLARATIONS = ":(exclude,glob)**/*.d.ts";
+const TYPESCRIPT = [":(glob)**/*.ts", ":(glob)**/*.tsx", DECLARATIONS];
+const JSCPD_FINISHED = 0;
+const REPORT = "jscpd-report.json";
+
+const Location = Schema.Struct({ name: Schema.String, start: Schema.Int, end: Schema.Int });
+
+const decodeReport = Schema.decodeUnknownEffect(
+  Schema.fromJsonString(Schema.Struct({ duplicates: Schema.Array(Schema.Struct({ firstFile: Location, secondFile: Location })) })),
+);
+
+function fragmentOf({ name, start, end }: typeof Location.Type): Fragment {
+  return { file: name, start, end };
+}
+
+export function repeatedLines(clones: readonly Clone[]): ReadonlyMap<string, number> {
+  const lines = new Map<string, Set<number>>();
+  for (const { file, start, end } of clones.flat()) {
+    const covered = lines.get(file) ?? new Set<number>();
+    for (let line = start; line <= end; line += 1) covered.add(line);
+    lines.set(file, covered);
+  }
+  return new Map([...lines].map(([file, covered]) => [file, covered.size]));
+}
+
+// jscpd reads files from disk, and the working tree need not hold the revision: a merge checkout or an uncommitted edit.
+const scan = Effect.fn("scan")(
+  function* (root: string, rev: string, files: readonly string[]) {
+    if (files.length === 0) return [];
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const scratch = yield* fs.makeTempDirectoryScoped({ prefix: "checks-repetition-" });
+    const tree = yield* checkoutFiles(rev, files, scratch, root);
+    const output = path.join(scratch, "report");
+    const flags = ["--silent", "--no-tips", "--no-gitignore", "--reporters", "json", "--output", output];
+    const thresholds = ["--min-tokens", `${MIN_TOKENS}`, "--min-lines", `${MIN_LINES}`];
+
+    const run = yield* collect("jscpd", [...flags, ...thresholds, "."], tree).pipe(
+      Effect.mapError((cause) => new JscpdUnreadable({ message: `cannot run jscpd: ${cause.message}` })),
+    );
+    if (run.exitCode !== JSCPD_FINISHED) {
+      return yield* new JscpdUnreadable({ message: `jscpd exited ${run.exitCode}: ${run.stderr.trim() || run.stdout.trim()}` });
+    }
+    const { duplicates } = yield* fs.readFileString(path.join(output, REPORT)).pipe(
+      Effect.flatMap(decodeReport),
+      Effect.mapError((cause) => new JscpdUnreadable({ message: `cannot read jscpd's report: ${cause.message}` })),
+    );
+    return duplicates.map(({ firstFile, secondFile }): Clone => [fragmentOf(firstFile), fragmentOf(secondFile)]);
+  },
+  Effect.scoped,
+);
+
+function clonesOf(file: string, clones: readonly Clone[]): readonly Clone[] {
+  return clones
+    .flatMap(([first, second]): Clone[] => {
+      if (first.file === file) return [[first, second]];
+      return second.file === file ? [[second, first]] : [];
+    })
+    .toSorted(([a], [b]) => a.start - b.start);
+}
+
+const runHold = Effect.fn("runHold")(function* (root: string, production: readonly string[], base: string, head: string) {
+  const pathspecs = [...production.map((glob) => `:(glob)${glob}`), DECLARATIONS];
+  const held = yield* pathsAt(head, pathspecs, root);
+  const holds = new Set(held);
+  const others = (yield* pathsAt(head, TYPESCRIPT, root)).filter((file) => !holds.has(file));
+  const formerPath = new Map(
+    (yield* changedPaths(base, head, pathspecs, root)).flatMap((change) => (change.kind === "renamed" ? [[change.path, change.from]] : [])),
+  );
+
+  const before = repeatedLines(yield* scan(root, base, yield* pathsAt(base, pathspecs, root)));
+  const clones = yield* scan(root, head, held);
+  const after = repeatedLines(clones);
+  const rises = held.flatMap((file): Rise[] => {
+    const was = before.get(formerPath.get(file) ?? file) ?? 0;
+    const is = after.get(file) ?? 0;
+    return is > was ? [{ file, before: was, after: is, clones: clonesOf(file, clones) }] : [];
+  });
+  const risen = new Set(rises.map((rise) => rise.file));
+  const advisory = [...after].filter(([file]) => !risen.has(file)).concat([...repeatedLines(yield* scan(root, head, others))]);
+  return {
+    measured: held.length,
+    rises: rises.toSorted((a, b) => a.file.localeCompare(b.file)),
+    advisory: new Map(advisory.toSorted(([a], [b]) => a.localeCompare(b))),
+  } satisfies Held;
+});
+
+function describe({ file, before, after, clones }: Rise): readonly string[] {
+  return [
+    `  ${file}: ${after} repeated line(s), up from ${before}`,
+    ...clones.map(([own, other]) => `    ${own.file}:${own.start}-${own.end} repeats ${other.file}:${other.start}-${other.end}`),
+  ];
+}
+
+function report({ measured, rises, advisory }: Held): string {
+  const verdict =
+    rises.length === 0
+      ? [`${NAME}: ${measured} production file(s) repeat no more lines than where the range starts, ${MEASURE}`]
+      : [`${NAME}: ${rises.length} production file(s) repeat more lines than where the range starts, ${MEASURE}:`, ...rises.flatMap(describe)];
+  const notice =
+    advisory.size === 0
+      ? []
+      : [
+          `${NAME}: advisory, ${advisory.size} file(s) repeat lines the hold does not fail:`,
+          ...[...advisory].map(([file, lines]) => `  ${file}: ${lines} repeated line(s)`),
+        ];
+  return [...verdict, ...notice].join("\n");
+}
+
+const hold = Effect.gen(function* () {
+  const root = (yield* git(["rev-parse", "--show-toplevel"])).trim();
+  const { base, head } = yield* rangeFromArgs(process.argv.slice(2), USAGE, root);
+  const { source, quality } = yield* readQuality(root);
+  const production = quality.sources?.production ?? [];
+  if (production.length === 0) {
+    yield* Console.log(`${NAME}: ${source} declares no sources.production`);
+    return true;
+  }
+  const held = yield* runHold(root, production, base, head);
+
+  yield* Console.log(report(held));
+  return held.rises.length === 0;
+});
+
+if (import.meta.main) runMain(NAME, hold);