@codesema/speccov 0.0.0-stage → 0.2.1

package/src/run.ts

@@ -0,0 +1,70 @@
+import type { Measurement, Rule, Verdict } from './rule';
+
+/**
+ * Runs each rule and classifies the result.
+ *
+ * A written rule that is violated is a violation — a bug. A default rule that is violated
+ * is a gap — a question, not an accusation: nobody ever wrote that this behaviour was
+ * forbidden.
+ *
+ * And a rule held without ever being exercised is vacuous: it proved nothing.
+ */
+export async function run(rules: readonly Rule[]): Promise<readonly Verdict[]> {
+    const verdicts: Verdict[] = [];
+    for (const rule of rules) {
+        const measurements = await rule.measure();
+        const { held, detail, magnitude, exercised } = rule.judge(measurements);
+
+        if (held && !exercised) {
+            // Green having seen nothing: the textbook case of vacuity. It cannot sit with
+            // the rules held, it held nothing.
+            verdicts.push({
+                kind: 'vacuous',
+                rule,
+                measurements,
+                magnitude,
+                detail,
+                question: `${detail}. The rule was never put to the test — what was it meant to check?`,
+            });
+        } else if (held) {
+            verdicts.push(
+                atBoundary(rule, measurements)
+                    ? {
+                          kind: 'borderline',
+                          rule,
+                          measurements,
+                          magnitude,
+                          detail,
+                          question: `${detail}. One notch stricter and the rule would fall — was the threshold chosen, or fitted to what we observed?`,
+                      }
+                    : { kind: 'held', rule, measurements, magnitude, detail },
+            );
+        } else if (rule.origin === 'declared') {
+            verdicts.push({ kind: 'infraction', rule, measurements, magnitude, detail });
+        } else {
+            verdicts.push({
+                kind: 'lacuna',
+                rule,
+                measurements,
+                magnitude,
+                detail,
+                question: `${detail}. No written rule forbids it. Is this intended?`,
+            });
+        }
+    }
+    return verdicts;
+}
+
+/**
+ * Rejudges the same measurements with the rule one notch stricter.
+ *
+ * Nothing is re-run: `judge()` is pure on its argument, so the probe costs a single call.
+ * A rule with no stricter variant has no margin to question.
+ */
+function atBoundary(rule: Rule, measurements: readonly Measurement[]): boolean {
+    const stricter = rule.tighten?.();
+    if (stricter === undefined) {
+        return false;
+    }
+    return !stricter.judge(measurements).held;
+}

package/src/scan/answers.ts

@@ -0,0 +1,152 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import type { ShapeKind, Site } from './shapes';
+
+/**
+ * A human answer to a candidate site, made mechanisable.
+ *
+ * `scan` is the cheapest entry point, so it is the one that runs in CI and in an agent loop —
+ * and until now it was the only one with nowhere to write an answer down. A site ruled out by
+ * a human came back identical at the next pass, carrying the same weight as a real N+1. For an
+ * agent that is worse than noise: the README's own worst case, "degrading code that was fine",
+ * needs exactly this to happen twice.
+ *
+ * What is remembered is a **shape**, never a line. A site moves down a file every time
+ * something is inserted above it, and an answer keyed on a line number would expire on the
+ * next commit, which is a way of remembering nothing.
+ */
+export type SiteAnswer = {
+    /** The file the site sits in, relative to the scanned root. */
+    readonly file: string;
+    /** Which question was answered: the same code can carry two. */
+    readonly kind: ShapeKind;
+    /** The code that was ruled on, whitespace normalised. */
+    readonly shape: string;
+    /** Date of the acceptance. */
+    readonly at: string;
+    /** Why it is not a defect. Optional, but it is what gets reread in six months. */
+    readonly why?: string;
+};
+
+export const DEFAULT_SCAN_ANSWERS_PATH = '.speccov/scan-answers.json';
+
+type SiteAnswerFile = {
+    readonly version: 1;
+    readonly sites: readonly SiteAnswer[];
+};
+
+/**
+ * Loads the answered sites. A missing file is normal — it is the first pass.
+ *
+ * An unreadable file **throws**, for the same reason `answers.ts` throws: read as empty it
+ * makes every pass noisy, read as "all accepted" it makes the pass blind. Both are worse than
+ * a halt.
+ */
+export async function loadSiteAnswers(path: string): Promise<readonly SiteAnswer[]> {
+    let raw: string;
+    try {
+        raw = await readFile(path, 'utf8');
+    } catch (cause) {
+        if ((cause as NodeJS.ErrnoException).code === 'ENOENT') {
+            return [];
+        }
+        throw cause;
+    }
+    return parseSiteAnswers(JSON.parse(raw), path);
+}
+
+export async function saveSiteAnswers(path: string, answers: readonly SiteAnswer[]): Promise<void> {
+    const sorted = [...answers].sort((left, right) => keyOf(left).localeCompare(keyOf(right)));
+    const file: SiteAnswerFile = { version: 1, sites: sorted };
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, `${JSON.stringify(file, null, 4)}\n`, 'utf8');
+}
+
+export function parseSiteAnswers(value: unknown, path: string): readonly SiteAnswer[] {
+    const file = value as Partial<SiteAnswerFile> | null;
+    if (file === null || typeof file !== 'object' || file.version !== 1 || !Array.isArray(file.sites)) {
+        throw new Error(`speccov: ${path} is not a valid scan answers file (expected { version: 1, sites: [] }).`);
+    }
+    return file.sites.map((answer, index) => parseSiteAnswer(answer, path, index));
+}
+
+function parseSiteAnswer(value: unknown, path: string, index: number): SiteAnswer {
+    const answer = value as Partial<SiteAnswer> | null;
+    const valid =
+        answer !== null &&
+        typeof answer === 'object' &&
+        typeof answer.file === 'string' &&
+        typeof answer.kind === 'string' &&
+        typeof answer.shape === 'string' &&
+        typeof answer.at === 'string';
+    if (!valid) {
+        throw new Error(`speccov: answered site ${index} is unreadable in ${path}.`);
+    }
+    const kept: SiteAnswer = { file: answer.file, kind: answer.kind as ShapeKind, shape: answer.shape, at: answer.at };
+    return typeof answer.why === 'string' ? { ...kept, why: answer.why } : kept;
+}
+
+export type AppliedSites = {
+    /** What the report still has to ask about. */
+    readonly sites: readonly Site[];
+    /** Sites kept quiet because a human already ruled. Counted, never invisible. */
+    readonly accepted: readonly Site[];
+};
+
+/**
+ * Applies the known answers to what this pass found.
+ *
+ * A site whose shape has not moved goes quiet, and stays counted. A site whose code changed
+ * comes back: the answer bore on the code as it read that day, and it is not the same question
+ * any more. That is the same refusal as an exceeded threshold in `answers.ts` — an acceptance
+ * is a rule the build defends, not a permanent silence.
+ */
+export function applySiteAnswers(sites: readonly Site[], answers: readonly SiteAnswer[]): AppliedSites {
+    const known = new Set(answers.map(keyOf));
+    const kept: Site[] = [];
+    const accepted: Site[] = [];
+    for (const site of sites) {
+        (known.has(keyOf(asAnswer(site))) ? accepted : kept).push(site);
+    }
+    return { sites: kept, accepted };
+}
+
+/** Turns a site into a "this is not a defect" answer, on the code as it reads today. */
+export function acceptSite(site: Site, at: string, why?: string): SiteAnswer {
+    const answer = { ...asAnswer(site), at };
+    return why === undefined || why.trim() === '' ? answer : { ...answer, why: why.trim() };
+}
+
+function asAnswer(site: Site): Omit<SiteAnswer, 'at'> {
+    return { file: site.file, kind: site.kind, shape: shapeOf(site.excerpt) };
+}
+
+/**
+ * What makes two sites the same one.
+ *
+ * The line is left out on purpose: it moves with every insertion above. What is left is the
+ * file, the question asked, and the code itself — which is what a human actually ruled on.
+ */
+function keyOf(answer: Omit<SiteAnswer, 'at'>): string {
+    return `${answer.file} · ${answer.kind} · ${canonical(answer.shape)}`;
+}
+
+/**
+ * The code with its layout dropped: reformatting is not a new question.
+ *
+ * Kept readable on purpose, because the answers file is read in review — the harsher form
+ * below is for comparing, never for storing. Note the one thing this cannot absorb: the
+ * excerpt is truncated, so a reformat long enough to move the cut asks again. The cautious
+ * direction, and the same one as everywhere else.
+ */
+export function shapeOf(excerpt: string): string {
+    return excerpt.replace(/\s+/gu, ' ').trim();
+}
+
+/**
+ * What a formatter is allowed to change without it becoming another site: whitespace, and the
+ * trailing comma prettier adds when it breaks a call across lines.
+ */
+function canonical(shape: string): string {
+    return shape.replace(/\s+/gu, '').replace(/,(?=[)\]}])/gu, '');
+}

package/src/scan/climb.ts

@@ -0,0 +1,26 @@
+import { existsSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+
+/**
+ * The nearest file of that name, from `start` upwards.
+ *
+ * We stop at the first one found, and never above the repository root: a file two levels
+ * above the project no longer describes it.
+ */
+export function climbFor(name: string, start: string): string | undefined {
+    let current = resolve(start);
+    for (;;) {
+        const candidate = join(current, name);
+        if (existsSync(candidate)) {
+            return candidate;
+        }
+        if (existsSync(join(current, '.git'))) {
+            return undefined;
+        }
+        const parent = dirname(current);
+        if (parent === current) {
+            return undefined;
+        }
+        current = parent;
+    }
+}

package/src/scan/flow.ts

@@ -0,0 +1,65 @@
+import ts from 'typescript';
+
+export function returnsPromise(call: ts.CallExpression, checker: ts.TypeChecker): boolean {
+    const signature = checker.getResolvedSignature(call);
+    if (signature === undefined) {
+        return false;
+    }
+    const returned = checker.getReturnTypeOfSignature(signature);
+    return checker.typeToString(returned).startsWith('Promise<');
+}
+
+/**
+ * Does this parameter leave the function through a call that returns a promise?
+ *
+ * "Exported" was the only definition of a frontier, and in a codebase of ES modules
+ * everything is exported: measured on a real monorepo, the question was asked 744 times, six
+ * times more than the four others together. A value that reaches an asynchronous call goes
+ * somewhere the compiler does not follow, a query, a request, a file. A value that only feeds
+ * a computation stays where the types still hold.
+ *
+ * One hop, and no more: the parameter is read inside the arguments of the call, never traced
+ * through a variable or a helper. So this can only take sites away, never invent one. And
+ * where there is nothing to read, a declaration without a body, the question stays.
+ */
+export function reachesAsyncCall(
+    parameter: ts.ParameterDeclaration,
+    body: ts.Node | undefined,
+    checker: ts.TypeChecker,
+): boolean {
+    const symbol = checker.getSymbolAtLocation(parameter.name);
+    if (body === undefined || symbol === undefined) {
+        return true;
+    }
+    let reached = false;
+    const visit = (node: ts.Node): void => {
+        if (reached) {
+            return;
+        }
+        if (
+            ts.isCallExpression(node) &&
+            returnsPromise(node, checker) &&
+            node.arguments.some((argument) => mentions(argument, symbol, checker))
+        ) {
+            reached = true;
+            return;
+        }
+        ts.forEachChild(node, visit);
+    };
+    visit(body);
+    return reached;
+}
+
+function mentions(node: ts.Node, symbol: ts.Symbol, checker: ts.TypeChecker): boolean {
+    if (ts.isIdentifier(node) && symbolOf(node, checker) === symbol) {
+        return true;
+    }
+    return ts.forEachChild(node, (child) => mentions(child, symbol, checker)) ?? false;
+}
+
+/** In `{ id }` the identifier names the property: the value it carries is asked for separately. */
+function symbolOf(identifier: ts.Identifier, checker: ts.TypeChecker): ts.Symbol | undefined {
+    return ts.isShorthandPropertyAssignment(identifier.parent)
+        ? checker.getShorthandAssignmentValueSymbol(identifier.parent)
+        : checker.getSymbolAtLocation(identifier);
+}

package/src/scan/guarantees.ts

@@ -0,0 +1,234 @@
+import { createRequire } from 'node:module';
+import { dirname, join, resolve } from 'node:path';
+import ts from 'typescript';
+
+export type Guarantee = {
+    readonly flag: string;
+    /** What the flag proves, in one sentence. */
+    readonly proves: string;
+    readonly enabled: boolean;
+};
+
+/**
+ * `impliedByStrict` matters: `parseJsonConfigFileContent` keeps `strict: true` as-is and
+ * leaves the flags it implies at `undefined`. Read naively, they would be called off.
+ */
+const FLAGS: readonly { flag: string; proves: string; impliedByStrict: boolean }[] = [
+    { flag: 'strictNullChecks', proves: 'no value is implicitly nullable', impliedByStrict: true },
+    { flag: 'noImplicitAny', proves: 'no parameter falls back to the any type', impliedByStrict: true },
+    { flag: 'useUnknownInCatchVariables', proves: 'a caught error is never typed any', impliedByStrict: true },
+    { flag: 'strictFunctionTypes', proves: 'function types are checked contravariantly', impliedByStrict: true },
+    { flag: 'strictPropertyInitialization', proves: 'no class property stays uninitialised', impliedByStrict: true },
+    { flag: 'noUncheckedIndexedAccess', proves: 'every indexed access is treated as possibly absent', impliedByStrict: false },
+    { flag: 'exactOptionalPropertyTypes', proves: 'an optional property cannot receive undefined', impliedByStrict: false },
+    { flag: 'noFallthroughCasesInSwitch', proves: 'no switch case falls through in silence', impliedByStrict: false },
+    { flag: 'noImplicitOverride', proves: 'no method overrides a parent by accident', impliedByStrict: false },
+    { flag: 'noPropertyAccessFromIndexSignature', proves: 'no dot access on an index signature', impliedByStrict: false },
+];
+
+/**
+ * The result of a read. The `unresolved` case exists because an unresolved `extends` makes
+ * TypeScript fall back to its defaults **in silence**: concluding "nothing is proven" then
+ * fires a broadside of false alarms on a project that is in fact full strict.
+ *
+ * A tool you trust does not guess. It says it could not.
+ */
+/**
+ * TS18003, "No inputs were found in config file".
+ *
+ * The only diagnostic ignored, and by name: it bears on the files to compile, never on
+ * the `compilerOptions` we came to read. Every other code stays blocking — an unknown
+ * diagnostic must not become a green light.
+ */
+const NO_INPUTS_FOUND = 18_003;
+
+/**
+ * TS6046, "Argument for '--x' option must be: …".
+ *
+ * A **value** error, not a resolution one: the file was found, the `extends` was followed,
+ * the other options are read right. Only the value of one option is unrecognised —
+ * typically a `lib` newer than the TypeScript shipped here.
+ *
+ * The distinction is fundamental. An `extends` that cannot be found makes EVERY option
+ * fall back to its default in silence: blocking. An unknown value on an option we do not
+ * read touches none of the flags: not blocking — but declared, never swallowed.
+ */
+const OPTION_VALUE = 6046;
+
+const OPTION_NAMED = /Argument for '--(?<option>[a-zA-Z]+)' option/u;
+
+export type Guarantees =
+    | {
+          readonly kind: 'read';
+          readonly flags: readonly Guarantee[];
+          /** Options whose value speccov's compiler did not recognise. Declared, never silenced. */
+          readonly unrecognised: readonly string[];
+      }
+    | { readonly kind: 'unresolved'; readonly path: string; readonly reason: string };
+
+/**
+ * Reads what the compiler already proves, `extends` resolved.
+ *
+ * These guarantees cost nothing: no rule, no test, no question. Naming them is what
+ * makes the rest of the report short.
+ */
+export function readGuarantees(tsconfigPath: string): Guarantees {
+    return readScope(tsconfigPath, new Set());
+}
+
+function readScope(tsconfigPath: string, seen: Set<string>): Guarantees {
+    if (seen.has(tsconfigPath)) {
+        return { kind: 'unresolved', path: tsconfigPath, reason: 'circular reference between projects' };
+    }
+    seen.add(tsconfigPath);
+
+    const { config, error } = ts.readConfigFile(tsconfigPath, ts.sys.readFile);
+    if (error !== undefined) {
+        return { kind: 'unresolved', path: tsconfigPath, reason: flatten(error) };
+    }
+    const parsed = ts.parseJsonConfigFileContent(config, ts.sys, dirname(tsconfigPath));
+    const unrecognised: string[] = [];
+    for (const diagnostic of parsed.errors) {
+        if (diagnostic.code === NO_INPUTS_FOUND) {
+            continue;
+        }
+        const option = harmless(diagnostic);
+        if (option === undefined) {
+            return { kind: 'unresolved', path: tsconfigPath, reason: flatten(diagnostic) };
+        }
+        unrecognised.push(option);
+    }
+
+    const references = parsed.projectReferences ?? [];
+    if (references.length > 0 && parsed.fileNames.length === 0) {
+        return delegated(tsconfigPath, references, seen, unrecognised);
+    }
+    return {
+        kind: 'read',
+        flags: flagsOf(parsed.options, strictByDefault(dirname(tsconfigPath))),
+        unrecognised: [...new Set(unrecognised)],
+    };
+}
+
+/**
+ * TypeScript 6 turned `strict` on by default. The default that counts is the one of the
+ * compiler the project runs, not the one shipped here: read with ours, a TS 6 project that
+ * never writes `strict` was called lukewarm in every scope.
+ */
+function strictByDefault(directory: string): boolean {
+    return Number(projectTypeScriptVersion(directory).split('.')[0]) >= 6;
+}
+
+/** The version of the TypeScript the project resolves, or ours when it resolves none. */
+function projectTypeScriptVersion(directory: string): string {
+    try {
+        const manifest: unknown = createRequire(join(resolve(directory), 'tsconfig.json'))('typescript/package.json');
+        const version = (manifest as { version?: unknown }).version;
+        return typeof version === 'string' ? version : ts.version;
+    } catch {
+        // TODO(ceiling): no TypeScript resolvable from the project, ours stands in and can call a TS 6 project lukewarm: declare it in "Not measured".
+        return ts.version;
+    }
+}
+
+/**
+ * The first diagnostic that forbids concluding, or `undefined` if all of them are harmless.
+ *
+ * Shared with building the typed program: two parallel disciplines would have drifted, and
+ * that is exactly what happened — type checking accepted a broken `extends` that the flag
+ * read refused, then announced its types as sound.
+ */
+export function firstBlocking(errors: readonly ts.Diagnostic[]): ts.Diagnostic | undefined {
+    return errors.find((diagnostic) => diagnostic.code !== NO_INPUTS_FOUND && harmless(diagnostic) === undefined);
+}
+
+/**
+ * The option name, if the diagnostic cannot skew what we read. `undefined` otherwise —
+ * and `undefined` stays blocking: a diagnostic we cannot classify is not a green light.
+ */
+function harmless(diagnostic: ts.Diagnostic): string | undefined {
+    if (diagnostic.code !== OPTION_VALUE) {
+        return undefined;
+    }
+    const option = OPTION_NAMED.exec(flatten(diagnostic))?.groups?.['option'];
+    // The second term cannot fire today — TS6046 only bears on options with enumerated
+    // values, and the ten flags are booleans. It stays on principle: the day the table
+    // takes something else, it would be a green light granted on a flag.
+    if (option === undefined || FLAGS.some(({ flag }) => flag === option)) {
+        return undefined;
+    }
+    return option;
+}
+
+/**
+ * A "solution" tsconfig compiles nothing itself: `files: []` plus `references`. It declares
+ * no option, so reading it naively would call everything off — while the referenced
+ * projects can be full strict. That is the case of every Nuxt project.
+ *
+ * So we descend into the references. A guarantee holds for the scope only if it holds in
+ * **each** delegated project: a single lukewarm link takes it away.
+ *
+ * An unreadable reference makes the whole scope unreadable. The opposite — ignoring it and
+ * concluding on the others — would recreate the silent hole this type exists to close.
+ */
+function delegated(
+    tsconfigPath: string,
+    references: readonly ts.ProjectReference[],
+    seen: Set<string>,
+    inherited: readonly string[],
+): Guarantees {
+    const parts: Extract<Guarantees, { kind: 'read' }>[] = [];
+    for (const reference of references) {
+        const resolved = resolveReference(reference.path);
+        if (resolved === undefined) {
+            return {
+                kind: 'unresolved',
+                path: tsconfigPath,
+                reason: `referenced project ${reference.path} was not found`,
+            };
+        }
+        const part = readScope(resolved, seen);
+        if (part.kind === 'unresolved') {
+            return { kind: 'unresolved', path: tsconfigPath, reason: `${part.path}: ${part.reason}` };
+        }
+        parts.push(part);
+    }
+
+    // `parts` holds successful reads only: every other outcome has returned above.
+    const [first, ...rest] = parts;
+    if (first === undefined) {
+        return { kind: 'unresolved', path: tsconfigPath, reason: 'no referenced project could be read' };
+    }
+    return {
+        kind: 'read',
+        unrecognised: [...new Set([...inherited, ...parts.flatMap((part) => part.unrecognised)])],
+        flags: first.flags.map((guarantee, index) => ({
+            ...guarantee,
+            enabled: guarantee.enabled && rest.every((part) => part.flags[index]?.enabled === true),
+        })),
+    };
+}
+
+/** A reference points either at the tsconfig, or at the directory holding it. */
+function resolveReference(path: string): string | undefined {
+    if (ts.sys.directoryExists(path)) {
+        const inside = join(path, 'tsconfig.json');
+        return ts.sys.fileExists(inside) ? inside : undefined;
+    }
+    return ts.sys.fileExists(path) ? path : undefined;
+}
+
+function flagsOf(options: ts.CompilerOptions, strictDefault: boolean): readonly Guarantee[] {
+    const strict = options.strict ?? strictDefault;
+    const read = (flag: string): unknown => (options as Record<string, unknown>)[flag];
+
+    return FLAGS.map(({ flag, proves, impliedByStrict }) => ({
+        flag,
+        proves,
+        enabled: read(flag) === true || (impliedByStrict && strict && read(flag) !== false),
+    }));
+}
+
+function flatten(diagnostic: ts.Diagnostic): string {
+    return ts.flattenDiagnosticMessageText(diagnostic.messageText, ' ');
+}

package/src/scan/ignored.ts

@@ -0,0 +1,40 @@
+import { spawnSync } from 'node:child_process';
+
+/**
+ * Filters out the paths the repository itself sets aside.
+ *
+ * A `.gitignore` already says what is not the project's code: generated directories,
+ * working copies, artefacts. Without it, a scan trips over a worktree dropped inside the
+ * repository and counts the same code twice — then reports it as two distinct scopes.
+ *
+ * Nothing is reimplemented: `git check-ignore` answers, with git's exact rules, index
+ * included — a tracked file is never called ignored, even if a pattern would target it.
+ * Outside a git repository there is no answer to be had: we keep everything.
+ */
+export async function dropIgnored(root: string, paths: readonly string[]): Promise<readonly string[]> {
+    if (paths.length === 0) {
+        return paths;
+    }
+    const ignored = askGit(root, paths);
+    return ignored === undefined ? paths : paths.filter((path) => !ignored.has(path));
+}
+
+const NUL = '\0';
+
+function askGit(root: string, paths: readonly string[]): Set<string> | undefined {
+    const { status, stdout } = spawnSync('git', ['-C', root, 'check-ignore', '--stdin', '-z'], {
+        input: paths.join(NUL) + NUL,
+        encoding: 'utf8',
+        stdio: ['pipe', 'pipe', 'ignore'],
+        // The answer grows with the repository: the default megabyte would cut it short.
+        maxBuffer: Infinity,
+    });
+
+    // 0: at least one path ignored. 1: none. Everything else (128 outside a repository, no
+    // status at all without git) is a refusal to answer, and a refusal to answer must not
+    // read as "nothing is ignored".
+    if (status !== 0 && status !== 1) {
+        return undefined;
+    }
+    return new Set(stdout.split(NUL).filter((line) => line !== ''));
+}