@codesema/speccov 0.0.0-stage → 0.2.1

package/src/draft/trace.ts

@@ -0,0 +1,219 @@
+import ts from 'typescript';
+
+/**
+ * Where the iterated collection comes from.
+ *
+ * Two origins, and one thing really tells them apart: the one the draft can **feed**. An
+ * injected dependency is driven through its double; a parameter of the entry point is driven
+ * more simply still — we are the ones passing it.
+ */
+export type CollectionOrigin =
+    | {
+          readonly kind: 'dependency';
+          /** The injected dependency that produces the collection, e.g. `users`. */
+          readonly property: string;
+          /** The method called on it, e.g. `list`. */
+          readonly method: string;
+          /** The path read on its result, e.g. `['items']`. Empty when the collection is returned as is. */
+          readonly path: readonly string[];
+      }
+    | {
+          /** The collection is an argument of the entry point: the draft passes it itself. */
+          readonly kind: 'argument';
+          readonly index: number;
+          readonly name: string;
+      };
+
+export type TraceScope = {
+    /** What contains the site: the class, or the file when the function is free-standing. */
+    readonly container: ts.ClassDeclaration | ts.SourceFile;
+    /** What the draft calls. Its parameters are under our direct control. */
+    readonly entry: ts.SignatureDeclarationBase;
+    readonly owned: ReadonlySet<string>;
+    readonly parsed: ts.SourceFile;
+};
+
+/**
+ * The calls that do not change **where the collection comes from**.
+ *
+ * `concat` and `flat` are excluded, for a reason different from the one that keeps them out of
+ * the width computation in `shapes.ts`: here they have **two origins**, not one.
+ * `a.concat(b)` would trace back to `a` and say nothing of `b`; the draft would then feed only
+ * one of the two sources and measure a fan-out over a half-empty collection — a green light
+ * obtained on the wrong thing. Two origins is a refusal, never a choice.
+ *
+ * `sort` and `reverse` stay: they reorder a single source, they do not add to it.
+ */
+const PASS_THROUGH = new Set(['filter', 'map', 'slice', 'sort', 'reverse']);
+
+/** Safety bound: a round trip between two methods must not loop forever. */
+const MAX_HOPS = 24;
+
+/**
+ * Traces the iterated collection back to what produces it.
+ *
+ * The site says *who* is called per item; it does not say *where the items come from*.
+ * Without this trace, the draft does not know what to feed, and the size never reaches the
+ * subject — the measurement is then flat and the rule lies.
+ *
+ * Every hop is a local declaration, a call argument or a property access: AST reading, never a
+ * guess. What cannot be traced back returns `undefined`, and the hole stays written in the
+ * draft.
+ */
+export function traceCollection(collection: ts.Expression, scope: TraceScope): CollectionOrigin | undefined {
+    const path: string[] = [];
+    let current: ts.Expression | undefined = collection;
+
+    for (let hop = 0; hop < MAX_HOPS && current !== undefined; hop += 1) {
+        const node: ts.Expression = current;
+
+        if (
+            ts.isAwaitExpression(node) ||
+            ts.isParenthesizedExpression(node) ||
+            ts.isNonNullExpression(node) ||
+            ts.isAsExpression(node)
+        ) {
+            current = node.expression;
+            continue;
+        }
+
+        // `this.users.list(…)`: the source, if `users` really is an injected dependency.
+        const source = injectedCall(node, scope.owned);
+        if (source !== undefined) {
+            return { kind: 'dependency', ...source, path: [...path].reverse() };
+        }
+
+        if (
+            ts.isCallExpression(node) &&
+            ts.isPropertyAccessExpression(node.expression) &&
+            PASS_THROUGH.has(node.expression.name.text)
+        ) {
+            current = node.expression.expression;
+            continue;
+        }
+
+        // `page.items`: note the property and carry on with `page`.
+        if (ts.isPropertyAccessExpression(node)) {
+            path.push(node.name.text);
+            current = node.expression;
+            continue;
+        }
+
+        if (ts.isIdentifier(node)) {
+            // A parameter of the entry point is the safest origin there is: the draft has
+            // nothing to double, it passes the collection itself. We require it returned as is —
+            // rebuilding a carrier object would invent its other fields.
+            const index = parameterIndex(scope.entry, node.text);
+            if (index !== -1 && path.length === 0 && enclosingCallable(node) === scope.entry) {
+                return { kind: 'argument', index, name: node.text };
+            }
+            current = boundTo(node, scope);
+            continue;
+        }
+
+        return undefined;
+    }
+    return undefined;
+}
+
+/** `this.<dependency>.<method>(…)`, when the dependency comes from the constructor. */
+function injectedCall(node: ts.Expression, owned: ReadonlySet<string>): { property: string; method: string } | undefined {
+    if (!ts.isCallExpression(node) || !ts.isPropertyAccessExpression(node.expression)) {
+        return undefined;
+    }
+    const target = node.expression.expression;
+    if (!ts.isPropertyAccessExpression(target) || target.expression.kind !== ts.SyntaxKind.ThisKeyword) {
+        return undefined;
+    }
+    const property = target.name.text;
+    return owned.has(property) ? { property, method: node.expression.name.text } : undefined;
+}
+
+function parameterIndex(callable: ts.SignatureDeclarationBase, name: string): number {
+    return callable.parameters.findIndex((parameter) => ts.isIdentifier(parameter.name) && parameter.name.text === name);
+}
+
+/**
+ * What an identifier is bound to: the initialiser of its `const`, or — when it is a parameter —
+ * the argument passed at the single call to its method.
+ *
+ * "Single" is not a detail. Two callers pass two different collections; picking one would amount
+ * to measuring one path while believing another is measured.
+ */
+function boundTo(name: ts.Identifier, scope: TraceScope): ts.Expression | undefined {
+    const callable = enclosingCallable(name);
+    if (callable === undefined) {
+        return undefined;
+    }
+
+    const local = localDeclaration(callable, name.text);
+    if (local !== undefined) {
+        return local;
+    }
+
+    const index = parameterIndex(callable, name.text);
+    const owner = ts.isMethodDeclaration(callable) || ts.isFunctionDeclaration(callable) ? callable.name : undefined;
+    if (index === -1 || owner === undefined) {
+        return undefined;
+    }
+    const calls = callsTo(scope.container, owner.getText(scope.parsed), scope.parsed);
+    return calls.length === 1 ? calls[0]?.arguments[index] : undefined;
+}
+
+function localDeclaration(callable: ts.SignatureDeclarationBase, name: string): ts.Expression | undefined {
+    let found: ts.Expression | undefined;
+    const visit = (node: ts.Node): void => {
+        if (
+            found === undefined &&
+            ts.isVariableDeclaration(node) &&
+            ts.isIdentifier(node.name) &&
+            node.name.text === name &&
+            node.initializer !== undefined
+        ) {
+            found = node.initializer;
+            return;
+        }
+        ts.forEachChild(node, visit);
+    };
+    const { body } = callable as ts.SignatureDeclarationBase & { body?: ts.Node };
+    if (body !== undefined) {
+        visit(body);
+    }
+    return found;
+}
+
+/** Every call to `<name>` in the container: `this.<name>(…)` or `<name>(…)`. */
+function callsTo(container: ts.ClassDeclaration | ts.SourceFile, name: string, parsed: ts.SourceFile): ts.CallExpression[] {
+    const calls: ts.CallExpression[] = [];
+    const visit = (node: ts.Node): void => {
+        if (ts.isCallExpression(node)) {
+            const callee = node.expression;
+            const onThis =
+                ts.isPropertyAccessExpression(callee) &&
+                callee.expression.kind === ts.SyntaxKind.ThisKeyword &&
+                callee.name.getText(parsed) === name;
+            if (onThis || (ts.isIdentifier(callee) && callee.text === name)) {
+                calls.push(node);
+            }
+        }
+        ts.forEachChild(node, visit);
+    };
+    visit(container);
+    return calls;
+}
+
+function enclosingCallable(node: ts.Node): ts.SignatureDeclarationBase | undefined {
+    let current: ts.Node | undefined = node.parent;
+    while (current !== undefined) {
+        if (
+            ts.isMethodDeclaration(current) ||
+            ts.isFunctionDeclaration(current) ||
+            ts.isArrowFunction(current) ||
+            ts.isFunctionExpression(current)
+        ) {
+            return current;
+        }
+        current = current.parent;
+    }
+    return undefined;
+}

package/src/forbidden.ts

@@ -0,0 +1,77 @@
+import { requireIntention, type Judgement, type Measurement, type Origin, type Rule } from './rule';
+
+export type ForbiddenOptions<T> = {
+    readonly name: string;
+    /** Mandatory: a ban is domain knowledge, no default sentence can state it. */
+    readonly intention: string;
+    /** The inputs to exercise. Hand-written list, fast-check, faker — take your pick. */
+    readonly cases: Iterable<T> | (() => Iterable<T>);
+    /** True when the input produces what is forbidden. */
+    readonly forbids: (value: T) => Promise<boolean> | boolean;
+    /** How to name a counterexample in the report. */
+    readonly describe?: (value: T) => string;
+    readonly origin?: Origin;
+};
+
+/**
+ * Ban rule: no input must ever produce this behaviour.
+ *
+ * It is the most demanding kind of rule, because it quantifies over the whole input space
+ * instead of illustrating a few happy cases.
+ */
+export function forbidden<T>(options: ForbiddenOptions<T>): Rule {
+    const intention = requireIntention(options.name, options.intention);
+    const describe = options.describe ?? ((value: T) => JSON.stringify(value));
+
+    /** Filled by `measure()`, read by `judge()`: `run()` guarantees that order. */
+    let firstCounterexample: string | undefined;
+
+    return {
+        name: options.name,
+        intention,
+        origin: options.origin ?? 'declared',
+
+        async measure(): Promise<readonly Measurement[]> {
+            firstCounterexample = undefined;
+            const cases = typeof options.cases === 'function' ? options.cases() : options.cases;
+            let tried = 0;
+            let violations = 0;
+
+            for (const value of cases) {
+                tried += 1;
+                if (await options.forbids(value)) {
+                    violations += 1;
+                    firstCounterexample ??= describe(value);
+                }
+            }
+            if (tried === 0) {
+                throw new Error(`speccov: “${options.name}” tried no input. A ban without cases proves nothing.`);
+            }
+            return [{ size: tried, cost: violations }];
+        },
+
+        judge(measurements): Judgement {
+            const [measurement] = measurements;
+            if (measurement === undefined) {
+                throw new Error(`speccov: no measurement for “${options.name}”.`);
+            }
+            const { size, cost } = measurement;
+            // `measure()` already refuses zero input: the predicate necessarily judged something real.
+            const exercised = size > 0;
+            if (cost === 0) {
+                return {
+                    held: true,
+                    exercised,
+                    magnitude: 0,
+                    detail: `No counterexample across ${size} input(s) tried`,
+                };
+            }
+            return {
+                held: false,
+                exercised,
+                magnitude: cost,
+                detail: `${cost} counterexample(s) across ${size} input(s), including ${firstCounterexample ?? 'unknown'}`,
+            };
+        },
+    };
+}

package/src/gauge.ts

@@ -0,0 +1,48 @@
+/**
+ * Concurrency probe: tracks the number of operations in flight and keeps the peak.
+ *
+ * The counterpart of {@link Counter}: one counts the total, the other measures the peak.
+ */
+export class Gauge {
+    #current = 0;
+    #peak = 0;
+
+    enter(): void {
+        this.#current += 1;
+        this.#peak = Math.max(this.#peak, this.#current);
+    }
+
+    /**
+     * One `exit()` too many throws rather than dropping below zero: an unbalanced probe
+     * underestimates the peak, so it would let through a rule that does not hold.
+     */
+    exit(): void {
+        if (this.#current === 0) {
+            throw new Error('speccov: exit() without a matching enter(); the concurrency probe is unbalanced.');
+        }
+        this.#current -= 1;
+    }
+
+    /** Wraps a piece of asynchronous work. The `exit()` happens even on failure. */
+    async track<T>(work: () => Promise<T>): Promise<T> {
+        this.enter();
+        try {
+            return await work();
+        } finally {
+            this.exit();
+        }
+    }
+
+    get current(): number {
+        return this.#current;
+    }
+
+    get peak(): number {
+        return this.#peak;
+    }
+
+    reset(): void {
+        this.#current = 0;
+        this.#peak = 0;
+    }
+}

package/src/growth.ts

@@ -0,0 +1,64 @@
+import type { Measurement } from './rule';
+
+export type Growth = {
+    /** Extra cost per input item, estimated by least squares. */
+    readonly perItem: number;
+    /** The measured series, ready to display: "1→3, 2→4, 4→6". */
+    readonly series: string;
+};
+
+/** Float margin. Costs are integers: a genuinely flat slope is 0. */
+export const EPSILON = 1e-9;
+
+/**
+ * Estimates how the cost grows with the input size.
+ *
+ * Least squares rather than a gap between the extremes: a single noisy measurement must not
+ * decide on its own. Any unusable data throws — a tool you trust never concludes "held" for
+ * want of a measurement.
+ */
+export function measureGrowth(measurements: readonly Measurement[]): Growth {
+    assertUsable(measurements);
+
+    const meanSize = mean(measurements.map(({ size }) => size));
+    const meanCost = mean(measurements.map(({ cost }) => cost));
+
+    let covariance = 0;
+    let variance = 0;
+    for (const { size, cost } of measurements) {
+        const deltaSize = size - meanSize;
+        covariance += deltaSize * (cost - meanCost);
+        variance += deltaSize * deltaSize;
+    }
+
+    if (variance <= EPSILON) {
+        throw new Error('speccov: every measurement carries the same input size; growth is undecidable.');
+    }
+
+    return {
+        perItem: covariance / variance,
+        series: measurements.map(({ size, cost }) => `${size}→${cost}`).join(', '),
+    };
+}
+
+function assertUsable(measurements: readonly Measurement[]): void {
+    if (measurements.length < 2) {
+        throw new Error(`speccov: judging growth needs at least two measurements, got ${measurements.length}.`);
+    }
+    for (const { size, cost } of measurements) {
+        if (!Number.isFinite(size) || size <= 0) {
+            throw new Error(`speccov: invalid input size (${size}), expected a finite number greater than zero.`);
+        }
+        if (!Number.isFinite(cost) || cost < 0) {
+            throw new Error(`speccov: invalid cost (${cost}), expected a finite number of zero or more.`);
+        }
+    }
+}
+
+function mean(values: readonly number[]): number {
+    let total = 0;
+    for (const value of values) {
+        total += value;
+    }
+    return total / values.length;
+}

package/src/index.ts

@@ -0,0 +1,21 @@
+export {
+    accept,
+    applyAnswers,
+    DEFAULT_ANSWERS_PATH,
+    loadAnswers,
+    parseAnswers,
+    saveAnswers,
+    type Answer,
+    type Applied,
+} from './answers';
+export { complexity, type ComplexityOptions } from './complexity';
+export { Counter } from './counter';
+export { allOf, type Domain } from './domain';
+export { forbidden, type ForbiddenOptions } from './forbidden';
+export { Gauge } from './gauge';
+export { metamorphic, type MetamorphicOptions } from './metamorphic';
+export { resource, type ResourceOptions } from './resource';
+export { measureGrowth, type Growth } from './growth';
+export { exitCode, report } from './report';
+export type { Held, Infraction, Judgement, Lacuna, Measurement, Origin, Rule, Verdict } from './rule';
+export { run } from './run';

package/src/json.ts

@@ -0,0 +1,121 @@
+import type { MutateResult } from './mutate/run';
+import { measurementWindow } from './report';
+import type { Verdict } from './rule';
+import { CANDIDATES, flagCoverage, type ScanResult } from './scan/report';
+import type { Site } from './scan/shapes';
+
+/**
+ * Machine-readable output.
+ *
+ * This is the real product: speccov's thesis is that an agent reads the verdict and acts.
+ * Making it parse prose wrapped at 88 columns would ask it to guess — exactly the brittle
+ * thing speccov exists to replace.
+ *
+ * The schema carries the same refusals as the prose. An unresolved scope shows up here too:
+ * without it, the absence of an alert would read as a green light, which is the one lie
+ * this tool cannot afford.
+ */
+const SCHEMA = 1;
+
+export function scanToJson(result: ScanResult, accepted: readonly Site[] = []): string {
+    const coverage = flagCoverage(result);
+    const [scopesEvaluated = 0] = coverage.map(({ scopesEvaluated: n }) => n);
+
+    return stringify({
+        speccov: SCHEMA,
+        command: 'scan',
+        filesScanned: result.filesScanned,
+        scopes: { evaluated: scopesEvaluated, found: result.guarantees.length },
+        provenByConstruction: {
+            flags: coverage.filter(({ offIn }) => offIn.length === 0).map(({ flag, proves }) => ({ flag, proves })),
+            boundedFanOut: result.sites.filter(({ kind }) => kind === 'bounded-fanout').length,
+            awaitedInFlight: result.sites.filter(({ kind }) => kind === 'in-flight-drain').length,
+        },
+        alreadyAnswered: {
+            count: accepted.length,
+            sites: accepted.map(({ file, line, excerpt }) => ({ file, line, excerpt })),
+        },
+        guaranteedByNobody: {
+            flags: coverage.filter(({ offIn }) => offIn.length > 0).map(({ flag, proves, offIn }) => ({ flag, proves, offIn })),
+            candidates: CANDIDATES.flatMap(({ kind, rule, question }) => {
+                const sites = result.sites.filter((site) => site.kind === kind);
+                return sites.length === 0
+                    ? []
+                    : [{ rule, question, sites: sites.map(({ file, line, excerpt }) => ({ file, line, excerpt })) }];
+            }),
+        },
+        notMeasured: {
+            behaviour: 'this pass is static; nothing was executed',
+            // Two distinct failures stop a scope from being read: the refused program
+            // (`program.ts`) and the unresolved tsconfig (`guarantees.ts`). Looking at only one
+            // would announce "types resolved" next to a full `unresolvedScopes` — a green light.
+            typesResolved:
+                result.refused.length === 0 && result.guarantees.every(({ guarantees }) => guarantees.kind !== 'unresolved'),
+            unresolvedScopes: result.guarantees.flatMap((scope) =>
+                scope.guarantees.kind === 'unresolved' ? [{ path: scope.path, reason: scope.guarantees.reason }] : [],
+            ),
+            unrecognisedOptions: result.guarantees.flatMap((scope) =>
+                scope.guarantees.kind === 'read' && scope.guarantees.unrecognised.length > 0
+                    ? [{ path: scope.path, options: [...scope.guarantees.unrecognised] }]
+                    : [],
+            ),
+            filesNotParsed: Object.fromEntries(result.skipped),
+            testsExcluded: result.testsExcluded,
+            ignored: { files: result.ignored.files, sites: result.ignored.sites },
+            unreadable: [...result.unreadable],
+            refusedScopes: result.refused.map(({ path, reason }) => ({ path, reason })),
+        },
+    });
+}
+
+export function verdictsToJson(verdicts: readonly Verdict[], accepted: readonly Verdict[], exitCode: number): string {
+    const window = measurementWindow(verdicts);
+
+    return stringify({
+        speccov: SCHEMA,
+        command: 'rules',
+        exitCode,
+        verdicts: verdicts.map((verdict) => ({
+            kind: verdict.kind,
+            rule: verdict.rule.name,
+            intention: verdict.rule.intention,
+            origin: verdict.rule.origin,
+            detail: verdict.detail,
+            magnitude: verdict.magnitude,
+            ...('question' in verdict ? { question: verdict.question } : {}),
+            measurements: verdict.measurements.map(({ size, cost }) => ({ size, cost })),
+        })),
+        alreadyAccepted: accepted.map((verdict) => verdict.rule.name),
+        notMeasured: {
+            inputSizesTried: window.sizes,
+            largestSizeTried: window.largest,
+            rulesWithoutMeasurement: window.unobserved,
+            dataShape: 'the rule file builds its own data; no production shape was observed',
+        },
+    });
+}
+
+export function mutantsToJson(result: MutateResult, exitCode: number): string {
+    const survivors = result.outcomes.filter(({ fate }) => fate === 'survived');
+
+    return stringify({
+        speccov: SCHEMA,
+        command: 'mutate',
+        exitCode,
+        tried: result.outcomes.length,
+        killed: result.outcomes.length - survivors.length,
+        survivors: survivors.map(({ mutant }) => ({
+            file: mutant.file,
+            line: mutant.line,
+            mutator: mutant.mutator,
+            original: mutant.original,
+            replacement: mutant.replacement,
+        })),
+        notMeasured: { generatedButNotTried: result.skipped },
+    });
+}
+
+/** Indented: a human sometimes rereads what their agent read, and a CI diff has to stay legible. */
+function stringify(payload: unknown): string {
+    return JSON.stringify(payload, undefined, 2);
+}

package/src/metamorphic.ts

@@ -0,0 +1,68 @@
+import { requireIntention, type Judgement, type Measurement, type Origin, type Rule } from './rule';
+
+export type MetamorphicOptions<V, O> = {
+    readonly name: string;
+    /** Mandatory: the expected relation cannot be guessed. */
+    readonly intention: string;
+    /** The variants to compare. At least two — comparing a thing to nothing proves nothing. */
+    readonly variants: readonly V[];
+    /** What is observed, and what must stay identical from one variant to the next. */
+    readonly observe: (variant: V) => Promise<O> | O;
+    readonly describe?: (variant: V) => string;
+    readonly origin?: Origin;
+};
+
+/**
+ * Metamorphic rule: one observation stays invariant from one variant to the next.
+ *
+ * It does not say which value is the right one — often we do not know. It only says the
+ * value must not change. This is what catches a `total` that follows the page instead of
+ * the full result set.
+ */
+export function metamorphic<V, O>(options: MetamorphicOptions<V, O>): Rule {
+    const intention = requireIntention(options.name, options.intention);
+    const describe = options.describe ?? ((variant: V) => JSON.stringify(variant));
+
+    if (options.variants.length < 2) {
+        throw new Error(`speccov: “${options.name}” needs at least two variants, got ${options.variants.length}.`);
+    }
+
+    /** Filled by `measure()`, read by `judge()`. */
+    let observed: readonly string[] = [];
+
+    return {
+        name: options.name,
+        intention,
+        origin: options.origin ?? 'declared',
+
+        async measure(): Promise<readonly Measurement[]> {
+            const seen: string[] = [];
+            for (const variant of options.variants) {
+                seen.push(`${describe(variant)} → ${JSON.stringify(await options.observe(variant))}`);
+            }
+            observed = seen;
+            const distinct = new Set(seen.map((entry) => entry.split(' → ').at(-1))).size;
+            return [{ size: options.variants.length, cost: distinct }];
+        },
+
+        judge(measurements): Judgement {
+            const [measurement] = measurements;
+            if (measurement === undefined) {
+                throw new Error(`speccov: no measurement for “${options.name}”.`);
+            }
+            const { size, cost } = measurement;
+            // The constructor already refuses fewer than two variants: an invariance has
+            // always had something to compare against. The hole is closed upstream, not here.
+            const exercised = true;
+            if (cost <= 1) {
+                return { held: true, exercised, magnitude: 0, detail: `Same observation across ${size} variants` };
+            }
+            return {
+                held: false,
+                exercised,
+                magnitude: cost - 1,
+                detail: `${cost} different values across ${size} variants: ${observed.join(' ; ')}`,
+            };
+        },
+    };
+}