@crvouga/atlas 0.11.0 → 0.12.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog: @crvouga/atlas
 
+## 0.12.0 (2026-10-28)
+
+### Features
+
+- diagnose spec identity, domain and coverage count inconsistencies (#44) ([4a66a62](https://github.com/crvouga/atlas/commit/4a66a627fc62a3f38ad7ce9af927427a7b522f9a))
+
 ## 0.11.0 (2026-10-25)
 
 ### Features

package/dist/reports/src/catalog-health.d.ts

@@ -0,0 +1,142 @@
+import type { CatalogModel } from './test-catalog.js';
+/** The version of the `CatalogHealth` document. Only optional fields are added within it. */
+export declare const CATALOG_HEALTH_VERSION = 1;
+/** The most paths listed in one finding; its `count` is always the whole. */
+export declare const HEALTH_ITEM_LIMIT = 25;
+/** Declares that paths under `from` are the same specs as paths under `to`, e.g. `{ from: '', to: 'specs/' }`. Empty `from` matches every path. */
+export type RootMapping = {
+    readonly from: string;
+    readonly to: string;
+};
+/** One thing worth a look. */
+export type HealthItem = {
+    readonly path: string;
+    /** Other paths it could be the same promise as, when any; Atlas has not merged them. */
+    readonly candidates?: readonly {
+        readonly path: string;
+        readonly reason: string;
+    }[];
+};
+/** Codes of `HealthFinding.code`. The set grows; a reader treats an unknown code as informational. */
+export type HealthCode = 'ambiguous-spec-path' | 'unmatched-spec-path' | 'duplicate-spec-content' | 'duplicate-spec-title' | 'generic-domain' | 'tests-without-spec' | 'no-spec-definitions' | 'fallback-identity' | 'unused-root-mapping' | 'spec-without-execution' | 'historical-only';
+/** A finding: what is inconsistent, how many it affects, and what to change. */
+export type HealthFinding = {
+    readonly code: HealthCode;
+    /** `warning` is something that makes a count or a filter misleading; `info` explains a number. */
+    readonly severity: 'warning' | 'info';
+    readonly message: string;
+    /** How many paths, specs or tests it affects. */
+    readonly count: number;
+    /** Up to `HEALTH_ITEM_LIMIT` of them. */
+    readonly items: readonly HealthItem[];
+};
+/** A spec that reached Atlas under more than one path and was joined by a declared root mapping. */
+export type SpecAlias = {
+    /** The logical spec's path after mapping. */
+    readonly path: string;
+    /** Every original path it was seen under, sorted. */
+    readonly sources: readonly string[];
+};
+/** One step from the Specs count to the Tests count. */
+export type ReconciliationStep = {
+    readonly code: 'merged-by-mapping' | 'defined-without-results' | 'results-without-definition';
+    readonly label: string;
+    /** The change to the running total: negative removes specs, positive adds them. */
+    readonly delta: number;
+};
+/** The `GET /api/reports/tests/health` document. */
+export type CatalogHealth = {
+    readonly schemaVersion: typeof CATALOG_HEALTH_VERSION;
+    readonly version: string;
+    readonly generatedAtMs: number;
+    /** True when there is no warning. Informational findings do not count. */
+    readonly ok: boolean;
+    readonly counts: {
+        readonly specs: {
+            /** Spec files the report is laid over (before mapping). */
+            readonly files: number;
+            /** Logical specs among them: files joined by a root mapping count once. */
+            readonly defined: number;
+            /** Logical specs, defined or not, with at least one test. */
+            readonly withTests: number;
+            /** Defined logical specs with a test that has a passed or failed result. They are the denominator of execution coverage. */
+            readonly executed: number;
+            /** Defined logical specs with no test that has a passed or failed result: the defined coverage gaps. */
+            readonly withoutExecution: number;
+        };
+        readonly tests: {
+            readonly total: number;
+            readonly withSpec: number;
+            readonly withoutSpec: number;
+            /** Tests whose current status is `passed`. A definition with no test never adds to it. */
+            readonly passed: number;
+            /** Tests the history holds but the report has no current result for. */
+            readonly historicalOnly: number;
+        };
+        readonly scenarios: {
+            /** Outlines: tests sharing the label `outline` count once; a test without it is its own outline. */
+            readonly outlines: number;
+            /** Concrete examples: tests (a check identity), however many environments ran them. */
+            readonly examples: number;
+            /** Current results per producer, environment and parameter set. */
+            readonly observations: number;
+            /** Observations that passed or failed. */
+            readonly executedObservations: number;
+            readonly passedObservations: number;
+        };
+    };
+    readonly reconciliation: {
+        /** What the Specs view counts: spec files. */
+        readonly specs: number;
+        /** What the Tests view groups by: logical specs with at least one test. */
+        readonly testSpecs: number;
+        readonly steps: readonly ReconciliationStep[];
+        /** Tests that name no spec; they are in no group above. */
+        readonly testsWithoutSpec: number;
+        /** In words, for a person or an agent. */
+        readonly explanation: readonly string[];
+    };
+    readonly identity: {
+        readonly checks: number;
+        /** Checks whose id equals their title: no stable id was given. */
+        readonly fallbackIdentity: number;
+        readonly basis: string;
+    };
+    readonly domains: readonly {
+        readonly name: string;
+        readonly specs: number;
+    }[];
+    readonly roots: readonly {
+        readonly from: string;
+        readonly to: string;
+        readonly matched: number;
+    }[];
+    readonly aliases: readonly SpecAlias[];
+    readonly findings: readonly HealthFinding[];
+};
+/** Parses `from>to` (as `?root=` gives it) into a mapping. Returns null when there is no `>`. Whitespace around each side is trimmed. */
+export declare function parseRootMapping(text: string): RootMapping | null;
+/** Reads the repeatable `root` search parameter: each value `from>to`, several may be comma-separated. Values without `>` are ignored. */
+export declare function parseRootMappings(params: URLSearchParams): RootMapping[];
+/**
+ * The logical path of a spec path: the path Atlas already normalises (`specPath`), after the first mapping whose `from`
+ * is a prefix of it (the longest `from` wins). A path already under `to` is left alone. Returns the path and the mapping
+ * that changed it, if any.
+ */
+export declare function canonicalSpecPath(path: string, roots: readonly RootMapping[]): {
+    readonly path: string;
+    readonly mapping: RootMapping | null;
+};
+/**
+ * Diagnoses a report's spec identity, domains and counts. `roots` are the declared path-root mappings. Pure: the same
+ * model and mappings give the same document, and nothing in it changes what the report says passed.
+ *
+ * @example
+ * ```ts
+ * const health = catalogHealth(model, [{ from: '', to: 'specs/' }]);
+ * if (!health.ok) console.log(healthMarkdown(health));
+ * ```
+ */
+export declare function catalogHealth(model: CatalogModel, roots?: readonly RootMapping[]): CatalogHealth;
+/** The health document as Markdown, for an agent or a terminal. */
+export declare function healthMarkdown(health: CatalogHealth): string;

package/dist/reports/src/catalog-health.js

@@ -0,0 +1,257 @@
+import { specPath } from './layer-merge.js';
+/** The version of the `CatalogHealth` document. Only optional fields are added within it. */
+export const CATALOG_HEALTH_VERSION = 1;
+/** The most paths listed in one finding; its `count` is always the whole. */
+export const HEALTH_ITEM_LIMIT = 25;
+/** The most candidates named for one ambiguous path. */
+const CANDIDATE_LIMIT = 5;
+/** Folder names that say where a spec lives, not what it is about. A domain with one of these names is likely a root folder. */
+const GENERIC_DOMAINS = new Set(['specs', 'spec', 'product', 'features', 'feature', 'tests', 'test', 'e2e', 'src', 'app', 'docs', 'overview', 'root', 'default']);
+/** Parses `from>to` (as `?root=` gives it) into a mapping. Returns null when there is no `>`. Whitespace around each side is trimmed. */
+export function parseRootMapping(text) {
+    const at = text.indexOf('>');
+    if (at < 0)
+        return null;
+    return { from: text.slice(0, at).trim(), to: text.slice(at + 1).trim() };
+}
+/** Reads the repeatable `root` search parameter: each value `from>to`, several may be comma-separated. Values without `>` are ignored. */
+export function parseRootMappings(params) {
+    return params.getAll('root').flatMap(value => value.split(',')).map(parseRootMapping).filter((mapping) => mapping !== null);
+}
+const slashes = (path) => path.replaceAll('\\', '/').replace(/^\.\//, '');
+const folder = (prefix) => (prefix === '' || prefix.endsWith('/') ? prefix : `${prefix}/`);
+/**
+ * The logical path of a spec path: the path Atlas already normalises (`specPath`), after the first mapping whose `from`
+ * is a prefix of it (the longest `from` wins). A path already under `to` is left alone. Returns the path and the mapping
+ * that changed it, if any.
+ */
+export function canonicalSpecPath(path, roots) {
+    const raw = slashes(path);
+    for (const mapping of roots) {
+        const from = folder(mapping.from);
+        const to = folder(mapping.to);
+        for (const candidate of [raw, specPath(raw)]) {
+            if (!candidate.startsWith(from) || (to !== '' && candidate.startsWith(to)))
+                continue;
+            const mapped = specPath(to + candidate.slice(from.length));
+            if (mapped !== specPath(raw))
+                return { path: mapped, mapping };
+        }
+    }
+    return { path: specPath(raw), mapping: null };
+}
+const byText = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+const executed = (status) => status === 'passed' || status === 'failed';
+const tail = (path) => path.split('/').at(-1).toLowerCase();
+/** Why two paths with the same file name might be one promise: one ends with the other, or only the name is shared. */
+function relation(a, b) {
+    return a.endsWith(`/${b}`) || b.endsWith(`/${a}`) ? 'one path ends with the other' : 'same file name';
+}
+function finding(code, severity, message, items, count = items.length) {
+    return { code, severity, message, count, items: items.slice(0, HEALTH_ITEM_LIMIT) };
+}
+/**
+ * Diagnoses a report's spec identity, domains and counts. `roots` are the declared path-root mappings. Pure: the same
+ * model and mappings give the same document, and nothing in it changes what the report says passed.
+ *
+ * @example
+ * ```ts
+ * const health = catalogHealth(model, [{ from: '', to: 'specs/' }]);
+ * if (!health.ok) console.log(healthMarkdown(health));
+ * ```
+ */
+export function catalogHealth(model, roots = []) {
+    const ordered = [...roots].sort((a, b) => b.from.length - a.from.length);
+    const matched = new Map(ordered.map(mapping => [mapping, new Set()]));
+    const groups = new Map();
+    const group = (path) => {
+        const { path: key, mapping } = canonicalSpecPath(path, ordered);
+        if (mapping)
+            matched.get(mapping)?.add(path);
+        let found = groups.get(key);
+        if (!found)
+            groups.set(key, found = { key, defined: new Map(), results: new Set(), tests: [] });
+        return found;
+    };
+    const specs = model.specs ?? [];
+    for (const spec of specs)
+        group(spec.path).defined.set(spec.path, spec);
+    for (const test of model.tests) {
+        if (test.specPath === null)
+            continue;
+        const found = group(test.specPath);
+        found.results.add(test.specPath);
+        found.tests.push(test);
+    }
+    const all = [...groups.values()].sort((a, b) => byText(a.key, b.key));
+    const definedGroups = all.filter(item => item.defined.size > 0);
+    const withTests = all.filter(item => item.tests.length > 0);
+    const resultsOnly = all.filter(item => item.defined.size === 0);
+    const definedOnly = definedGroups.filter(item => item.tests.length === 0);
+    const withoutExecution = definedGroups.filter(item => !item.tests.some(test => test.outcomes.some(outcome => executed(outcome.status))));
+    const files = specs.length;
+    const hasOutline = files > 0;
+    const findings = [];
+    // Aliases: one logical spec under several original paths. Joined only because a mapping said so.
+    const aliases = all
+        .map(item => ({ path: item.key, sources: [...new Set([...item.defined.keys(), ...item.results])].sort(byText) }))
+        .filter(item => item.sources.length > 1);
+    // Ambiguity: a path with results and no definition, beside a definition with no results, that might be the same promise.
+    if (hasOutline) {
+        const byName = new Map();
+        for (const item of definedOnly)
+            for (const path of item.defined.keys()) {
+                const name = tail(path);
+                byName.set(name, [...(byName.get(name) ?? []), item]);
+            }
+        const ambiguous = [];
+        const unmatched = [];
+        for (const item of resultsOnly) {
+            for (const path of [...item.results].sort(byText)) {
+                const candidates = new Map();
+                for (const near of byName.get(tail(path)) ?? [])
+                    for (const other of near.defined.keys())
+                        if (tail(other) === tail(path))
+                            candidates.set(other, relation(path, other));
+                if (candidates.size)
+                    ambiguous.push({ path, candidates: [...candidates].sort(([a], [b]) => byText(a, b)).slice(0, CANDIDATE_LIMIT).map(([candidate, reason]) => ({ path: candidate, reason })) });
+                else
+                    unmatched.push({ path });
+            }
+        }
+        if (ambiguous.length)
+            findings.push(finding('ambiguous-spec-path', 'warning', 'These paths have results but no definition, and a defined spec with no results looks like the same promise. Atlas has not merged them. If they are one, declare a root mapping (`?root=<from>><to>`); if not, nothing to do.', ambiguous));
+        if (unmatched.length)
+            findings.push(finding('unmatched-spec-path', 'warning', 'These paths have results but no definition in the specs the report is laid over, and nothing looks like them. Publish the spec set that defines them, or correct the path the producer records.', unmatched));
+    }
+    else if (withTests.length) {
+        findings.push(finding('no-spec-definitions', 'info', 'No specs are defined for this report, so each test\'s spec and domain come from the path of its results alone. Publish the specs with the results (`--specs <dir>`) to define them.', [], withTests.length));
+    }
+    // Duplicates among definitions: the same content, or the same title, under different logical paths.
+    const byFingerprint = new Map();
+    const byTitle = new Map();
+    for (const item of definedGroups) {
+        const spec = [...item.defined.values()].sort((a, b) => byText(a.path, b.path))[0];
+        if (spec.fingerprint)
+            byFingerprint.set(spec.fingerprint, [...(byFingerprint.get(spec.fingerprint) ?? []), item.key]);
+        const title = spec.title.trim().toLowerCase();
+        if (title && title !== tail(item.key))
+            byTitle.set(title, [...(byTitle.get(title) ?? []), { path: item.key, fingerprint: spec.fingerprint ?? '' }]);
+    }
+    const sameContent = [...byFingerprint.values()].filter(paths => paths.length > 1);
+    if (sameContent.length)
+        findings.push(finding('duplicate-spec-content', 'warning', 'These specs have identical content under different paths. They are counted separately; if they are one promise, declare a root mapping.', sameContent.flatMap(paths => paths.map(path => ({ path, candidates: paths.filter(other => other !== path).slice(0, CANDIDATE_LIMIT).map(other => ({ path: other, reason: 'identical content' })) })))));
+    const sameTitle = [...byTitle.values()].filter(list => list.length > 1 && new Set(list.map(item => item.fingerprint)).size > 1);
+    if (sameTitle.length)
+        findings.push(finding('duplicate-spec-title', 'info', 'These specs share a title but differ in content or path. Atlas keeps them apart; the title alone is never an identity.', sameTitle.flatMap(list => list.map(item => ({ path: item.path, candidates: list.filter(other => other.path !== item.path).slice(0, CANDIDATE_LIMIT).map(other => ({ path: other.path, reason: 'same title' })) })))));
+    // Domains over logical specs.
+    const domainCounts = new Map();
+    for (const item of all) {
+        const domain = [...item.defined.values()].sort((a, b) => byText(a.path, b.path))[0]?.domain || item.tests[0]?.spec?.domain || '';
+        if (domain)
+            domainCounts.set(domain, (domainCounts.get(domain) ?? 0) + 1);
+    }
+    const domains = [...domainCounts].map(([name, specs]) => ({ name, specs })).sort((a, b) => b.specs - a.specs || byText(a.name, b.name));
+    const withDomain = domains.reduce((sum, domain) => sum + domain.specs, 0);
+    const top = domains[0];
+    if (top && withDomain >= 3 && top.specs / withDomain >= 0.8) {
+        const generic = GENERIC_DOMAINS.has(top.name.toLowerCase());
+        findings.push(finding('generic-domain', 'warning', `${top.specs} of ${withDomain} specs are in the domain "${top.name}"${generic ? ', which looks like a root folder rather than an area of the product' : ''}, so filtering by domain will not separate them. Give specs an explicit domain (the spec set's \`domain\`, or each spec entry's); explicit metadata always wins over the folder.`, [], top.specs));
+    }
+    const testsWithoutSpec = model.tests.filter(test => test.specPath === null);
+    if (testsWithoutSpec.length)
+        findings.push(finding('tests-without-spec', 'info', 'These tests name no spec, so they are in no spec or domain group (the Tests view shows them under "(no spec)").', testsWithoutSpec.map(test => ({ path: test.id })), testsWithoutSpec.length));
+    const fallback = model.tests.filter(test => test.id === test.title);
+    if (fallback.length)
+        findings.push(finding('fallback-identity', 'info', 'These checks use their title as their id. A check is its spec path and id, so renaming one starts a new check and splits its history. Have the producer send a stable `id`.', fallback.map(test => ({ path: test.id })), fallback.length));
+    const unused = ordered.filter(mapping => !matched.get(mapping)?.size);
+    if (unused.length)
+        findings.push(finding('unused-root-mapping', 'warning', 'These root mappings matched no spec path. Check `from` against the paths in the report.', unused.map(mapping => ({ path: `${mapping.from}>${mapping.to}` }))));
+    if (withoutExecution.length)
+        findings.push(finding('spec-without-execution', 'info', 'These defined specs have no test with a passed or failed result. They are coverage gaps; they add nothing to any passed count.', withoutExecution.map(item => ({ path: item.key })), withoutExecution.length));
+    const known = new Set(model.tests.map(test => test.key));
+    const historical = (model.history?.tests() ?? []).filter(test => !known.has(test.key));
+    if (historical.length)
+        findings.push(finding('historical-only', 'info', 'The history holds results for these tests, but the report has no current result for them (removed, renamed or not yet re-run). The Tests view does not count them.', historical.map(test => ({ path: test.specPath ? `${test.specPath}  ${test.id}` : test.id })), historical.length));
+    // Scenarios: outlines, examples, observations.
+    const outlines = new Set(model.tests.map(test => (test.labels.outline ? `${test.specPath ?? ''}\u0000${test.labels.outline}` : test.key)));
+    let observations = 0, executedObservations = 0, passedObservations = 0;
+    for (const test of model.tests)
+        for (const outcome of test.outcomes) {
+            observations += 1;
+            if (executed(outcome.status))
+                executedObservations += 1;
+            if (outcome.status === 'passed')
+                passedObservations += 1;
+        }
+    const mergedDefined = files - definedGroups.length;
+    const steps = [];
+    if (mergedDefined)
+        steps.push({ code: 'merged-by-mapping', label: `${mergedDefined} spec files are the same promise as another under a declared root mapping`, delta: -mergedDefined });
+    if (definedOnly.length)
+        steps.push({ code: 'defined-without-results', label: `${definedOnly.length} defined specs have no test`, delta: -definedOnly.length });
+    if (resultsOnly.length)
+        steps.push({ code: 'results-without-definition', label: `${resultsOnly.length} specs have tests but no definition in the specs the report is laid over`, delta: resultsOnly.length });
+    const explanation = [
+        `The Specs view lists ${files} spec files; the Tests view groups ${model.tests.length - testsWithoutSpec.length} tests into ${withTests.length} specs.`,
+        ...steps.map(step => `${step.delta < 0 ? 'Minus' : 'Plus'} ${Math.abs(step.delta)}: ${step.label}.`),
+        ...(testsWithoutSpec.length ? [`${testsWithoutSpec.length} tests name no spec and belong to neither count.`] : []),
+        ...(historical.length ? [`${historical.length} more tests exist only in history and are in neither view.`] : []),
+        `${model.tests.length} tests have ${observations} current results (one per producer, environment and parameter set); ${executedObservations} were executed and ${passedObservations} passed.`
+    ];
+    return {
+        schemaVersion: CATALOG_HEALTH_VERSION,
+        version: model.version,
+        generatedAtMs: model.generatedAtMs,
+        ok: findings.every(item => item.severity !== 'warning'),
+        counts: {
+            specs: { files, defined: definedGroups.length, withTests: withTests.length, executed: definedGroups.length - withoutExecution.length, withoutExecution: withoutExecution.length },
+            tests: { total: model.tests.length, withSpec: model.tests.length - testsWithoutSpec.length, withoutSpec: testsWithoutSpec.length, passed: model.tests.filter(test => test.status === 'passed').length, historicalOnly: historical.length },
+            scenarios: { outlines: outlines.size, examples: model.tests.length, observations, executedObservations, passedObservations }
+        },
+        reconciliation: { specs: files, testSpecs: withTests.length, steps, testsWithoutSpec: testsWithoutSpec.length, explanation },
+        identity: {
+            checks: model.tests.length,
+            fallbackIdentity: fallback.length,
+            basis: 'A check is its spec path and its id. A check whose producer sent no id is known by its title, so a rename is a new check; a spec is known by its path, joined to another only by a declared root mapping.'
+        },
+        domains,
+        roots: ordered.map(mapping => ({ from: mapping.from, to: mapping.to, matched: matched.get(mapping)?.size ?? 0 })),
+        aliases,
+        findings
+    };
+}
+const cell = (text) => text.replaceAll('|', '\\|').replace(/\s+/g, ' ');
+/** The health document as Markdown, for an agent or a terminal. */
+export function healthMarkdown(health) {
+    const { counts } = health;
+    const lines = [
+        '# Catalog health',
+        '',
+        health.ok ? 'No warnings.' : `${health.findings.filter(item => item.severity === 'warning').length} warnings.`,
+        '',
+        '## Counts',
+        '',
+        `- Specs: ${counts.specs.files} files, ${counts.specs.defined} defined, ${counts.specs.withTests} with tests, ${counts.specs.executed} executed, ${counts.specs.withoutExecution} without an executed test.`,
+        `- Tests: ${counts.tests.total} (${counts.tests.passed} passed, ${counts.tests.withoutSpec} without a spec, ${counts.tests.historicalOnly} only in history).`,
+        `- Scenarios: ${counts.scenarios.outlines} outlines, ${counts.scenarios.examples} examples, ${counts.scenarios.observations} observations (${counts.scenarios.executedObservations} executed, ${counts.scenarios.passedObservations} passed).`,
+        '',
+        '## Why Specs and Tests differ',
+        '',
+        ...health.reconciliation.explanation.map(line => `- ${line}`)
+    ];
+    if (health.aliases.length)
+        lines.push('', '## Specs under more than one path', '', ...health.aliases.slice(0, HEALTH_ITEM_LIMIT).map(alias => `- ${cell(alias.path)} <- ${alias.sources.map(cell).join(', ')}`));
+    if (health.findings.length) {
+        lines.push('', '## Findings');
+        for (const item of health.findings) {
+            lines.push('', `### ${item.code} (${item.severity}, ${item.count})`, '', item.message, '');
+            for (const entry of item.items)
+                lines.push(`- ${cell(entry.path)}${entry.candidates?.length ? ` ~ ${entry.candidates.map(candidate => `${cell(candidate.path)} (${candidate.reason})`).join('; ')}` : ''}`);
+            if (item.count > item.items.length)
+                lines.push(`- ... ${item.count - item.items.length} more`);
+        }
+    }
+    lines.push('', `Identity: ${health.identity.basis}`);
+    return `${lines.join('\n')}\n`;
+}

package/dist/reports/src/test-catalog.d.ts

@@ -311,6 +311,12 @@ export declare class TestHistory {
     get rev(): number;
     /** The specs the added layers recorded. */
     get specs(): RecordedSpecs;
+    /** Every test the history holds a result for (keyed as `TestSummary.key`), including tests the report no longer has a current result for. */
+    tests(): readonly {
+        readonly key: string;
+        readonly id: string;
+        readonly specPath: string | null;
+    }[];
     /** How many layers were added. */
     get layers(): number;
     /** Takes in one layer: its header, its rows, the specs it recorded and the backend it was read from. A layer id already added is ignored. */
@@ -353,6 +359,8 @@ export type CatalogModel = {
     readonly facets: CatalogFacets;
     readonly haystacks: ReadonlyMap<string, string>;
     readonly history: TestHistory | null;
+    /** The specs the report is laid over, sorted by path: the given ones, else those the layers recorded. What `catalogHealth` calls defined. Absent on a model built from a server's catalog document. */
+    readonly specs?: readonly SpecEntry[];
 };
 /** The lowercase text a search matches a test against: title, id, spec, domain, tags, labels, environments and producers. */
 export declare function haystackOf(test: TestSummary): string;

package/dist/reports/src/test-catalog.js

@@ -80,6 +80,10 @@ export class TestHistory {
     get specs() {
         return this.#specs;
     }
+    /** Every test the history holds a result for (keyed as `TestSummary.key`), including tests the report no longer has a current result for. */
+    tests() {
+        return [...this.#idents.values()].map(({ slot, id, specPath }) => ({ key: slot, id, specPath }));
+    }
     /** How many layers were added. */
     get layers() {
         return this.#seen.size;
@@ -462,7 +466,7 @@ export function catalogModel(layers, sources = {}) {
         branches: facet(tests, test => test.stats.branches),
         labels: Object.fromEntries([...labelKeys].sort().map(key => [key, facet(tests, test => (test.labels[key] === undefined ? [] : [test.labels[key]]))])),
     };
-    return { version: sources.version ?? '', generatedAtMs: sources.now ?? Date.now(), tests, byKey, summary, facets, haystacks, history };
+    return { version: sources.version ?? '', generatedAtMs: sources.now ?? Date.now(), tests, byKey, summary, facets, haystacks, history, specs: recorded.entries() };
 }
 const SORTERS = {
     spec: (a, b) => compareText(a.spec?.domain ?? '\uffff', b.spec?.domain ?? '\uffff') || compareText(a.spec?.title ?? '\uffff', b.spec?.title ?? '\uffff') || compareText(a.title, b.title),

package/dist/schema/src/api.d.ts

@@ -1153,6 +1153,85 @@ export declare const TestCatalogSchema: z.ZodObject<{
         live: z.ZodBoolean;
     }, z.core.$loose>>;
 }, z.core.$loose>;
+/**
+ * `GET /api/reports/tests/health`: why the report's Specs, domains and counts may disagree. Counts keep the denominators
+ * apart (spec files, defined specs, specs with an executed test, tests, outlines, examples, observations), `reconciliation`
+ * walks from one to another, and `findings` name what to fix. It reads the report; it never changes what passed.
+ */
+export declare const CatalogHealthSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<1>;
+    version: z.ZodString;
+    generatedAtMs: z.ZodNumber;
+    ok: z.ZodBoolean;
+    counts: z.ZodObject<{
+        specs: z.ZodObject<{
+            files: z.ZodNumber;
+            defined: z.ZodNumber;
+            withTests: z.ZodNumber;
+            executed: z.ZodNumber;
+            withoutExecution: z.ZodNumber;
+        }, z.core.$loose>;
+        tests: z.ZodObject<{
+            total: z.ZodNumber;
+            withSpec: z.ZodNumber;
+            withoutSpec: z.ZodNumber;
+            passed: z.ZodNumber;
+            historicalOnly: z.ZodNumber;
+        }, z.core.$loose>;
+        scenarios: z.ZodObject<{
+            outlines: z.ZodNumber;
+            examples: z.ZodNumber;
+            observations: z.ZodNumber;
+            executedObservations: z.ZodNumber;
+            passedObservations: z.ZodNumber;
+        }, z.core.$loose>;
+    }, z.core.$loose>;
+    reconciliation: z.ZodObject<{
+        specs: z.ZodNumber;
+        testSpecs: z.ZodNumber;
+        steps: z.ZodArray<z.ZodObject<{
+            code: z.ZodString;
+            label: z.ZodString;
+            delta: z.ZodNumber;
+        }, z.core.$loose>>;
+        testsWithoutSpec: z.ZodNumber;
+        explanation: z.ZodArray<z.ZodString>;
+    }, z.core.$loose>;
+    identity: z.ZodObject<{
+        checks: z.ZodNumber;
+        fallbackIdentity: z.ZodNumber;
+        basis: z.ZodString;
+    }, z.core.$loose>;
+    domains: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        specs: z.ZodNumber;
+    }, z.core.$loose>>;
+    roots: z.ZodArray<z.ZodObject<{
+        from: z.ZodString;
+        to: z.ZodString;
+        matched: z.ZodNumber;
+    }, z.core.$loose>>;
+    aliases: z.ZodArray<z.ZodObject<{
+        path: z.ZodString;
+        sources: z.ZodArray<z.ZodString>;
+    }, z.core.$loose>>;
+    findings: z.ZodArray<z.ZodObject<{
+        code: z.ZodString;
+        severity: z.ZodEnum<{
+            info: "info";
+            warning: "warning";
+        }>;
+        message: z.ZodString;
+        count: z.ZodNumber;
+        items: z.ZodArray<z.ZodObject<{
+            path: z.ZodString;
+            candidates: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                path: z.ZodString;
+                reason: z.ZodString;
+            }, z.core.$loose>>>;
+        }, z.core.$loose>>;
+    }, z.core.$loose>>;
+}, z.core.$loose>;
 /** `GET /api/reports/tests/history`: one test and every result recorded for it (at most 100), newest first. */
 export declare const TestHistorySchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<1>;

package/dist/schema/src/api.js

@@ -275,6 +275,56 @@ export const TestCatalogSchema = z.looseObject({
     limit: z.number().int().nonnegative(),
     tests: z.array(TestSummarySchema)
 });
+const nonneg = z.number().int().nonnegative();
+const HealthItemSchema = z.looseObject({
+    path: z.string(),
+    candidates: z.array(z.looseObject({ path: z.string(), reason: z.string() })).optional().describe('Other paths it could be the same promise as. Atlas has not merged them')
+});
+/**
+ * `GET /api/reports/tests/health`: why the report's Specs, domains and counts may disagree. Counts keep the denominators
+ * apart (spec files, defined specs, specs with an executed test, tests, outlines, examples, observations), `reconciliation`
+ * walks from one to another, and `findings` name what to fix. It reads the report; it never changes what passed.
+ */
+export const CatalogHealthSchema = z.looseObject({
+    schemaVersion: z.literal(1),
+    version: z.string().describe('Changes whenever the report or its history does'),
+    generatedAtMs: z.number(),
+    ok: z.boolean().describe('True when there is no warning-severity finding'),
+    counts: z.looseObject({
+        specs: z.looseObject({
+            files: nonneg.describe('Spec files the report is laid over, before any root mapping'),
+            defined: nonneg.describe('Logical specs: files joined by a declared root mapping count once'),
+            withTests: nonneg.describe('Logical specs, defined or not, that have at least one test'),
+            executed: nonneg.describe('Defined logical specs with a test that passed or failed'),
+            withoutExecution: nonneg.describe('Defined logical specs with no passed or failed test: the defined coverage gaps')
+        }),
+        tests: z.looseObject({ total: nonneg, withSpec: nonneg, withoutSpec: nonneg, passed: nonneg.describe('Tests whose current status is passed'), historicalOnly: nonneg.describe('Tests only the history holds a result for') }),
+        scenarios: z.looseObject({
+            outlines: nonneg.describe('Tests sharing the label `outline` count once; a test without it is its own outline'),
+            examples: nonneg.describe('Tests (check identities), however many environments ran them'),
+            observations: nonneg.describe('Current results per producer, environment and parameter set'),
+            executedObservations: nonneg, passedObservations: nonneg
+        })
+    }),
+    reconciliation: z.looseObject({
+        specs: nonneg.describe('What the Specs view counts: spec files'),
+        testSpecs: nonneg.describe('What the Tests view groups by: logical specs with at least one test'),
+        steps: z.array(z.looseObject({ code: z.string(), label: z.string(), delta: z.number().int() })).describe('From `specs` to `testSpecs`: each step adds its `delta`'),
+        testsWithoutSpec: nonneg,
+        explanation: z.array(z.string())
+    }),
+    identity: z.looseObject({ checks: nonneg, fallbackIdentity: nonneg.describe('Checks whose id equals their title'), basis: z.string() }),
+    domains: z.array(z.looseObject({ name: z.string(), specs: nonneg })).describe('Specs per domain, most first'),
+    roots: z.array(z.looseObject({ from: z.string(), to: z.string(), matched: nonneg.describe('Distinct paths the mapping changed') })),
+    aliases: z.array(z.looseObject({ path: z.string(), sources: z.array(z.string()) })).describe('Logical specs seen under more than one original path, joined by a root mapping'),
+    findings: z.array(z.looseObject({
+        code: z.string().describe('An open set: treat a code you do not know as informational'),
+        severity: z.enum(['warning', 'info']),
+        message: z.string(),
+        count: nonneg,
+        items: z.array(HealthItemSchema).describe('At most 25; `count` is the whole')
+    }))
+});
 /** `GET /api/reports/tests/history`: one test and every result recorded for it (at most 100), newest first. */
 export const TestHistorySchema = z.looseObject({
     schemaVersion: z.literal(1),

package/dist/server/src/app.js

@@ -25,6 +25,7 @@ import { reportStatus } from '../../reports/src/report-status.js';
 import { ReportIndex } from '../../reports/src/report-index.js';
 import { listRunSummaries, readRunFile } from '../../reports/src/run-bundle.js';
 import { latestSpecSet, readSpecBody } from '../../reports/src/spec-set.js';
+import { catalogHealth, healthMarkdown, parseRootMappings } from '../../reports/src/catalog-health.js';
 import { catalogMarkdown, catalogModel, historyMarkdown, parseTestQuery, runDetail, runList, runMarkdown, runsMarkdown, testCatalog, testHistory } from '../../reports/src/test-catalog.js';
 import { llmsTxt } from './llms.js';
 import { openApiDocument } from './openapi.js';
@@ -399,6 +400,14 @@ function createProjectApplication(options) {
         const document = () => testCatalog(found, query);
         return answer(c, found.version, document, () => catalogMarkdown(document()), `tests|${new URL(c.req.url).search}|${wantsMarkdown(c)}`);
     });
+    app.get('/api/reports/tests/health', async (c) => {
+        if (!index.ready)
+            return notReady(c);
+        const found = await catalog();
+        const params = new URL(c.req.url).searchParams;
+        const document = () => catalogHealth(found, parseRootMappings(params));
+        return answer(c, found.version, document, () => healthMarkdown(document()), `health|${params.getAll('root').join('\u0001')}|${wantsMarkdown(c)}`);
+    });
     app.get('/api/reports/tests/history', async (c) => {
         if (!index.ready)
             return notReady(c);

package/dist/server/src/llms.js

@@ -68,6 +68,7 @@ Every endpoint below answers JSON, or Markdown with \`format=markdown\`. Prefix
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests?format=markdown&limit=50"                 # summary, environments, domains, failing, flaky, slowest, then the tests
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests?status=failed,flaky&environment=desktop-web" # filter: q, status, environment, producer, domain, spec, tag, run; sort, offset, limit
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests/history?key=<url-encoded test key>"         # one test: every recorded result, durations, branches, CI runs
+curl "${url}/api/projects/$ATLAS_PROJECT/reports/tests/health?format=markdown"                      # why Specs, domains and counts disagree; add root=<from>><to> to declare a path-root mapping
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/runs?format=markdown"                             # CI runs, newest first, with their results by status
 curl "${url}/api/projects/$ATLAS_PROJECT/reports/runs/detail?key=<url-encoded run key>"            # every test one run recorded
 \`\`\`