@crvouga/atlas 1.14.0 → 1.16.0

package/dist/reports/src/run-compare.js

@@ -0,0 +1,329 @@
+import { compareLatency, groupOf, latencyObservations, latencyReview } from './latency.js';
+/** Where a pair of results (or one result with no partner) stands, in the order a review reads them. */
+export const COMPARE_GROUPS = ['regressed', 'recovered', 'still-failing', 'not-comparable', 'unknown', 'removed', 'added', 'unchanged'];
+/** What each group is called and what it does and does not say. */
+export const COMPARE_GROUP_TEXT = {
+    regressed: { label: 'Newly failing', meaning: 'Passed in the baseline run and failed in the candidate run, under the same conditions and the same wording.' },
+    recovered: { label: 'Recovered', meaning: 'Failed in the baseline run and passed in the candidate run. One later pass is not a proven fix.' },
+    'still-failing': { label: 'Still failing', meaning: 'Failed in both runs.' },
+    'not-comparable': { label: 'Promise or conditions differ', meaning: 'The behavior is worded differently, or the two results were recorded under different conditions, so the two outcomes are not the same experiment and no regression or recovery is claimed.' },
+    unknown: { label: 'Not run or unfinished', meaning: 'One side did not run it or it is still running. That is unknown: not a failure and not a pass.' },
+    removed: { label: 'Not in the candidate run', meaning: 'Only the baseline run recorded it. The candidate may not have run it, or it was renamed or dropped: unknown, not a regression and not a pass.' },
+    added: { label: 'Not in the baseline run', meaning: 'Only the candidate run recorded it. It may be new, or the baseline may not have run it.' },
+    unchanged: { label: 'Same outcome', meaning: 'Passed in both runs, or not run in both.' }
+};
+/** The condition of a result. A server that does not send `kind` leaves it null: unknown, equal only to another unknown. */
+export function conditionOf(result) {
+    return { kind: result.kind ?? null, producer: result.producer, environment: result.environment, parameters: result.parameters ?? {} };
+}
+/** A stable text for a condition, equal exactly when two conditions are. */
+export function conditionId(condition) {
+    const parameters = Object.keys(condition.parameters).sort().map(name => `${name}=${condition.parameters[name]}`).join(',');
+    return [condition.kind ?? '', condition.producer, condition.environment ?? '', parameters].join('/');
+}
+/** How two conditions differ, in words; empty when they are the same. */
+export function conditionDifferences(baseline, candidate) {
+    const out = [];
+    if (baseline.environment !== candidate.environment)
+        out.push(`environment ${baseline.environment ?? 'not declared'} against ${candidate.environment ?? 'not declared'}`);
+    if (baseline.producer !== candidate.producer)
+        out.push(`producer ${baseline.producer} against ${candidate.producer}`);
+    if ((baseline.kind ?? '') !== (candidate.kind ?? ''))
+        out.push(`kind ${baseline.kind ?? 'not declared'} against ${candidate.kind ?? 'not declared'}`);
+    for (const name of [...new Set([...Object.keys(baseline.parameters), ...Object.keys(candidate.parameters)])].sort()) {
+        if (baseline.parameters[name] !== candidate.parameters[name])
+            out.push(`${name} ${baseline.parameters[name] ?? 'not set'} against ${candidate.parameters[name] ?? 'not set'}`);
+    }
+    return out;
+}
+const completed = (status) => status === 'passed' || status === 'failed';
+const compareText = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+/** Short form of a spec fingerprint, to show. */
+export function shortRevision(revision) {
+    return revision ? revision.slice(0, 8) : 'not recorded';
+}
+/** The wording a run recorded for a check: what that run's layer called it, else the check's newest title. */
+export function recordedTitle(result) {
+    return result.recordedTitle ?? result.title;
+}
+function readPromise(baseline, candidate) {
+    const baselineTitle = baseline ? recordedTitle(baseline) : null;
+    const candidateTitle = candidate ? recordedTitle(candidate) : null;
+    const baselineRevision = baseline?.specRevision ?? null;
+    const candidateRevision = candidate?.specRevision ?? null;
+    const reasons = [];
+    const base = { baselineTitle, candidateTitle, baselineRevision, candidateRevision };
+    if (!baseline || !candidate)
+        return { ...base, state: 'unknown', reasons: ['Only one run recorded this behavior, so its promise cannot be compared.'] };
+    const titleDiffers = baselineTitle !== candidateTitle;
+    const revisionKnown = baselineRevision !== null && candidateRevision !== null;
+    const revisionDiffers = revisionKnown && baselineRevision !== candidateRevision;
+    if (titleDiffers)
+        reasons.push(`The behavior is worded differently: “${baselineTitle}” became “${candidateTitle}”.`);
+    if (revisionDiffers)
+        reasons.push(`The spec file was recorded at revision ${shortRevision(baselineRevision)} and then ${shortRevision(candidateRevision)}. A revision is of the whole file, so it does not say that this behavior's own promise changed.`);
+    if (titleDiffers)
+        return { ...base, state: 'changed', reasons };
+    if (revisionDiffers)
+        return { ...base, state: 'revision-differs', reasons };
+    if (!revisionKnown) {
+        reasons.push(`${baselineRevision === null && candidateRevision === null ? 'Neither run' : baselineRevision === null ? 'The baseline run' : 'The candidate run'} recorded no spec revision, so whether the promise is the same is unknown.`);
+        return { ...base, state: 'unknown', reasons };
+    }
+    return { ...base, state: 'same', reasons };
+}
+function movement(baseline, candidate) {
+    const b = baseline.status;
+    const c = candidate.status;
+    if (!completed(b) || !completed(c)) {
+        const where = !completed(c) ? 'candidate' : 'baseline';
+        const what = (status) => (status === 'running' ? 'still running' : 'not run');
+        const side = !completed(c) ? c : b;
+        return { group: 'unknown', summary: `${what(side)[0].toUpperCase()}${what(side).slice(1)} in the ${where} run, so there is no outcome to compare. That is unknown, not a failure or a pass.` };
+    }
+    if (b === 'passed' && c === 'failed')
+        return { group: 'regressed', summary: 'Passed in the baseline run, failed in the candidate run.' };
+    if (b === 'failed' && c === 'passed')
+        return { group: 'recovered', summary: 'Failed in the baseline run, passed in the candidate run.' };
+    if (b === 'failed')
+        return { group: 'still-failing', summary: 'Failed in both runs.' };
+    return { group: 'unchanged', summary: 'Passed in both runs.' };
+}
+function pairOf(key, baseline, candidate, differences) {
+    const promise = readPromise(baseline, candidate);
+    const id = [key, baseline ? conditionId(conditionOf(baseline)) : '', candidate ? conditionId(conditionOf(candidate)) : ''].join('\u0001');
+    const title = candidate ? recordedTitle(candidate) : recordedTitle(baseline);
+    const specPath = (candidate ?? baseline).specPath;
+    const caveats = [];
+    const common = { id, key, title, specPath, baseline, candidate, promise, conditions: differences, caveats };
+    if (!baseline)
+        return { ...common, group: 'added', summary: 'Only the candidate run recorded it. It may be new, or the baseline may not have run it.' };
+    if (!candidate)
+        return { ...common, group: 'removed', summary: 'The candidate run recorded no result for it. It may not have run it, or it was renamed or dropped: unknown, not a regression.' };
+    if (promise.state === 'changed' || differences.length > 0) {
+        const why = [...(promise.state === 'changed' ? ['the behavior is worded differently'] : []), ...(differences.length ? [`it was recorded under different conditions (${differences.join('; ')})`] : [])];
+        return { ...common, group: 'not-comparable', summary: `Not the same experiment: ${why.join(' and ')}. Both outcomes are shown; no regression or recovery is claimed.` };
+    }
+    const moved = movement(baseline, candidate);
+    if (promise.state === 'revision-differs')
+        caveats.push(...promise.reasons);
+    else if (promise.state === 'unknown' && moved.group !== 'unknown')
+        caveats.push(...promise.reasons);
+    return { ...common, ...moved };
+}
+/**
+ * Pairs the results of two runs and reads what changed. Results are paired by check identity and then by condition (see
+ * the module comment); titles never pair two results. Each side's results are those the run recorded last for each
+ * check and condition (`RunDetail.results`).
+ *
+ * @param baseline - The run to compare against.
+ * @param candidate - The run under review. Both are used as given: a newer run never replaces either.
+ * @returns Every result of either run, grouped by what changed, with the reasons.
+ * @example
+ * ```ts
+ * const comparison = compareRuns(await runDetail('run:ci-100'), await runDetail('run:ci-101'));
+ * comparison.results.filter(item => item.group === 'regressed');
+ * ```
+ */
+export function compareRuns(baseline, candidate) {
+    const byKey = (results) => {
+        const map = new Map();
+        for (const result of results) {
+            const list = map.get(result.key);
+            if (list)
+                list.push(result);
+            else
+                map.set(result.key, [result]);
+        }
+        return map;
+    };
+    const left = byKey(baseline.results);
+    const right = byKey(candidate.results);
+    const pairs = [];
+    for (const key of [...new Set([...left.keys(), ...right.keys()])]) {
+        const sides = { base: [...(left.get(key) ?? [])], cand: [...(right.get(key) ?? [])] };
+        const order = (a, b) => compareText(conditionId(conditionOf(a)), conditionId(conditionOf(b))) || compareText(a.layer, b.layer);
+        sides.base.sort(order);
+        sides.cand.sort(order);
+        // First the same condition on both sides, then whatever is left of the same check under other conditions.
+        const restBase = [];
+        for (const result of sides.base) {
+            const at = sides.cand.findIndex(other => conditionId(conditionOf(other)) === conditionId(conditionOf(result)));
+            if (at < 0)
+                restBase.push(result);
+            else
+                pairs.push(pairOf(key, result, sides.cand.splice(at, 1)[0], []));
+        }
+        while (restBase.length && sides.cand.length) {
+            const result = restBase.shift();
+            const other = sides.cand.shift();
+            pairs.push(pairOf(key, result, other, conditionDifferences(conditionOf(result), conditionOf(other))));
+        }
+        for (const result of restBase)
+            pairs.push(pairOf(key, result, null, []));
+        for (const result of sides.cand)
+            pairs.push(pairOf(key, null, result, []));
+    }
+    pairs.sort((a, b) => COMPARE_GROUPS.indexOf(a.group) - COMPARE_GROUPS.indexOf(b.group) ||
+        compareText(a.specPath ?? '', b.specPath ?? '') ||
+        compareText(a.title, b.title) ||
+        compareText(a.id, b.id));
+    const counts = Object.fromEntries(COMPARE_GROUPS.map(group => [group, 0]));
+    for (const pair of pairs)
+        counts[pair.group] += 1;
+    const notes = [];
+    if (baseline.run.key === candidate.run.key)
+        notes.push('The same run is on both sides, so nothing can differ.');
+    if (baseline.run.branch !== candidate.run.branch)
+        notes.push(`The runs are on different branches (${baseline.run.branch ?? 'none recorded'} and ${candidate.run.branch ?? 'none recorded'}).`);
+    if (candidate.run.endedAtMs < baseline.run.endedAtMs)
+        notes.push('The candidate run ended before the baseline run.');
+    for (const [name, side] of [['baseline', baseline], ['candidate', candidate]]) {
+        if (side.run.freshness === 'modified')
+            notes.push(`The ${name} run was recorded from a checkout with uncommitted changes.`);
+        if (side.run.counts.running > 0)
+            notes.push(`The ${name} run still has results running, so its outcomes may change.`);
+    }
+    return { baseline: baseline.run, candidate: candidate.run, results: pairs, counts, notes };
+}
+const devicesOf = (entries) => [...new Set(latencyObservations(entries).map(o => groupOf(o, 'device')))].sort();
+/**
+ * Compares the product latency recorded in two sets of results of one check, one per side. A metric is compared only
+ * when it is the same measurement (name, unit, boundaries) and both sides were measured on the same devices, in the
+ * same environments, with the same instrumentation and other conditions; otherwise it is `incompatible` and no change
+ * is computed, with the reasons. A metric only one side recorded is named as such. A side without any latency is unknown,
+ * not fast: it contributes no metric. `conditionsDiffer` names differences of the results' own conditions, which make
+ * every metric incompatible.
+ */
+export function compareProductLatency(baseline, candidate, conditionsDiffer = []) {
+    const left = latencyReview(baseline).metrics;
+    const right = latencyReview(candidate).metrics;
+    const sameDevices = devicesOf(baseline).join('\u0001') === devicesOf(candidate).join('\u0001');
+    const out = [];
+    for (const { metric, groups } of left) {
+        const base = groups[0];
+        const other = right.find(item => item.metric.key === metric.key)?.groups[0];
+        if (!other) {
+            const renamed = right.find(item => item.metric.name === metric.name);
+            out.push({ state: 'baseline-only', name: metric.name, summary: base, reasons: [renamed ? `The candidate measures ${metric.name} over a different interval (${renamed.metric.boundary.label}), so the two are not the same measurement.` : 'The candidate recorded no such measurement.'] });
+            continue;
+        }
+        const change = compareLatency(base, other);
+        const reasons = [...conditionsDiffer.map(line => `Recorded under different conditions: ${line}.`)];
+        if (!sameDevices)
+            reasons.push(`Devices differ: ${devicesOf(baseline).join(', ')} against ${devicesOf(candidate).join(', ')}.`);
+        if (change.comparable) {
+            reasons.push(...change.disclosures.filter(line => /^(Environments differ|Test conditions differ|Instrumentation changed)/.test(line)));
+        }
+        else
+            reasons.push(...change.reasons);
+        if (reasons.length)
+            out.push({ state: 'incompatible', name: metric.name, baseline: base, candidate: other, change: null, reasons });
+        else
+            out.push({ state: 'compared', name: metric.name, baseline: base, candidate: other, change, reasons: change.disclosures });
+    }
+    for (const { metric, groups } of right) {
+        if (!left.some(item => item.metric.key === metric.key) && !left.some(item => item.metric.name === metric.name)) {
+            out.push({ state: 'candidate-only', name: metric.name, summary: groups[0], reasons: ['The baseline recorded no such measurement.'] });
+        }
+    }
+    return out;
+}
+/**
+ * The scope that counts the independent executions of one result's check up to the end of its run: the same
+ * environment, producer and parameters, and the spec revision that result was recorded against (executions against
+ * another revision are other promises). A result that recorded no revision is counted across revisions, which
+ * {@link compareReliability} will not compare. Use it for each side, with that side's result and run.
+ */
+export function reliabilityScopeAt(result, run) {
+    return {
+        ...(result.environment ? { environment: [result.environment] } : {}),
+        producer: [result.producer],
+        ...(result.parameters && Object.keys(result.parameters).length ? { parameter: Object.entries(result.parameters).map(([name, value]) => `${name}=${value}`).sort() } : {}),
+        ...(result.specRevision ? { specRevision: [result.specRevision] } : { revision: 'any' }),
+        until: run.endedAtMs
+    };
+}
+/**
+ * Sets the reliability of a check up to the baseline run against its reliability up to the candidate run. A change is
+ * stated only when both samples are established (at least `minimumExecutions` independent executions), both counted the
+ * same recorded spec revision (see {@link reliabilityScopeAt}) and the conditions match; whether the difference exceeds the uncertainty is
+ * said by `direction`. Otherwise each figure stands alone and the reasons say why.
+ */
+export function compareReliability(baseline, candidate, conditionsDiffer = []) {
+    const reasons = conditionsDiffer.map(line => `Counted under different conditions: ${line}.`);
+    for (const [name, document] of [['baseline', baseline], ['candidate', candidate]]) {
+        if (document.sample === 'none')
+            reasons.push(`The ${name} figure has no completed execution, so it is unknown, not reliable.`);
+        else if (document.sample === 'insufficient')
+            reasons.push(`The ${name} figure rests on ${document.eventual.executions} independent ${document.eventual.executions === 1 ? 'execution' : 'executions'}; at least ${document.minimumExecutions} are needed.`);
+    }
+    const revisionOf = (document) => document.scope.specRevision?.join(',') ?? null;
+    if (revisionOf(baseline) === null || revisionOf(candidate) === null)
+        reasons.push('A side recorded no spec revision, so whether both count the same promise is unknown.');
+    else if (revisionOf(baseline) !== revisionOf(candidate))
+        reasons.push('The executions were counted against different revisions of the spec, so they describe different promises.');
+    const base = baseline.eventual;
+    const cand = candidate.eventual;
+    if (reasons.length || base.rate === null || cand.rate === null)
+        return { comparable: false, baseline: base, candidate: cand, reasons };
+    const separate = baseline.interval && candidate.interval ? baseline.interval.upper < candidate.interval.lower || candidate.interval.upper < baseline.interval.lower : false;
+    const points = Math.round((cand.rate - base.rate) * 1000) / 10;
+    return {
+        comparable: true,
+        baseline: base,
+        candidate: cand,
+        points,
+        direction: !separate ? 'indistinguishable' : cand.rate > base.rate ? 'higher' : 'lower',
+        reasons: separate ? [] : ['The two intervals overlap, so the difference may be chance. It is not shown as an improvement or a decline.']
+    };
+}
+/**
+ * Lines up the recorded steps of two results by name (the n-th step of a name on one side with the n-th of that name on
+ * the other), keeping each side's own time origin: an offset is milliseconds after that side's check began. Nothing is
+ * aligned by position or by fuzzy wording. When nothing lines up, `explanation` says why.
+ */
+export function alignSteps(baseline, candidate) {
+    const keyed = (steps) => {
+        const seen = new Map();
+        return steps.map(step => {
+            const n = seen.get(step.name) ?? 0;
+            seen.set(step.name, n + 1);
+            return { step, id: `${n}\u0001${step.name}` };
+        });
+    };
+    const left = keyed(baseline);
+    const right = keyed(candidate);
+    const rightById = new Map(right.map(item => [item.id, item.step]));
+    const leftIds = new Set(left.map(item => item.id));
+    const steps = [];
+    for (const { step, id } of left) {
+        const other = rightById.get(id);
+        steps.push({
+            name: step.name,
+            ...(step.offsetMs === undefined ? {} : { baselineOffsetMs: step.offsetMs }),
+            ...(other?.offsetMs === undefined ? {} : { candidateOffsetMs: other.offsetMs }),
+            onBaseline: true,
+            onCandidate: other !== undefined
+        });
+    }
+    for (const { step, id } of right) {
+        if (!leftIds.has(id))
+            steps.push({ name: step.name, ...(step.offsetMs === undefined ? {} : { candidateOffsetMs: step.offsetMs }), onBaseline: false, onCandidate: true });
+    }
+    const seekable = steps.filter((step) => step.baselineOffsetMs !== undefined && step.candidateOffsetMs !== undefined);
+    const shared = steps.filter(step => step.onBaseline && step.onCandidate).length;
+    const differs = steps.some(step => !(step.onBaseline && step.onCandidate));
+    let explanation;
+    if (seekable.length > 0)
+        explanation = `${seekable.length} recorded ${seekable.length === 1 ? 'step is' : 'steps are'} on both sides with a time. Seeking uses each side's own step time from the start of its own check; the recordings are otherwise not synchronized.`;
+    else if (!baseline.length && !candidate.length)
+        explanation = 'Neither side recorded steps, so the recordings cannot be aligned. Each plays from its own start.';
+    else if (!baseline.length || !candidate.length)
+        explanation = `Only the ${baseline.length ? 'baseline' : 'candidate'} recorded steps, so the recordings cannot be aligned. Each plays from its own start.`;
+    else if (shared === 0)
+        explanation = 'The two sides share no step name, so the recordings cannot be aligned. Each plays from its own start.';
+    else
+        explanation = 'The shared steps carry no time on both sides, so the recordings cannot be aligned. Each plays from its own start.';
+    return { state: seekable.length > 0 ? 'aligned' : 'unaligned', steps, seekable, differs, explanation };
+}

package/dist/reports/src/test-catalog.d.ts

@@ -322,6 +322,14 @@ export type RunResult = {
     readonly detail: string;
     /** Every video its evidence references, with who holds it (check, retry attempt or step). Absent when there is none. */
     readonly videos?: readonly VideoRef[];
+    /** The kind of the part that held it (`behavior`, `checks`…). With the producer, environment and parameters it is the condition the result was recorded under. Absent from an older server. */
+    readonly kind?: string;
+    /** The other choices that made it its own result (a device size, a locale…). Absent when there are none, and from an older server. */
+    readonly parameters?: Readonly<Record<string, string>>;
+    /** The fingerprint of its spec file the layer recorded: results with different ones were recorded against different revisions of the spec. Absent means not recorded, never "the same". */
+    readonly specRevision?: string;
+    /** The title this run's layer gave the check. `title` is the newest title the check has in the whole history, so a check that was reworded shows its newest wording in both runs; this is the wording as recorded. Absent from an older server. */
+    readonly recordedTitle?: string;
 };
 /**
  * One layer of a run, as the run's artifact hub names it: who recorded it, how many checks it holds (zero is possible

package/dist/reports/src/test-catalog.js

@@ -105,6 +105,8 @@ export class TestHistory {
     #idents = new Map();
     #runs = new Map();
     #specs = new RecordedSpecs();
+    /** Titles seen, so the many results that share one hold one string. */
+    #titles = new Map();
     #seen = new Set();
     #rev = 0;
     #cells = null;
@@ -187,9 +189,16 @@ export class TestHistory {
                 ident.title = row.title;
                 ident.titledAt = layer.recordedAtMs;
             }
-            this.#remember(slot, { o: origin, s: row.status, d: row.durationMs, k: row.kind, t: row.detail, e: row.evidence, m: productLatency(row.measurements), f: row.specPath ? fingerprints[row.specPath] : undefined });
+            this.#remember(slot, { o: origin, s: row.status, d: row.durationMs, k: row.kind, t: row.detail, e: row.evidence, m: productLatency(row.measurements), f: row.specPath ? fingerprints[row.specPath] : undefined, ti: this.#title(row.title) });
         }
     }
+    #title(title) {
+        const held = this.#titles.get(title);
+        if (held !== undefined)
+            return held;
+        this.#titles.set(title, title);
+        return title;
+    }
     /** Sorted insert, newest first; past the bound, the oldest is dropped. */
     #remember(slot, entry) {
         const list = this.#bySlot.get(slot);
@@ -262,6 +271,10 @@ export class TestHistory {
             key: i.slot, id: i.id, title: i.title, specPath: i.specPath, environment: r.o.environment, producer: r.o.producer,
             status: r.s, durationMs: r.d, at: r.o.at, layer: r.o.layer, detail: shortDetail(r.t),
             ...(r.e?.videos ? { videos: r.e.videos } : {}),
+            kind: r.k,
+            ...(Object.keys(r.o.parameters).length ? { parameters: r.o.parameters } : {}),
+            ...(r.f ? { specRevision: r.f } : {}),
+            ...(r.ti !== undefined ? { recordedTitle: r.ti } : {}),
         }));
         results.sort((a, b) => FAILURE_RANK[b.status] - FAILURE_RANK[a.status] || compareText(a.specPath ?? '', b.specPath ?? '') || compareText(a.title, b.title) || compareText(a.environment ?? '', b.environment ?? '') || compareText(a.producer, b.producer) || compareText(a.key, b.key));
         const layers = [...run.members].sort((a, b) => a.recordedAtMs - b.recordedAtMs || compareText(a.id, b.id));

package/dist/schema/src/api.d.ts

@@ -2808,6 +2808,8 @@ export declare const CatalogHealthSchema: z.ZodObject<{
     identity: z.ZodObject<{
         checks: z.ZodNumber;
         fallbackIdentity: z.ZodNumber;
+        declaredBehaviorIds: z.ZodOptional<z.ZodNumber>;
+        declaredExampleIds: z.ZodOptional<z.ZodNumber>;
         basis: z.ZodString;
     }, z.core.$loose>;
     domains: z.ZodArray<z.ZodObject<{
@@ -2818,6 +2820,10 @@ export declare const CatalogHealthSchema: z.ZodObject<{
         from: z.ZodString;
         to: z.ZodString;
         matched: z.ZodNumber;
+        source: z.ZodOptional<z.ZodEnum<{
+            request: "request";
+            configuration: "configuration";
+        }>>;
     }, z.core.$loose>>;
     aliases: z.ZodArray<z.ZodObject<{
         path: z.ZodString;
@@ -3624,6 +3630,10 @@ export declare const RunDetailSchema: z.ZodObject<{
                 extensions: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
             }, z.core.$loose>>;
         }, z.core.$loose>>>;
+        kind: z.ZodOptional<z.ZodString>;
+        parameters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        specRevision: z.ZodOptional<z.ZodString>;
+        recordedTitle: z.ZodOptional<z.ZodString>;
     }, z.core.$loose>>;
     layers: z.ZodOptional<z.ZodArray<z.ZodObject<{
         id: z.ZodString;

package/dist/schema/src/api.js

@@ -70,14 +70,14 @@ export const VideoRefSchema = z.looseObject({
     step: z.string().optional().describe('The name of the step that holds it'),
     tags: z.array(z.string()).optional().describe('The normalized tags of this attachment itself. Absent when it has none (and in documents of older servers)'),
     recording: z.string().optional().describe('What identifies the bytes; references with the same value are one recording. Absent when it cannot be played'),
-    available: z.boolean().describe('False when the reference names no bytes (no bundle key, address or inline text)'),
+    available: z.boolean().describe('False when the reference names no bytes: no bundle key, address or inline text, or a bundle key that the layer\'s own bundle does not hold'),
     scope: EvidenceScopeSchema.optional().describe('What this recording shows as its producer declared it: its own `scope`, else its attempt\'s, check\'s, part\'s or layer\'s. Absent: the scope is unknown, never live, physical or audible')
 });
 /** Video references as counts: how many are referenced, how many distinct recordings can be played, how many cannot. */
 export const VideoCountsSchema = z.looseObject({
     references: z.number().int().nonnegative().describe('Every reference to a video; identical bytes referenced twice count twice'),
     recordings: z.number().int().nonnegative().describe('Distinct playable recordings'),
-    unavailable: z.number().int().nonnegative().describe('References that name no bytes')
+    unavailable: z.number().int().nonnegative().describe('References that name no bytes, or bytes the layer does not hold')
 });
 /** What a result carries beyond its outcome, as counts and short facts. Present only when there is something. */
 export const EvidenceSummarySchema = z.looseObject({
@@ -533,9 +533,15 @@ export const CatalogHealthSchema = z.looseObject({
         testsWithoutSpec: nonneg,
         explanation: z.array(z.string())
     }),
-    identity: z.looseObject({ checks: nonneg, fallbackIdentity: nonneg.describe('Checks whose id equals their title'), basis: z.string() }),
+    identity: z.looseObject({
+        checks: nonneg,
+        fallbackIdentity: nonneg.describe('Checks whose id equals their title'),
+        declaredBehaviorIds: nonneg.optional().describe('Checks that declare a `behavior-id` label'),
+        declaredExampleIds: nonneg.optional().describe('Checks that declare an `example-id` label'),
+        basis: z.string()
+    }),
     domains: z.array(z.looseObject({ name: z.string(), specs: nonneg })).describe('Specs per domain, most first'),
-    roots: z.array(z.looseObject({ from: z.string(), to: z.string(), matched: nonneg.describe('Distinct paths the mapping changed') })),
+    roots: z.array(z.looseObject({ from: z.string(), to: z.string(), matched: nonneg.describe('Distinct paths the mapping changed'), source: z.enum(['configuration', 'request']).optional().describe('Where it was declared: the server\'s `catalog.roots` setting or this request\'s `root` parameter') })),
     aliases: z.array(z.looseObject({ path: z.string(), sources: z.array(z.string()) })).describe('Logical specs seen under more than one original path, joined by a root mapping'),
     findings: z.array(z.looseObject({
         code: z.string().describe('An open set: treat a code you do not know as informational'),
@@ -671,7 +677,11 @@ export const RunDetailSchema = z.looseObject({
     results: z.array(z.looseObject({
         key: z.string(), id: z.string(), title: z.string(), specPath: z.string().nullable(), environment: z.string().nullable(), producer: z.string(),
         status: EvidenceStatusSchema, durationMs: z.number().nullable(), at: z.number(), layer: z.string(), detail: z.string(),
-        videos: z.array(VideoRefSchema).optional().describe('Every video its evidence references, with who holds it')
+        videos: z.array(VideoRefSchema).optional().describe('Every video its evidence references, with who holds it'),
+        kind: z.string().optional().describe('The kind of the part that held it. With the producer, environment and parameters it is the condition the result was recorded under. Absent from an older server'),
+        parameters: z.record(z.string(), z.string()).optional().describe('The other choices that made it its own result (a device size, a locale…). Absent when there are none, and from an older server'),
+        specRevision: z.string().optional().describe('The fingerprint of its spec file the layer recorded. Absent means not recorded, never "the same"'),
+        recordedTitle: z.string().optional().describe('The title this run\'s layer gave the check. `title` is the check\'s newest title in the whole history; this is the wording as recorded. Absent from an older server')
     })),
     layers: z.array(z.looseObject({
         id: z.string().describe('The layer id; read its files with `/api/reports/evidence/detail/{backend}/{id}`'),

package/dist/schema/src/server-config.d.ts

@@ -66,6 +66,9 @@ export declare const ServerConfigSchema: z.ZodObject<{
         branches: z.ZodOptional<z.ZodArray<z.ZodString>>;
         ignoreModified: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strict>>;
+    catalog: z.ZodDefault<z.ZodObject<{
+        roots: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    }, z.core.$strict>>;
     refreshSeconds: z.ZodDefault<z.ZodNumber>;
     limits: z.ZodDefault<z.ZodObject<{
         maxObjectMiB: z.ZodDefault<z.ZodNumber>;

package/dist/schema/src/server-config.js

@@ -50,6 +50,11 @@ export const ServerConfigSchema = z.strictObject({
         ignoreModified: z.boolean().optional().describe('Leave out layers from modified checkouts and unknown origins')
     })
         .default({}),
+    catalog: z
+        .strictObject({
+        roots: z.array(z.string().refine((value) => value.includes('>'), { message: 'Write a root mapping as from>to, e.g. ">specs/" (paths under `from` are the same specs as paths under `to`)' })).default([]).describe('Root mappings `from>to` that relate spec paths a producer publishes to the paths the specs are recorded under (an empty `from` matches every path). The catalog health report applies them to every request, beside any `root` the request adds')
+    })
+        .default({ roots: [] }),
     refreshSeconds: z.number().min(1).default(10).describe('How often the backends are looked at for layers written by something other than this server'),
     limits: z
         .strictObject({

package/dist/server/src/app.d.ts

@@ -30,6 +30,8 @@ export type AtlasServerOptions = {
     /** The built viewer. Without it, only the API is served. */
     readonly site?: string;
     readonly outlineBranch?: string;
+    /** Root mappings `from>to` the catalog health report always applies (the `catalog.roots` setting), beside any `root` a request adds. */
+    readonly catalogRoots?: readonly string[];
     readonly include?: (layer: LayerHeader) => boolean;
     readonly refreshMs?: number;
     readonly limits?: {

package/dist/server/src/app.js

@@ -26,7 +26,7 @@ import { reportStatus } from '../../reports/src/report-status.js';
 import { ReportIndex } from '../../reports/src/report-index.js';
 import { listRunSummaries, readRunFile } from '../../reports/src/run-bundle.js';
 import { latestSpecSet, readSpecBody } from '../../reports/src/spec-set.js';
-import { catalogHealth, healthMarkdown, parseRootMappings } from '../../reports/src/catalog-health.js';
+import { catalogHealth, healthMarkdown, parseRootMapping, parseRootMappings } from '../../reports/src/catalog-health.js';
 import { parseArtifactFilter } from '../../reports/src/artifact-query.js';
 import { filterUiCatalog, pageUiCatalog, parseUiCatalogQuery, uiCatalog, uiCatalogMarkdown } from '../../reports/src/ui-catalog.js';
 import { parseReliabilityScope, reliability, reliabilityMarkdown } from '../../reports/src/reliability.js';
@@ -451,12 +451,16 @@ function createProjectApplication(options) {
         const document = () => testCatalog(found, query);
         return answer(c, found.version, document, () => catalogMarkdown(document()), `tests|${new URL(c.req.url).search}|${wantsMarkdown(c)}`);
     });
+    const configuredRoots = (options.catalogRoots ?? []).flatMap(text => parseRootMapping(text) ?? []).map(mapping => ({ ...mapping, source: 'configuration' }));
     app.get('/api/reports/tests/health', async (c) => {
         if (!index.ready)
             return notReady(c);
         const found = await catalog();
         const params = new URL(c.req.url).searchParams;
-        const document = () => catalogHealth(found, parseRootMappings(params));
+        // The mappings the operator configured, then those this request adds (one already configured is not counted twice).
+        const requested = parseRootMappings(params).filter(mapping => !configuredRoots.some(item => item.from === mapping.from && item.to === mapping.to));
+        const roots = [...configuredRoots, ...requested.map(mapping => ({ ...mapping, source: 'request' }))];
+        const document = () => catalogHealth(found, roots);
         return answer(c, found.version, document, () => healthMarkdown(document()), `health|${params.getAll('root').join('\u0001')}|${wantsMarkdown(c)}`);
     });
     app.get('/api/reports/tests/history', async (c) => {

package/dist/server/src/config.js

@@ -57,6 +57,7 @@ export const SERVER_SETTINGS = [
     { summary: 'Whose specs the report is laid over', key: 'outlineBranch', flag: '--outline-branch <branch>', env: ['ATLAS_OUTLINE_BRANCH'], default: '`main`, then `master`, then the newest', example: { value: 'trunk', read: c => c.outlineBranch, expected: 'trunk' } },
     { summary: 'Only layers recorded on these branches take part', key: 'include.branches', flag: '--branch <name>` (repeatable)', env: ['ATLAS_ONLY_BRANCH'], default: 'all', example: { value: 'main', read: c => c.include.branches, expected: ['main'] } },
     { summary: 'Leave out layers from modified checkouts and unknown origins', key: 'include.ignoreModified', flag: '--ignore-modified', env: ['ATLAS_IGNORE_MODIFIED'], default: 'no', example: { value: '1', read: c => c.include.ignoreModified, expected: true } },
+    { summary: 'Root mappings `from>to` that join spec paths a producer publishes to the paths the specs are recorded under, for the catalog health report (comma-separated; an empty `from` matches every path)', key: 'catalog.roots', flag: null, env: ['ATLAS_CATALOG_ROOTS'], default: 'none', example: { value: '>specs/,apps/>specs/web/', read: c => c.catalog.roots, expected: ['>specs/', 'apps/>specs/web/'] } },
     { summary: 'How often (seconds) the backends are looked at for layers written by something else', key: 'refreshSeconds', flag: '--refresh <seconds>', env: ['ATLAS_REFRESH_SECONDS'], default: '`10`', example: { value: '30', read: c => c.refreshSeconds, expected: 30 } },
     { summary: 'Let anyone look, not upload', key: 'auth.anonymousRead', flag: '--anonymous-read', env: ['ATLAS_ANONYMOUS_READ'], default: 'no', example: { value: '1', read: c => c.anonymousRead, expected: true } },
     { summary: 'Listen publicly with no tokens (something in front signs people in)', key: 'auth.insecure', flag: '--insecure', env: ['ATLAS_INSECURE'], default: 'no', example: { value: '1', read: c => c.insecure, expected: true } },
@@ -151,6 +152,9 @@ export function loadServerConfig(argv, env, cwd, read = file => readFileSync(fil
             limits[key] = Number(env[variable]);
     }
     merged.limits = limits;
+    const roots = list(env.ATLAS_CATALOG_ROOTS);
+    if (roots.length)
+        merged.catalog = { ...raw.catalog, roots };
     if (flags.insecure || truthy(env.ATLAS_INSECURE))
         auth.insecure = true;
     if (flags['anonymous-read'] || truthy(env.ATLAS_ANONYMOUS_READ))

package/dist/server/src/main.js

@@ -56,6 +56,7 @@ export function serverOptionsOf(config, backends, log) {
         backends, write: S3_BACKEND_ID,
         tokens: config.tokens, anonymousRead: config.anonymousRead,
         ...(config.outlineBranch ? { outlineBranch: config.outlineBranch } : {}), ...(include ? { include } : {}),
+        ...(config.catalog.roots.length ? { catalogRoots: config.catalog.roots } : {}),
         runTimeoutMs: config.runTimeoutSeconds * 1000, runStallMs: config.runStallSeconds * 1000, refreshMs: config.refreshSeconds * 1000, limits: config.limits, heartbeatMs: config.heartbeatSeconds * 1000,
         ...(config.log && log ? { log } : {})
     };

package/dist/server/src/openapi.js

@@ -313,10 +313,10 @@ const BASE_OPERATIONS = [
             '',
             '`counts` keeps the denominators apart: spec files, defined specs, specs with an executed test (and those without: the defined coverage gaps), tests, outlines, examples and per-environment observations. `reconciliation` walks from the Specs count to the Tests count. `findings` name ambiguous and unmatched spec paths, identical or same-titled specs, a domain shared by nearly every spec, tests with no spec, checks that fall back to their title as identity and root mappings that matched nothing.',
             '',
-            'Two paths are one logical spec only when a `root` mapping relates them (`from>to`: paths under `from` are the same specs as paths under `to`; an empty `from` matches every path). Without one, similar paths are reported as ambiguous and never merged. A check\'s label `outline` names the outline its example belongs to. Send `format=markdown` for Markdown.'
+            'Two paths are one logical spec only when a `root` mapping relates them (`from>to`: paths under `from` are the same specs as paths under `to`; an empty `from` matches every path). Without one, similar paths are reported as ambiguous and never merged. A check\'s label `outline` names the outline its example belongs to; the labels `behavior-id` and `example-id` declare stable ids, which are counted and checked for ids shared by several checks. Send `format=markdown` for Markdown.'
         ].join('\n'),
         parameters: [
-            { name: 'root', in: 'query', description: 'A root mapping `from>to`, URL-encoded. Repeatable or comma-separated.', schema: str, example: '>specs/' },
+            { name: 'root', in: 'query', description: 'A root mapping `from>to`, URL-encoded. Repeatable or comma-separated. It is added to the mappings the server is configured with (`catalog.roots`, `ATLAS_CATALOG_ROOTS`); `roots[].source` says which is which.', schema: str, example: '>specs/' },
             { name: 'format', in: 'query', description: '`markdown` for a Markdown report instead of JSON.', schema: { type: 'string', enum: ['json', 'markdown'] } }
         ],
         responses: { '200': { ...json('The diagnostics.', 'CatalogHealth'), headers: { ETag: { description: 'Changes with the report, its history and the mappings.', schema: str } } }, '304': { description: 'Unchanged since the `If-None-Match` ETag.' }, '401': UNAUTHORIZED, '503': NOT_READY }

package/docs/extending.md

@@ -30,7 +30,7 @@ carry any of these. Every one is optional.
 | `annotations[]` | Notes about the result: a warning, a deprecation, an accessibility finding, a coverage figure. |
 | `attempts[]` | Every try, when the check was retried. |
 | `tags[]` | Free labels to filter by: `smoke`, `payments`. |
-| `labels{}` | Facts about the result that are not its identity: `owner`, `severity`, `browser`, `build`. The viewer reads `build`, `platform`, `applies-to` and `applies-to-by` to tell whether evidence applies to a release (see [the viewer conventions](ui.md#behaviors-read-watch-inspect)). |
+| `labels{}` | Facts about the result that are not its identity: `owner`, `severity`, `browser`, `build`. The viewer reads `build`, `platform`, `applies-to` and `applies-to-by` to tell whether evidence applies to a release (see [the viewer conventions](ui.md#behaviors-read-watch-inspect)). `behavior-id` and `example-id` declare a stable id for the behavior and for one example of an outline, and `outline` names the outline; the [catalog health](server.md#reading-the-report-as-an-agent) report counts them and names an id shared by several checks. They never change which check a result belongs to. |
 | `scope{}` | What the evidence does and does not show: surface, device kind, synthetic data, scripted dependencies, audio, pacing, what was asserted and its limits. Unknown when absent. See [Evidence scope](rich-evidence.md#evidence-scope). |
 | `links[]` | Where to read more: the CI run, the issue, the trace client. |
 | `extensions{}` | Structured data of your own or a tool's, under a name you own. |

package/docs/reference/http-api.md

@@ -550,9 +550,9 @@ Diagnoses the report's metadata, so a domain filter or a coverage percentage can
 
 `counts` keeps the denominators apart: spec files, defined specs, specs with an executed test (and those without: the defined coverage gaps), tests, outlines, examples and per-environment observations. `reconciliation` walks from the Specs count to the Tests count. `findings` name ambiguous and unmatched spec paths, identical or same-titled specs, a domain shared by nearly every spec, tests with no spec, checks that fall back to their title as identity and root mappings that matched nothing.
 
-Two paths are one logical spec only when a `root` mapping relates them (`from>to`: paths under `from` are the same specs as paths under `to`; an empty `from` matches every path). Without one, similar paths are reported as ambiguous and never merged. A check's label `outline` names the outline its example belongs to. Send `format=markdown` for Markdown.
+Two paths are one logical spec only when a `root` mapping relates them (`from>to`: paths under `from` are the same specs as paths under `to`; an empty `from` matches every path). Without one, similar paths are reported as ambiguous and never merged. A check's label `outline` names the outline its example belongs to; the labels `behavior-id` and `example-id` declare stable ids, which are counted and checked for ids shared by several checks. Send `format=markdown` for Markdown.
 
-- `root` (query): A root mapping `from>to`, URL-encoded. Repeatable or comma-separated.
+- `root` (query): A root mapping `from>to`, URL-encoded. Repeatable or comma-separated. It is added to the mappings the server is configured with (`catalog.roots`, `ATLAS_CATALOG_ROOTS`); `roots[].source` says which is which.
 - `format` (query): `markdown` for a Markdown report instead of JSON.
 - `project` (query): Project slug. Alternatively send X-Atlas-Project or use the project-prefixed route. Omit for the default project. Conflicting selectors are refused.
 
@@ -566,12 +566,12 @@ Diagnoses the report's metadata, so a domain filter or a coverage percentage can
 
 `counts` keeps the denominators apart: spec files, defined specs, specs with an executed test (and those without: the defined coverage gaps), tests, outlines, examples and per-environment observations. `reconciliation` walks from the Specs count to the Tests count. `findings` name ambiguous and unmatched spec paths, identical or same-titled specs, a domain shared by nearly every spec, tests with no spec, checks that fall back to their title as identity and root mappings that matched nothing.
 
-Two paths are one logical spec only when a `root` mapping relates them (`from>to`: paths under `from` are the same specs as paths under `to`; an empty `from` matches every path). Without one, similar paths are reported as ambiguous and never merged. A check's label `outline` names the outline its example belongs to. Send `format=markdown` for Markdown.
+Two paths are one logical spec only when a `root` mapping relates them (`from>to`: paths under `from` are the same specs as paths under `to`; an empty `from` matches every path). Without one, similar paths are reported as ambiguous and never merged. A check's label `outline` names the outline its example belongs to; the labels `behavior-id` and `example-id` declare stable ids, which are counted and checked for ids shared by several checks. Send `format=markdown` for Markdown.
 
 Scoped to one project. The same project may be selected on the unprefixed API using ?project= or X-Atlas-Project. Unknown projects return 404 on reads and are created by an authenticated upload.
 
 - `project` (path, required): Project slug. Alternatively send X-Atlas-Project or use the project-prefixed route. Omit for the default project. Conflicting selectors are refused.
-- `root` (query): A root mapping `from>to`, URL-encoded. Repeatable or comma-separated.
+- `root` (query): A root mapping `from>to`, URL-encoded. Repeatable or comma-separated. It is added to the mappings the server is configured with (`catalog.roots`, `ATLAS_CATALOG_ROOTS`); `roots[].source` says which is which.
 - `format` (query): `markdown` for a Markdown report instead of JSON.
 
 Responses: `200` The diagnostics.; `304` Unchanged since the `If-None-Match` ETag.; `401` No token, or an unknown one. Send `Authorization: Bearer <token>`.; `503` The server is still reading its backends. Retry after the number of seconds in `Retry-After`.