e2e 0.21.1 → 0.21.2

package/dist/report/failure-text.js

@@ -8,6 +8,7 @@
  * each block to its page. Both read the facts from here, so the two never disagree.
  */
 import path from 'node:path';
+import { modelLabel } from '../config/agent.js';
 import { isLoopbackHost } from '../internal/urls.js';
 import { cell, code, formatDuration, link, MAX_CELL_CHARS, MAX_ID_CHARS, MAX_LABEL_CHARS, MAX_PATH_CHARS, MAX_TITLE_CHARS, plural } from './markdown-text.js';
 import { repeatSuffix } from './format.js';
@@ -86,11 +87,25 @@ export function toldAttempt(result, final) {
     }
     return final.final;
 }
-/** The step a failure happened at: the first that did not pass, with its position. */
-export function failedStepOf(steps) {
-    const index = steps.findIndex((step) => step.status !== 'passed');
-    const step = steps[index];
-    return step === undefined || step.status === 'passed' ? undefined : { index, step: step };
+/**
+ * The step whose error ended the attempt, with its position: the step in the
+ * error's phase whose own error carries its code and line, else the last with
+ * its code, since a caught failure comes earlier, else the step a timeout or
+ * interrupt cut.
+ */
+export function failedStepOf(attempt) {
+    const error = attempt.error;
+    if (error === undefined)
+        return undefined;
+    const phase = error.phase;
+    const candidates = attempt.steps.flatMap((step, index) => step.status !== 'passed' && (phase === undefined || (step.phase ?? 'body') === phase) ? [{ index, step: step }] : []);
+    const alike = candidates.filter(({ step }) => step.error?.code === error.code);
+    const line = error.source;
+    if (line !== undefined && alike.length > 0) {
+        // The step's own error, not its source: that is the call site, a line below the `await` of a split call.
+        return alike.findLast(({ step }) => step.error?.source?.file === line.file && step.error.source.line === line.line);
+    }
+    return alike.at(-1) ?? candidates.findLast(({ step }) => step.status === 'cancelled');
 }
 /**
  * How the attempts compare, when there was more than one: the same code at
@@ -105,7 +120,7 @@ export function attemptsLine(result, final) {
         .filter((attempt) => attempt.status !== 'passed' && attempt.status !== 'skipped' && attempt.status !== 'interrupted' && (attempt.error !== undefined || attempt.steps.length > 0))
         .map((attempt) => ({
         code: cell(attempt.error?.code ?? attempt.status, 128),
-        step: failedStepOf(attempt.steps)?.index,
+        step: failedStepOf(attempt)?.index,
     }));
     const [first] = failed;
     if (first === undefined || final.attempts.length < 2 || failed.length < 2)
@@ -139,7 +154,7 @@ export function failureSource(result, told) {
     const fromError = told.error?.source;
     if (fromError !== undefined)
         return fromError;
-    const step = failedStepOf(told.steps)?.step;
+    const step = failedStepOf(told)?.step;
     if (step !== undefined && step.source.file !== 'unknown')
         return step.source;
     return result.source;
@@ -315,6 +330,7 @@ function cacheLine(cache, cacheDir) {
             : `no replay: ${CACHE_REASON_TEXT[cache.reason ?? 'no-entry']}`;
     const parts = [
         `cache: ${how}`,
+        ...(cache.recordedBy === undefined ? [] : [`recorded with ${code(modelLabel({ provider: cache.recordedBy.provider, id: cache.recordedBy.model }), MAX_DETAIL_CHARS)}`]),
         ...(cache.detail === undefined ? [] : [cell(cache.detail, MAX_DETAIL_CHARS)]),
         ...(cache.write === undefined ? [] : [CACHE_WRITE_TEXT[cache.write]]),
         ...(cache.notRecorded === 'param-collision' ? ['nothing recorded: a unique() value collides with another param'] : []),
@@ -450,7 +466,7 @@ export function renderTracePage(report, result, final, options = {}) {
         lines.push(`Passed after ${plural(final.failedAttempts, 'failed attempt')}; this page tells the last failure.  `);
     if (lines.at(-1) !== '')
         lines.push('');
-    const at = failedStepOf(told.steps);
+    const at = failedStepOf(told);
     if (told.steps.length > 0) {
         const videos = told.artifacts
             .filter((artifact) => artifact.kind === 'video' && artifact.startedAt !== undefined)

package/dist/report/list-model.d.ts

@@ -4,6 +4,7 @@
  * each consumer renders at its own indent and width when it prints.
  */
 import type { ResultStatus } from '../run/records.ts';
+import type { RecordedModel } from '../cache/trace.ts';
 import type { StepActivity, StepEvent, StepKind, StepProgress } from '../run/steps.ts';
 import type { AiUsage } from './format.ts';
 /** Step events worth a glance in the live window; polls and policy decisions stay quiet. */
@@ -60,6 +61,17 @@ export interface RunningTest {
      */
     readonly steps: FinishedStep[];
 }
+/** The model and agent named in the run header. */
+export interface ConfiguredModel {
+    readonly label: string;
+    readonly agent: string;
+}
+/** One recording mismatch, rendered with its test's finished steps. */
+export interface ReplayNotice {
+    readonly api: string;
+    readonly label: string;
+    readonly source: RecordedModel;
+}
 /** One finished pair, held until its file's block prints. */
 export interface TestLine {
     readonly title: string;
@@ -71,6 +83,7 @@ export interface TestLine {
     readonly firstErrorLine: string | undefined;
     readonly skipReason: string | undefined;
     readonly steps: readonly FinishedStep[];
+    readonly replays: readonly ReplayNotice[];
 }
 /**
  * Every pair of one test file on one target: vitest's "test module". Its

package/dist/report/list-steps.d.ts

@@ -3,8 +3,9 @@
  * finished step prints in a CI log, in a file block, and in the live window,
  * each at its own indent, so nothing here knows where the line lands.
  */
+import type { StepRecord } from '../run/steps.ts';
 import { type Colors } from './format.ts';
-import type { FinishedStep, ShownEvent } from './list-model.ts';
+import type { ConfiguredModel, FinishedStep, ReplayNotice, ShownEvent } from './list-model.ts';
 /** One-line, quoted step label; `maxColumns` clips it further to fit a row. */
 export declare function stepLabel(label: string, maxColumns?: number): string;
 export interface StepLineOptions {
@@ -18,6 +19,10 @@ export interface StepLineOptions {
  * model calls, and a non-passed status.
  */
 export declare function stepLine(pc: Colors, step: FinishedStep, options?: StepLineOptions): string;
+/** Recording mismatches under one test, deduplicated across its attempts. */
+export declare function replayNotices(attempts: readonly (readonly StepRecord[])[], configured: ConfiguredModel | undefined): ReplayNotice[];
+/** A recording mismatch under its test in the file block. */
+export declare function replayLine(pc: Colors, notice: ReplayNotice): string;
 /**
  * One model turn or tool call under the current step. A tool call reads as
  * the act it performed when the engine described it, else as the tool name

package/dist/report/list-steps.js

@@ -3,11 +3,14 @@
  * finished step prints in a CI log, in a file block, and in the live window,
  * each at its own indent, so nothing here knows where the line lands.
  */
+import { modelLabel } from '../config/agent.js';
 import { sanitizeText, truncateUtf8 } from '../internal/errors.js';
 import { collapseText } from '../internal/text.js';
 import { F_CHECK, F_CROSS, fitColumns, formatTime, formatTokens, terminalColumns, visibleWidth, } from './format.js';
 /** Step labels stay one glanceable line; the report holds the full text. */
 const MAX_STEP_LABEL_COLUMNS = 72;
+/** Recording model labels stay on one bounded terminal line. */
+const MAX_RECORDED_MODEL_COLUMNS = 120;
 /** A clipped step line keeps at least this much of its label, however narrow the terminal. */
 const MIN_STEP_LABEL_COLUMNS = 24;
 /** Columns a step line spends around its label: the quotes and the space before the tail. */
@@ -42,6 +45,32 @@ export function stepLine(pc, step, options = {}) {
         : options.maxWidth - visibleWidth(head) - visibleWidth(tail) - LABEL_CHROME;
     return `${head}${stepLabel(step.label, room)} ${tail}`;
 }
+/** Recording mismatches under one test, deduplicated across its attempts. */
+export function replayNotices(attempts, configured) {
+    if (configured === undefined)
+        return [];
+    const notices = new Map();
+    for (const steps of attempts) {
+        for (const step of steps) {
+            if (step.agent !== undefined && step.agent !== configured.agent)
+                continue;
+            const source = step.cache?.recordedBy;
+            if (source === undefined || step.cache?.replayedActions === 0)
+                continue;
+            const label = modelLabel({ provider: source.provider, id: source.model });
+            if (label === configured.label)
+                continue;
+            const key = JSON.stringify([step.cache?.entry ?? [step.index, step.api, step.label], source.provider, source.model]);
+            notices.set(key, { api: step.api, label: step.label, source });
+        }
+    }
+    return [...notices.values()];
+}
+/** A recording mismatch under its test in the file block. */
+export function replayLine(pc, notice) {
+    const label = modelLabel({ provider: notice.source.provider, id: notice.source.model });
+    return `${pc.dim(notice.api)} ${stepLabel(notice.label)} ${pc.dim(`replayed · recorded with ${fitColumns(collapseText(label), MAX_RECORDED_MODEL_COLUMNS)}`)}`;
+}
 /**
  * `(↑in ↓out)` for one model call, the total when only that is known, or
  * nothing when the provider reported no usage.

package/dist/report/list.d.ts

@@ -88,6 +88,7 @@ export declare class ListReporter implements Reporter {
      * there only when no step named one.
      */
     private models;
+    private configuredModel;
     /**
      * The exploration an `e2e explore` run reports through `explore` events;
      * while present, its view stands in for the one synthetic test's file

package/dist/report/list.js

@@ -14,7 +14,7 @@ import { codeFrame, userFrame } from './code-frame.js';
 import { addCacheTally, addModelTally, addUsage, aiSegment, bounded, cacheText, durationText, fitColumns, emptyCacheTally, emptyCounters, emptyUsage, F_CHECK, F_CROSS, F_POINTER, fileOutcome, formatClock, formatTime, modelsText, padTitle, rule, stateString, statusBucket, stepsCacheTally, repeatSuffix, stepsModelTally, stepsUsage, sumUsage, tally, terminalColumns, usageText, } from './format.js';
 import { ExploreView } from './list-explore.js';
 import { isShownEvent } from './list-model.js';
-import { stepLine } from './list-steps.js';
+import { replayLine, replayNotices, stepLine } from './list-steps.js';
 import { LiveWindow } from './live-window.js';
 import { repeatGroups, repeatLine, repeatSummary } from './repeats.js';
 import { RunningTree } from './running-tree.js';
@@ -53,7 +53,7 @@ function appLogTally(appLog) {
     return parts.length === 0 ? undefined : parts.join(', ');
 }
 /** Details of an ordinary result: summed over its attempts, the error from the last. */
-function attemptDetails(attempts) {
+function attemptDetails(attempts, configured) {
     const steps = attempts.map((attempt) => attempt.steps);
     // A retry the run interrupted tells nothing; the failure before it is the verdict.
     const told = attempts.findLast((attempt) => attempt.status !== 'interrupted') ?? attempts.at(-1);
@@ -67,6 +67,7 @@ function attemptDetails(attempts) {
         ...failureOf(told),
         appLog: appLogTally(told?.appLog ?? []),
         skipReason: told?.status === 'skipped' ? undefined : told?.skip?.reason,
+        replays: replayNotices(steps, configured),
     };
 }
 /** The failure evidence of one attempt, and the path of the screen text it points at. */
@@ -96,7 +97,7 @@ function videoPaths(attempts) {
  * that attempt never ran (skipped, or absent after a launch failure) inherits
  * the attempt's error, since that is what stopped it.
  */
-function serialMemberDetails(group, testId) {
+function serialMemberDetails(group, testId, configured) {
     const runs = group.attempts.map((attempt) => ({
         attempt,
         member: attempt.members.find((member) => member.testId === testId),
@@ -116,6 +117,7 @@ function serialMemberDetails(group, testId) {
         ...failureOf(own === undefined ? undefined : { failure: own.failure, artifacts: last?.attempt.artifacts ?? [] }),
         appLog: appLogTally(own?.appLog ?? []),
         skipReason: neverRan ? undefined : own.skip?.reason,
+        replays: replayNotices(steps, configured),
     };
 }
 const DEFAULT_OUTPUT = {
@@ -192,6 +194,7 @@ export class ListReporter {
      * there only when no step named one.
      */
     models;
+    configuredModel;
     /**
      * The exploration an `e2e explore` run reports through `explore` events;
      * while present, its view stands in for the one synthetic test's file
@@ -378,6 +381,7 @@ export class ListReporter {
     }
     runStarted(event) {
         const { pc } = this;
+        this.configuredModel = event.model === undefined ? undefined : { label: event.model, agent: event.agents?.[0] ?? 'default' };
         this.projectRoot = event.projectRoot;
         this.artifactsRoot = event.artifactsRoot;
         this.targets = event.targets;
@@ -568,15 +572,16 @@ export class ListReporter {
      * rather than failing the run.
      */
     detailsOf(result) {
+        const configured = this.configuredModel?.agent === result.agent ? this.configuredModel : undefined;
         if (result.serialGroupId === undefined)
-            return attemptDetails(result.attempts);
+            return attemptDetails(result.attempts, configured);
         const pending = this.pendingSerial.get(result.serialGroupId);
         if (pending === undefined)
-            return attemptDetails([]);
+            return attemptDetails([], configured);
         pending.remaining -= 1;
         if (pending.remaining <= 0)
             this.pendingSerial.delete(result.serialGroupId);
-        return serialMemberDetails(pending.group, result.test.id);
+        return serialMemberDetails(pending.group, result.test.id, configured);
     }
     testFinished(result) {
         // Unselected pairs are report-only: they never print and the plan never
@@ -591,7 +596,7 @@ export class ListReporter {
         // Read before `detailsOf` releases the result's serial group.
         const skippedAfter = failureBeforeSkip(result, (id) => this.pendingSerial.get(id)?.group);
         const id = resultId(result.test.id, result.target.name, result.agent, result.repeat);
-        const { durationMs, usage, models, cache, error, videos, failure, screenPath, appLog, skipReason } = this.detailsOf(result);
+        const { durationMs, usage, models, cache, error, videos, failure, screenPath, appLog, skipReason, replays } = this.detailsOf(result);
         addUsage(this.runUsage, usage);
         addModelTally(this.runModels, models);
         addCacheTally(this.runCache, cache);
@@ -614,6 +619,7 @@ export class ListReporter {
             firstErrorLine: error === undefined ? undefined : bounded(error.message).split('\n')[0],
             skipReason: result.skip === undefined ? undefined : bounded(result.skip.reason),
             steps,
+            replays,
         };
         group.lines.push(line);
         this.runs.push({
@@ -697,6 +703,7 @@ export class ListReporter {
             counts.interrupted > 0 ||
             counts.flaky > 0 ||
             this.groups.size === 1 ||
+            group.lines.some((line) => line.replays.length > 0) ||
             (this.live && group.lines.some((line) => line.steps.length > 0));
         if (!verbose)
             return;
@@ -706,6 +713,8 @@ export class ListReporter {
                 this.print(rendered);
             for (const step of line.steps)
                 this.print(`${STEP_INDENT}${stepLine(pc, step)}`);
+            for (const replay of line.replays)
+                this.print(`${STEP_INDENT}${replayLine(pc, replay)}`);
         }
     }
     /** One test row under its file line, plus the first error line for a failure. */

package/dist/report/live-window.js

@@ -25,9 +25,10 @@ const SYNC_END = `${ESC}[?2026l`;
  * when counting and never cutting inside a grapheme cluster: a glyph or emoji
  * sequence that would cross the limit is dropped whole. The erase sequence
  * counts physical rows, so a line that wraps would leave stale fragments
- * behind on every repaint.
+ * behind on every repaint. An entry with a line break still paints as one row.
  */
 function clampToWidth(text, columns) {
+    text = text.replace(/\r\n?|\n/g, ' ');
     const max = Math.max(4, columns - WIDTH_MARGIN);
     let width = 0;
     let out = '';

package/dist/report/markdown.js

@@ -210,7 +210,7 @@ function runErrorLine(error) {
 function failureBlock({ result, final }, manyTargets, options) {
     const kind = statusBucket(result.status);
     const told = toldAttempt(result, final);
-    const at = failedStepOf(told.steps);
+    const at = failedStepOf(told);
     const error = told.error;
     const name = error === undefined ? `**${cell(result.status, MAX_ID_CHARS)}**` : errorName(error);
     const lead = at === undefined ? name : `${name} ${stepClause(told.steps, at)}`;

package/dist/run/fixtures.js

@@ -102,11 +102,13 @@ export function createFixtures(environment) {
         engineCapabilities: environment.target.engine?.capabilities ?? new Set(),
         steps: environment.steps,
         select,
+        reportUnservedModel: environment.models.unservedModel,
         config: environment.config,
         target: {
             name: environment.target.name,
             platform: environment.target.platform,
             verbs: environment.session.verbs,
+            tapModifiers: environment.session.tapModifiers,
         },
         app: environment.target.app,
         attempt: environment.attempt,

package/dist/run/host-trace.d.ts

@@ -1,5 +1,6 @@
 /** The per-session `trace.md` a hosted attempt writes at close, through the run report's renderer. */
 import type { ResolvedConfig, ResolvedTarget } from '../config/resolve.ts';
+import type { Report } from '../types.ts';
 import type { AttemptRecord } from './records.ts';
 export interface HostTraceOptions {
     readonly config: ResolvedConfig;
@@ -17,5 +18,5 @@ export interface HostTraceOptions {
     readonly tracePath: string;
     readonly redact: (text: string) => string;
 }
-/** Writes one session's trace page as a single-result run, so it reads like `e2e run`'s. */
-export declare function writeHostTrace(options: HostTraceOptions): Promise<void>;
+/** Writes one session's trace page as a single-result run and returns its Report. */
+export declare function writeHostTrace(options: HostTraceOptions): Promise<Report>;

package/dist/run/host-trace.js

@@ -5,7 +5,7 @@ import { buildReport } from '../report/build.js';
 import { renderTracePage } from '../report/failure-text.js';
 import { outcome } from '../report/outcome.js';
 import { toPosixPath, writeTextReport } from '../report/write.js';
-/** Writes one session's trace page as a single-result run, so it reads like `e2e run`'s. */
+/** Writes one session's trace page as a single-result run and returns its Report. */
 export async function writeHostTrace(options) {
     const { config, attempt, title } = options;
     const status = attempt.status;
@@ -34,4 +34,5 @@ export async function writeHostTrace(options) {
         },
     });
     await writeTextReport(options.tracePath, options.redact(page));
+    return report;
 }

package/dist/run/last-run.d.ts

@@ -52,4 +52,13 @@ export declare function carryForward(lastRun: Report1Document, current: Report1D
  * `--last-failed` rerun keeps of the tree when it starts.
  */
 export declare function reportArtifactPaths(document: Report1Document): ReadonlySet<string>;
+/**
+ * How long each file took for each target in the previous run, keyed by
+ * `target::file`: every attempt of every result and serial group summed. A
+ * file with no attempt (skipped, or never reached) has no entry. No readable
+ * report is no history, never an error.
+ */
+export declare function previousFileDurations(reportPath: string): Promise<ReadonlyMap<string, number>>;
+/** The key `previousFileDurations` files one target's file under. */
+export declare function fileDurationKey(targetName: string, file: string): string;
 export {};

package/dist/run/last-run.js

@@ -232,3 +232,31 @@ function isError(value) {
                 typeof scope.targetId === 'string' &&
                 Array.isArray(scope.titlePath))));
 }
+/**
+ * How long each file took for each target in the previous run, keyed by
+ * `target::file`: every attempt of every result and serial group summed. A
+ * file with no attempt (skipped, or never reached) has no entry. No readable
+ * report is no history, never an error.
+ */
+export async function previousFileDurations(reportPath) {
+    const durations = new Map();
+    let report;
+    try {
+        report = await readLastRun(reportPath);
+    }
+    catch {
+        return durations;
+    }
+    for (const unit of [...report.run.results, ...report.run.serialGroups]) {
+        if (unit.attempts.length === 0)
+            continue;
+        const key = fileDurationKey(unit.targetId, unit.file);
+        const ms = unit.attempts.reduce((sum, attempt) => sum + attempt.durationMs, 0);
+        durations.set(key, (durations.get(key) ?? 0) + ms);
+    }
+    return durations;
+}
+/** The key `previousFileDurations` files one target's file under. */
+export function fileDurationKey(targetName, file) {
+    return `${targetName}::${file}`;
+}

package/dist/run/open-session.d.ts

@@ -1,6 +1,6 @@
 /** Hosted test runners keep a worker alive while opening isolated per-test attempts. */
 import { type SerializedError } from '../internal/errors.ts';
-import type { E2EConfig, TestFixtures } from '../types.ts';
+import type { E2EConfig, Report, TestFixtures } from '../types.ts';
 import type { StepProgress, StepRecord } from './steps.ts';
 export interface OpenHostOptions {
     /** Config search directory, or project root of a config value; default `process.cwd()`. */
@@ -48,6 +48,8 @@ export interface E2ESession<Fixtures extends object = Record<string, unknown>> e
     /** Absolute per-session trace path. Written before close settles successfully. */
     readonly tracePath: string;
     steps(): readonly StepRecord[];
+    /** The session Report after close and a successful trace write; otherwise undefined. */
+    report(): Report | undefined;
     /** Idempotent. Returns soft assertion, abandoned-operation, and cleanup failures. Disposal throws them as AggregateError. */
     close(): Promise<readonly SerializedError[]>;
 }

package/dist/run/open-session.js

@@ -50,7 +50,7 @@ async function openHostWorker(options) {
                     runId: attempt.runId, attemptId: attempt.attemptId, target: target.name,
                     fixtures: attempt.fixtures,
                     artifactsDir: attempt.artifactsDir, tracePath: attempt.tracePath,
-                    steps: () => attempt.steps.all(), close: attempt.close,
+                    steps: () => attempt.steps.all(), report: attempt.report, close: attempt.close,
                     [Symbol.asyncDispose]: () => dispose(attempt.close, 'session'),
                 };
             }),

package/dist/run/runner.js

@@ -21,12 +21,12 @@ import { allocateAppPorts } from './app-ports.js';
 import { inProcessSpawner } from './in-process.js';
 import { someSkippedAfterFailure, tracedResultIds } from './records.js';
 import { runUnits } from './scheduler.js';
-import { buildWorkPlans, plannedSlots } from './units.js';
+import { buildWorkPlans, longestFirst, plannedSlots } from './units.js';
 import { SessionStore } from './sessions.js';
 import { outputLayout } from './output.js';
 import { registerStaticSecrets } from './secrecy.js';
 import { nextRerunDir, pruneArtifacts } from './artifacts.js';
-import { carryForward, lastFailedIds, readLastRun, reportArtifactPaths } from './last-run.js';
+import { carryForward, fileDurationKey, lastFailedIds, previousFileDurations, readLastRun, reportArtifactPaths } from './last-run.js';
 import { childProcessSpawner } from './worker/handle.js';
 import { setSecretRegistry } from '../secrets.js';
 import { withAbort } from '../internal/time.js';
@@ -461,6 +461,10 @@ export async function run(options = {}) {
         const workerConfigPath = options.tests === undefined ? config.configPath : undefined;
         const inProcess = workerConfigPath === undefined;
         const runWorkers = inProcess ? 1 : config.workers;
+        if (runWorkers > 1) {
+            const previous = await previousFileDurations(outputLayout(config.output).report);
+            plans = longestFirst(plans, (targetName, file) => previous.get(fileDurationKey(targetName, file)));
+        }
         const timeLimit = options.maxDuration !== undefined
             ? { ms: options.maxDuration, source: '--max-duration' }
             : config.maxDuration === undefined

package/dist/run/sessions.js

@@ -287,12 +287,12 @@ function parseEnvelope(rawText, name) {
 /** Parses an authenticated payload; one this runner did not write is SESSION_INVALID. */
 function parsePayload(text, name) {
     const parsed = JSON.parse(text);
-    const secrets = parsed?.secrets;
     if (typeof parsed !== 'object' ||
         parsed === null ||
         !('data' in parsed) ||
-        !Array.isArray(secrets) ||
-        !secrets.every((entry) => Array.isArray(entry) && entry.length === 2 && entry.every((part) => typeof part === 'string'))) {
+        !('secrets' in parsed) ||
+        !Array.isArray(parsed.secrets) ||
+        !parsed.secrets.every((entry) => Array.isArray(entry) && entry.length === 2 && entry.every((part) => typeof part === 'string'))) {
         throw new ConfigurationError('SESSION_INVALID', `session "${name}" has an unexpected payload`);
     }
     return parsed;

package/dist/run/standalone-attempt.d.ts

@@ -6,7 +6,7 @@ import { PollScope } from '../expect/poll-scope.ts';
 import { SoftFailures } from '../expect/soft.ts';
 import type { DebugTrace } from '../internal/debug.ts';
 import { type SerializedError } from '../internal/errors.ts';
-import type { TestFixtures } from '../types.ts';
+import type { Report, TestFixtures } from '../types.ts';
 import { AttemptBudget } from './budget.ts';
 import { type TargetExecutor } from './execute.ts';
 import type { AttemptRecord } from './records.ts';
@@ -37,6 +37,7 @@ export interface StandaloneAttempt {
     readonly budget: AttemptBudget;
     readonly artifactsDir: string;
     readonly tracePath: string;
+    report(): Report | undefined;
     close(): Promise<readonly SerializedError[]>;
 }
 /** The prepared worker an attempt runs on. */

package/dist/run/standalone-attempt.js

@@ -124,6 +124,7 @@ export async function launchAttempt(worker, reservation, onClosed) {
     const startedMs = Date.now();
     const tracePath = path.join(artifacts.dir, 'trace.md');
     let ending;
+    let report;
     const close = () => {
         ending ??= (async () => {
             clearTimeout(expiry);
@@ -167,7 +168,7 @@ export async function launchAttempt(worker, reservation, onClosed) {
             if (worker.kind === 'hosted') {
                 try {
                     const { error, status } = verdict();
-                    await writeHostTrace({
+                    report = await writeHostTrace({
                         config, target, runId, testId: identity.testId, agent, tracePath, redact,
                         title: processSecrets.redact(redact(options.name ?? target.name)),
                         resultsDir: worker.resultsDir, segments: identity.segments,
@@ -195,6 +196,7 @@ export async function launchAttempt(worker, reservation, onClosed) {
     return {
         runId, attemptId, fixtures: acquired.fixtures, agentRuntime: acquired.agentRuntime, session: opened, steps, budget,
         artifactsDir: artifacts.dir, tracePath, close,
+        report: () => report && structuredClone(report),
     };
 }
 /**

package/dist/run/steps.d.ts

@@ -2,7 +2,7 @@
 import type { ReplayHandOffReason } from '../agent/executor.ts';
 import type { CacheWrite } from '../cache/context.ts';
 import type { TraceReplayMissReason } from '../cache/decide.ts';
-import type { DerivedReason } from '../cache/trace.ts';
+import type { RecordedModel, DerivedReason } from '../cache/trace.ts';
 import { TestError, type SerializedError } from '../internal/errors.ts';
 import { type SourceLocation } from '../internal/source.ts';
 import type { AppLogEntry, AppLogSource } from '../engine/index.ts';
@@ -123,6 +123,8 @@ export interface StepCacheInfo {
      * screen the recording started on against the one the step did.
      */
     detail?: string;
+    /** Configured model when the consulted entry was recorded; absent on older entries. */
+    recordedBy?: RecordedModel;
 }
 /** The cache detail as the report keeps it: what `agent.act()` returns, and what became of the recording once the attempt settled. */
 export type StepCacheRecord = StepCacheInfo & {

package/dist/run/units.d.ts

@@ -37,6 +37,14 @@ export interface TargetWorkPlan {
 }
 /** Builds per-target work plans from a selection, preserving report order. */
 export declare function buildWorkPlans(selection: Selection, collection: Collection, projectRoot: string): TargetWorkPlan[];
+/**
+ * Orders each plan's file units longest first by the time the previous run
+ * spent in them, so a long file starts while short ones fill the other
+ * workers instead of starting last and running alone. Files without a
+ * timing go first, being of unknown length; ties keep collection order.
+ * Report order does not depend on dispatch order.
+ */
+export declare function longestFirst(plans: readonly TargetWorkPlan[], durationOf: (targetName: string, file: string) => number | undefined): TargetWorkPlan[];
 /**
  * The worker slots the run will start for a plan, `0` to `slots - 1`: the
  * run's worker cap, the engine's declared `workers`, and the units to

package/dist/run/units.js

@@ -55,6 +55,20 @@ export function buildWorkPlans(selection, collection, projectRoot) {
         return { target, setupUnits, fileUnits, immediate };
     });
 }
+/**
+ * Orders each plan's file units longest first by the time the previous run
+ * spent in them, so a long file starts while short ones fill the other
+ * workers instead of starting last and running alone. Files without a
+ * timing go first, being of unknown length; ties keep collection order.
+ * Report order does not depend on dispatch order.
+ */
+export function longestFirst(plans, durationOf) {
+    return plans.map((plan) => ({
+        ...plan,
+        fileUnits: plan.fileUnits.toSorted((a, b) => (durationOf(b.targetName, b.file) ?? Number.POSITIVE_INFINITY) -
+            (durationOf(a.targetName, a.file) ?? Number.POSITIVE_INFINITY) || 0),
+    }));
+}
 /**
  * The worker slots the run will start for a plan, `0` to `slots - 1`: the
  * run's worker cap, the engine's declared `workers`, and the units to

package/dist/run/worker-models.d.ts

@@ -22,6 +22,8 @@ export declare class WorkerModels {
      * need none, and judgment methods still fail on first use.
      */
     readonly preflight: (agent: ResolvedAgentConfig, name: string) => void;
+    /** A model the provider refused, met by a step's call: reported to the run once, and every later preflight rethrows it. */
+    readonly unservedModel: (error: E2EError) => void;
     /** Adapter for one resolved model; each distinct model is built once per worker. A missing one names the agent `name`. */
     readonly build: (model: ResolvedModel | undefined, name: string) => ModelAdapter;
 }

package/dist/run/worker-models.js

@@ -40,6 +40,13 @@ export class WorkerModels {
         if (this.failure !== undefined)
             throw this.failure;
     };
+    /** A model the provider refused, met by a step's call: reported to the run once, and every later preflight rethrows it. */
+    unservedModel = (error) => {
+        if (this.failure !== undefined)
+            return;
+        this.failure = error;
+        this.onFailure(error);
+    };
     /** Adapter for one resolved model; each distinct model is built once per worker. A missing one names the agent `name`. */
     build = (model, name) => {
         if (model === undefined)

package/dist/types.d.ts

@@ -292,7 +292,7 @@ export interface Screen {
     getByText(text: TextMatch, options?: TextMatchOptions): Locator;
     /** Creates a lazy displayed-value query. */
     getByDisplayValue(value: TextMatch, options?: TextMatchOptions): Locator;
-    /** Creates a lazy test-id query: a string matches the whole id, case-sensitive; a RegExp tests it. */
+    /** Creates a lazy test-id query: a string matches the whole id, case-sensitive; a RegExp tests it. An engine may also match an id without its platform's namespacing. */
     getByTestId(id: TextMatch, options?: {
         visible?: boolean;
     }): Locator;
@@ -1336,7 +1336,7 @@ export interface E2EConfig {
     /**
      * The directory a run writes its results to, relative to the project root;
      * default `.e2e`. It holds `report.json`, `junit.xml`, `summary.md`,
-     * `ai-trace.json`, `artifacts/` (cleared at the start of every run),
+     * `ai-trace.json`, `results/` (cleared once the run's tests start, except on a `--last-failed` rerun),
      * `sessions/`, and the `e2e mcp` session videos under `videos/`.
      * `--output <dir>` overrides it for one run. It must be inside the project
      * root and not the root itself, may not hold a test glob's directory, and