e2e 0.16.0 → 0.18.0

package/dist/run/scheduler.js

@@ -7,14 +7,23 @@
  * gating, and reporting logic covers both child-process and in-process
  * execution.
  */
-import { InfrastructureError, serializeError } from '../internal/errors.js';
+import { InfrastructureError, TestTimeoutError, serializeError } from '../internal/errors.js';
 import { timestamp, uuidv7 } from '../internal/ids.js';
+import { retryVerdict } from './retry.js';
+import { SerialGroupProgress, serialGroupId } from './serial.js';
 import { INTERRUPTED_BEFORE_START, countsTowardFailureLimit, failureLimitSkip, nonRunResult, pairResult, setupFailedSkip, unstartedResult, pairKey, } from './units.js';
 import { decodeResult, } from './worker/protocol.js';
 /** Consecutive worker-boot failures per target before its units are failed. */
 const MAX_INIT_FAILURES = 2;
 /** How long a retiring or draining worker gets before it is force-killed. */
 const SHUTDOWN_GRACE_MS = 10_000;
+/** Why a pair of a target whose workers cannot boot never ran. */
+const WORKER_START_FAILED = {
+    cause: 'infrastructure-unavailable',
+    reason: 'worker process failed to start',
+};
+/** The least time a worker past an attempt deadline gets to answer a ping, however small `cleanupTimeout` is. */
+const MIN_PING_REPLY_MS = 5_000;
 /** Whether a worker process died of a signal that interrupts the run, as a Ctrl-C to its process group does. */
 function isInterruptSignal(signal) {
     return signal === 'SIGINT' || signal === 'SIGTERM' || signal === 'SIGHUP';
@@ -23,6 +32,16 @@ function isInterruptSignal(signal) {
 export async function runUnits(options) {
     await new Scheduler(options).run();
 }
+/** The result of a pair from the attempts it finished, with the verdict they reach. */
+function finishedResult(pair, attempts) {
+    const last = attempts.at(-1);
+    return pairResult(pair, {
+        status: retryVerdict(attempts),
+        selected: true,
+        attempts,
+        ...(last?.status === 'skipped' ? { skip: last.skip } : {}),
+    });
+}
 /**
  * One worker and the unit it is executing. Owns its own lifecycle so the
  * scheduler never has to keep a separate record in sync with the transport.
@@ -39,13 +58,22 @@ class SchedulerWorker {
     reported = new Set();
     /** The pair (`pairKey`) executing now, when one is. */
     inFlightPair;
+    /** Attempts of `unit`'s pairs heard ahead of the pair's result, by `pairKey`: what a crash keeps. */
+    attempts = new Map();
+    /** What was heard of `unit`'s serial groups still running, by group report id: what a crash keeps. */
+    serialGroups = new Map();
     sawFailure = false;
     becameReady = false;
     /** Told to tear down at once by a forced interrupt; its exit is then the one asked for. */
     terminated = false;
     /** Killed by the scheduler, once a grace budget ran out or over a fatal it reported; its exit is then the one asked for too. */
     killed = false;
+    /** The attempt whose worker stopped answering past its deadline, once the watchdog killed the worker over it. */
+    hung;
     killTimer;
+    watched;
+    watchTimer;
+    awaitingPong = false;
     constructor(targetName, workerSlot, spawn, onMessage, onExit) {
         this.targetName = targetName;
         this.workerSlot = workerSlot;
@@ -53,6 +81,7 @@ class SchedulerWorker {
             onMessage: (message) => onMessage(this, message),
             onExit: (detail, signal) => {
                 this.clearKillTimer();
+                this.unwatch();
                 onExit(this, detail, signal ?? null);
             },
         });
@@ -62,6 +91,8 @@ class SchedulerWorker {
         this.state = 'busy';
         this.unit = unit;
         this.reported.clear();
+        this.attempts.clear();
+        this.serialGroups.clear();
         this.inFlightPair = undefined;
         this.sawFailure = false;
         const pairs = unit.pairs.map((pair) => ({
@@ -104,12 +135,79 @@ class SchedulerWorker {
         }, graceMs);
         this.killTimer.unref();
     }
+    /** The attempts heard of one of `unit`'s pairs, started on first word of them. */
+    pairAttempts(key) {
+        let attempts = this.attempts.get(key);
+        if (attempts === undefined) {
+            attempts = { finished: [], running: undefined };
+            this.attempts.set(key, attempts);
+        }
+        return attempts;
+    }
+    /** The progress of one of `unit`'s serial groups, started on first word of it. */
+    serialGroup(groupId) {
+        let progress = this.serialGroups.get(groupId);
+        if (progress === undefined) {
+            progress = new SerialGroupProgress();
+            this.serialGroups.set(groupId, progress);
+        }
+        return progress;
+    }
     clearKillTimer() {
         if (this.killTimer === undefined)
             return;
         clearTimeout(this.killTimer);
         this.killTimer = undefined;
     }
+    /**
+     * The watchdog over one attempt. The worker enforces the test timeout with
+     * timers in its own process, which a body that blocks the event loop
+     * (`while (true) {}`) never lets fire. From the deadline on, the worker is
+     * pinged once per cleanup budget (at least `MIN_PING_REPLY_MS`) for as
+     * long as the attempt lasts, teardown included: one still tearing it down
+     * answers and goes on, one that has not answered by the next ping is
+     * killed, and the attempt is the one it hung in.
+     */
+    watch(message) {
+        this.unwatch();
+        this.watched = {
+            pair: pairKey(message.testId, message.agent, message.repeat),
+            index: message.attempt,
+            id: message.attemptId,
+            startedAt: message.startedAt,
+            timeoutMs: message.timeoutMs,
+        };
+        const check = () => {
+            if (this.watched === undefined || !this.runner.alive)
+                return;
+            if (this.awaitingPong) {
+                this.hung = this.watched;
+                this.killed = true;
+                this.runner.kill();
+                return;
+            }
+            this.awaitingPong = true;
+            this.watchAfter(Math.max(message.graceMs, MIN_PING_REPLY_MS), check);
+            this.runner.send({ type: 'ping' });
+        };
+        this.watchAfter(message.timeoutMs, check);
+    }
+    /** The worker answered a ping: its event loop turns. */
+    pong() {
+        this.awaitingPong = false;
+    }
+    /** Stops watching the attempt: it ended, or the worker is gone. */
+    unwatch() {
+        if (this.watchTimer !== undefined)
+            clearTimeout(this.watchTimer);
+        this.watchTimer = undefined;
+        this.watched = undefined;
+        this.awaitingPong = false;
+    }
+    watchAfter(delayMs, check) {
+        this.watchTimer = setTimeout(check, delayMs);
+        this.watchTimer.unref();
+    }
 }
 class Scheduler {
     options;
@@ -356,8 +454,17 @@ class Scheduler {
             return runnable.length === unit.pairs.length ? unit : { ...unit, pairs: runnable };
         }
     }
-    /** Puts an untaken unit back so capacity pressure never drops work. */
+    /**
+     * Puts an untaken unit back so capacity pressure never drops work. A
+     * failed target's queues were already drained and nothing takes from them
+     * again, so its unit skips instead.
+     */
     returnUnit(state, unit) {
+        if (state.failed) {
+            for (const pair of unit.pairs)
+                this.report(unstartedResult(pair, WORKER_START_FAILED));
+            return;
+        }
         if (unit.kind === 'setup')
             state.setupQueue.unshift(unit);
         else
@@ -478,10 +585,27 @@ class Scheduler {
             }
             case 'pair-start': {
                 worker.inFlightPair = pairKey(message.testId, message.agent, message.repeat);
+                if (message.serialId !== undefined) {
+                    worker
+                        .serialGroup(serialGroupId(message.serialId, worker.targetName, message.agent, message.repeat))
+                        .memberStarted(message.testId);
+                }
                 const { type: _type, ...start } = message;
                 this.options.events.onTestStart?.(start, worker.targetName);
                 break;
             }
+            case 'attempt-deadline': {
+                worker.watch(message);
+                break;
+            }
+            case 'attempt-end': {
+                worker.unwatch();
+                break;
+            }
+            case 'pong': {
+                worker.pong();
+                break;
+            }
             case 'progress': {
                 this.options.events.onProgress?.({ testId: message.testId, agent: message.agent, repeat: message.repeat }, worker.targetName, message.progress);
                 break;
@@ -498,7 +622,9 @@ class Scheduler {
             case 'result': {
                 const state = this.targetState(worker);
                 const result = decodeResult(message.result, state.target);
-                worker.reported.add(pairKey(result.test.id, result.agent, result.repeat));
+                const key = pairKey(result.test.id, result.agent, result.repeat);
+                worker.reported.add(key);
+                worker.attempts.delete(key);
                 if (result.status !== 'passed' && result.status !== 'flaky' && result.status !== 'skipped') {
                     worker.sawFailure = true;
                 }
@@ -508,9 +634,28 @@ class Scheduler {
                 break;
             }
             case 'serial-group': {
+                worker.serialGroups.delete(message.group.id);
                 this.options.events.onSerialGroup(message.group);
                 break;
             }
+            case 'attempt-start': {
+                worker.pairAttempts(pairKey(message.testId, message.agent, message.repeat)).running = message.index;
+                break;
+            }
+            case 'attempt': {
+                const attempts = worker.pairAttempts(pairKey(message.testId, message.agent, message.repeat));
+                attempts.finished.push(message.attempt);
+                attempts.running = undefined;
+                break;
+            }
+            case 'serial-member': {
+                worker.serialGroup(message.groupId).memberFinished(message.attempt, message.member);
+                break;
+            }
+            case 'serial-attempt': {
+                worker.serialGroup(message.groupId).attemptFinished(message.run);
+                break;
+            }
             case 'unit-done': {
                 for (const runError of message.runErrors)
                     this.options.events.onRunError(runError);
@@ -610,50 +755,121 @@ class Scheduler {
         this.signalledWave ??= new Set([...this.workers, worker]);
         return this.signalledWave.has(worker);
     }
-    /** Emits records for a unit whose worker died before reporting it done. */
+    /**
+     * Emits records for a unit whose worker died before reporting it done.
+     * Attempts the worker finished stand. The test in flight gets a
+     * `WORKER_CRASH` attempt after them when the crash landed during an
+     * attempt; one between attempts (an `afterAll` after the last) is recorded
+     * on the last attempt instead, whose cleanup it forced. A serial group in
+     * flight gets a failed group attempt after its finished ones. A worker the
+     * run's interrupt stopped charges no crash: a test keeps the attempts it
+     * finished and the verdict they reach, as a retry the interrupt cut short
+     * does, and a serial group's attempt in flight is recorded interrupted.
+     *
+     * A worker the watchdog killed hung in one attempt: that attempt timed out
+     * (`TEST_TIMEOUT`, after the attempts before it, never retried), and the
+     * unit's tests it never reached run on a fresh worker, as they would have
+     * after a timeout the worker enforced itself. Only the rest of a serial
+     * group it hung in is lost with it.
+     */
     synthesizeCrashResults(state, worker, unit) {
-        const interrupted = this.interrupting;
+        const interrupted = this.interrupting && (worker.terminated || worker.killed);
+        const crash = serializeError(new InfrastructureError('WORKER_CRASH', 'worker process exited during this test'));
+        const hung = worker.hung;
+        const hungPair = hung === undefined ? undefined : unit.pairs.find((pair) => pairKey(pair.test.id, pair.agent, pair.repeat) === hung.pair);
+        const hungGroupId = hungPair?.test.serialId === undefined
+            ? undefined
+            : serialGroupId(hungPair.test.serialId, worker.targetName, hungPair.agent, hungPair.repeat);
+        const settledGroups = new Set();
+        const rescheduled = [];
         for (const pair of unit.pairs) {
             const key = pairKey(pair.test.id, pair.agent, pair.repeat);
             if (worker.reported.has(key))
                 continue;
+            const { serialId } = pair.test;
+            const groupId = serialId === undefined ? undefined : serialGroupId(serialId, worker.targetName, pair.agent, pair.repeat);
+            if (groupId !== undefined) {
+                if (settledGroups.has(groupId))
+                    continue;
+                const members = unit.pairs.filter((member) => member.test.serialId === serialId && member.agent === pair.agent && member.repeat === pair.repeat);
+                const progress = worker.serialGroups.get(groupId);
+                let settled;
+                if (hung !== undefined && groupId === hungGroupId)
+                    settled = progress?.crashed(members, state.target, hangError(hung), 'timed-out');
+                else if (interrupted)
+                    settled = progress?.interrupted(members, state.target, this.stopSkip ?? INTERRUPTED_BEFORE_START);
+                else
+                    settled = progress?.crashed(members, state.target, crash, 'failed');
+                if (settled !== undefined) {
+                    settledGroups.add(groupId);
+                    this.options.events.onSerialGroup(settled.group);
+                    for (const result of settled.results)
+                        this.report(result);
+                    continue;
+                }
+            }
+            const { finished, running } = worker.attempts.get(key) ?? { finished: [], running: undefined };
+            if (hung !== undefined && key === hung.pair) {
+                this.markSessionsFailed(state, pair);
+                this.report(finishedResult(pair, [...finished, hungAttempt(hung)]));
+                continue;
+            }
             if (interrupted) {
-                this.report(unstartedResult(pair, this.stopSkip ?? INTERRUPTED_BEFORE_START));
+                this.report(finished.length === 0
+                    ? unstartedResult(pair, this.stopSkip ?? INTERRUPTED_BEFORE_START)
+                    : finishedResult(pair, finished));
                 continue;
             }
-            // A crashed setup never persisted its sessions; dependents must skip.
-            if (pair.test.kind === 'setup') {
-                for (const session of pair.test.sessions) {
-                    state.failedSessions.set(session, pair.test.id);
-                }
+            if (hung !== undefined && (hungGroupId === undefined || groupId !== hungGroupId)) {
+                rescheduled.push(pair);
+                continue;
             }
-            const wasRunning = key === worker.inFlightPair && pair.test.serialId === undefined;
-            if (!wasRunning) {
+            this.markSessionsFailed(state, pair);
+            if (key !== worker.inFlightPair) {
                 this.report(unstartedResult(pair, {
                     cause: 'infrastructure-unavailable',
                     reason: 'worker process exited before this test started',
                 }));
                 continue;
             }
+            const last = finished.at(-1);
+            if (last !== undefined && running === undefined) {
+                this.report(finishedResult(pair, [
+                    ...finished.slice(0, -1),
+                    { ...last, secondaryErrors: [...last.secondaryErrors, crash], cleanup: 'forced' },
+                ]));
+                continue;
+            }
             this.report(pairResult(pair, {
                 status: 'failed',
                 selected: true,
                 attempts: [
+                    ...finished,
                     {
                         id: uuidv7(),
-                        index: 0,
+                        index: running ?? finished.length,
                         status: 'failed',
                         startedAt: timestamp(),
                         durationMs: 0,
                         steps: [],
                         artifacts: [],
-                        error: serializeError(new InfrastructureError('WORKER_CRASH', 'worker process exited during this test')),
+                        error: crash,
                         secondaryErrors: [],
                         cleanup: 'forced',
                     },
                 ],
             }));
         }
+        if (rescheduled.length > 0)
+            this.returnUnit(state, { ...unit, pairs: rescheduled });
+    }
+    /** A crashed setup never persisted its sessions; dependents must skip. */
+    markSessionsFailed(state, pair) {
+        if (pair.test.kind !== 'setup')
+            return;
+        for (const session of pair.test.sessions) {
+            state.failedSessions.set(session, pair.test.id);
+        }
     }
     /**
      * Fails a target whose workers cannot boot; drains its queues with skips.
@@ -667,10 +883,7 @@ class Scheduler {
                 error: serializeError(new InfrastructureError('WORKER_INIT_FAILED', `workers for target "${state.target.name}" failed to start ${MAX_INIT_FAILURES} times; remaining units skipped`)),
             });
         }
-        this.skipQueues(state, {
-            cause: 'infrastructure-unavailable',
-            reason: 'worker process failed to start',
-        });
+        this.skipQueues(state, WORKER_START_FAILED);
     }
     /** Shuts every remaining worker down, force-killing stragglers. */
     async drain() {
@@ -682,3 +895,26 @@ class Scheduler {
         await Promise.all(remaining.map((worker) => worker.runner.exit));
     }
 }
+/** Why the watchdog killed a worker that hung in `hung`. */
+function hangError(hung) {
+    return serializeError(new TestTimeoutError(`test ran past its ${hung.timeoutMs} ms timeout and its worker stopped responding, so the worker was killed`));
+}
+/**
+ * The attempt a worker the watchdog killed hung in. It ends the pair's
+ * attempts: a body that blocked the event loop once does so again, so it is
+ * not retried.
+ */
+function hungAttempt(hung) {
+    return {
+        id: hung.id,
+        index: hung.index,
+        status: 'timed-out',
+        startedAt: hung.startedAt,
+        durationMs: Date.now() - Date.parse(hung.startedAt),
+        steps: [],
+        artifacts: [],
+        error: hangError(hung),
+        secondaryErrors: [],
+        cleanup: 'forced',
+    };
+}

package/dist/run/secrecy.d.ts

@@ -6,40 +6,43 @@ import type { Secret } from '../types.ts';
 export interface SessionSecrecy {
     /** Every secret value the session may have seen; live, so a provider's value joins the moment it exists. */
     readonly ledger: SecretLedger;
-    /** How far a secret reached the session, and what that withholds and rewrites. */
+    /** Whether a secret was filled into the session, and what that withholds. */
     readonly exposure: SecretExposure;
 }
 /**
- * How far a secret reached one session. It only rises, and is never cleared:
- * `none`, no secret reached it; `engine`, the engine resolved one for an
- * option it hands the app (basic-auth credentials), so the app has the value
- * and a page can render it (an echo of the `Authorization` header) though
- * nothing was typed; `filled`, a secret was typed into the app.
+ * Whether a secret was typed into one session's app. It only rises, and is
+ * never cleared: `none`, nothing was filled; `filled`, a secret was typed
+ * into the app.
  */
-export type SecretExposureLevel = 'none' | 'engine' | 'filled';
+export type SecretExposureLevel = 'none' | 'filled';
 /**
  * One session's exposure level, and the one place that maps it to what each
- * consumer does. Either level above `none` rewrites what text can carry the
- * value (a trace's text entries, a text-like download) through the ledger,
- * as every report and observation already is. Only a fill withholds pixels
- * (failure screenshots, `app.screenshot()`, assert evidence, model input,
- * and a trace's screencast frames): an engine-held secret is protected as
- * text only, so a page that renders it on screen is not masked. Only a fill
+ * consumer withholds. Only a fill withholds pixels (failure screenshots,
+ * `app.screenshot()`, assert evidence, model input, and a trace's
+ * screencast frames): a value that reached the app otherwise (an engine
+ * option, a URL or a `fill` the test spelled it into) is protected as text
+ * only, so a page that renders it on screen is not masked. Only a fill
  * carries into a saved session too: the value then lives in the app's
- * state, while an engine-held one is resolved again by whatever engine opens
- * the restoring session.
+ * state. Text is not decided here: see `redactsRecordings`.
  */
 export declare class SecretExposure {
     private current;
-    /** Records that a secret reached the session at `level`; a lower level never replaces a higher one. */
+    /** Records that a secret was filled into the session. */
     raise(level: Exclude<SecretExposureLevel, 'none'>): void;
     /** Whether no pixels may leave the session: no screenshot or screencast frame is taken, kept, or handed to a model. */
     get withholdsPixels(): boolean;
-    /** Whether a text recording of the session (a trace, a text-like download) is rewritten through the ledger before it is kept. */
-    get redactsRecordings(): boolean;
     /** Whether a session saved from this one carries the taint to the sessions that restore it. */
     get carriesTaint(): boolean;
 }
+/**
+ * Whether a text recording of the session (a trace, a text-like download) is
+ * rewritten through its ledger before it is kept: whenever the ledger holds
+ * a value, filled or not. A value reaches the app in ways the runner never
+ * sees (a URL or a `fill` the test spelled it into, the app's own config),
+ * so recordings are redacted like every report and observation, not only
+ * after a fill.
+ */
+export declare function redactsRecordings(secrecy: SessionSecrecy): boolean;
 /**
  * Every secret value any session in this process has seen, for text the
  * process itself emits rather than a session: a test's console output leaves
@@ -56,7 +59,7 @@ export declare const processSecrets: SecretLedger;
  * handed out read-only: nothing registers into it.
  */
 export declare function staticSecretLedger(secrets: ResolvedConfig['allSecrets']): Pick<SecretLedger, 'redact' | 'entries'>;
-/** Seeds `processSecrets` with the static values of `secrets`, so output before any session opens is covered too. */
+/** Seeds `processSecrets` with the static values of `secrets` and makes it `serializeError`'s default redactor, so output before any session opens is covered too. */
 export declare function registerStaticSecrets(secrets: ResolvedConfig['allSecrets']): void;
 /**
  * The plaintext of one configured secret, a provider's value computed fresh.

package/dist/run/secrecy.js

@@ -1,41 +1,44 @@
 /** What one live session knows about secrets: the values to redact, and whether one reached it. */
 import { MIN_SECRET_LENGTH, secretLength } from '../config/secrets.js';
-import { ConfigurationError } from '../internal/errors.js';
+import { ConfigurationError, setErrorRedactor } from '../internal/errors.js';
 import { SecretLedger } from '../internal/redact.js';
 import { unavailableCode } from '../secrets.js';
-const EXPOSURE_RANK = { none: 0, engine: 1, filled: 2 };
 /**
  * One session's exposure level, and the one place that maps it to what each
- * consumer does. Either level above `none` rewrites what text can carry the
- * value (a trace's text entries, a text-like download) through the ledger,
- * as every report and observation already is. Only a fill withholds pixels
- * (failure screenshots, `app.screenshot()`, assert evidence, model input,
- * and a trace's screencast frames): an engine-held secret is protected as
- * text only, so a page that renders it on screen is not masked. Only a fill
+ * consumer withholds. Only a fill withholds pixels (failure screenshots,
+ * `app.screenshot()`, assert evidence, model input, and a trace's
+ * screencast frames): a value that reached the app otherwise (an engine
+ * option, a URL or a `fill` the test spelled it into) is protected as text
+ * only, so a page that renders it on screen is not masked. Only a fill
  * carries into a saved session too: the value then lives in the app's
- * state, while an engine-held one is resolved again by whatever engine opens
- * the restoring session.
+ * state. Text is not decided here: see `redactsRecordings`.
  */
 export class SecretExposure {
     current = 'none';
-    /** Records that a secret reached the session at `level`; a lower level never replaces a higher one. */
+    /** Records that a secret was filled into the session. */
     raise(level) {
-        if (EXPOSURE_RANK[level] > EXPOSURE_RANK[this.current])
-            this.current = level;
+        this.current = level;
     }
     /** Whether no pixels may leave the session: no screenshot or screencast frame is taken, kept, or handed to a model. */
     get withholdsPixels() {
         return this.current === 'filled';
     }
-    /** Whether a text recording of the session (a trace, a text-like download) is rewritten through the ledger before it is kept. */
-    get redactsRecordings() {
-        return this.current !== 'none';
-    }
     /** Whether a session saved from this one carries the taint to the sessions that restore it. */
     get carriesTaint() {
         return this.current === 'filled';
     }
 }
+/**
+ * Whether a text recording of the session (a trace, a text-like download) is
+ * rewritten through its ledger before it is kept: whenever the ledger holds
+ * a value, filled or not. A value reaches the app in ways the runner never
+ * sees (a URL or a `fill` the test spelled it into, the app's own config),
+ * so recordings are redacted like every report and observation, not only
+ * after a fill.
+ */
+export function redactsRecordings(secrecy) {
+    return !secrecy.ledger.isEmpty;
+}
 /**
  * Every secret value any session in this process has seen, for text the
  * process itself emits rather than a session: a test's console output leaves
@@ -61,8 +64,9 @@ export function staticSecretLedger(secrets) {
     }
     return ledger;
 }
-/** Seeds `processSecrets` with the static values of `secrets`, so output before any session opens is covered too. */
+/** Seeds `processSecrets` with the static values of `secrets` and makes it `serializeError`'s default redactor, so output before any session opens is covered too. */
 export function registerStaticSecrets(secrets) {
+    setErrorRedactor(processSecrets.redact);
     for (const [name, { value }] of secrets) {
         if (typeof value === 'string')
             processSecrets.register(name, value);

package/dist/run/serial.d.ts

@@ -2,14 +2,14 @@
 import type { TargetSession } from '../engine/surface.ts';
 import { type SerializedError } from '../internal/errors.ts';
 import { type RegisteredTest } from '../collect/registry.ts';
-import { type TestTargetPair } from '../collect/select.ts';
+import { type SkipInfo, type TestTargetPair } from '../collect/select.ts';
 import type { ResolvedTarget } from '../config/resolve.ts';
 import type { ArtifactStore } from '../types.ts';
 import type { AttemptContext, ClosingRecord, SessionClose, SessionPlan } from './execute.ts';
 import type { ArtifactSink } from './fixtures.ts';
 import type { StepRecord } from './steps.ts';
 import { type FileRef, type Realm, RealmManager } from './realm.ts';
-import type { AttemptRecord, ResultRecord, SerialGroupRecord } from './records.ts';
+import type { AttemptRecord, ResultRecord, SerialAttemptRecord, SerialGroupRecord, SerialMemberRecord } from './records.ts';
 /**
  * One shared session for a serial-group attempt: members preserve app state,
  * so a screen an earlier member reached stays current for later members.
@@ -37,6 +37,8 @@ export interface SharedSerialSession {
 export interface SerialHost {
     readonly target: ResolvedTarget;
     readonly artifactsRoot: string;
+    /** See `TargetExecutorOptions.rerunDir`. */
+    readonly rerunDir: string | undefined;
     readonly runId: string;
     /** The configured artifact store, so group-owned artifacts (the shared trace) upload like any other. */
     readonly artifactStore: ArtifactStore | undefined;
@@ -54,6 +56,77 @@ export interface SerialHost {
      * steps, duration, and error from the group it has already seen.
      */
     emitSerialGroup(group: SerialGroupRecord): void;
+    /**
+     * Streams one member that finished in the group attempt running now, so a
+     * worker that dies before the group ends does not take it along.
+     */
+    serialMemberFinished(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void;
+    /** Streams one finished group attempt, for the same reason. */
+    serialAttemptFinished(groupId: string, run: SerialAttemptRun): void;
 }
+/** The identity of a group attempt, known from its start. */
+export type SerialAttemptStart = Pick<SerialAttemptRecord, 'id' | 'index' | 'startedAt'>;
+/** One group attempt as it ended, and whether it got as far as its members. */
+export interface SerialAttemptRun {
+    readonly record: SerialAttemptRecord;
+    /**
+     * False when the attempt stopped before any member ran (its file would not
+     * load, its session would not launch). Such a retry keeps the members'
+     * verdicts from the attempt before it.
+     */
+    readonly reachedMembers: boolean;
+}
+/** The report id of one serial group variant: a group runs once per agent and per repeat. */
+export declare function serialGroupId(serialId: string, targetName: string, agent: string, repeat: number): string;
 /** Runs one serial unit to completion and emits every member result. */
 export declare function runSerialUnit(host: SerialHost, members: readonly TestTargetPair[], file: FileRef): Promise<SerialGroupRecord>;
+/** A serial group decided without its worker's word: the record and its members' results. */
+interface SettledSerialGroup {
+    readonly group: SerialGroupRecord;
+    readonly results: readonly ResultRecord[];
+}
+/**
+ * What the runner heard of one serial group a worker is running: the
+ * attempts it finished, and the members finished so far in the attempt
+ * running now. When the worker dies before reporting the group, this is
+ * what the report keeps.
+ */
+export declare class SerialGroupProgress {
+    private readonly finished;
+    private current;
+    private currentMembers;
+    /** The member whose body is running now, by test id, until its record arrives. */
+    private running;
+    /** A member of the group announced its start. */
+    memberStarted(testId: string): void;
+    /** A member finished in the attempt running now. */
+    memberFinished(attempt: SerialAttemptStart, member: SerialMemberRecord): void;
+    /** A group attempt finished; the next one starts empty. */
+    attemptFinished(run: SerialAttemptRun): void;
+    /**
+     * The group after its worker died with `error`, or undefined when the
+     * runner heard nothing of it running. The finished attempts stand, and the
+     * attempt in flight fails: the member whose body was running ends `status`
+     * with `error` (`timed-out` when the watchdog killed a worker that hung in
+     * it) and the ones after it skip. A crash between members (a hook,
+     * the session closing) fails the attempt and leaves its finished members
+     * as they were; one in a retry before any member ran fails the first, as a
+     * launch failure does. A crash after the last attempt, when no retry
+     * follows it, is recorded on that attempt.
+     */
+    crashed(members: readonly TestTargetPair[], target: ResolvedTarget, error: SerializedError, status: 'failed' | 'timed-out'): SettledSerialGroup | undefined;
+    /**
+     * The group after the run's interrupt stopped its worker, or undefined
+     * when the runner heard nothing of it running. The finished attempts
+     * stand, and an attempt in flight is interrupted: the members it finished
+     * keep their records and the rest skip with `skip`. The verdict is the one
+     * those attempts reach, as for a retry the interrupt cut short.
+     */
+    interrupted(members: readonly TestTargetPair[], target: ResolvedTarget, skip: SkipInfo): SettledSerialGroup | undefined;
+    /** Whether a member of the attempt in flight started or finished. */
+    private get inFlight();
+    /** The record of the attempt in flight as it stood, with the members it finished. */
+    private attemptInFlight;
+    private settled;
+}
+export {};