e2e 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build.json +1 -1
- package/dist/agent/act.js +7 -4
- package/dist/agent/action-dispatcher.js +4 -5
- package/dist/agent/actions.d.ts +32 -23
- package/dist/agent/actions.js +38 -24
- package/dist/agent/model/sdk.js +2 -0
- package/dist/agent/observation-feed.d.ts +5 -6
- package/dist/agent/observation-feed.js +5 -7
- package/dist/agent/observation.d.ts +40 -11
- package/dist/agent/observation.js +109 -59
- package/dist/agent/point-tap.d.ts +8 -8
- package/dist/agent/point-tap.js +1 -1
- package/dist/agent/replay.d.ts +19 -8
- package/dist/agent/replay.js +11 -14
- package/dist/agent/scroll-target.d.ts +2 -3
- package/dist/agent/step-cache.d.ts +41 -14
- package/dist/agent/step-cache.js +147 -62
- package/dist/agent/tool-loop.js +2 -0
- package/dist/cache/anchors.d.ts +72 -23
- package/dist/cache/anchors.js +233 -60
- package/dist/cache/context.d.ts +25 -14
- package/dist/cache/context.js +50 -8
- package/dist/cache/decide.d.ts +2 -2
- package/dist/cache/decide.js +4 -6
- package/dist/cache/identity.d.ts +18 -4
- package/dist/cache/identity.js +9 -4
- package/dist/cache/recorder.d.ts +15 -1
- package/dist/cache/recorder.js +36 -23
- package/dist/cache/rekeyed.d.ts +46 -0
- package/dist/cache/rekeyed.js +89 -0
- package/dist/cache/relocate.d.ts +14 -14
- package/dist/cache/relocate.js +33 -25
- package/dist/cache/route.d.ts +32 -33
- package/dist/cache/route.js +94 -50
- package/dist/cache/store.d.ts +6 -0
- package/dist/cache/store.js +21 -1
- package/dist/cache/trace.d.ts +53 -14
- package/dist/cache/trace.js +70 -28
- package/dist/cli/index.js +8 -8
- package/dist/cli/init/engines.d.ts +1 -3
- package/dist/cli/init/engines.js +2 -2
- package/dist/cli/init/gateways.d.ts +1 -1
- package/dist/cli/init/gateways.js +11 -1
- package/dist/cli/init/sibling-versions.json +1 -1
- package/dist/cli/init/versions.d.ts +0 -8
- package/dist/cli/init/versions.js +1 -10
- package/dist/config/resolve.d.ts +1 -0
- package/dist/config/resolve.js +33 -1
- package/dist/engine/index.d.ts +2 -2
- package/dist/expect/async.js +2 -1
- package/dist/expect/poll-scope.d.ts +52 -0
- package/dist/expect/poll-scope.js +113 -0
- package/dist/expect/poll.js +31 -4
- package/dist/expect/values.js +11 -4
- package/dist/internal/abandoned.d.ts +11 -0
- package/dist/internal/abandoned.js +23 -0
- package/dist/internal/brands.d.ts +2 -0
- package/dist/internal/brands.js +2 -0
- package/dist/internal/client-identity.d.ts +9 -0
- package/dist/internal/client-identity.js +14 -0
- package/dist/internal/redact.d.ts +7 -4
- package/dist/internal/redact.js +116 -23
- package/dist/internal/time.d.ts +6 -0
- package/dist/internal/time.js +25 -13
- package/dist/internal/urls.d.ts +9 -5
- package/dist/internal/urls.js +19 -10
- package/dist/locator/engine.d.ts +12 -2
- package/dist/locator/engine.js +8 -3
- package/dist/locator/screen.js +28 -4
- package/dist/mcp/catalog.d.ts +3 -2
- package/dist/mcp/catalog.js +6 -6
- package/dist/mcp/session.js +1 -1
- package/dist/oauth/chatgpt.js +3 -23
- package/dist/oauth/cli.js +1 -0
- package/dist/oauth/copilot.d.ts +5 -4
- package/dist/oauth/copilot.js +80 -7
- package/dist/oauth/grok.js +2 -1
- package/dist/oauth/models.js +2 -1
- package/dist/oauth/opencode-console.d.ts +10 -0
- package/dist/oauth/opencode-console.js +133 -0
- package/dist/oauth/providers/github-copilot.d.ts +12 -0
- package/dist/oauth/providers/github-copilot.js +77 -23
- package/dist/oauth/providers/opencode-console.d.ts +31 -0
- package/dist/oauth/providers/opencode-console.js +148 -0
- package/dist/oauth/providers.d.ts +2 -6
- package/dist/oauth/providers.js +3 -7
- package/dist/oauth/responses.d.ts +9 -0
- package/dist/oauth/responses.js +23 -0
- package/dist/report/build.d.ts +26 -0
- package/dist/report/build.js +16 -8
- package/dist/report/failure-text.d.ts +6 -1
- package/dist/report/failure-text.js +14 -4
- package/dist/report/format.d.ts +10 -5
- package/dist/report/format.js +12 -5
- package/dist/report/junit.js +11 -12
- package/dist/report/list.d.ts +1 -0
- package/dist/report/list.js +38 -18
- package/dist/report/markdown.d.ts +1 -1
- package/dist/report/markdown.js +37 -14
- package/dist/report/repeats.d.ts +6 -1
- package/dist/report/repeats.js +12 -3
- package/dist/run/artifacts.d.ts +18 -0
- package/dist/run/artifacts.js +68 -1
- package/dist/run/execute.d.ts +9 -0
- package/dist/run/execute.js +49 -24
- package/dist/run/extended-fixtures.js +6 -2
- package/dist/run/failure-evidence.js +4 -2
- package/dist/run/fixtures.js +6 -8
- package/dist/run/in-process.d.ts +2 -0
- package/dist/run/in-process.js +1 -0
- package/dist/run/last-run.d.ts +42 -8
- package/dist/run/last-run.js +121 -14
- package/dist/run/realm.d.ts +4 -1
- package/dist/run/realm.js +20 -6
- package/dist/run/records.d.ts +16 -0
- package/dist/run/records.js +26 -0
- package/dist/run/retry.d.ts +3 -1
- package/dist/run/retry.js +4 -2
- package/dist/run/runner.d.ts +1 -1
- package/dist/run/runner.js +69 -30
- package/dist/run/serial.d.ts +2 -0
- package/dist/run/serial.js +8 -5
- package/dist/run/steps.d.ts +0 -6
- package/dist/run/steps.js +3 -20
- package/dist/run/worker/entry.js +5 -4
- package/dist/run/worker/protocol.d.ts +2 -0
- package/dist/run/worker/session.d.ts +2 -0
- package/dist/run/worker/session.js +1 -0
- package/dist/telemetry/events.js +1 -0
- package/dist/types.d.ts +9 -3
- package/docs/agents.mdx +6 -3
- package/docs/assertions.mdx +94 -0
- package/docs/authentication.mdx +3 -1
- package/docs/browser.mdx +42 -6
- package/docs/cache.mdx +64 -18
- package/docs/ci.mdx +11 -7
- package/docs/debugging.mdx +6 -3
- package/docs/executors.mdx +4 -2
- package/docs/github.mdx +12 -4
- package/docs/{writing-tests.mdx → goals.mdx} +21 -101
- package/docs/index.mdx +4 -4
- package/docs/integrations/kernel.mdx +1 -1
- package/docs/locators.mdx +64 -0
- package/docs/migrate/cypress.mdx +2 -2
- package/docs/migrate/maestro.mdx +2 -2
- package/docs/migrate/playwright.mdx +19 -16
- package/docs/migrate/selenium.mdx +2 -2
- package/docs/mobile.mdx +21 -4
- package/docs/models.mdx +53 -7
- package/docs/physical-devices.mdx +76 -0
- package/docs/quickstart.mdx +4 -3
- package/docs/reference/app.mdx +1 -1
- package/docs/reference/cli.mdx +30 -13
- package/docs/reference/config.mdx +25 -7
- package/docs/reference/engine.mdx +3 -3
- package/docs/reference/environment.mdx +8 -1
- package/docs/reference/errors.mdx +5 -5
- package/docs/reference/expect.mdx +34 -12
- package/docs/reference/mobile.mdx +23 -13
- package/docs/reference/reporters.mdx +8 -2
- package/docs/reference/screen.mdx +30 -7
- package/docs/reference/test.mdx +11 -4
- package/docs/reference/web.mdx +125 -21
- package/docs/security.mdx +26 -16
- package/docs/subscriptions.mdx +91 -12
- package/docs/telemetry.mdx +1 -1
- package/package.json +16 -3
- package/schema/report-v1.schema.json +14 -1
- package/skills/e2e/SKILL.md +2 -2
- package/skills/e2e/references/agent.md +22 -11
- package/skills/e2e/references/debugging.md +4 -4
- package/skills/e2e/references/running.md +13 -8
- package/skills/e2e/references/setup.md +36 -15
- package/skills/e2e/references/writing-tests.md +34 -17
package/dist/report/repeats.d.ts
CHANGED
|
@@ -15,6 +15,11 @@ export interface RepeatGroup {
|
|
|
15
15
|
readonly runs: readonly RepeatRun[];
|
|
16
16
|
/** Runs that passed on their first attempt; a flaky run needed a retry and does not count. */
|
|
17
17
|
readonly passed: number;
|
|
18
|
+
/**
|
|
19
|
+
* Whether a run that reached a verdict did not pass: a flake. A run the
|
|
20
|
+
* run interrupted, or never started, says nothing about the test.
|
|
21
|
+
*/
|
|
22
|
+
readonly unstable: boolean;
|
|
18
23
|
}
|
|
19
24
|
/**
|
|
20
25
|
* Groups runs by `key`, keeping only tests that ran more than once and did
|
|
@@ -27,5 +32,5 @@ export declare function repeatGroups<T>(items: readonly T[], describe: (item: T)
|
|
|
27
32
|
}): RepeatGroup[];
|
|
28
33
|
/** `2 of 3 tests passed all 5 runs`; the run count is the most any test had. */
|
|
29
34
|
export declare function repeatSummary(groups: readonly RepeatGroup[]): string;
|
|
30
|
-
/** `3/5 passed · repeat 0 ASSERTION_FAILED · repeat 2 flaky`, naming each run that did not pass. */
|
|
35
|
+
/** `3/5 passed · repeat 0 ASSERTION_FAILED · repeat 2 flaky · repeat 4 interrupted`, naming each run that did not pass. */
|
|
31
36
|
export declare function repeatLine(group: RepeatGroup): string;
|
package/dist/report/repeats.js
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
* of one test, target, and agent side by side, so a flake reads as `3/5
|
|
4
4
|
* passed` with the runs that did not, instead of five results to count.
|
|
5
5
|
*/
|
|
6
|
+
/** Statuses of a run that reached no verdict: stopped by the run, or never started. */
|
|
7
|
+
const UNDECIDED = new Set(['interrupted', 'skipped']);
|
|
6
8
|
/**
|
|
7
9
|
* Groups runs by `key`, keeping only tests that ran more than once and did
|
|
8
10
|
* not skip every run, in first-seen order.
|
|
@@ -19,7 +21,12 @@ export function repeatGroups(items, describe) {
|
|
|
19
21
|
.filter((group) => group.runs.length > 1 && group.runs.some((run) => run.status !== 'skipped'))
|
|
20
22
|
.map((group) => {
|
|
21
23
|
const runs = group.runs.toSorted((a, b) => a.repeat - b.repeat);
|
|
22
|
-
return {
|
|
24
|
+
return {
|
|
25
|
+
label: group.label,
|
|
26
|
+
runs,
|
|
27
|
+
passed: runs.filter((run) => run.status === 'passed').length,
|
|
28
|
+
unstable: runs.some((run) => run.status !== 'passed' && !UNDECIDED.has(run.status)),
|
|
29
|
+
};
|
|
23
30
|
});
|
|
24
31
|
}
|
|
25
32
|
/** `2 of 3 tests passed all 5 runs`; the run count is the most any test had. */
|
|
@@ -28,13 +35,15 @@ export function repeatSummary(groups) {
|
|
|
28
35
|
const runs = Math.max(...groups.map((group) => group.runs.length));
|
|
29
36
|
return `${stable} of ${groups.length} test${groups.length === 1 ? '' : 's'} passed all ${runs} runs`;
|
|
30
37
|
}
|
|
31
|
-
/** A run that did not pass, in a word: its error code,
|
|
38
|
+
/** A run that did not pass, in a word: its error code, `flaky` with the code its retry recovered from, or why it reached no verdict. */
|
|
32
39
|
function outcomeOf(run) {
|
|
33
40
|
if (run.status === 'flaky')
|
|
34
41
|
return run.code === undefined ? 'flaky' : `flaky (${run.code})`;
|
|
42
|
+
if (UNDECIDED.has(run.status))
|
|
43
|
+
return run.status;
|
|
35
44
|
return run.code ?? run.status;
|
|
36
45
|
}
|
|
37
|
-
/** `3/5 passed · repeat 0 ASSERTION_FAILED · repeat 2 flaky`, naming each run that did not pass. */
|
|
46
|
+
/** `3/5 passed · repeat 0 ASSERTION_FAILED · repeat 2 flaky · repeat 4 interrupted`, naming each run that did not pass. */
|
|
38
47
|
export function repeatLine(group) {
|
|
39
48
|
const misses = group.runs
|
|
40
49
|
.filter((run) => run.status !== 'passed')
|
package/dist/run/artifacts.d.ts
CHANGED
|
@@ -56,6 +56,24 @@ export declare function createAttemptArtifacts(options: {
|
|
|
56
56
|
readonly attemptId?: string;
|
|
57
57
|
};
|
|
58
58
|
}): AttemptArtifacts;
|
|
59
|
+
/**
|
|
60
|
+
* Empties an artifact tree down to the files `keep` names, by their report
|
|
61
|
+
* paths, and removes the directories left empty. A `--last-failed` rerun
|
|
62
|
+
* starts this way rather than from an empty tree: the evidence the report it
|
|
63
|
+
* reruns names stays, since that report's results fold into the rerun's and
|
|
64
|
+
* its carried tests are not run again.
|
|
65
|
+
*/
|
|
66
|
+
export declare function pruneArtifacts(root: string, keep: ReadonlySet<string>): Promise<void>;
|
|
67
|
+
/**
|
|
68
|
+
* Claims the directory a `--last-failed` rerun's attempts write under, beside
|
|
69
|
+
* the evidence it kept, and returns its name: `rerun-<n>`, past every number
|
|
70
|
+
* already there, so the tree reads in run order. The directory is created
|
|
71
|
+
* exclusively, so no attempt of the rerun writes into a directory an earlier
|
|
72
|
+
* run's report still names.
|
|
73
|
+
*/
|
|
74
|
+
export declare function claimRerunDir(root: string): Promise<string>;
|
|
75
|
+
/** An attempt's report segments, under the rerun's directory on a `--last-failed` rerun. */
|
|
76
|
+
export declare function attemptSegments(rerunDir: string | undefined, segments: readonly string[]): readonly string[];
|
|
59
77
|
/**
|
|
60
78
|
* Restricts a report path segment to a safe filename alphabet and length. A
|
|
61
79
|
* value that is already safe and within the cap is unchanged. Any other value
|
package/dist/run/artifacts.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/** Attempt-scoped artifact directory and report registration. */
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
3
|
import { createReadStream, mkdirSync } from 'node:fs';
|
|
4
|
-
import { readFile, stat } from 'node:fs/promises';
|
|
4
|
+
import { mkdir, readdir, readFile, rm, rmdir, stat } from 'node:fs/promises';
|
|
5
5
|
import path from 'node:path';
|
|
6
6
|
import { pipeline } from 'node:stream/promises';
|
|
7
7
|
import { writeFileAtomic } from '../internal/atomic-write.js';
|
|
@@ -156,6 +156,73 @@ export function createAttemptArtifacts(options) {
|
|
|
156
156
|
};
|
|
157
157
|
return { dir, records, sink, settle };
|
|
158
158
|
}
|
|
159
|
+
/** What a `--last-failed` rerun's attempt directory is called, `rerun-<n>`, numbered from 1. */
|
|
160
|
+
const RERUN_DIR = /^rerun-([1-9][0-9]*)$/;
|
|
161
|
+
/**
|
|
162
|
+
* Empties an artifact tree down to the files `keep` names, by their report
|
|
163
|
+
* paths, and removes the directories left empty. A `--last-failed` rerun
|
|
164
|
+
* starts this way rather than from an empty tree: the evidence the report it
|
|
165
|
+
* reruns names stays, since that report's results fold into the rerun's and
|
|
166
|
+
* its carried tests are not run again.
|
|
167
|
+
*/
|
|
168
|
+
export async function pruneArtifacts(root, keep) {
|
|
169
|
+
const prune = async (dir, relative) => {
|
|
170
|
+
let entries;
|
|
171
|
+
try {
|
|
172
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
173
|
+
}
|
|
174
|
+
catch (cause) {
|
|
175
|
+
if (cause.code === 'ENOENT')
|
|
176
|
+
return true;
|
|
177
|
+
throw cause;
|
|
178
|
+
}
|
|
179
|
+
let empty = true;
|
|
180
|
+
for (const entry of entries) {
|
|
181
|
+
const reportPath = relative === '' ? entry.name : `${relative}/${entry.name}`;
|
|
182
|
+
const absolute = path.join(dir, entry.name);
|
|
183
|
+
if (entry.isDirectory()) {
|
|
184
|
+
if (await prune(absolute, reportPath))
|
|
185
|
+
await rmdir(absolute);
|
|
186
|
+
else
|
|
187
|
+
empty = false;
|
|
188
|
+
}
|
|
189
|
+
else if (keep.has(reportPath)) {
|
|
190
|
+
empty = false;
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
await rm(absolute, { force: true });
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return empty;
|
|
197
|
+
};
|
|
198
|
+
await prune(root, '');
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Claims the directory a `--last-failed` rerun's attempts write under, beside
|
|
202
|
+
* the evidence it kept, and returns its name: `rerun-<n>`, past every number
|
|
203
|
+
* already there, so the tree reads in run order. The directory is created
|
|
204
|
+
* exclusively, so no attempt of the rerun writes into a directory an earlier
|
|
205
|
+
* run's report still names.
|
|
206
|
+
*/
|
|
207
|
+
export async function claimRerunDir(root) {
|
|
208
|
+
await mkdir(root, { recursive: true });
|
|
209
|
+
const taken = (await readdir(root)).map((name) => Number(RERUN_DIR.exec(name)?.[1] ?? 0));
|
|
210
|
+
for (let n = Math.max(0, ...taken) + 1;; n += 1) {
|
|
211
|
+
const name = `rerun-${n}`;
|
|
212
|
+
try {
|
|
213
|
+
await mkdir(path.join(root, name));
|
|
214
|
+
return name;
|
|
215
|
+
}
|
|
216
|
+
catch (cause) {
|
|
217
|
+
if (cause.code !== 'EEXIST')
|
|
218
|
+
throw cause;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
/** An attempt's report segments, under the rerun's directory on a `--last-failed` rerun. */
|
|
223
|
+
export function attemptSegments(rerunDir, segments) {
|
|
224
|
+
return rerunDir === undefined ? segments : [rerunDir, ...segments];
|
|
225
|
+
}
|
|
159
226
|
/**
|
|
160
227
|
* Rewrites a text-like download through the session's ledger, the way a
|
|
161
228
|
* trace's text entries are, and returns the redaction the record can claim:
|
package/dist/run/execute.d.ts
CHANGED
|
@@ -34,6 +34,12 @@ export interface TargetExecutorOptions {
|
|
|
34
34
|
readonly target: ResolvedTarget;
|
|
35
35
|
readonly runId: string;
|
|
36
36
|
readonly artifactsRoot: string;
|
|
37
|
+
/**
|
|
38
|
+
* The directory under `artifactsRoot` a `--last-failed` rerun's attempts
|
|
39
|
+
* write in (`claimRerunDir`), beside the evidence of the run it reruns;
|
|
40
|
+
* undefined when the attempts fill the root themselves.
|
|
41
|
+
*/
|
|
42
|
+
readonly rerunDir?: string | undefined;
|
|
37
43
|
readonly sessionStore: SessionStore;
|
|
38
44
|
readonly headed: boolean;
|
|
39
45
|
/** This worker's slot among the target's workers; see `EngineInitInfo.workerSlot`. */
|
|
@@ -103,6 +109,7 @@ export declare class TargetExecutor implements SerialHost {
|
|
|
103
109
|
get runId(): string;
|
|
104
110
|
get artifactStore(): ArtifactStore | undefined;
|
|
105
111
|
readonly artifactsRoot: string;
|
|
112
|
+
readonly rerunDir: string | undefined;
|
|
106
113
|
readonly interruptSignal: AbortSignal;
|
|
107
114
|
readonly realms: RealmManager;
|
|
108
115
|
readonly debug: DebugTrace;
|
|
@@ -113,6 +120,8 @@ export declare class TargetExecutor implements SerialHost {
|
|
|
113
120
|
private lastAttemptTestId;
|
|
114
121
|
private readonly models;
|
|
115
122
|
private readonly sessionIdentity;
|
|
123
|
+
/** The file store's listing under `cache.strict`, read once for every attempt here. */
|
|
124
|
+
private readonly storedRecordings;
|
|
116
125
|
/** Resolves once the engine's init hook completed for this worker. */
|
|
117
126
|
private engineReady;
|
|
118
127
|
constructor(options: TargetExecutorOptions);
|
package/dist/run/execute.js
CHANGED
|
@@ -10,14 +10,16 @@ import { timestamp, uuidv7 } from '../internal/ids.js';
|
|
|
10
10
|
import { engineAppInfo } from '../config/app.js';
|
|
11
11
|
import { Deadline, NEVER_ABORTS, withAbort, withScopedBudget, withTimeout } from '../internal/time.js';
|
|
12
12
|
import { createAgentCacheContext, flushStagedTraces } from '../cache/context.js';
|
|
13
|
+
import { storedRecordingsFor } from '../cache/rekeyed.js';
|
|
13
14
|
import { pairRecordings } from '../collect/select.js';
|
|
14
|
-
import { createAttemptArtifacts, sanitizePathSegment } from './artifacts.js';
|
|
15
|
+
import { attemptSegments, createAttemptArtifacts, sanitizePathSegment } from './artifacts.js';
|
|
15
16
|
import { AttemptBudget } from './budget.js';
|
|
16
17
|
import { createEngineSession } from '../engine/session.js';
|
|
17
18
|
import { createExtendedFixtures } from './extended-fixtures.js';
|
|
18
19
|
import { captureFailureEvidence } from './failure-evidence.js';
|
|
19
20
|
import { createFixtures } from './fixtures.js';
|
|
20
21
|
import { publishAttempt } from '../expect/attempt.js';
|
|
22
|
+
import { PollScope, runInPollScope } from '../expect/poll-scope.js';
|
|
21
23
|
import { SoftFailures } from '../expect/soft.js';
|
|
22
24
|
import { findRegistered, RealmManager, runHook } from './realm.js';
|
|
23
25
|
import { isFailedStatus } from './records.js';
|
|
@@ -43,6 +45,7 @@ export class TargetExecutor {
|
|
|
43
45
|
return this.config.artifactStore;
|
|
44
46
|
}
|
|
45
47
|
artifactsRoot;
|
|
48
|
+
rerunDir;
|
|
46
49
|
interruptSignal;
|
|
47
50
|
realms;
|
|
48
51
|
debug;
|
|
@@ -53,12 +56,15 @@ export class TargetExecutor {
|
|
|
53
56
|
lastAttemptTestId;
|
|
54
57
|
models;
|
|
55
58
|
sessionIdentity;
|
|
59
|
+
/** The file store's listing under `cache.strict`, read once for every attempt here. */
|
|
60
|
+
storedRecordings;
|
|
56
61
|
/** Resolves once the engine's init hook completed for this worker. */
|
|
57
62
|
engineReady;
|
|
58
63
|
constructor(options) {
|
|
59
64
|
this.options = options;
|
|
60
65
|
this.target = options.target;
|
|
61
66
|
this.artifactsRoot = options.artifactsRoot;
|
|
67
|
+
this.rerunDir = options.rerunDir;
|
|
62
68
|
this.interruptSignal = options.interruptSignal;
|
|
63
69
|
this.debug = options.debug ?? new DebugTrace(false);
|
|
64
70
|
this.models = new WorkerModels((error) => {
|
|
@@ -74,6 +80,7 @@ export class TargetExecutor {
|
|
|
74
80
|
debug: this.debug,
|
|
75
81
|
});
|
|
76
82
|
this.sessionIdentity = targetIdentity(options.target);
|
|
83
|
+
this.storedRecordings = storedRecordingsFor(options.config.cache);
|
|
77
84
|
}
|
|
78
85
|
get config() {
|
|
79
86
|
return this.options.config;
|
|
@@ -588,7 +595,13 @@ export class TargetExecutor {
|
|
|
588
595
|
// agent and repeat segments keep a test run as several agents, or
|
|
589
596
|
// several times, from overwriting itself.
|
|
590
597
|
segments: shared?.artifactSegments ??
|
|
591
|
-
|
|
598
|
+
attemptSegments(this.rerunDir, [
|
|
599
|
+
this.target.name,
|
|
600
|
+
sanitizePathSegment(registered.artifactName ?? pair.test.id),
|
|
601
|
+
pair.agent,
|
|
602
|
+
...repeatSegment(pair.repeat),
|
|
603
|
+
`attempt-${attemptIndex}`,
|
|
604
|
+
]),
|
|
592
605
|
attemptId,
|
|
593
606
|
currentStepId: () => steps.currentStepId,
|
|
594
607
|
...(this.config.artifactStore === undefined ? {} : { store: this.config.artifactStore }),
|
|
@@ -614,6 +627,8 @@ export class TargetExecutor {
|
|
|
614
627
|
let skipped;
|
|
615
628
|
let phase = 'launch';
|
|
616
629
|
let timedOut = false;
|
|
630
|
+
// The polls the fixtures, the beforeEach hooks, and the body start.
|
|
631
|
+
const bodyPolls = new PollScope('the test body');
|
|
617
632
|
// Captured the moment the primary failure lands: steps that pass later —
|
|
618
633
|
// afterEach cleanup, teardown — must not confirm traces the failure
|
|
619
634
|
// implicated (a cleanup assertion says nothing about the failed flow).
|
|
@@ -664,6 +679,7 @@ export class TargetExecutor {
|
|
|
664
679
|
testId: pair.test.id,
|
|
665
680
|
target: this.sessionIdentity,
|
|
666
681
|
attemptIndex,
|
|
682
|
+
recordings: this.storedRecordings,
|
|
667
683
|
})
|
|
668
684
|
: undefined;
|
|
669
685
|
// A rejection nobody caught, handed in by the process while this attempt
|
|
@@ -722,22 +738,25 @@ export class TargetExecutor {
|
|
|
722
738
|
// object, whichever `test` they were registered through.
|
|
723
739
|
const extended = createExtendedFixtures(registered.fixtures, fixtures, this.target.engine?.name ?? 'none');
|
|
724
740
|
/**
|
|
725
|
-
* Steps the body left running when it settled were not
|
|
726
|
-
* is recorded as failed at its call, cancelled through
|
|
727
|
-
* signal the way a timeout cancels the body, and waited for
|
|
728
|
-
* cleanup budget, so teardown starts on a quiet session.
|
|
729
|
-
* such a step would fail once the session closed with
|
|
730
|
-
* it and take the worker down.
|
|
741
|
+
* Steps and polls the body left running when it settled were not
|
|
742
|
+
* awaited. A step is recorded as failed at its call, cancelled through
|
|
743
|
+
* the attempt signal the way a timeout cancels the body, and waited for
|
|
744
|
+
* within one cleanup budget, so teardown starts on a quiet session.
|
|
745
|
+
* Left alone, such a step would fail once the session closed with
|
|
746
|
+
* nobody to catch it and take the worker down. A poll is cancelled
|
|
747
|
+
* alone; left alone, its timeout would fail whatever ran then.
|
|
748
|
+
* Returns their errors, steps first.
|
|
731
749
|
*/
|
|
732
750
|
const abandonNotAwaited = async () => {
|
|
733
751
|
if (attemptEnd.signal.aborted)
|
|
734
|
-
return
|
|
735
|
-
const
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
752
|
+
return [];
|
|
753
|
+
const pollsNotAwaited = bodyPolls.close();
|
|
754
|
+
const stepsNotAwaited = steps.abandonRunning();
|
|
755
|
+
if (stepsNotAwaited !== undefined) {
|
|
756
|
+
attemptAbort.abort();
|
|
757
|
+
await withTimeout(steps.settleAbandoned(), this.config.cleanupTimeout, () => new Error('abandoned steps did not settle')).catch(() => undefined);
|
|
758
|
+
}
|
|
759
|
+
return [stepsNotAwaited, pollsNotAwaited].filter((error) => error !== undefined);
|
|
741
760
|
};
|
|
742
761
|
const mainWork = async () => {
|
|
743
762
|
try {
|
|
@@ -751,19 +770,22 @@ export class TargetExecutor {
|
|
|
751
770
|
}
|
|
752
771
|
catch (cause) {
|
|
753
772
|
// The body's own failure stays the verdict; the step it abandoned
|
|
754
|
-
// and the soft failures it kept are noted beside it.
|
|
773
|
+
// and the soft failures it kept are noted beside it. A skip keeps
|
|
774
|
+
// only the soft failures: a step or poll it left running is not one.
|
|
755
775
|
const notAwaited = await abandonNotAwaited();
|
|
756
776
|
const softFailure = soft.close();
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
}
|
|
777
|
+
const kept = isRuntimeSkip(cause) ? [softFailure] : [...notAwaited, softFailure];
|
|
778
|
+
for (const secondary of kept) {
|
|
779
|
+
if (secondary !== undefined) {
|
|
780
|
+
secondaryErrors.push(serializeError(secondary, { phase: 'body', projectRoot: this.config.projectRoot, redact }));
|
|
762
781
|
}
|
|
763
782
|
}
|
|
764
783
|
throw cause;
|
|
765
784
|
}
|
|
766
|
-
const notAwaited = await abandonNotAwaited();
|
|
785
|
+
const [notAwaited, ...moreNotAwaited] = await abandonNotAwaited();
|
|
786
|
+
for (const secondary of moreNotAwaited) {
|
|
787
|
+
secondaryErrors.push(serializeError(secondary, { phase: 'body', projectRoot: this.config.projectRoot, redact }));
|
|
788
|
+
}
|
|
767
789
|
if (notAwaited !== undefined)
|
|
768
790
|
throw notAwaited;
|
|
769
791
|
// Soft failures fail the attempt once the body has settled, before
|
|
@@ -779,8 +801,8 @@ export class TargetExecutor {
|
|
|
779
801
|
// down with the worker process; the attempt records the interrupt and
|
|
780
802
|
// moves to cleanup.
|
|
781
803
|
const work = this.options.isolated
|
|
782
|
-
? withAbort(mainWork
|
|
783
|
-
: mainWork
|
|
804
|
+
? withAbort(runInPollScope(bodyPolls, mainWork), this.interruptSignal, () => new E2EError('interrupted', 'INTERRUPTED', `run interrupted in phase ${phase}`))
|
|
805
|
+
: runInPollScope(bodyPolls, mainWork);
|
|
784
806
|
const body = Promise.race([
|
|
785
807
|
work,
|
|
786
808
|
new Promise((_, reject) => {
|
|
@@ -798,6 +820,8 @@ export class TargetExecutor {
|
|
|
798
820
|
// The race has settled: a rejection surfacing while the evidence is
|
|
799
821
|
// captured is recorded, not aimed at it.
|
|
800
822
|
cutBody = undefined;
|
|
823
|
+
// A body cut short may have been awaiting its polls; they end with it.
|
|
824
|
+
bodyPolls.close();
|
|
801
825
|
// `skipRunningTest` has already refused the cases that may not skip.
|
|
802
826
|
if (isRuntimeSkip(cause))
|
|
803
827
|
skipped = cause;
|
|
@@ -859,6 +883,7 @@ export class TargetExecutor {
|
|
|
859
883
|
recordFailure(cause, phase);
|
|
860
884
|
}
|
|
861
885
|
finally {
|
|
886
|
+
bodyPolls.close();
|
|
862
887
|
this.strayFailure = undefined;
|
|
863
888
|
this.lastAttemptTestId = pair.test.id;
|
|
864
889
|
attemptEnd.abort();
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
/** The fixtures a test's `test.extend()` chain adds to one attempt. */
|
|
2
|
+
import { pollLineage } from '../expect/poll-scope.js';
|
|
2
3
|
import { ConfigurationError } from '../internal/errors.js';
|
|
3
4
|
/**
|
|
4
5
|
* Binds a chain of fixture definitions to one attempt's fixture object. The
|
|
@@ -50,8 +51,10 @@ export function createExtendedFixtures(definitions, fixtures, engineName) {
|
|
|
50
51
|
ready();
|
|
51
52
|
return released;
|
|
52
53
|
};
|
|
54
|
+
// The function's polls are the body's until `use` returns, then its teardown's.
|
|
55
|
+
const polls = pollLineage();
|
|
53
56
|
const run = Promise.resolve()
|
|
54
|
-
.then(() => definition.fn(fixtures, use))
|
|
57
|
+
.then(() => polls.run(() => definition.fn(fixtures, use)))
|
|
55
58
|
.then(() => {
|
|
56
59
|
if (!used) {
|
|
57
60
|
failed(new ConfigurationError('TEST_SETUP_FAILED', `fixture "${name}" returned without calling use(); a fixture awaits use(value) to hand its value to the test`));
|
|
@@ -66,7 +69,7 @@ export function createExtendedFixtures(definitions, fixtures, engineName) {
|
|
|
66
69
|
run.catch(() => undefined);
|
|
67
70
|
await provided;
|
|
68
71
|
if (!abandoned)
|
|
69
|
-
active.push({ name, run, release, misuse: () => misuse });
|
|
72
|
+
active.push({ name, run, release, polls, misuse: () => misuse });
|
|
70
73
|
};
|
|
71
74
|
return {
|
|
72
75
|
async setUp() {
|
|
@@ -78,6 +81,7 @@ export function createExtendedFixtures(definitions, fixtures, engineName) {
|
|
|
78
81
|
return active.toReversed().map((state) => ({
|
|
79
82
|
name: state.name,
|
|
80
83
|
run: async () => {
|
|
84
|
+
state.polls.handOver();
|
|
81
85
|
state.release();
|
|
82
86
|
await state.run;
|
|
83
87
|
const misuse = state.misuse();
|
|
@@ -109,7 +109,9 @@ function locatorCandidates(error, observation, redact, appOrigin) {
|
|
|
109
109
|
return [];
|
|
110
110
|
if (error.code !== 'LOCATOR_NOT_FOUND' && error.code !== 'LOCATOR_AMBIGUOUS')
|
|
111
111
|
return [];
|
|
112
|
-
const
|
|
112
|
+
const details = error.details ?? {};
|
|
113
|
+
// The nodes are redacted; the query is compared in the same form, so a secret test id still finds its node.
|
|
114
|
+
const [role, testId, name] = [details.role, details.testId, details.name].map((value) => (value === undefined ? undefined : redact(value)));
|
|
113
115
|
const words = tokens(name ?? '');
|
|
114
116
|
if (role === undefined && testId === undefined && words.length === 0)
|
|
115
117
|
return [];
|
|
@@ -139,7 +141,7 @@ function locatorCandidates(error, observation, redact, appOrigin) {
|
|
|
139
141
|
return scored
|
|
140
142
|
.toSorted((a, b) => b.score - a.score)
|
|
141
143
|
.slice(0, MAX_CANDIDATES)
|
|
142
|
-
.map(({ node }) => truncateUtf8(formatNode(node, 0,
|
|
144
|
+
.map(({ node }) => truncateUtf8(formatNode(node, 0, appOrigin), MAX_CANDIDATE_BYTES));
|
|
143
145
|
}
|
|
144
146
|
/**
|
|
145
147
|
* Whether a word on screen is a near miss for one asked for: a typo or an
|
package/dist/run/fixtures.js
CHANGED
|
@@ -311,20 +311,18 @@ function createApp(environment, engine, exposure) {
|
|
|
311
311
|
* `app.open()` on a target with no URL and no `navigate`, a device: it
|
|
312
312
|
* launches the pinned app fresh, the way Maestro's `launchApp` does, and is
|
|
313
313
|
* what a test calls before it looks at the screen. A path has nowhere to
|
|
314
|
-
* go there; a link goes through `device.openLink`.
|
|
314
|
+
* go there; a link goes through `device.openLink`. An engine without a
|
|
315
|
+
* relaunch hook is refused here; one with a hook reports its own reason it
|
|
316
|
+
* cannot launch, such as a build not installed yet.
|
|
315
317
|
*/
|
|
316
318
|
const launchPinnedApp = (openPath) => steps.run('app', 'app.open', '', async () => {
|
|
317
319
|
if (openPath !== undefined) {
|
|
318
320
|
throw new ConfigurationError('APP_URL_REQUIRED', `app.open("${openPath}") needs an app URL and target "${target.name}" has none: on a device app.open() launches the pinned app, and a link goes through device.openLink`);
|
|
319
321
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
}
|
|
323
|
-
catch (cause) {
|
|
324
|
-
if (cause.code !== 'UNSUPPORTED_CAPABILITY')
|
|
325
|
-
throw cause;
|
|
326
|
-
throw new ConfigurationError('UNSUPPORTED_CAPABILITY', `app.open() has no app to launch on target "${target.name}": pin one with the target's app.bundleId or app.appPath, or bring one up with device.openApp`);
|
|
322
|
+
if (target.engine?.session?.restart === undefined) {
|
|
323
|
+
throw new ConfigurationError('UNSUPPORTED_CAPABILITY', `app.open() has nothing to open on target "${target.name}": it has no app URL, and engine ${target.engine?.name ?? 'none'} does not relaunch a pinned app`);
|
|
327
324
|
}
|
|
325
|
+
await engine.session.app.restart(engine.operation(config.timeout));
|
|
328
326
|
});
|
|
329
327
|
return {
|
|
330
328
|
baseUrl: target.app.base?.href,
|
package/dist/run/in-process.d.ts
CHANGED
|
@@ -28,6 +28,8 @@ export interface InProcessRunnerOptions {
|
|
|
28
28
|
}) | undefined;
|
|
29
29
|
readonly runId: string;
|
|
30
30
|
readonly artifactsRoot: string;
|
|
31
|
+
/** See `TargetExecutorOptions.rerunDir`. */
|
|
32
|
+
readonly rerunDir: string | undefined;
|
|
31
33
|
readonly sessionStore: SessionStore;
|
|
32
34
|
readonly headed: boolean;
|
|
33
35
|
readonly debug: DebugTrace;
|
package/dist/run/in-process.js
CHANGED
|
@@ -101,6 +101,7 @@ class InProcessRunner {
|
|
|
101
101
|
sessionStore: this.options.sessionStore,
|
|
102
102
|
runId: this.options.runId,
|
|
103
103
|
artifactsRoot: this.options.artifactsRoot,
|
|
104
|
+
rerunDir: this.options.rerunDir,
|
|
104
105
|
headed: this.options.headed,
|
|
105
106
|
workerSlot: this.workerSlot,
|
|
106
107
|
env: this.options.envFor(this.targetName),
|
package/dist/run/last-run.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/** What `--last-failed` reads back from the previous run's report. */
|
|
2
2
|
import type { Report1Document } from '../report/build.ts';
|
|
3
|
+
type ReportCarried = NonNullable<Report1Document['run']['carried']>;
|
|
3
4
|
/**
|
|
4
5
|
* The previous run's report, read from where this run will write its own. It
|
|
5
6
|
* is what `--last-failed` selects from, and reporters receive it as
|
|
@@ -9,13 +10,46 @@ import type { Report1Document } from '../report/build.ts';
|
|
|
9
10
|
export declare function readLastRun(reportPath: string): Promise<Report1Document>;
|
|
10
11
|
/**
|
|
11
12
|
* The result ids of a report's tests that did not pass: failed, timed out,
|
|
12
|
-
* interrupted, or skipped because a setup, a serial predecessor, a hook,
|
|
13
|
-
* the
|
|
14
|
-
* `afterAll` that failed, until a run where that hook
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
13
|
+
* interrupted, or skipped because a setup, a serial predecessor, a hook, the
|
|
14
|
+
* worker, or the failure limit stopped it, every selected test in the scope
|
|
15
|
+
* of a `beforeAll` or `afterAll` that failed, until a run where that hook
|
|
16
|
+
* passes, and every test the report carries from an earlier run that it did
|
|
17
|
+
* not run again. A report older than `selected` counts every result as
|
|
18
|
+
* selected. An explicit skip and a filtered test passed in the sense that
|
|
19
|
+
* matters here: nothing to run again. The ids are
|
|
20
|
+
* `resultId(testId, target, agent)`, so a test is named per target and agent
|
|
21
|
+
* whichever of its `--repeat-each` runs did not pass.
|
|
20
22
|
*/
|
|
21
23
|
export declare function lastFailedIds(document: Report1Document): ReadonlySet<string>;
|
|
24
|
+
/** What a rerun's collection found: the tests it collected by id, and the files it discovered but could not collect. */
|
|
25
|
+
export interface RerunCollection {
|
|
26
|
+
readonly testIds: ReadonlySet<string>;
|
|
27
|
+
readonly uncollectedFiles: ReadonlySet<string>;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* What a `--last-failed` rerun carries forward from the report it selected
|
|
31
|
+
* from: every result that report owed a rerun and this run did not run, as
|
|
32
|
+
* the run that last ran it reported it; the serial groups those results'
|
|
33
|
+
* attempts live on; and the suite hook failures whose scope still holds one
|
|
34
|
+
* of them. A result is owed when it did not pass, or when it is in the scope
|
|
35
|
+
* of a failed `beforeAll` or `afterAll` this run did not see pass: a hook
|
|
36
|
+
* whose scope this run ran again without it failing is resolved, and one
|
|
37
|
+
* that failed again is this run's own error, which still makes its left-out
|
|
38
|
+
* tests owed. A test this run did not run is one it collected (`collected`,
|
|
39
|
+
* by test id), on a target the config still has, that it did not select:
|
|
40
|
+
* left out by a filter, on a target or agent the run did not ask for, or a
|
|
41
|
+
* setup no selected test needed, which the report lists no row for. A test in
|
|
42
|
+
* a file the run found but could not collect (`uncollectedFiles`, a narrowed
|
|
43
|
+
* run past a file that fails to import) is still there and stays carried. A
|
|
44
|
+
* test the run no longer collects cannot run again and is dropped. A test
|
|
45
|
+
* this run selected is no longer carried, whatever its outcome: its own
|
|
46
|
+
* result says it now. Undefined when nothing is carried.
|
|
47
|
+
*/
|
|
48
|
+
export declare function carryForward(lastRun: Report1Document, current: Report1Document, collected: RerunCollection): ReportCarried | undefined;
|
|
49
|
+
/**
|
|
50
|
+
* The artifact paths a report names, relative to the artifacts root: every
|
|
51
|
+
* attempt's of its results and serial groups, and of what it carries. What a
|
|
52
|
+
* `--last-failed` rerun keeps of the tree when it starts.
|
|
53
|
+
*/
|
|
54
|
+
export declare function reportArtifactPaths(document: Report1Document): ReadonlySet<string>;
|
|
55
|
+
export {};
|