@specific.dev/spectest 0.54.1 → 0.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/daemon.js +10 -0
- package/dist/harness/raw-fetch.d.ts +7 -0
- package/dist/harness/raw-fetch.js +33 -0
- package/dist/ids.d.ts +9 -0
- package/dist/ids.js +11 -1
- package/dist/index.d.ts +43 -0
- package/dist/index.js +52 -3
- package/dist/locator-errors.d.ts +21 -0
- package/dist/locator-errors.js +142 -0
- package/dist/locator.js +18 -6
- package/dist/mcp-auth.d.ts +176 -0
- package/dist/mcp-auth.js +455 -0
- package/dist/mcp-transport.d.ts +130 -0
- package/dist/mcp-transport.js +330 -0
- package/dist/mcp.d.ts +246 -0
- package/dist/mcp.js +1060 -0
- package/dist/recorder.d.ts +5 -0
- package/package.json +1 -1
- package/src/daemon.ts +10 -0
- package/src/harness/raw-fetch.ts +36 -0
- package/src/ids.ts +12 -1
- package/src/index.ts +114 -3
- package/src/locator-errors.test.ts +125 -0
- package/src/locator-errors.ts +153 -0
- package/src/locator.ts +25 -6
- package/src/mcp-auth.ts +626 -0
- package/src/mcp-transport.ts +427 -0
- package/src/mcp.test.ts +94 -0
- package/src/mcp.ts +1435 -0
- package/src/recorder.ts +5 -2
package/dist/recorder.d.ts
CHANGED
|
@@ -599,6 +599,7 @@ export type StepBlock = {
|
|
|
599
599
|
value?: string;
|
|
600
600
|
error?: boolean;
|
|
601
601
|
}[];
|
|
602
|
+
label?: string;
|
|
602
603
|
} | {
|
|
603
604
|
type: "table";
|
|
604
605
|
columns: string[];
|
|
@@ -611,6 +612,10 @@ export type StepBlock = {
|
|
|
611
612
|
type: "chat";
|
|
612
613
|
messages: ChatBlockMessage[];
|
|
613
614
|
label?: string;
|
|
615
|
+
} | {
|
|
616
|
+
type: "details";
|
|
617
|
+
summary: string;
|
|
618
|
+
blocks: StepBlock[];
|
|
614
619
|
};
|
|
615
620
|
/** One message of a `chat` block. */
|
|
616
621
|
export interface ChatBlockMessage {
|
package/package.json
CHANGED
package/src/daemon.ts
CHANGED
|
@@ -118,6 +118,10 @@ import {
|
|
|
118
118
|
} from "./harness/methods.js";
|
|
119
119
|
import type { Mobile, MobileApp } from "./mobile.js";
|
|
120
120
|
import { openTerminal } from "./terminal.js";
|
|
121
|
+
// `ctx.mcp(url)`. One client per call, no registry: an authenticated
|
|
122
|
+
// client is passed to descendants as a test's return value.
|
|
123
|
+
import { openMcp } from "./mcp.js";
|
|
124
|
+
import { setRawFetch } from "./harness/raw-fetch.js";
|
|
121
125
|
import { readAnnotation, type RenderAnnotation } from "./annotate.js";
|
|
122
126
|
import {
|
|
123
127
|
pauseRecording,
|
|
@@ -4178,6 +4182,10 @@ function isTransportError(err: unknown): boolean {
|
|
|
4178
4182
|
*/
|
|
4179
4183
|
function installFetchWrapper(): () => void {
|
|
4180
4184
|
const original = globalThis.fetch;
|
|
4185
|
+
// SDK internals (the MCP client, its OAuth flow) must not see the
|
|
4186
|
+
// wrapper: a wrapped `res.ok` is an object, and the wrapper reads every
|
|
4187
|
+
// body to the end, which never finishes for an SSE stream.
|
|
4188
|
+
setRawFetch(original);
|
|
4181
4189
|
const wrappedFn = async (
|
|
4182
4190
|
input: Parameters<typeof fetch>[0],
|
|
4183
4191
|
init?: Parameters<typeof fetch>[1],
|
|
@@ -4934,6 +4942,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
|
4934
4942
|
openTerminal: recordedOpenTerminal,
|
|
4935
4943
|
browser: trackedOpenBrowser,
|
|
4936
4944
|
mobile: trackedOpenMobile,
|
|
4945
|
+
mcp: openMcp,
|
|
4937
4946
|
testName: testCase.name,
|
|
4938
4947
|
parent,
|
|
4939
4948
|
};
|
|
@@ -5455,6 +5464,7 @@ async function evalCode(
|
|
|
5455
5464
|
openTerminal: evalOpenTerminal,
|
|
5456
5465
|
browser: trackedOpenBrowser,
|
|
5457
5466
|
mobile: trackedOpenMobile,
|
|
5467
|
+
mcp: openMcp,
|
|
5458
5468
|
testName: "eval",
|
|
5459
5469
|
parent: undefined,
|
|
5460
5470
|
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// The real `fetch`, for SDK internals that must not go through the test
|
|
2
|
+
// recorder's wrapper.
|
|
3
|
+
//
|
|
4
|
+
// While a test (or an eval) runs, the daemon replaces `globalThis.fetch`
|
|
5
|
+
// with an instrumented one: it records an `http` event, and it hands back
|
|
6
|
+
// a WRAPPED response whose `ok` / `status` are provenance handles rather
|
|
7
|
+
// than a boolean and a number. That is exactly right for a test's own
|
|
8
|
+
// calls, and exactly wrong inside the SDK, in two ways:
|
|
9
|
+
//
|
|
10
|
+
// 1. `if (!res.ok)` is always false against a handle, because a handle
|
|
11
|
+
// is an object. An SDK client that checks it would read a `401` as a
|
|
12
|
+
// success. (`pauseRecording()` does not help: it stops the event, not
|
|
13
|
+
// the wrapper.)
|
|
14
|
+
// 2. The wrapper clones the response and reads the clone to the end so
|
|
15
|
+
// it can record the body. For a streaming reply — Server-Sent Events
|
|
16
|
+
// — that does not finish until the stream closes, so the caller is
|
|
17
|
+
// handed its response only after the stream it was waiting to read
|
|
18
|
+
// has already ended. A long-lived stream deadlocks.
|
|
19
|
+
//
|
|
20
|
+
// So SDK-internal HTTP goes through `rawFetch`. The daemon publishes the
|
|
21
|
+
// original here when it installs its wrapper.
|
|
22
|
+
|
|
23
|
+
let original: typeof fetch | undefined;
|
|
24
|
+
|
|
25
|
+
/** Called by the daemon with the real `fetch`, before it installs its
|
|
26
|
+
* instrumented one. */
|
|
27
|
+
export function setRawFetch(fn: typeof fetch): void {
|
|
28
|
+
original = fn;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** `fetch`, never the instrumented wrapper. Falls back to the global for
|
|
32
|
+
* a context where nothing was ever installed (unit tests, a plain
|
|
33
|
+
* script). */
|
|
34
|
+
export function rawFetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response> {
|
|
35
|
+
return (original ?? globalThis.fetch)(input as RequestInfo, init);
|
|
36
|
+
}
|
package/src/ids.ts
CHANGED
|
@@ -44,7 +44,18 @@ export function generateId(prefix: string): string {
|
|
|
44
44
|
return `${prefix}_0${base32Crockford(digest.subarray(0, 16)).slice(0, RANDOM_LEN)}`;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
/**
|
|
47
|
+
/**
|
|
48
|
+
* Read `n` bytes directly from /dev/urandom (NOT Bun's cached pool).
|
|
49
|
+
*
|
|
50
|
+
* Exported because every in-VM secret with a uniqueness requirement has
|
|
51
|
+
* the same problem as an id: Bun's pool is frozen into the snapshot, so
|
|
52
|
+
* sibling forks draw identical bytes. An OAuth PKCE verifier and `state`
|
|
53
|
+
* (`mcp-auth.ts`) must come from here, not from `crypto.getRandomValues`.
|
|
54
|
+
*/
|
|
55
|
+
export function secureRandomBytes(n: number): Uint8Array {
|
|
56
|
+
return urandom(n);
|
|
57
|
+
}
|
|
58
|
+
|
|
48
59
|
function urandom(n: number): Uint8Array {
|
|
49
60
|
const buf = new Uint8Array(n);
|
|
50
61
|
try {
|
package/src/index.ts
CHANGED
|
@@ -66,6 +66,34 @@ export type { Browser, BrowserOptions, Keyboard, Mouse, Touchscreen } from "./br
|
|
|
66
66
|
|
|
67
67
|
import type { Browser, BrowserOptions } from "./browser.js";
|
|
68
68
|
|
|
69
|
+
// The MCP client (`ctx.mcp`). Drives a Model Context Protocol server the
|
|
70
|
+
// way a real AI client would, with the OAuth flow a test runs through its
|
|
71
|
+
// own browser. See `mcp.ts`.
|
|
72
|
+
export {
|
|
73
|
+
McpHttpError,
|
|
74
|
+
McpRpcError,
|
|
75
|
+
McpAuthDeniedError,
|
|
76
|
+
type Mcp,
|
|
77
|
+
type McpOptions,
|
|
78
|
+
type McpAuthorization,
|
|
79
|
+
type McpChallenge,
|
|
80
|
+
type McpIdentity,
|
|
81
|
+
type McpToolResult,
|
|
82
|
+
type McpToolInfo,
|
|
83
|
+
type McpResourceInfo,
|
|
84
|
+
type McpResourceContents,
|
|
85
|
+
type McpPromptInfo,
|
|
86
|
+
type McpPromptResult,
|
|
87
|
+
type McpContent,
|
|
88
|
+
type McpNotification,
|
|
89
|
+
type McpSamplingRequest,
|
|
90
|
+
type McpSamplingResult,
|
|
91
|
+
type AuthorizeOptions,
|
|
92
|
+
type AuthorizationServerInfo,
|
|
93
|
+
} from "./mcp.js";
|
|
94
|
+
|
|
95
|
+
import type { Mcp, McpOptions } from "./mcp.js";
|
|
96
|
+
|
|
69
97
|
// Playwright-native locators — the select-then-act surface shared by
|
|
70
98
|
// `ctx.browser()` and `ctx.mobile()`. `Locator` mirrors playwright-core's
|
|
71
99
|
// Locator (getBy*/filter/first/nth/click/fill/textContent/…, STRICT mode).
|
|
@@ -87,6 +115,7 @@ import {
|
|
|
87
115
|
getBrowserProbe,
|
|
88
116
|
DEFAULT_ACTION_TIMEOUT_MS,
|
|
89
117
|
} from "./locator.js";
|
|
118
|
+
import { formatWaited, locatorFailureMessage } from "./locator-errors.js";
|
|
90
119
|
|
|
91
120
|
export type { UrlPattern } from "./url-match.js";
|
|
92
121
|
import type { UrlPattern } from "./url-match.js";
|
|
@@ -1497,6 +1526,47 @@ export interface TestContext<
|
|
|
1497
1526
|
* natural way to assert on the exit code.
|
|
1498
1527
|
*/
|
|
1499
1528
|
openTerminal(service: string, opts?: TerminalOpts): Promise<Terminal>;
|
|
1529
|
+
/**
|
|
1530
|
+
* Connect to a Model Context Protocol server over HTTP, as an AI client
|
|
1531
|
+
* would. Every tool call, resource read and prompt renders as its own
|
|
1532
|
+
* step in the timeline, with the arguments and the result.
|
|
1533
|
+
*
|
|
1534
|
+
* ```ts
|
|
1535
|
+
* const mcp = await ctx.mcp("https://api.test/mcp");
|
|
1536
|
+
* const res = await mcp.call("create_invoice", { customer: "Acme" });
|
|
1537
|
+
* expect(res.json<{ id: string }>().id).toMatch(/^inv_/);
|
|
1538
|
+
* ```
|
|
1539
|
+
*
|
|
1540
|
+
* **Each call returns a NEW client.** There is no per-name registry like
|
|
1541
|
+
* `ctx.browser("alice")` has, and no default session. A client that has
|
|
1542
|
+
* authenticated is handed to the tests that need it, as this test's
|
|
1543
|
+
* return value — `ctx.parent` crosses the fork in daemon memory, live
|
|
1544
|
+
* connections included:
|
|
1545
|
+
*
|
|
1546
|
+
* ```ts
|
|
1547
|
+
* export const signedIn = env.test("sign in", async (ctx) => {
|
|
1548
|
+
* const mcp = await ctx.mcp(URL);
|
|
1549
|
+
* const auth = await mcp.authorize();
|
|
1550
|
+
* const page = await ctx.browser();
|
|
1551
|
+
* await page.goto(auth.url);
|
|
1552
|
+
* await page.getByRole("button", { name: "Allow" }).click();
|
|
1553
|
+
* await auth.complete();
|
|
1554
|
+
* return mcp;
|
|
1555
|
+
* });
|
|
1556
|
+
*
|
|
1557
|
+
* env.test("call a tool", { dependsOn: signedIn }, async (ctx) => {
|
|
1558
|
+
* await ctx.parent.call("create_invoice", { amount: 250 });
|
|
1559
|
+
* });
|
|
1560
|
+
* ```
|
|
1561
|
+
*
|
|
1562
|
+
* Two identities are two calls to `ctx.mcp` — nothing to name.
|
|
1563
|
+
*
|
|
1564
|
+
* A server that needs authorization answers `401`, which is reported and
|
|
1565
|
+
* not hidden: `ctx.mcp(url)` returns an unauthenticated client carrying
|
|
1566
|
+
* the challenge, and a call on it fails with the server's own rejection.
|
|
1567
|
+
* That is what makes "this tool is protected" a test.
|
|
1568
|
+
*/
|
|
1569
|
+
mcp(url: string, opts?: McpOptions): Promise<Mcp>;
|
|
1500
1570
|
/**
|
|
1501
1571
|
* Open the headless browser. Backed by Chromium-over-CDP inside the VM.
|
|
1502
1572
|
*
|
|
@@ -2467,7 +2537,8 @@ function buildLocatorMatchers(
|
|
|
2467
2537
|
expected?: unknown,
|
|
2468
2538
|
): Promise<void> => {
|
|
2469
2539
|
const started = Date.now();
|
|
2470
|
-
const
|
|
2540
|
+
const budget = timeout ?? DEFAULT_ACTION_TIMEOUT_MS;
|
|
2541
|
+
const deadline = started + budget;
|
|
2471
2542
|
let actual: unknown;
|
|
2472
2543
|
// Whether this matcher's subject is meant to be on screen, i.e. whether
|
|
2473
2544
|
// the step should carry a reveal target for the replay (see
|
|
@@ -2479,19 +2550,55 @@ function buildLocatorMatchers(
|
|
|
2479
2550
|
(visibility && (matcher === "toBeHidden") !== negated) ||
|
|
2480
2551
|
(matcher === "toHaveCount" && expected === 0);
|
|
2481
2552
|
const settleOpts = { reveal: !expectsGone };
|
|
2553
|
+
// "That element is not there" said plainly, or `undefined` when the
|
|
2554
|
+
// element IS there and the matcher's own description is the better
|
|
2555
|
+
// sentence.
|
|
2556
|
+
//
|
|
2557
|
+
// Two ways to learn it. A matcher whose probe waits (textContent,
|
|
2558
|
+
// inputValue, isEnabled …) has already been told, by the playwright
|
|
2559
|
+
// timeout it caught. One whose probe answers instantly (isVisible) has
|
|
2560
|
+
// not — `toBeVisible()` on a selector that matches nothing and one that
|
|
2561
|
+
// matches a hidden element fail identically — so the count is read once,
|
|
2562
|
+
// at failure time only. `toHaveCount` is left alone: "expected count 2,
|
|
2563
|
+
// got 0" already says it, and better.
|
|
2564
|
+
const elementFailure = async (err: unknown): Promise<string | undefined> => {
|
|
2565
|
+
if (err !== undefined) return locatorFailureMessage(probe.label, err);
|
|
2566
|
+
if (expectsGone || matcher === "toHaveCount") return undefined;
|
|
2567
|
+
try {
|
|
2568
|
+
if ((await probe.count()) === 0) {
|
|
2569
|
+
return `No element matches ${probe.label} (waited ${formatWaited(budget)})`;
|
|
2570
|
+
}
|
|
2571
|
+
} catch {
|
|
2572
|
+
/* The page is gone or the chain is invalid — the matcher's own
|
|
2573
|
+
description still says what was expected. */
|
|
2574
|
+
}
|
|
2575
|
+
return undefined;
|
|
2576
|
+
};
|
|
2577
|
+
// The last error a probe threw, kept for the failure message: a matcher
|
|
2578
|
+
// whose read waits for the element (textContent, inputValue, isEnabled …)
|
|
2579
|
+
// reports a missing element as a playwright timeout, and that — not
|
|
2580
|
+
// "got <error: Timeout 5000ms exceeded …>" — is the sentence to fail with.
|
|
2581
|
+
let probeError: unknown;
|
|
2482
2582
|
for (;;) {
|
|
2483
2583
|
let satisfied: boolean;
|
|
2484
2584
|
try {
|
|
2485
2585
|
const r = await check();
|
|
2486
2586
|
satisfied = r.satisfied;
|
|
2487
2587
|
actual = r.actual;
|
|
2588
|
+
probeError = undefined;
|
|
2488
2589
|
} catch (err) {
|
|
2590
|
+
probeError = err;
|
|
2489
2591
|
if (Date.now() < deadline) {
|
|
2490
2592
|
await new Promise((r) => setTimeout(r, LOCATOR_POLL_MS));
|
|
2491
2593
|
continue;
|
|
2492
2594
|
}
|
|
2493
2595
|
satisfied = false;
|
|
2494
|
-
actual
|
|
2596
|
+
// The recorded "actual" is what the timeline shows next to the
|
|
2597
|
+
// matcher, so it gets the readable summary too — never the call-log
|
|
2598
|
+
// wall a playwright timeout carries.
|
|
2599
|
+
actual =
|
|
2600
|
+
locatorFailureMessage(probe.label, err) ??
|
|
2601
|
+
`<error: ${(err as Error)?.message ?? String(err)}>`;
|
|
2495
2602
|
}
|
|
2496
2603
|
const passed = satisfied !== negated;
|
|
2497
2604
|
if (passed) {
|
|
@@ -2508,7 +2615,11 @@ function buildLocatorMatchers(
|
|
|
2508
2615
|
return;
|
|
2509
2616
|
}
|
|
2510
2617
|
if (Date.now() >= deadline) {
|
|
2511
|
-
|
|
2618
|
+
// The deadline, not the stopwatch: every one of these polled until it
|
|
2619
|
+
// ran out, and "(waited 5s)" is both what the author set and the same
|
|
2620
|
+
// number twice in a row — a measured 4_987ms is neither.
|
|
2621
|
+
const msg = (await elementFailure(probeError)) ??
|
|
2622
|
+
`${describe(actual)} (waited ${formatWaited(budget)})`;
|
|
2512
2623
|
const sourceSeq = await probe.settle(matcher, Date.now() - started, msg, settleOpts);
|
|
2513
2624
|
recordAssertion({
|
|
2514
2625
|
matcher,
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { formatWaited, locatorFailureMessage, rewriteLocatorError } from "./locator-errors.js";
|
|
4
|
+
|
|
5
|
+
/** A playwright actionability timeout, shaped exactly as playwright-core
|
|
6
|
+
* 1.61 raises one: `name`, the message with the call log appended, the same
|
|
7
|
+
* log as an array, and a stack whose first lines ARE the message. Captured
|
|
8
|
+
* from a real run against headless Chromium. */
|
|
9
|
+
function timeoutError(message: string, log: string[]): Error {
|
|
10
|
+
const full = `${message}\nCall log:\n${log.join("\n")}\n`;
|
|
11
|
+
const err = new Error(full) as Error & { log: string[] };
|
|
12
|
+
err.name = "TimeoutError";
|
|
13
|
+
err.log = log;
|
|
14
|
+
err.stack = `${full}\n at /project/spectest/tests/todo.ts:12:34`;
|
|
15
|
+
return err;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
describe("locatorFailureMessage", () => {
|
|
19
|
+
test("names the element that was never found", () => {
|
|
20
|
+
const err = timeoutError("click: Timeout 5000ms exceeded.", [
|
|
21
|
+
" - waiting for getByTestId('add')",
|
|
22
|
+
]);
|
|
23
|
+
expect(locatorFailureMessage('testid "add"', err)).toBe(
|
|
24
|
+
'No element matches testid "add" (waited 5s)',
|
|
25
|
+
);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("reports the state of an element that was found", () => {
|
|
29
|
+
const err = timeoutError("click: Timeout 5000ms exceeded.", [
|
|
30
|
+
" - waiting for locator('#dis')",
|
|
31
|
+
" - locator resolved to <button id=\"dis\" disabled>Nope</button>",
|
|
32
|
+
" - attempting click action",
|
|
33
|
+
" 2 × waiting for element to be visible, enabled and stable",
|
|
34
|
+
" - element is not enabled",
|
|
35
|
+
" - retrying click action",
|
|
36
|
+
" - waiting 100ms",
|
|
37
|
+
]);
|
|
38
|
+
expect(locatorFailureMessage('css "#dis"', err)).toBe(
|
|
39
|
+
'Element css "#dis" is not enabled (waited 5s)',
|
|
40
|
+
);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("names what covered the element", () => {
|
|
44
|
+
const err = timeoutError("click: Timeout 1000ms exceeded.", [
|
|
45
|
+
" - waiting for locator('#ok')",
|
|
46
|
+
" - locator resolved to <button id=\"ok\">OK</button>",
|
|
47
|
+
" - attempting click action",
|
|
48
|
+
" - <div id=\"cover\"></div> intercepts pointer events",
|
|
49
|
+
]);
|
|
50
|
+
expect(locatorFailureMessage('css "#ok"', err)).toBe(
|
|
51
|
+
'Element css "#ok" is covered by <div id="cover"></div> (waited 1s)',
|
|
52
|
+
);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("a wait for the element to go away says it is still there", () => {
|
|
56
|
+
const err = timeoutError("waitFor: Timeout 2500ms exceeded.", [
|
|
57
|
+
" - waiting for locator('#ok') to be hidden",
|
|
58
|
+
" 11 × locator resolved to visible <button id=\"ok\">OK</button>",
|
|
59
|
+
]);
|
|
60
|
+
expect(locatorFailureMessage('css "#ok"', err)).toBe(
|
|
61
|
+
'Element css "#ok" is still visible (waited 2.5s)',
|
|
62
|
+
);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("falls back when the element matched for a reason we cannot read", () => {
|
|
66
|
+
const err = timeoutError("click: Timeout 5000ms exceeded.", [
|
|
67
|
+
" - waiting for locator('#ok')",
|
|
68
|
+
" - locator resolved to <button id=\"ok\">OK</button>",
|
|
69
|
+
" - something new playwright started logging",
|
|
70
|
+
]);
|
|
71
|
+
expect(locatorFailureMessage('css "#ok"', err)).toBe(
|
|
72
|
+
'Element css "#ok" never became ready for this action (waited 5s)',
|
|
73
|
+
);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test("reads the call log out of the message when the log array is gone", () => {
|
|
77
|
+
const err = timeoutError("click: Timeout 5000ms exceeded.", [
|
|
78
|
+
" - waiting for getByTestId('add')",
|
|
79
|
+
]);
|
|
80
|
+
delete (err as { log?: unknown }).log;
|
|
81
|
+
expect(locatorFailureMessage('testid "add"', err)).toBe(
|
|
82
|
+
'No element matches testid "add" (waited 5s)',
|
|
83
|
+
);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("leaves errors that are not locator timeouts alone", () => {
|
|
87
|
+
const strict = new Error(
|
|
88
|
+
"click: Error: strict mode violation: locator('p') resolved to 2 elements",
|
|
89
|
+
);
|
|
90
|
+
expect(locatorFailureMessage('css "p"', strict)).toBeUndefined();
|
|
91
|
+
expect(locatorFailureMessage('css "p"', new Error("page closed"))).toBeUndefined();
|
|
92
|
+
expect(locatorFailureMessage('css "p"', "not an error at all")).toBeUndefined();
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
describe("rewriteLocatorError", () => {
|
|
97
|
+
test("rewrites the message AND the copy embedded in the stack", () => {
|
|
98
|
+
const err = timeoutError("click: Timeout 5000ms exceeded.", [
|
|
99
|
+
" - waiting for getByTestId('add')",
|
|
100
|
+
]);
|
|
101
|
+
const out = rewriteLocatorError(err, 'testid "add"') as Error;
|
|
102
|
+
|
|
103
|
+
expect(out).toBe(err); // same error: the author's own frames survive
|
|
104
|
+
expect(out.message).toBe('No element matches testid "add" (waited 5s)');
|
|
105
|
+
expect(out.stack).toStartWith('No element matches testid "add" (waited 5s)');
|
|
106
|
+
expect(out.stack).not.toContain("Timeout 5000ms exceeded");
|
|
107
|
+
expect(out.stack).toContain("todo.ts:12:34");
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test("hands back anything it does not understand untouched", () => {
|
|
111
|
+
const err = new Error("page closed");
|
|
112
|
+
expect(rewriteLocatorError(err, 'css "p"')).toBe(err);
|
|
113
|
+
expect(err.message).toBe("page closed");
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
describe("formatWaited", () => {
|
|
118
|
+
test("reads in the units a reader thinks in", () => {
|
|
119
|
+
expect(formatWaited(250)).toBe("250ms");
|
|
120
|
+
expect(formatWaited(5000)).toBe("5s");
|
|
121
|
+
expect(formatWaited(5040)).toBe("5s");
|
|
122
|
+
expect(formatWaited(1450)).toBe("1.5s");
|
|
123
|
+
expect(formatWaited(30_000)).toBe("30s");
|
|
124
|
+
});
|
|
125
|
+
});
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// Readable failures for locator steps.
|
|
2
|
+
//
|
|
3
|
+
// Playwright reports every unmet actionability wait the same way: a
|
|
4
|
+
// `TimeoutError` whose message is "Timeout 5000ms exceeded." with the real
|
|
5
|
+
// story — did the element exist at all? was it disabled? did something cover
|
|
6
|
+
// it? — buried in a call log below it. A test that clicks a button that is not
|
|
7
|
+
// on the page therefore fails with a sentence about OUR deadline, which reads
|
|
8
|
+
// as an infrastructure problem and says nothing about the page.
|
|
9
|
+
//
|
|
10
|
+
// This module turns those into one sentence that names the element and what
|
|
11
|
+
// was wrong with it. Anything it does not recognise (a strict-mode violation,
|
|
12
|
+
// a closed page, an assertion of ours) is passed through untouched — the rule
|
|
13
|
+
// is that a message is only ever replaced when we have something better to
|
|
14
|
+
// say.
|
|
15
|
+
//
|
|
16
|
+
// The call log is read from the error's own `log` array (playwright attaches
|
|
17
|
+
// it) and falls back to parsing the message, since only the message survives
|
|
18
|
+
// a serialize/deserialize round trip.
|
|
19
|
+
|
|
20
|
+
/** A wait's duration, in the units a reader thinks in: "800ms", "5s", "1.5s". */
|
|
21
|
+
export function formatWaited(ms: number): string {
|
|
22
|
+
if (!Number.isFinite(ms) || ms < 0) return "0ms";
|
|
23
|
+
if (ms < 1000) return `${Math.round(ms)}ms`;
|
|
24
|
+
// Rounded off the millisecond count, not through toFixed — 1450ms is
|
|
25
|
+
// "1.5s", and binary floating point renders 1.45 as 1.4.
|
|
26
|
+
return `${Math.round(ms / 100) / 10}s`;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The states `waitFor` (and the mobile tap's own pre-wait) can ask for. Read
|
|
30
|
+
* off the call log rather than threaded through the callers, so a wait nested
|
|
31
|
+
* inside some other action is described just as well. */
|
|
32
|
+
type WaitState = "attached" | "detached" | "visible" | "hidden";
|
|
33
|
+
|
|
34
|
+
interface TimeoutDetail {
|
|
35
|
+
/** The deadline playwright reported — the exact number the author set. */
|
|
36
|
+
timeoutMs: number;
|
|
37
|
+
/** Call-log lines, leading "- " and "N × " repeat counts stripped. */
|
|
38
|
+
log: string[];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Recognise playwright's actionability timeout and pull it apart. Returns
|
|
42
|
+
* `undefined` for every other error. */
|
|
43
|
+
function parseTimeout(err: unknown): TimeoutDetail | undefined {
|
|
44
|
+
const e = err as { name?: string; message?: unknown; log?: unknown } | null;
|
|
45
|
+
if (!e || e.name !== "TimeoutError" || typeof e.message !== "string") return undefined;
|
|
46
|
+
const m = /Timeout (\d+)ms exceeded/.exec(e.message);
|
|
47
|
+
if (!m) return undefined;
|
|
48
|
+
const raw = Array.isArray(e.log)
|
|
49
|
+
? (e.log as unknown[]).filter((l): l is string => typeof l === "string")
|
|
50
|
+
: logFromMessage(e.message);
|
|
51
|
+
return { timeoutMs: Number(m[1]), log: raw.map(cleanLogLine).filter(Boolean) };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function logFromMessage(message: string): string[] {
|
|
55
|
+
const at = message.indexOf("Call log:");
|
|
56
|
+
if (at === -1) return [];
|
|
57
|
+
return message.slice(at + "Call log:".length).split("\n");
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Strip a line's leading bullet and repeat count ("2 × waiting for …"). */
|
|
61
|
+
function cleanLogLine(line: string): string {
|
|
62
|
+
return line.trim().replace(/^-\s*/, "").replace(/^\d+\s*×\s*/, "");
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Did the locator ever match anything? Playwright logs "locator resolved to
|
|
66
|
+
* <html>" the moment it does, and nothing of the sort when it never did. */
|
|
67
|
+
function resolved(log: string[]): boolean {
|
|
68
|
+
return log.some((l) => l.startsWith("locator resolved to"));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The state a `waitFor`-style wait was after, from its opening log line. */
|
|
72
|
+
function waitedForState(log: string[]): WaitState | undefined {
|
|
73
|
+
for (const line of log) {
|
|
74
|
+
const m = /^waiting for .* to be (attached|detached|visible|hidden)$/.exec(line);
|
|
75
|
+
if (m) return m[1] as WaitState;
|
|
76
|
+
}
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Why an element that DID match still could not be acted on. Read from the
|
|
81
|
+
* last attempt, since that is the state the deadline caught it in. */
|
|
82
|
+
function actionabilityReason(log: string[]): string | undefined {
|
|
83
|
+
for (let i = log.length - 1; i >= 0; i--) {
|
|
84
|
+
const line = log[i]!;
|
|
85
|
+
// "element is not visible" / "not enabled" / "not stable" / "not editable"
|
|
86
|
+
const missing = /^element is not (.+)$/.exec(line);
|
|
87
|
+
if (missing) return `is not ${missing[1]}`;
|
|
88
|
+
// "<div id="overlay"></div> intercepts pointer events"
|
|
89
|
+
const covered = /^(<.+>) intercepts pointer events$/.exec(line);
|
|
90
|
+
if (covered) return `is covered by ${covered[1]}`;
|
|
91
|
+
}
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The sentence to fail with, or `undefined` when `err` is not a locator
|
|
97
|
+
* timeout we understand (in which case the caller must leave it alone).
|
|
98
|
+
*
|
|
99
|
+
* `label` is the locator's human chain label, the same string the timeline
|
|
100
|
+
* step shows.
|
|
101
|
+
*/
|
|
102
|
+
export function locatorFailureMessage(label: string, err: unknown): string | undefined {
|
|
103
|
+
const detail = parseTimeout(err);
|
|
104
|
+
if (!detail) return undefined;
|
|
105
|
+
const waited = `(waited ${formatWaited(detail.timeoutMs)})`;
|
|
106
|
+
if (!resolved(detail.log)) return `No element matches ${label} ${waited}`;
|
|
107
|
+
|
|
108
|
+
// It matched, so the failure is about the element's state.
|
|
109
|
+
switch (waitedForState(detail.log)) {
|
|
110
|
+
case "hidden":
|
|
111
|
+
return `Element ${label} is still visible ${waited}`;
|
|
112
|
+
case "detached":
|
|
113
|
+
return `Element ${label} is still attached to the page ${waited}`;
|
|
114
|
+
case "visible":
|
|
115
|
+
return `Element ${label} is not visible ${waited}`;
|
|
116
|
+
default:
|
|
117
|
+
break;
|
|
118
|
+
}
|
|
119
|
+
const reason = actionabilityReason(detail.log);
|
|
120
|
+
return reason
|
|
121
|
+
? `Element ${label} ${reason} ${waited}`
|
|
122
|
+
: `Element ${label} never became ready for this action ${waited}`;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Replace a locator timeout's message in place and hand the error back, so the
|
|
127
|
+
* caller can `throw` it unchanged in every other respect.
|
|
128
|
+
*
|
|
129
|
+
* The error is mutated rather than wrapped: its stack holds the frames of the
|
|
130
|
+
* author's own call, which a fresh Error would lose. The stack string embeds
|
|
131
|
+
* the old message (it is built at construction), so that copy is rewritten
|
|
132
|
+
* too — otherwise the CLI's failure block would print the friendly message and
|
|
133
|
+
* then the timeout wall right under it.
|
|
134
|
+
*/
|
|
135
|
+
export function rewriteLocatorError(err: unknown, label: string): unknown {
|
|
136
|
+
const message = locatorFailureMessage(label, err);
|
|
137
|
+
if (message === undefined) return err;
|
|
138
|
+
const e = err as Error;
|
|
139
|
+
const old = e.message;
|
|
140
|
+
if (typeof e.stack === "string") {
|
|
141
|
+
for (const [header, replacement] of [
|
|
142
|
+
[`${e.name}: ${old}`, `${e.name}: ${message}`],
|
|
143
|
+
[old, message],
|
|
144
|
+
] as const) {
|
|
145
|
+
if (e.stack.startsWith(header)) {
|
|
146
|
+
e.stack = replacement + e.stack.slice(header.length);
|
|
147
|
+
break;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
e.message = message;
|
|
152
|
+
return e;
|
|
153
|
+
}
|
package/src/locator.ts
CHANGED
|
@@ -26,6 +26,7 @@ import { Buffer } from "node:buffer";
|
|
|
26
26
|
import type { Page, Locator as PWLocator } from "playwright-core";
|
|
27
27
|
import type { RecordableFields } from "./browser.js";
|
|
28
28
|
import type { Wrapped } from "./inspect.js";
|
|
29
|
+
import { rewriteLocatorError } from "./locator-errors.js";
|
|
29
30
|
import { resolveExistingProjectPath } from "./project-files.js";
|
|
30
31
|
import { truncateUtf8 } from "./recorder.js";
|
|
31
32
|
|
|
@@ -701,6 +702,19 @@ export function makeLocator(
|
|
|
701
702
|
const extend = (step: Step): Locator =>
|
|
702
703
|
makeLocator(backend, strategy, { steps: [...chain.steps, step] });
|
|
703
704
|
|
|
705
|
+
// Every terminal op runs through this: playwright's actionability timeouts
|
|
706
|
+
// say "Timeout 5000ms exceeded" and hide what actually went wrong in a call
|
|
707
|
+
// log, so they are rewritten into a sentence naming the element and its
|
|
708
|
+
// state (see locator-errors.ts). Applied INSIDE `pageOp`, so the recorded
|
|
709
|
+
// step carries the readable message too — not just the thrown error.
|
|
710
|
+
const readable = async <T>(fn: () => Promise<T>): Promise<T> => {
|
|
711
|
+
try {
|
|
712
|
+
return await fn();
|
|
713
|
+
} catch (err) {
|
|
714
|
+
throw rewriteLocatorError(err, label);
|
|
715
|
+
}
|
|
716
|
+
};
|
|
717
|
+
|
|
704
718
|
// One recorded event, result NOT wrapped (void/action).
|
|
705
719
|
const act = <T>(
|
|
706
720
|
action: string,
|
|
@@ -708,7 +722,7 @@ export function makeLocator(
|
|
|
708
722
|
fn: (loc: PWLocator, page: Page) => Promise<T>,
|
|
709
723
|
): Promise<T> =>
|
|
710
724
|
backend.pageOp(action, { selector: label, ...fields }, (page) =>
|
|
711
|
-
fn(lower(page, chain), page),
|
|
725
|
+
readable(() => fn(lower(page, chain), page)),
|
|
712
726
|
);
|
|
713
727
|
|
|
714
728
|
// One recorded event whose fields the action itself finishes filling in:
|
|
@@ -720,7 +734,9 @@ export function makeLocator(
|
|
|
720
734
|
fn: (loc: PWLocator, rec: Partial<RecordableFields>) => Promise<T>,
|
|
721
735
|
): Promise<T> => {
|
|
722
736
|
const rec: Partial<RecordableFields> = { selector: label };
|
|
723
|
-
return backend.pageOp(action, rec, (page) =>
|
|
737
|
+
return backend.pageOp(action, rec, (page) =>
|
|
738
|
+
readable(() => fn(lower(page, chain), rec)),
|
|
739
|
+
);
|
|
724
740
|
};
|
|
725
741
|
|
|
726
742
|
// One recorded event, result provenance-wrapped for expect().
|
|
@@ -729,9 +745,12 @@ export function makeLocator(
|
|
|
729
745
|
fn: (loc: PWLocator) => Promise<T>,
|
|
730
746
|
fields: Partial<RecordableFields> = {},
|
|
731
747
|
): Promise<Wrapped<T>> =>
|
|
732
|
-
backend.pageOp(
|
|
733
|
-
|
|
734
|
-
|
|
748
|
+
backend.pageOp(
|
|
749
|
+
action,
|
|
750
|
+
{ selector: label, ...fields },
|
|
751
|
+
(page) => readable(() => fn(lower(page, chain))),
|
|
752
|
+
{ wrap: true },
|
|
753
|
+
) as Promise<Wrapped<T>>;
|
|
735
754
|
|
|
736
755
|
// Silent, non-recorded reads for expect(locator) matchers to poll.
|
|
737
756
|
const probe: LocatorProbe = {
|
|
@@ -848,7 +867,7 @@ export function makeLocator(
|
|
|
848
867
|
backend.pageOp(
|
|
849
868
|
"evaluate",
|
|
850
869
|
{ selector: label, description },
|
|
851
|
-
(page) => lower(page, chain).evaluate(fn as never, arg),
|
|
870
|
+
(page) => readable(() => lower(page, chain).evaluate(fn as never, arg)),
|
|
852
871
|
{ wrap: true },
|
|
853
872
|
) as Promise<Wrapped<never>>,
|
|
854
873
|
|