@ultimat3/scraping 20.1.6 → 20.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/cdp-a11y.ts +125 -0
- package/src/cdp-arm.ts +173 -0
- package/src/cdp-fake.ts +131 -2
- package/src/cdp-port.ts +46 -0
- package/src/cdp-snapshot.ts +11 -0
- package/src/cdp-target.ts +99 -166
- package/src/error-throws.ts +14 -0
- package/src/errors.ts +6 -0
- package/src/html-target.ts +49 -0
- package/src/index.ts +15 -1
- package/src/key-chord.ts +60 -0
- package/src/page-over-target.ts +30 -1
- package/src/page.ts +42 -1
- package/src/target.ts +51 -0
package/src/cdp-target.ts
CHANGED
|
@@ -5,25 +5,24 @@ import { isUltimateError } from '@ultimat3/core';
|
|
|
5
5
|
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
6
6
|
import { parse, t } from '@ultimat3/schema';
|
|
7
7
|
import { browserRecord } from './browser-record';
|
|
8
|
-
import
|
|
9
|
-
import {
|
|
10
|
-
import
|
|
8
|
+
import { axNodesFor } from './cdp-a11y';
|
|
9
|
+
import type { CdpArmInit } from './cdp-arm';
|
|
10
|
+
import { arm } from './cdp-arm';
|
|
11
|
+
import type { CdpBrowserLike, CdpFrameLike } from './cdp-port';
|
|
12
|
+
import {
|
|
13
|
+
clearExpression,
|
|
14
|
+
focusExpression,
|
|
15
|
+
parseSnapshots,
|
|
16
|
+
snapshotExpression,
|
|
17
|
+
} from './cdp-snapshot';
|
|
11
18
|
import { type ColorScheme, colorSchemeFeatures } from './color-scheme';
|
|
12
19
|
import { browserUnreachable, pageCrashed, scrapeNotImplemented } from './error-throws';
|
|
13
|
-
import
|
|
14
|
-
import {
|
|
15
|
-
import
|
|
16
|
-
ConsoleLine,
|
|
17
|
-
ConsoleRing,
|
|
18
|
-
NetworkEntry,
|
|
19
|
-
NetworkRing,
|
|
20
|
-
PageError,
|
|
21
|
-
PageErrorRing,
|
|
22
|
-
ResourceType,
|
|
23
|
-
} from './rings';
|
|
24
|
-
import { createRing, pageErrorEntry, RESOURCE_TYPES } from './rings';
|
|
20
|
+
import { parseKeyChord } from './key-chord';
|
|
21
|
+
import type { ConsoleLine, NetworkEntry, PageError } from './rings';
|
|
22
|
+
import { createRing } from './rings';
|
|
25
23
|
import type { SessionSnapshot } from './session-state';
|
|
26
24
|
import type {
|
|
25
|
+
AxNode,
|
|
27
26
|
CaptureOptions,
|
|
28
27
|
FrameRef,
|
|
29
28
|
GotoOptions,
|
|
@@ -49,83 +48,6 @@ const cookieSchema = t.array(
|
|
|
49
48
|
}),
|
|
50
49
|
) as unknown as StandardSchemaV1<unknown, ScrapeCookie[]>;
|
|
51
50
|
|
|
52
|
-
const asResourceType = (raw: string): ResourceType =>
|
|
53
|
-
(RESOURCE_TYPES as readonly string[]).includes(raw) ? (raw as ResourceType) : 'other';
|
|
54
|
-
|
|
55
|
-
/** The library's event payloads are `unknown` here — read structurally, never cast. */
|
|
56
|
-
const asRequest = (payload: unknown): CdpRequestLike | undefined => {
|
|
57
|
-
if (typeof payload !== 'object' || payload === null) return undefined;
|
|
58
|
-
const candidate = payload as Partial<CdpRequestLike>;
|
|
59
|
-
return typeof candidate.url === 'function' && typeof candidate.abort === 'function'
|
|
60
|
-
? (candidate as CdpRequestLike)
|
|
61
|
-
: undefined;
|
|
62
|
-
};
|
|
63
|
-
|
|
64
|
-
/**
|
|
65
|
-
* CDP's console levels, mapped onto this package's five. `warning` is the library's spelling of
|
|
66
|
-
* `warn`, `verbose` of `debug`, and everything structural (`table`, `startGroup`, `dir`) is a log
|
|
67
|
-
* line with a shape — never its own level, because `ConsoleLine.level` is what an author filters on.
|
|
68
|
-
*/
|
|
69
|
-
const CONSOLE_LEVELS: Readonly<Record<string, ConsoleLine['level']>> = {
|
|
70
|
-
error: 'error',
|
|
71
|
-
assert: 'error',
|
|
72
|
-
warning: 'warn',
|
|
73
|
-
warn: 'warn',
|
|
74
|
-
info: 'info',
|
|
75
|
-
debug: 'debug',
|
|
76
|
-
verbose: 'debug',
|
|
77
|
-
};
|
|
78
|
-
|
|
79
|
-
/**
|
|
80
|
-
* `Object.hasOwn`, never the read alone: the type word arrives off the WIRE, so
|
|
81
|
-
* `CONSOLE_LEVELS['__proto__']` answered `Object.prototype` and `['constructor']` the `Object`
|
|
82
|
-
* function — neither of which a `?? 'log'` fallback can rescue, because neither is `undefined`.
|
|
83
|
-
* `ConsoleLine.level` would then hold a value its own type says is one of five words, so the
|
|
84
|
-
* `level === 'error'` filter this ring exists for matched nothing and `JSON.stringify` dropped
|
|
85
|
-
* the field from a snapshot outright. Lowercasing is not the guard: `__proto__` and `constructor`
|
|
86
|
-
* are already lowercase. Same discriminator as `packages/flags/src/subject.ts`.
|
|
87
|
-
*/
|
|
88
|
-
const consoleLevel = (type: string): ConsoleLine['level'] => {
|
|
89
|
-
const word = type.toLowerCase();
|
|
90
|
-
return Object.hasOwn(CONSOLE_LEVELS, word) ? (CONSOLE_LEVELS[word] ?? 'log') : 'log';
|
|
91
|
-
};
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* Reads a string out of somebody else's event payload, calling an accessor THROUGH ITS OWNER.
|
|
95
|
-
*
|
|
96
|
-
* `HTTPRequest.method()` and `ConsoleMessage.type()`/`.text()` read `this` — they are methods on
|
|
97
|
-
* the library's own objects, not closures over a value. Handing the bare function to a helper
|
|
98
|
-
* (`readString(request.method)`) drops the receiver, so the accessor answers against `undefined`:
|
|
99
|
-
* on one build that throws inside the interception handler, on another it answers wrong.
|
|
100
|
-
*/
|
|
101
|
-
const readStringFrom = (owner: unknown, key: string): string | undefined => {
|
|
102
|
-
if (typeof owner !== 'object' || owner === null) return undefined;
|
|
103
|
-
const value = (owner as Record<string, unknown>)[key];
|
|
104
|
-
if (typeof value === 'string') return value;
|
|
105
|
-
if (typeof value !== 'function') return undefined;
|
|
106
|
-
const answer = (value as (this: unknown) => unknown).call(owner);
|
|
107
|
-
return typeof answer === 'string' ? answer : undefined;
|
|
108
|
-
};
|
|
109
|
-
|
|
110
|
-
/**
|
|
111
|
-
* A `pageerror` payload, read defensively — never cast, and never assumed to be an `Error`.
|
|
112
|
-
*
|
|
113
|
-
* `readStringFrom`, the same reader the console handler uses, because the payload has the same
|
|
114
|
-
* problem: `message` and `stack` are an own property on one build and an accessor on another, and
|
|
115
|
-
* a schema parse cannot call an accessor. A page can also `throw 'a string'` or throw a frozen
|
|
116
|
-
* object with no `message` at all — both reach here, and both are recorded as SOMETHING having
|
|
117
|
-
* thrown, because an entry with a poor message is still the difference between "the island threw"
|
|
118
|
-
* and silence.
|
|
119
|
-
*/
|
|
120
|
-
const readPageError = (payload: unknown, at: number): PageError => {
|
|
121
|
-
if (typeof payload === 'string') return pageErrorEntry({ message: payload, at });
|
|
122
|
-
return pageErrorEntry({
|
|
123
|
-
message: readStringFrom(payload, 'message') ?? '',
|
|
124
|
-
stack: readStringFrom(payload, 'stack'),
|
|
125
|
-
at,
|
|
126
|
-
});
|
|
127
|
-
};
|
|
128
|
-
|
|
129
51
|
/**
|
|
130
52
|
* The failures `guard()` must NOT re-label. Two codes, each because a SECOND attempt reaches the
|
|
131
53
|
* identical answer — never because the error looked coded.
|
|
@@ -167,81 +89,8 @@ const isStructuralRefusal = (thrown: unknown): boolean =>
|
|
|
167
89
|
* of three drivers a bound the other two ignore, which is precisely the divergence
|
|
168
90
|
* `driver-parity.test.ts` exists to refuse. `DEFAULT_RING_CAPACITY` is the one bound.
|
|
169
91
|
*/
|
|
170
|
-
export interface CdpTargetInit {
|
|
171
|
-
readonly page: CdpPageLike;
|
|
92
|
+
export interface CdpTargetInit extends CdpArmInit {
|
|
172
93
|
readonly browser: CdpBrowserLike;
|
|
173
|
-
readonly rules: InterceptRules;
|
|
174
|
-
readonly clock: ScrapeClock;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
/**
|
|
178
|
-
* Everything `arm()` writes into. Named rather than positional: three rings of near-identical
|
|
179
|
-
* type plus a latch is a call site nobody can read, and swapping two of them is a mistake the
|
|
180
|
-
* compiler cannot catch.
|
|
181
|
-
*/
|
|
182
|
-
interface CdpSinks {
|
|
183
|
-
readonly network: NetworkRing;
|
|
184
|
-
readonly console: ConsoleRing;
|
|
185
|
-
readonly pageErrors: PageErrorRing;
|
|
186
|
-
readonly crashed: { value: string | undefined };
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
/**
|
|
190
|
-
* Interception is armed BEFORE the first navigation and refuses at the request, not after the
|
|
191
|
-
* response — an `allowHosts` that reported afterwards would be a log line about bytes that
|
|
192
|
-
* already left the container.
|
|
193
|
-
*/
|
|
194
|
-
async function arm(init: CdpTargetInit, sinks: CdpSinks): Promise<void> {
|
|
195
|
-
const { network, console: console_, pageErrors, crashed } = sinks;
|
|
196
|
-
await init.page.setRequestInterception(true);
|
|
197
|
-
init.page.on('request', (payload) => {
|
|
198
|
-
const request = asRequest(payload);
|
|
199
|
-
if (request === undefined) return;
|
|
200
|
-
const url = request.url();
|
|
201
|
-
const type = asResourceType(request.resourceType());
|
|
202
|
-
// The METHOD the browser is actually sending. Recording every request as a GET made
|
|
203
|
-
// `page.network()` — which `X_SCRAPE_HTTP_FAILED`'s own fix line tells the reader to open —
|
|
204
|
-
// misreport every POST and PUT the page made.
|
|
205
|
-
const method = readStringFrom(request, 'method') ?? 'GET';
|
|
206
|
-
const verdict = interceptVerdict(url, type, init.rules);
|
|
207
|
-
const at = init.clock.now().getTime();
|
|
208
|
-
if (verdict === 'allow') {
|
|
209
|
-
network.push({ method, url, resourceType: type, at });
|
|
210
|
-
void request.continue();
|
|
211
|
-
return;
|
|
212
|
-
}
|
|
213
|
-
network.push(refusalEntry(url, type, verdict, at, method));
|
|
214
|
-
void request.abort();
|
|
215
|
-
});
|
|
216
|
-
init.page.on('console', (payload) => {
|
|
217
|
-
console_.push({
|
|
218
|
-
level: consoleLevel(readStringFrom(payload, 'type') ?? ''),
|
|
219
|
-
text: readStringFrom(payload, 'text') ?? '',
|
|
220
|
-
at: init.clock.now().getTime(),
|
|
221
|
-
});
|
|
222
|
-
});
|
|
223
|
-
/**
|
|
224
|
-
* The page threw and nothing caught it. THE gap this ring closes: a screenshot of an island
|
|
225
|
-
* that threw during hydration is a picture of the server-rendered markup, indistinguishable
|
|
226
|
-
* from a page that worked — and `console` does not carry it, because throwing calls no console
|
|
227
|
-
* method. Subscribed here, beside the others, so a target is observing before its first
|
|
228
|
-
* navigation: an exception raised during load has no second chance to be recorded.
|
|
229
|
-
*
|
|
230
|
-
* NOT the same event as `error` below, and the difference is the whole reason this is a
|
|
231
|
-
* separate handler: puppeteer's `pageerror` is "an uncaught exception happens within the page"
|
|
232
|
-
* and its `error` is "the page crashes" (`PageEvent.PageError` / `PageEvent.Error`). One is the
|
|
233
|
-
* app being broken and the session is fine; the other is the tab being gone. Recording a
|
|
234
|
-
* `pageerror` into `crashed` would make every scrape of a page with one bad island answer
|
|
235
|
-
* X_SCRAPE_PAGE_CRASHED — a code registered `terminal` — for a page still perfectly usable.
|
|
236
|
-
*/
|
|
237
|
-
init.page.on('pageerror', (payload) => {
|
|
238
|
-
pageErrors.push(readPageError(payload, init.clock.now().getTime()));
|
|
239
|
-
});
|
|
240
|
-
// A renderer that dies must be a CODE, not a hang: every later call answers X_SCRAPE_PAGE_CRASHED
|
|
241
|
-
// instead of waiting out its own timeout against a tab that is gone.
|
|
242
|
-
init.page.on('error', (payload) => {
|
|
243
|
-
crashed.value = readStringFrom(payload, 'message') ?? 'renderer crashed';
|
|
244
|
-
});
|
|
245
94
|
}
|
|
246
95
|
|
|
247
96
|
export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
|
|
@@ -319,7 +168,26 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
|
|
|
319
168
|
await frame.select(selector, ...values);
|
|
320
169
|
}),
|
|
321
170
|
evaluate: (expression) => guard('evaluate', () => frame.evaluate(expression)),
|
|
171
|
+
// The PAGE's, listed rather than inherited so the next reader sees it was decided: a browser
|
|
172
|
+
// has one keyboard, and the focused element — set with the frame's own `focus` below — is what
|
|
173
|
+
// routes the chord into the frame's document.
|
|
174
|
+
press: (chord) => parent.press(chord),
|
|
322
175
|
frames: () => Promise.resolve([]),
|
|
176
|
+
focus: (selector) =>
|
|
177
|
+
guard('focus', async () => {
|
|
178
|
+
await frame.evaluate(focusExpression(selector));
|
|
179
|
+
}),
|
|
180
|
+
// `Accessibility.getPartialAXTree` is addressed by backend node id and `DOM.querySelectorAll`
|
|
181
|
+
// by a document's node id, so a frame read needs the frame's own document — a second
|
|
182
|
+
// `DOM.getDocument` scoped by frame id that `cdp-a11y.ts` does not perform. Refused by name
|
|
183
|
+
// rather than answered from the PARENT document, which is what the spread alone would do.
|
|
184
|
+
accessibility: (_selector, _max): Promise<readonly AxNode[]> =>
|
|
185
|
+
Promise.reject(
|
|
186
|
+
scrapeNotImplemented(
|
|
187
|
+
'accessibility() on a frame of the puppeteer driver',
|
|
188
|
+
'read the accessibility tree through the page — page.accessibility(selector) — which covers the top-level document only',
|
|
189
|
+
),
|
|
190
|
+
),
|
|
323
191
|
});
|
|
324
192
|
|
|
325
193
|
const target: ScrapeTarget = {
|
|
@@ -351,6 +219,59 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
|
|
|
351
219
|
await init.page.select(selector, ...values);
|
|
352
220
|
}),
|
|
353
221
|
evaluate: (expression) => guard('evaluate', () => init.page.evaluate(expression)),
|
|
222
|
+
// `async`, and the parse OUTSIDE `guard()`: the chord is the caller's literal, not the
|
|
223
|
+
// browser's answer, so a bad one is `X_SCRAPE_KEY_INVALID` and never re-labelled "the browser
|
|
224
|
+
// went away" — and it is parsed before the keyboard is touched, because a refusal halfway
|
|
225
|
+
// through the sequence below would leave a modifier held for every later verb on this page.
|
|
226
|
+
press: async (chord: string): Promise<void> => {
|
|
227
|
+
const { modifiers, key } = parseKeyChord(chord);
|
|
228
|
+
await guard('press', async () => {
|
|
229
|
+
const keyboard = init.page.keyboard;
|
|
230
|
+
if (keyboard === undefined) {
|
|
231
|
+
throw scrapeNotImplemented(
|
|
232
|
+
'press() on a CDP page with no keyboard',
|
|
233
|
+
'upgrade the launcher to a puppeteer-core that exposes page.keyboard, or dispatch the key from the app under test instead of the browser',
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
// Down in the order written, the key, then up in REVERSE — the order a hand releases
|
|
237
|
+
// them, and the order the browser's own `press` with modifiers performs. `.call`-free:
|
|
238
|
+
// `keyboard` is read as an object and its methods are called through it.
|
|
239
|
+
for (const modifier of modifiers) await keyboard.down(modifier);
|
|
240
|
+
try {
|
|
241
|
+
await keyboard.press(key);
|
|
242
|
+
} finally {
|
|
243
|
+
for (const modifier of [...modifiers].reverse()) await keyboard.up(modifier);
|
|
244
|
+
}
|
|
245
|
+
});
|
|
246
|
+
},
|
|
247
|
+
focus: (selector: string): Promise<void> =>
|
|
248
|
+
guard('focus', async () => {
|
|
249
|
+
const focus = init.page.focus;
|
|
250
|
+
if (typeof focus !== 'function') {
|
|
251
|
+
throw scrapeNotImplemented(
|
|
252
|
+
'focus() on a CDP page with no focus() method',
|
|
253
|
+
'upgrade the launcher to a puppeteer-core that exposes page.focus(), or click the element instead',
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
// `.call`, for `setColorScheme`'s reason: the member is read off the object.
|
|
257
|
+
await focus.call(init.page, selector);
|
|
258
|
+
}),
|
|
259
|
+
accessibility: (selector: string, max: number): Promise<readonly AxNode[]> =>
|
|
260
|
+
guard('accessibility', async () => {
|
|
261
|
+
const createSession = init.page.createCDPSession;
|
|
262
|
+
if (typeof createSession !== 'function') {
|
|
263
|
+
throw scrapeNotImplemented(
|
|
264
|
+
'accessibility() on a CDP page with no createCDPSession() method',
|
|
265
|
+
'upgrade the launcher to a puppeteer-core that exposes page.createCDPSession(), or assert on the markup with page.query() — which reads attributes, not what the browser computed',
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
const session = await createSession.call(init.page);
|
|
269
|
+
try {
|
|
270
|
+
return await axNodesFor(session, selector, max);
|
|
271
|
+
} finally {
|
|
272
|
+
await session.detach();
|
|
273
|
+
}
|
|
274
|
+
}),
|
|
354
275
|
setOfflineMode: (enabled: boolean): Promise<void> =>
|
|
355
276
|
guard('setOfflineMode', async () => {
|
|
356
277
|
const source = init.page as { setOfflineMode?: (value: boolean) => Promise<void> };
|
|
@@ -379,6 +300,18 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
|
|
|
379
300
|
// puppeteer's own `Page` needs it.
|
|
380
301
|
await emulate.call(init.page, colorSchemeFeatures(scheme));
|
|
381
302
|
}),
|
|
303
|
+
prepare: (expression: string): Promise<void> =>
|
|
304
|
+
guard('prepare', async () => {
|
|
305
|
+
// Read off the object and called through it, for `setColorScheme`'s reason.
|
|
306
|
+
const evaluateOnNewDocument = init.page.evaluateOnNewDocument;
|
|
307
|
+
if (typeof evaluateOnNewDocument !== 'function') {
|
|
308
|
+
throw scrapeNotImplemented(
|
|
309
|
+
'prepare() on a CDP page with no evaluateOnNewDocument() method',
|
|
310
|
+
'upgrade the launcher to a puppeteer-core that exposes page.evaluateOnNewDocument(), or seed the state from the app under test instead of the browser',
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
await evaluateOnNewDocument.call(init.page, expression);
|
|
314
|
+
}),
|
|
382
315
|
screenshot: (options: CaptureOptions) =>
|
|
383
316
|
guard('screenshot', async () => {
|
|
384
317
|
// `fullPage` is OMITTED when a clip is given rather than sent as `false`: the two are
|
package/src/error-throws.ts
CHANGED
|
@@ -437,3 +437,17 @@ export const captureClipOnPdf = (clip: CaptureClip): ScrapeError =>
|
|
|
437
437
|
fix: 'call page.screenshot({ clip }) for one component, or page.pdf() with no clip for the document',
|
|
438
438
|
meta: { clip: { ...clip } },
|
|
439
439
|
});
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* A chord that cannot be pressed. Refused before ANY key goes down, which is the reason it is a
|
|
443
|
+
* parse and not a try: `press()` holds every modifier, presses the key, then releases — and a
|
|
444
|
+
* chord refused halfway through would leave `Control` held on the page for every later verb.
|
|
445
|
+
* Terminal for `X_SCRAPE_CAPTURE_INVALID`'s reason: the chord is the caller's own literal.
|
|
446
|
+
*/
|
|
447
|
+
export const keyInvalid = (chord: string, reason: string): ScrapeError =>
|
|
448
|
+
new ScrapeError({
|
|
449
|
+
code: 'X_SCRAPE_KEY_INVALID',
|
|
450
|
+
cause: `the key chord ${JSON.stringify(chord)} ${reason}`,
|
|
451
|
+
fix: "spell the chord as zero or more of Meta, Control, Alt, Shift joined by '+' and then the key — 'Meta+K', 'Escape', 'Shift+Tab' — using the browser's own key names",
|
|
452
|
+
meta: { chord },
|
|
453
|
+
});
|
package/src/errors.ts
CHANGED
|
@@ -38,6 +38,7 @@ export const SCRAPE_OWNED_ERROR_CODES = [
|
|
|
38
38
|
'X_SCRAPE_SESSION_EXPIRED',
|
|
39
39
|
'X_SCRAPE_PROMPT_UNANSWERED',
|
|
40
40
|
'X_SCRAPE_BLOCKED',
|
|
41
|
+
'X_SCRAPE_KEY_INVALID',
|
|
41
42
|
] as const;
|
|
42
43
|
|
|
43
44
|
/**
|
|
@@ -89,6 +90,7 @@ export const SCRAPE_ERROR_TITLES: Readonly<Record<ScrapeOwnedErrorCode, string>>
|
|
|
89
90
|
X_SCRAPE_SESSION_EXPIRED: 'the restored session is no longer valid and nothing can renew it',
|
|
90
91
|
X_SCRAPE_PROMPT_UNANSWERED: 'a login step asked for a code and nothing answered',
|
|
91
92
|
X_SCRAPE_BLOCKED: 'the site refused this client — the identity is spent',
|
|
93
|
+
X_SCRAPE_KEY_INVALID: 'the key chord names no key a browser can press',
|
|
92
94
|
};
|
|
93
95
|
|
|
94
96
|
// One unconditional call, so a second package claiming one of these codes throws
|
|
@@ -186,6 +188,10 @@ export const SCRAPE_ERROR_RETRY = {
|
|
|
186
188
|
X_SCRAPE_AUTH_FAILED: 'terminal',
|
|
187
189
|
X_SCRAPE_SESSION_EXPIRED: 'terminal',
|
|
188
190
|
X_SCRAPE_PROMPT_UNANSWERED: 'terminal',
|
|
191
|
+
// A declaration error, for `X_SCRAPE_CAPTURE_INVALID`'s reason: the chord is the caller's own
|
|
192
|
+
// literal, attempt 2 passes the identical string, and a retry is a browser launch for no chance
|
|
193
|
+
// of a different parse. Refused before any key goes down, so no modifier is left held.
|
|
194
|
+
X_SCRAPE_KEY_INVALID: 'terminal',
|
|
189
195
|
} as const satisfies Readonly<Record<ScrapeOwnedErrorCode, 'retryable' | 'terminal'>>;
|
|
190
196
|
|
|
191
197
|
registerErrorRetry(SCRAPE_ERROR_RETRY);
|
package/src/html-target.ts
CHANGED
|
@@ -19,6 +19,7 @@ import { queryHtml } from './html-query';
|
|
|
19
19
|
import { markupRequests } from './html-requests';
|
|
20
20
|
import type { InterceptRules } from './intercept';
|
|
21
21
|
import { interceptVerdict, refusalEntry } from './intercept';
|
|
22
|
+
import { parseKeyChord } from './key-chord';
|
|
22
23
|
import type { PageRecording } from './recording';
|
|
23
24
|
import { splitDownload } from './recording';
|
|
24
25
|
import type { ConsoleLine, NetworkEntry, PageError } from './rings';
|
|
@@ -26,6 +27,7 @@ import { createRing } from './rings';
|
|
|
26
27
|
import type { SessionSnapshot } from './session-state';
|
|
27
28
|
import { EMPTY_SESSION } from './session-state';
|
|
28
29
|
import type {
|
|
30
|
+
AxNode,
|
|
29
31
|
CaptureOptions,
|
|
30
32
|
ElementSnapshot,
|
|
31
33
|
FrameRef,
|
|
@@ -296,6 +298,12 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
|
|
|
296
298
|
type: (selector, text) => typeIn(document, selector, text),
|
|
297
299
|
clear: (selector) => setIn(document, selector, ''),
|
|
298
300
|
select: (selector, values) => setIn(document, selector, values[0] ?? ''),
|
|
301
|
+
// The frame's OWN document, for `click`'s reason: inheriting `base.focus` would refuse a
|
|
302
|
+
// selector that exists only in the frame, and accept one that exists only in the parent.
|
|
303
|
+
async focus(selector: string): Promise<void> {
|
|
304
|
+
const element = await atIn(document, selector, 0);
|
|
305
|
+
if (element === undefined) throw fixtureMissing(`${url} ${selector}`, init.source);
|
|
306
|
+
},
|
|
299
307
|
frames: () => Promise.resolve([]),
|
|
300
308
|
};
|
|
301
309
|
};
|
|
@@ -326,6 +334,35 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
|
|
|
326
334
|
clear: (selector: string): Promise<void> => setIn(pageDocument, selector, ''),
|
|
327
335
|
select: (selector: string, values: readonly string[]): Promise<void> =>
|
|
328
336
|
setIn(pageDocument, selector, values[0] ?? ''),
|
|
337
|
+
/**
|
|
338
|
+
* PARSED and then resolved, for `setColorScheme`'s reason and not `setOfflineMode`'s. There is
|
|
339
|
+
* no JS engine here, so no `keydown` listener can hear the chord — but the one thing this
|
|
340
|
+
* driver CAN be wrong about, the chord's spelling, is checked by the same `parseKeyChord` the
|
|
341
|
+
* real driver runs, so `'Ctrl+K'` is `X_SCRAPE_KEY_INVALID` offline exactly as it is live. What
|
|
342
|
+
* it cannot prove is that the page reacted; a test that needs that runs on `localBrowser()`.
|
|
343
|
+
*/
|
|
344
|
+
press(chord: string): Promise<void> {
|
|
345
|
+
live();
|
|
346
|
+
parseKeyChord(chord);
|
|
347
|
+
return Promise.resolve();
|
|
348
|
+
},
|
|
349
|
+
/** The selector must exist, as `click` requires — a focus on nothing is a fixture gap. */
|
|
350
|
+
async focus(selector: string): Promise<void> {
|
|
351
|
+
const element = await at(selector, 0);
|
|
352
|
+
if (element === undefined) throw fixtureMissing(`${page.url} ${selector}`, init.source);
|
|
353
|
+
},
|
|
354
|
+
/**
|
|
355
|
+
* REFUSED, and `async` so it REJECTS. No accessibility engine runs here, and the honest
|
|
356
|
+
* alternative — reading `role=` and `aria-label=` off the markup — is the defect the read
|
|
357
|
+
* exists to catch: a `<div onclick>` has no computed role, and a fake that answered the
|
|
358
|
+
* attribute would pass every test against the element a screen reader cannot reach.
|
|
359
|
+
*/
|
|
360
|
+
async accessibility(_selector: string, _max: number): Promise<readonly AxNode[]> {
|
|
361
|
+
throw scrapeNotImplemented(
|
|
362
|
+
`accessibility() on the ${init.driver} driver`,
|
|
363
|
+
'run this assertion on localBrowser()/remoteBrowser(), whose accessibility() reads the tree the browser computed — an offline driver parses markup and computes no role',
|
|
364
|
+
);
|
|
365
|
+
},
|
|
329
366
|
evaluate(expression: string): Promise<unknown> {
|
|
330
367
|
live();
|
|
331
368
|
const answer = recorded(page.evaluate, expression);
|
|
@@ -375,6 +412,18 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
|
|
|
375
412
|
colorScheme = scheme === 'no-preference' ? null : scheme;
|
|
376
413
|
return Promise.resolve();
|
|
377
414
|
},
|
|
415
|
+
/**
|
|
416
|
+
* ACCEPTED, for `setColorScheme`'s reason and not `setOfflineMode`'s. This driver parses
|
|
417
|
+
* markup and executes none of it, so there is no document script for the expression to run
|
|
418
|
+
* ahead of — and no assertion a resolved promise could let through: what a prepared script
|
|
419
|
+
* seeds is the INPUT to a boot this driver never runs. Refusing would cost `x shot` and every
|
|
420
|
+
* `ui.*` tool their unit tests on a machine with no Chrome, for an outcome that does not exist
|
|
421
|
+
* here. `live()` still refuses a closed target, which is the one thing this can be wrong about
|
|
422
|
+
* — and `async`, so that refusal REJECTS rather than escaping a caller's `.catch()`.
|
|
423
|
+
*/
|
|
424
|
+
async prepare(_expression: string): Promise<void> {
|
|
425
|
+
live();
|
|
426
|
+
},
|
|
378
427
|
screenshot: (options: CaptureOptions): Promise<Uint8Array> =>
|
|
379
428
|
Promise.resolve(framedPng(options.clip, colorScheme)),
|
|
380
429
|
pdf: (_options: CaptureOptions): Promise<Uint8Array> => Promise.resolve(FAKE_PDF),
|
package/src/index.ts
CHANGED
|
@@ -21,15 +21,23 @@ export { burnSession, createPrompt, ensureAuthenticated, restorableSession } fro
|
|
|
21
21
|
export { browserRecord } from './browser-record';
|
|
22
22
|
export type { CaptureClip, CaptureFraming } from './capture-clip';
|
|
23
23
|
export { assertCaptureFraming } from './capture-clip';
|
|
24
|
+
export { axNodesFor } from './cdp-a11y';
|
|
24
25
|
export type {
|
|
25
26
|
CdpBrowserLike,
|
|
26
27
|
CdpFrameLike,
|
|
28
|
+
CdpKeyboardLike,
|
|
27
29
|
CdpLauncherLike,
|
|
28
30
|
CdpPageLike,
|
|
29
31
|
CdpRequestLike,
|
|
30
32
|
CdpScreenshotOptions,
|
|
33
|
+
CdpSessionLike,
|
|
31
34
|
} from './cdp-port';
|
|
32
|
-
export {
|
|
35
|
+
export {
|
|
36
|
+
clearExpression,
|
|
37
|
+
focusExpression,
|
|
38
|
+
parseSnapshots,
|
|
39
|
+
snapshotExpression,
|
|
40
|
+
} from './cdp-snapshot';
|
|
33
41
|
export type { CdpTargetInit } from './cdp-target';
|
|
34
42
|
export { CDP_DRIVER, cdpTarget } from './cdp-target';
|
|
35
43
|
export type { Deadline, ScrapeClock, TestScrapeClock } from './clock';
|
|
@@ -62,6 +70,7 @@ export {
|
|
|
62
70
|
fixtureStale,
|
|
63
71
|
hostBlocked,
|
|
64
72
|
httpFailed,
|
|
73
|
+
keyInvalid,
|
|
65
74
|
notActionable,
|
|
66
75
|
outputInvalid,
|
|
67
76
|
pageCrashed,
|
|
@@ -117,9 +126,12 @@ export type { RedirectHop } from './http-redirect';
|
|
|
117
126
|
export { MAX_REDIRECT_HOPS, redirectHop } from './http-redirect';
|
|
118
127
|
export type { InterceptRules, InterceptVerdict } from './intercept';
|
|
119
128
|
export { interceptVerdict, refusalEntry } from './intercept';
|
|
129
|
+
export type { KeyChord, KeyModifier } from './key-chord';
|
|
130
|
+
export { KEY_MODIFIERS, parseKeyChord } from './key-chord';
|
|
120
131
|
export type { OfflineSessionInit } from './offline-session';
|
|
121
132
|
export { openOfflineSession } from './offline-session';
|
|
122
133
|
export type {
|
|
134
|
+
AccessibilityOptions,
|
|
123
135
|
CaptureRequest,
|
|
124
136
|
DownloadRequest,
|
|
125
137
|
ElementValue,
|
|
@@ -127,6 +139,7 @@ export type {
|
|
|
127
139
|
ScrapePage,
|
|
128
140
|
WaitOptions,
|
|
129
141
|
} from './page';
|
|
142
|
+
export { DEFAULT_ACCESSIBILITY_MAX } from './page';
|
|
130
143
|
export type { PageContext } from './page-over-target';
|
|
131
144
|
export { pageOverTarget } from './page-over-target';
|
|
132
145
|
export type { Pacer } from './rate';
|
|
@@ -197,6 +210,7 @@ export {
|
|
|
197
210
|
storageSessionStore,
|
|
198
211
|
} from './session-state';
|
|
199
212
|
export type {
|
|
213
|
+
AxNode,
|
|
200
214
|
CaptureOptions,
|
|
201
215
|
ElementBox,
|
|
202
216
|
ElementSnapshot,
|
package/src/key-chord.ts
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// A key chord, parsed ONCE and refused before any key goes down.
|
|
2
|
+
//
|
|
3
|
+
// `'Meta+K'` is the spelling every scraper in the audit wrote by hand, and every one of them split
|
|
4
|
+
// on `+` and handed the pieces to the browser unchecked — so `'Ctrl+K'` held nothing (the browser
|
|
5
|
+
// knows `Control`, not `Ctrl`), pressed `K`, and the test went green with the palette closed. The
|
|
6
|
+
// grammar is small enough to state in full: zero or more modifiers, joined by `+`, then one key,
|
|
7
|
+
// spelled the way the browser spells it. The parse is the whole check, and it runs on every driver
|
|
8
|
+
// — an offline driver has no keyboard, but it can still tell a caller the chord is wrong.
|
|
9
|
+
|
|
10
|
+
import { keyInvalid } from './error-throws';
|
|
11
|
+
|
|
12
|
+
/** The four modifiers a browser holds, under the names the browser's own keyboard uses. */
|
|
13
|
+
export const KEY_MODIFIERS = ['Meta', 'Control', 'Alt', 'Shift'] as const;
|
|
14
|
+
|
|
15
|
+
export type KeyModifier = (typeof KEY_MODIFIERS)[number];
|
|
16
|
+
|
|
17
|
+
export interface KeyChord {
|
|
18
|
+
/** In the order written, which is the order they go down; they come up in reverse. */
|
|
19
|
+
readonly modifiers: readonly KeyModifier[];
|
|
20
|
+
/** The browser's own key name — `K`, `Enter`, `Escape`, `ArrowDown`. Never empty. */
|
|
21
|
+
readonly key: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const isModifier = (word: string): word is KeyModifier =>
|
|
25
|
+
(KEY_MODIFIERS as readonly string[]).includes(word);
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* `'Meta+K'` → `{ modifiers: ['Meta'], key: 'K' }`. Refuses, with `X_SCRAPE_KEY_INVALID`, an
|
|
29
|
+
* unknown modifier, an empty key, a trailing `+`, and a modifier written twice — each is a chord
|
|
30
|
+
* the browser would hold and release differently from what the caller meant, and a chord is the
|
|
31
|
+
* caller's own literal, so the refusal is terminal.
|
|
32
|
+
*
|
|
33
|
+
* The literal `+` key has no spelling in this grammar: `'+'` splits to nothing. Type it with
|
|
34
|
+
* `page.type()`, which is what a character is for; a chord is for a key the page listens to.
|
|
35
|
+
*/
|
|
36
|
+
export function parseKeyChord(chord: string): KeyChord {
|
|
37
|
+
if (chord === '') throw keyInvalid(chord, 'is empty, so there is no key to press');
|
|
38
|
+
const parts = chord.split('+');
|
|
39
|
+
const key = parts.at(-1) ?? '';
|
|
40
|
+
if (key === '') {
|
|
41
|
+
throw keyInvalid(
|
|
42
|
+
chord,
|
|
43
|
+
"ends in '+', so it names modifiers and no key — the last segment is the key to press",
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
const modifiers: KeyModifier[] = [];
|
|
47
|
+
for (const word of parts.slice(0, -1)) {
|
|
48
|
+
if (!isModifier(word)) {
|
|
49
|
+
throw keyInvalid(
|
|
50
|
+
chord,
|
|
51
|
+
`names ${JSON.stringify(word)} as a modifier, and the browser holds only ${KEY_MODIFIERS.join(', ')}`,
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
if (modifiers.includes(word)) {
|
|
55
|
+
throw keyInvalid(chord, `holds ${word} twice, and a key held twice comes up once`);
|
|
56
|
+
}
|
|
57
|
+
modifiers.push(word);
|
|
58
|
+
}
|
|
59
|
+
return { modifiers, key };
|
|
60
|
+
}
|
package/src/page-over-target.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// driver a test uses and the driver production uses.
|
|
5
5
|
|
|
6
6
|
import type { Secret } from '@ultimat3/core';
|
|
7
|
-
import { isSecret, revealSecret } from '@ultimat3/core';
|
|
7
|
+
import { finiteCount, isSecret, revealSecret } from '@ultimat3/core';
|
|
8
8
|
import type { ActionabilityState } from './actionability';
|
|
9
9
|
import { awaitActionable } from './actionability';
|
|
10
10
|
import { assertCaptureFraming } from './capture-clip';
|
|
@@ -14,6 +14,7 @@ import type { ColorScheme } from './color-scheme';
|
|
|
14
14
|
import { hostBlocked, secretExposed, selectorMissing } from './error-throws';
|
|
15
15
|
import { hostDecision } from './hosts';
|
|
16
16
|
import type {
|
|
17
|
+
AccessibilityOptions,
|
|
17
18
|
CaptureRequest,
|
|
18
19
|
DownloadRequest,
|
|
19
20
|
ElementValue,
|
|
@@ -21,10 +22,12 @@ import type {
|
|
|
21
22
|
ScrapePage,
|
|
22
23
|
WaitOptions,
|
|
23
24
|
} from './page';
|
|
25
|
+
import { DEFAULT_ACCESSIBILITY_MAX } from './page';
|
|
24
26
|
import type { RobotsGate } from './robots';
|
|
25
27
|
import type { ScrapeSecrets } from './secrets';
|
|
26
28
|
import { safeConsole, safeHtml, safeNetwork, safePageErrors } from './secrets';
|
|
27
29
|
import type {
|
|
30
|
+
AxNode,
|
|
28
31
|
CaptureOptions,
|
|
29
32
|
ElementSnapshot,
|
|
30
33
|
ScrapeCookie,
|
|
@@ -131,6 +134,28 @@ function frameOver(
|
|
|
131
134
|
await wait(selector, options, 'actionable');
|
|
132
135
|
await (await resolve()).select(selector, values);
|
|
133
136
|
},
|
|
137
|
+
// `actionable`, like `click`: focus moved to a hidden or disabled control is focus the page
|
|
138
|
+
// will not keep, and the chord that follows lands on the body.
|
|
139
|
+
async focus(selector, options): Promise<void> {
|
|
140
|
+
await wait(selector, options, 'actionable');
|
|
141
|
+
await (await resolve()).focus(selector);
|
|
142
|
+
},
|
|
143
|
+
// `async`, for `download()`'s reason: a third-party target that throws synchronously from a
|
|
144
|
+
// promise-typed method must still reach the caller's `.catch()`.
|
|
145
|
+
async press(chord): Promise<void> {
|
|
146
|
+
await (await resolve()).press(chord);
|
|
147
|
+
},
|
|
148
|
+
async accessibility(selector, options?: AccessibilityOptions): Promise<readonly AxNode[]> {
|
|
149
|
+
// Screened, never `??`-defaulted alone: `NaN` is not nullish, and `slice(0, NaN)` is
|
|
150
|
+
// `slice(0, 0)` — an empty answer that reads as "nothing matched" (`finiteCount`, core).
|
|
151
|
+
const max = finiteCount(
|
|
152
|
+
'page.accessibility()',
|
|
153
|
+
'max',
|
|
154
|
+
options?.max ?? DEFAULT_ACCESSIBILITY_MAX,
|
|
155
|
+
1,
|
|
156
|
+
);
|
|
157
|
+
return await (await resolve()).accessibility(selector, max);
|
|
158
|
+
},
|
|
134
159
|
// Resolved through `resolve()` like every other verb, which is what makes a FRAME's `query`
|
|
135
160
|
// read the frame's document: `resolve` is `resolveChild` there, and both drivers override
|
|
136
161
|
// `query` on the frame target. A forward that reached for a captured page target instead
|
|
@@ -249,6 +274,10 @@ export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePa
|
|
|
249
274
|
async colorScheme(scheme: ColorScheme): Promise<void> {
|
|
250
275
|
await target.setColorScheme(scheme);
|
|
251
276
|
},
|
|
277
|
+
// `async`, for `offline()`'s reason.
|
|
278
|
+
async prepare(expression: string): Promise<void> {
|
|
279
|
+
await target.prepare(expression);
|
|
280
|
+
},
|
|
252
281
|
cookies: (): Promise<readonly ScrapeCookie[]> => target.cookies(),
|
|
253
282
|
session: () => target.session(),
|
|
254
283
|
// Redacted BY VALUE on the way out, the same pass `html()` makes. A console line and a request
|