@ultimat3/scraping 23.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -5
- package/package.json +6 -6
- package/src/artifacts.ts +2 -4
- package/src/auth.ts +93 -17
- package/src/cdp-arm.ts +65 -1
- package/src/cdp-port.ts +21 -0
- package/src/cdp-snapshot.ts +11 -3
- package/src/cdp-target.ts +82 -58
- package/src/clock.ts +4 -9
- package/src/driver-cdp.ts +12 -3
- package/src/driver.ts +1 -1
- package/src/error-throws-session.ts +38 -1
- package/src/error-throws.ts +6 -16
- package/src/errors.ts +8 -2
- package/src/event-prompt.ts +3 -3
- package/src/expect.ts +21 -4
- package/src/html-target.ts +14 -19
- package/src/http-recorded.ts +3 -1
- package/src/http.ts +47 -10
- package/src/index.ts +22 -17
- package/src/intercept.ts +2 -2
- package/src/launch-args.ts +51 -0
- package/src/page-over-target.ts +14 -6
- package/src/pinned-host.ts +65 -0
- package/src/rate.ts +1 -1
- package/src/recover.ts +4 -3
- package/src/rings.ts +3 -3
- package/src/robots-fetch.ts +105 -21
- package/src/robots.ts +50 -12
- package/src/scrape-run.ts +42 -25
- package/src/scrape.ts +5 -4
- package/src/secrets.ts +61 -9
- package/src/session-state.ts +11 -0
- package/src/target.ts +14 -2
- package/src/url-secrets.ts +68 -7
- package/src/usage.ts +1 -1
- package/src/cdp-fake.ts +0 -420
- package/src/hosts.ts +0 -4
package/src/html-target.ts
CHANGED
|
@@ -5,16 +5,11 @@
|
|
|
5
5
|
// through to the network would make a green offline suite that is secretly hitting production —
|
|
6
6
|
// the exact failure an offline driver exists to prevent.
|
|
7
7
|
|
|
8
|
+
import { NotImplementedError } from '@ultimat3/core';
|
|
8
9
|
import type { CaptureClip } from './capture-clip';
|
|
9
10
|
import type { ScrapeClock } from './clock';
|
|
10
11
|
import type { ColorScheme } from './color-scheme';
|
|
11
|
-
import {
|
|
12
|
-
browserUnreachable,
|
|
13
|
-
downloadTimeout,
|
|
14
|
-
fixtureMissing,
|
|
15
|
-
fixtureStale,
|
|
16
|
-
scrapeNotImplemented,
|
|
17
|
-
} from './error-throws';
|
|
12
|
+
import { browserUnreachable, downloadTimeout, fixtureMissing, fixtureStale } from './error-throws';
|
|
18
13
|
import { queryHtml } from './html-query';
|
|
19
14
|
import { markupRequests } from './html-requests';
|
|
20
15
|
import type { InterceptRules } from './intercept';
|
|
@@ -23,7 +18,7 @@ import { parseKeyChord } from './key-chord';
|
|
|
23
18
|
import type { PageRecording } from './recording';
|
|
24
19
|
import { splitDownload } from './recording';
|
|
25
20
|
import type { ConsoleLine, NetworkEntry, PageError } from './rings';
|
|
26
|
-
import {
|
|
21
|
+
import { boundedRing } from './rings';
|
|
27
22
|
import type { SessionSnapshot } from './session-state';
|
|
28
23
|
import { EMPTY_SESSION } from './session-state';
|
|
29
24
|
import type {
|
|
@@ -124,14 +119,14 @@ interface OfflineDocument {
|
|
|
124
119
|
}
|
|
125
120
|
|
|
126
121
|
export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
|
|
127
|
-
const consoleRing =
|
|
128
|
-
const networkRing =
|
|
122
|
+
const consoleRing = boundedRing<ConsoleLine>();
|
|
123
|
+
const networkRing = boundedRing<NetworkEntry>();
|
|
129
124
|
// Built and never pushed to, deliberately: this target parses markup and executes none of it, so
|
|
130
125
|
// there is no uncaught exception for it to have. It ANSWERS rather than omitting the ring —
|
|
131
126
|
// `page.pageErrors()` returning `[]` here is the honest "nothing threw, and nothing could",
|
|
132
127
|
// where a missing ring would be a page method that throws on two of the three drivers.
|
|
133
128
|
// `driver-parity.test.ts` pins the divergence, beside the box/hit-target one it already carries.
|
|
134
|
-
const pageErrorRing =
|
|
129
|
+
const pageErrorRing = boundedRing<PageError>();
|
|
135
130
|
const overlay = new Map<string, string>();
|
|
136
131
|
/**
|
|
137
132
|
* One overlay per FRAME, held here rather than in the frame target, because `frames()` builds a
|
|
@@ -358,10 +353,10 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
|
|
|
358
353
|
* attribute would pass every test against the element a screen reader cannot reach.
|
|
359
354
|
*/
|
|
360
355
|
async accessibility(_selector: string, _max: number): Promise<readonly AxNode[]> {
|
|
361
|
-
throw
|
|
362
|
-
`accessibility() on the ${init.driver} driver`,
|
|
363
|
-
'run this assertion on localBrowser()/remoteBrowser(), whose accessibility() reads the tree the browser computed — an offline driver parses markup and computes no role',
|
|
364
|
-
);
|
|
356
|
+
throw new NotImplementedError({
|
|
357
|
+
cause: `accessibility() on the ${init.driver} driver is not implemented by this driver`,
|
|
358
|
+
fix: 'run this assertion on localBrowser()/remoteBrowser(), whose accessibility() reads the tree the browser computed — an offline driver parses markup and computes no role',
|
|
359
|
+
});
|
|
365
360
|
},
|
|
366
361
|
evaluate(expression: string): Promise<unknown> {
|
|
367
362
|
live();
|
|
@@ -381,10 +376,10 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
|
|
|
381
376
|
// `async`, so the refusal REJECTS: the method is typed `Promise<void>` and a synchronous
|
|
382
377
|
// `throw` from one jumps straight over `page.offline(true).catch(…)` at every caller.
|
|
383
378
|
async setOfflineMode(_enabled: boolean): Promise<void> {
|
|
384
|
-
throw
|
|
385
|
-
`setOfflineMode() on the ${init.driver} driver`,
|
|
386
|
-
'run this assertion on localBrowser()/remoteBrowser(), whose setOfflineMode() reaches a real browser — an offline driver has no network to cut, so it cannot prove an offline behaviour',
|
|
387
|
-
);
|
|
379
|
+
throw new NotImplementedError({
|
|
380
|
+
cause: `setOfflineMode() on the ${init.driver} driver is not implemented by this driver`,
|
|
381
|
+
fix: 'run this assertion on localBrowser()/remoteBrowser(), whose setOfflineMode() reaches a real browser — an offline driver has no network to cut, so it cannot prove an offline behaviour',
|
|
382
|
+
});
|
|
388
383
|
},
|
|
389
384
|
/**
|
|
390
385
|
* ACCEPTED and recorded, where `setOfflineMode` refuses — and the line between them is which
|
package/src/http-recorded.ts
CHANGED
|
@@ -64,7 +64,9 @@ export function recordedHttp(init: RecordedHttpInit): ScrapeHttp {
|
|
|
64
64
|
if (found === undefined) throw fixtureMissing(`${method} ${url}`, init.source);
|
|
65
65
|
if (init.maxAgeMs !== undefined && found.recordedAt !== undefined) {
|
|
66
66
|
const age = init.clock.now().getTime() - new Date(found.recordedAt).getTime();
|
|
67
|
-
|
|
67
|
+
// `!(age <= limit)`, never `age > limit`: an unparseable `recordedAt` makes `age` NaN, and
|
|
68
|
+
// `NaN > limit` is false — a recording of unknown age replayed as fresh. `auth.ts` the same.
|
|
69
|
+
if (!(age <= init.maxAgeMs)) throw fixtureStale(`${method} ${url}`, age, init.maxAgeMs);
|
|
68
70
|
}
|
|
69
71
|
init.network.push({
|
|
70
72
|
method,
|
package/src/http.ts
CHANGED
|
@@ -20,6 +20,8 @@ import type { RedirectHop } from './http-redirect';
|
|
|
20
20
|
import { MAX_REDIRECT_HOPS, redirectHop } from './http-redirect';
|
|
21
21
|
import type { InterceptRules } from './intercept';
|
|
22
22
|
import { interceptVerdict } from './intercept';
|
|
23
|
+
import type { DialTarget, HostResolve } from './pinned-host';
|
|
24
|
+
import { dialTarget, resolveHost } from './pinned-host';
|
|
23
25
|
import type { NetworkRing } from './rings';
|
|
24
26
|
import type { RobotsGate } from './robots';
|
|
25
27
|
import type { ScrapeSecrets } from './secrets';
|
|
@@ -43,6 +45,11 @@ export type ScrapeFetch = (input: string, init: ScrapeFetchInit) => Promise<Resp
|
|
|
43
45
|
export interface ScrapeFetchInit extends RequestInit {
|
|
44
46
|
/** The session's exit. A different exit IP mid-session is a different client to an anti-bot. */
|
|
45
47
|
readonly proxy?: string | undefined;
|
|
48
|
+
/**
|
|
49
|
+
* The NAME a pinned connection proves (`pinned-host.ts`): the url names the approved address,
|
|
50
|
+
* and Bun's `fetch` verifies the certificate against this instead.
|
|
51
|
+
*/
|
|
52
|
+
readonly tls?: { readonly serverName?: string | undefined } | undefined;
|
|
46
53
|
}
|
|
47
54
|
|
|
48
55
|
export interface HttpRequestInit {
|
|
@@ -114,6 +121,8 @@ export interface HttpTransportInit {
|
|
|
114
121
|
/** The run's meter: one `httpRequest` per hop on the wire, with the body bytes read for it. */
|
|
115
122
|
readonly usage?: UsageMeter | undefined;
|
|
116
123
|
readonly fetch?: ScrapeFetch | undefined;
|
|
124
|
+
/** The resolver a wildcard-admitted name is checked and pinned with. Defaults to `Bun.dns`. */
|
|
125
|
+
readonly resolve?: HostResolve | undefined;
|
|
117
126
|
}
|
|
118
127
|
|
|
119
128
|
/**
|
|
@@ -196,6 +205,22 @@ const withoutCredentials = (headers: Readonly<Record<string, string>>): Record<s
|
|
|
196
205
|
Object.entries(headers).filter(([name]) => !CROSS_ORIGIN_STRIPPED.has(name.toLowerCase())),
|
|
197
206
|
);
|
|
198
207
|
|
|
208
|
+
/**
|
|
209
|
+
* The request's headers from every source, LATER sources winning, keyed by the lower-cased name.
|
|
210
|
+
* HTTP names are case-insensitive and `Headers` APPENDS two spellings of one name: a caller's
|
|
211
|
+
* `User-Agent: Mine` beside the session's `user-agent` went out as `BrowserUA, Mine`, and a
|
|
212
|
+
* declared `Cookie` was joined to the jar's instead of replacing it.
|
|
213
|
+
*/
|
|
214
|
+
const composeHeaders = (
|
|
215
|
+
...sources: readonly Readonly<Record<string, string>>[]
|
|
216
|
+
): Record<string, string> => {
|
|
217
|
+
const out = new Map<string, string>();
|
|
218
|
+
for (const source of sources) {
|
|
219
|
+
for (const [name, value] of Object.entries(source)) out.set(name.toLowerCase(), value);
|
|
220
|
+
}
|
|
221
|
+
return Object.fromEntries(out);
|
|
222
|
+
};
|
|
223
|
+
|
|
199
224
|
/** Unparseable is not same-origin: a URL this package cannot read is one it cannot vouch for. */
|
|
200
225
|
const sameOrigin = (left: string, right: string): boolean => {
|
|
201
226
|
try {
|
|
@@ -251,12 +276,21 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
|
|
|
251
276
|
* The three gates, in one place, so the initial URL and hop seven are screened by the same code
|
|
252
277
|
* in the same order. A second copy for redirects is how the two drift.
|
|
253
278
|
*/
|
|
254
|
-
const
|
|
279
|
+
const resolve = init.resolve ?? resolveHost;
|
|
280
|
+
const screen = async (target: string): Promise<DialTarget> => {
|
|
255
281
|
if (interceptVerdict(target, 'fetch', init.rules) !== 'allow') {
|
|
256
282
|
throw hostBlocked(target, init.rules.allowHosts);
|
|
257
283
|
}
|
|
284
|
+
// Through a proxy the PROXY resolves the name, from its own network, so an answer from this
|
|
285
|
+
// box's resolver says nothing about where the request lands — and pinning would hand the
|
|
286
|
+
// proxy an address where it expects the name.
|
|
287
|
+
const dial =
|
|
288
|
+
init.proxy === undefined
|
|
289
|
+
? await dialTarget(target, init.rules.allowHosts, resolve)
|
|
290
|
+
: { url: target };
|
|
258
291
|
await init.robots?.assertAllowed(target);
|
|
259
292
|
await init.pace?.(init.signal);
|
|
293
|
+
return dial;
|
|
260
294
|
};
|
|
261
295
|
return {
|
|
262
296
|
async request(url: string, request: HttpRequestInit = {}): Promise<ScrapeResponse> {
|
|
@@ -284,7 +318,7 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
|
|
|
284
318
|
1,
|
|
285
319
|
);
|
|
286
320
|
init.onActivity?.();
|
|
287
|
-
await screen(url);
|
|
321
|
+
let dial = await screen(url);
|
|
288
322
|
const session = await init.session();
|
|
289
323
|
// `AbortSignal.timeout` and NOT `clock.sleep`: this is a deadline handed to the platform's
|
|
290
324
|
// own fetch, not a wait this package performs — and under a test clock a slept deadline
|
|
@@ -319,14 +353,17 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
|
|
|
319
353
|
const declared = credentialsInScope
|
|
320
354
|
? request.headers
|
|
321
355
|
: withoutCredentials(request.headers ?? {});
|
|
322
|
-
const response = await call(
|
|
356
|
+
const response = await call(dial.url, {
|
|
323
357
|
method: hop.method,
|
|
324
|
-
headers:
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
358
|
+
headers: composeHeaders(
|
|
359
|
+
carried,
|
|
360
|
+
session.userAgent === '' ? {} : { 'user-agent': session.userAgent },
|
|
361
|
+
cookies === undefined ? {} : { cookie: cookies },
|
|
362
|
+
declared ?? {},
|
|
363
|
+
// Last: a pinned request is reached AS the name that was screened, whatever was declared.
|
|
364
|
+
dial.host === undefined ? {} : { host: dial.host },
|
|
365
|
+
),
|
|
366
|
+
...(dial.serverName === undefined ? {} : { tls: { serverName: dial.serverName } }),
|
|
330
367
|
...(hop.body === undefined ? {} : { body: hop.body }),
|
|
331
368
|
signal: AbortSignal.any(signals),
|
|
332
369
|
// The whole point: the platform's own `follow` is what made the four lines above
|
|
@@ -360,7 +397,7 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
|
|
|
360
397
|
await discardHopBody(response);
|
|
361
398
|
if (followed >= MAX_REDIRECT_HOPS) throw redirectLoop(url, next.url, followed + 1);
|
|
362
399
|
init.onActivity?.();
|
|
363
|
-
await screen(next.url);
|
|
400
|
+
dial = await screen(next.url);
|
|
364
401
|
if (!sameOrigin(hop.url, next.url)) credentialsInScope = false;
|
|
365
402
|
hop = next;
|
|
366
403
|
}
|
package/src/index.ts
CHANGED
|
@@ -5,10 +5,7 @@
|
|
|
5
5
|
export type { ActionabilityState, ActionabilityWait } from './actionability';
|
|
6
6
|
export { awaitActionable } from './actionability';
|
|
7
7
|
export type { ArtifactRef, ArtifactWriter, ArtifactWriterInit } from './artifacts';
|
|
8
|
-
export {
|
|
9
|
-
createArtifactWriter,
|
|
10
|
-
DEFAULT_CONTENT_TYPE,
|
|
11
|
-
} from './artifacts';
|
|
8
|
+
export { artifactWriter } from './artifacts';
|
|
12
9
|
export type {
|
|
13
10
|
AuthContext,
|
|
14
11
|
PromptHandler,
|
|
@@ -16,12 +13,14 @@ export type {
|
|
|
16
13
|
PromptRequest,
|
|
17
14
|
ScrapeAuth,
|
|
18
15
|
} from './auth';
|
|
19
|
-
export { burnSession,
|
|
16
|
+
export { burnSession, ensureAuthenticated, restorableSession, scrapePrompt } from './auth';
|
|
20
17
|
export { browserRecord } from './browser-record';
|
|
21
18
|
export type { CaptureClip, CaptureFraming } from './capture-clip';
|
|
22
19
|
export { assertCaptureFraming } from './capture-clip';
|
|
23
20
|
export type {
|
|
24
21
|
CdpBrowserLike,
|
|
22
|
+
CdpBrowserSessionLike,
|
|
23
|
+
CdpBrowserTargetLike,
|
|
25
24
|
CdpFrameLike,
|
|
26
25
|
CdpKeyboardLike,
|
|
27
26
|
CdpLauncherLike,
|
|
@@ -42,8 +41,7 @@ export {
|
|
|
42
41
|
scrapeClock,
|
|
43
42
|
setScrapeClock,
|
|
44
43
|
systemScrapeClock,
|
|
45
|
-
|
|
46
|
-
throwIfAborted,
|
|
44
|
+
testScrapeClock,
|
|
47
45
|
} from './clock';
|
|
48
46
|
export type { ColorScheme } from './color-scheme';
|
|
49
47
|
export { COLOR_SCHEMES, isColorScheme } from './color-scheme';
|
|
@@ -79,11 +77,10 @@ export {
|
|
|
79
77
|
redirectLoop,
|
|
80
78
|
remoteRequired,
|
|
81
79
|
robotsDisallowed,
|
|
82
|
-
|
|
80
|
+
scrapeSessionExpired,
|
|
83
81
|
scrapeTimeout,
|
|
84
82
|
secretExposed,
|
|
85
83
|
selectorMissing,
|
|
86
|
-
sessionExpired,
|
|
87
84
|
wedged,
|
|
88
85
|
yieldCollapsed,
|
|
89
86
|
} from './error-throws';
|
|
@@ -121,8 +118,6 @@ export {
|
|
|
121
118
|
memoryYieldHistory,
|
|
122
119
|
} from './expect';
|
|
123
120
|
export { BURNS_SESSION, errorCode, NEVER_RETRIED } from './failures';
|
|
124
|
-
export type { HostDecision, HostRule } from './hosts';
|
|
125
|
-
export { ANY_HOST, hostDecision, hostMatches } from './hosts';
|
|
126
121
|
export { queryHtml } from './html-query';
|
|
127
122
|
export type { MarkupRequest } from './html-requests';
|
|
128
123
|
export type { HtmlTargetInit, RecordingLookup } from './html-target';
|
|
@@ -151,7 +146,7 @@ export type {
|
|
|
151
146
|
export type { PageContext } from './page-over-target';
|
|
152
147
|
export { pageOverTarget } from './page-over-target';
|
|
153
148
|
export type { Pacer } from './rate';
|
|
154
|
-
export {
|
|
149
|
+
export { scrapePacer } from './rate';
|
|
155
150
|
export type { HttpRecording, PageRecording } from './recording';
|
|
156
151
|
export {
|
|
157
152
|
httpRecordingSchema,
|
|
@@ -172,13 +167,21 @@ export type {
|
|
|
172
167
|
Ring,
|
|
173
168
|
} from './rings';
|
|
174
169
|
export {
|
|
175
|
-
|
|
170
|
+
boundedRing,
|
|
176
171
|
MAX_PAGE_ERROR_CHARS,
|
|
177
172
|
pageErrorEntry,
|
|
178
173
|
RESOURCE_TYPES,
|
|
179
174
|
} from './rings';
|
|
180
|
-
export type {
|
|
181
|
-
|
|
175
|
+
export type {
|
|
176
|
+
RobotsAnswer,
|
|
177
|
+
RobotsFetch,
|
|
178
|
+
RobotsGate,
|
|
179
|
+
RobotsGateInit,
|
|
180
|
+
RobotsPolicy,
|
|
181
|
+
RobotsRules,
|
|
182
|
+
RobotsUnreachable,
|
|
183
|
+
} from './robots';
|
|
184
|
+
export { parseRobots, robotsAllows, robotsGate } from './robots';
|
|
182
185
|
export type { RobotsFetchInit } from './robots-fetch';
|
|
183
186
|
export {
|
|
184
187
|
DEFAULT_ROBOTS_MAX_BYTES,
|
|
@@ -196,13 +199,13 @@ export { scrape } from './scrape';
|
|
|
196
199
|
export { DEFAULT_PAGE_TIMEOUT_MS, runScrape } from './scrape-run';
|
|
197
200
|
export type { ScrapeSecrets, SecretResolver } from './secrets';
|
|
198
201
|
export {
|
|
199
|
-
createSecretBag,
|
|
200
202
|
MIN_REDACTABLE_LENGTH,
|
|
201
203
|
redactSecrets,
|
|
202
204
|
safeConsole,
|
|
203
205
|
safeHtml,
|
|
204
206
|
safeNetwork,
|
|
205
207
|
safePageErrors,
|
|
208
|
+
secretBag,
|
|
206
209
|
} from './secrets';
|
|
207
210
|
export type {
|
|
208
211
|
ScrapeSessionStore,
|
|
@@ -213,6 +216,7 @@ export type {
|
|
|
213
216
|
export {
|
|
214
217
|
memorySessionStore,
|
|
215
218
|
parseSessionState,
|
|
219
|
+
recordVersion,
|
|
216
220
|
SESSION_SEAL_PURPOSE,
|
|
217
221
|
sessionDigest,
|
|
218
222
|
sessionKeyFor,
|
|
@@ -225,12 +229,13 @@ export type {
|
|
|
225
229
|
ElementSnapshot,
|
|
226
230
|
FrameRef,
|
|
227
231
|
GotoOptions,
|
|
232
|
+
QueryOptions,
|
|
228
233
|
ScrapeCookie,
|
|
229
234
|
ScrapeDownloadFile,
|
|
230
235
|
ScrapeTarget,
|
|
231
236
|
} from './target';
|
|
232
237
|
export { endpointLabel, urlSecretValues } from './url-secrets';
|
|
233
238
|
export type { RunUsageMeter, ScrapeUsage, UsageMeter } from './usage';
|
|
234
|
-
export {
|
|
239
|
+
export { usageMeter } from './usage';
|
|
235
240
|
export type { WedgeGuard, WedgeGuardInit } from './watchdog';
|
|
236
241
|
export { DEFAULT_GRACE_MS } from './watchdog';
|
package/src/intercept.ts
CHANGED
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
// differently — `driver-parity.test.ts` asserts the same refusal on all three, and a rule
|
|
6
6
|
// re-implemented per driver is a rule that holds only on the driver nobody ships.
|
|
7
7
|
|
|
8
|
-
import type { HostRule } from '
|
|
9
|
-
import { hostDecision } from '
|
|
8
|
+
import type { HostRule } from '@ultimat3/core';
|
|
9
|
+
import { hostDecision } from '@ultimat3/core';
|
|
10
10
|
import type { NetworkEntry, ResourceType } from './rings';
|
|
11
11
|
|
|
12
12
|
export interface InterceptRules {
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// The argument list a launched browser gets: the caller's own, then the exit. The exit is the
|
|
2
|
+
// session's — the driver's `proxy` or the run's `egress` — so it is appended LAST and a caller
|
|
3
|
+
// switch that would decide the route instead is refused rather than ranked.
|
|
4
|
+
|
|
5
|
+
import { launchArgsInvalid } from './error-throws-session';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Every switch Chrome reads a proxy decision from. Matched on the switch NAME, case-folded and
|
|
9
|
+
* with either prefix POSIX Chrome accepts (`--` and `-`), so a value that merely mentions a proxy
|
|
10
|
+
* (`--user-agent=proxy-server-test`) is not refused.
|
|
11
|
+
*/
|
|
12
|
+
const ROUTE_SWITCHES = new Set([
|
|
13
|
+
'proxy-server',
|
|
14
|
+
'proxy-bypass-list',
|
|
15
|
+
'proxy-pac-url',
|
|
16
|
+
'proxy-auto-detect',
|
|
17
|
+
'no-proxy-server',
|
|
18
|
+
]);
|
|
19
|
+
|
|
20
|
+
const switchName = (arg: string): string | undefined => {
|
|
21
|
+
const match = /^--?([^=]+)/.exec(arg);
|
|
22
|
+
return match?.[1]?.toLowerCase();
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* `undefined` when the launch carries no args at all — the launcher's own defaults stay untouched
|
|
27
|
+
* then. Refused with `X_SCRAPE_LAUNCH_ARGS_INVALID` before anything starts: a browser launched on a
|
|
28
|
+
* second route is a session that reports one exit and leaves the worker by another.
|
|
29
|
+
*/
|
|
30
|
+
export function launchArgs(
|
|
31
|
+
callerArgs: unknown,
|
|
32
|
+
proxyArg: string | undefined,
|
|
33
|
+
): string[] | undefined {
|
|
34
|
+
if (callerArgs !== undefined) {
|
|
35
|
+
if (!Array.isArray(callerArgs) || !callerArgs.every((arg) => typeof arg === 'string')) {
|
|
36
|
+
throw launchArgsInvalid('is not an array of strings', false);
|
|
37
|
+
}
|
|
38
|
+
for (const arg of callerArgs as readonly string[]) {
|
|
39
|
+
const name = switchName(arg);
|
|
40
|
+
if (name !== undefined && ROUTE_SWITCHES.has(name)) {
|
|
41
|
+
throw launchArgsInvalid(
|
|
42
|
+
`sets --${name}, which decides the route this session leaves by`,
|
|
43
|
+
true,
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const args = [...((callerArgs as readonly string[] | undefined) ?? [])];
|
|
49
|
+
if (proxyArg !== undefined) args.push(`--proxy-server=${proxyArg}`);
|
|
50
|
+
return args.length === 0 ? undefined : args;
|
|
51
|
+
}
|
package/src/page-over-target.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// driver a test uses and the driver production uses.
|
|
5
5
|
|
|
6
6
|
import type { Secret } from '@ultimat3/core';
|
|
7
|
-
import { finiteCount, isSecret, revealSecret } from '@ultimat3/core';
|
|
7
|
+
import { finiteCount, hostDecision, isSecret, revealSecret } from '@ultimat3/core';
|
|
8
8
|
import type { ActionabilityState } from './actionability';
|
|
9
9
|
import { awaitActionable } from './actionability';
|
|
10
10
|
import { assertCaptureFraming } from './capture-clip';
|
|
@@ -12,7 +12,6 @@ import type { ScrapeClock } from './clock';
|
|
|
12
12
|
import { deadline } from './clock';
|
|
13
13
|
import type { ColorScheme } from './color-scheme';
|
|
14
14
|
import { hostBlocked, secretExposed, selectorMissing } from './error-throws';
|
|
15
|
-
import { hostDecision } from './hosts';
|
|
16
15
|
import type {
|
|
17
16
|
AccessibilityOptions,
|
|
18
17
|
CaptureRequest,
|
|
@@ -30,6 +29,7 @@ import type {
|
|
|
30
29
|
AxNode,
|
|
31
30
|
CaptureOptions,
|
|
32
31
|
ElementSnapshot,
|
|
32
|
+
QueryOptions,
|
|
33
33
|
ScrapeCookie,
|
|
34
34
|
ScrapeDownloadFile,
|
|
35
35
|
ScrapeTarget,
|
|
@@ -80,9 +80,13 @@ const toValue = (snapshot: ElementSnapshot): ElementValue => ({
|
|
|
80
80
|
attrs: snapshot.attrs,
|
|
81
81
|
});
|
|
82
82
|
|
|
83
|
-
async function first(
|
|
83
|
+
async function first(
|
|
84
|
+
resolve: Resolve,
|
|
85
|
+
selector: string,
|
|
86
|
+
options?: QueryOptions,
|
|
87
|
+
): Promise<ElementSnapshot | undefined> {
|
|
84
88
|
const target = await resolve();
|
|
85
|
-
return (await target.query(selector))[0];
|
|
89
|
+
return (await target.query(selector, options))[0];
|
|
86
90
|
}
|
|
87
91
|
|
|
88
92
|
function frameOver(
|
|
@@ -103,14 +107,18 @@ function frameOver(
|
|
|
103
107
|
): Promise<ElementSnapshot> => {
|
|
104
108
|
const target = await resolve();
|
|
105
109
|
lastUrl = target.url();
|
|
110
|
+
const state = options?.state ?? fallback;
|
|
111
|
+
// Only `actionable` reads `hitTarget`, so only it pays for moving the page — on every poll,
|
|
112
|
+
// because a layout shift between two can carry the element back out of view.
|
|
113
|
+
const query: QueryOptions = { reveal: state === 'actionable' };
|
|
106
114
|
return awaitActionable({
|
|
107
115
|
selector,
|
|
108
116
|
url: lastUrl,
|
|
109
|
-
state
|
|
117
|
+
state,
|
|
110
118
|
timeoutMs: timeoutFor(options),
|
|
111
119
|
clock: ctx.clock,
|
|
112
120
|
signal: ctx.signal,
|
|
113
|
-
snapshot: () => first(resolve, selector),
|
|
121
|
+
snapshot: () => first(resolve, selector, query),
|
|
114
122
|
});
|
|
115
123
|
};
|
|
116
124
|
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// The DNS half of `allowHosts`, for the two legs this package dials itself — the HTTP leg and the
|
|
2
|
+
// robots read. Core's `hostDecision` is synchronous and judges NAMES: a wildcard never admits an
|
|
3
|
+
// address literal inside the network, but `allowHosts: ['*']` still admits a hostname that
|
|
4
|
+
// resolves inward. So a name admitted by a wildcard alone is resolved here, every address
|
|
5
|
+
// classified by core's one classifier, and the connection PINNED to the approved address — the
|
|
6
|
+
// shape `@ultimat3/jobs`' `webhook-target.ts` takes for a webhook.
|
|
7
|
+
|
|
8
|
+
import type { HostRule } from '@ultimat3/core';
|
|
9
|
+
import { classifyAddress, hostMatches } from '@ultimat3/core';
|
|
10
|
+
import { hostResolvesInward } from './error-throws-session';
|
|
11
|
+
|
|
12
|
+
/** Every address a hostname answers. Injected in tests: the network is sealed in this repo. */
|
|
13
|
+
export type HostResolve = (hostname: string) => Promise<readonly string[]>;
|
|
14
|
+
|
|
15
|
+
/** `Bun.dns.lookup` — the runtime's own resolver, every family, every address. */
|
|
16
|
+
export const resolveHost: HostResolve = async (hostname) =>
|
|
17
|
+
(await Bun.dns.lookup(hostname)).map((entry) => entry.address);
|
|
18
|
+
|
|
19
|
+
/** What the transport connects to: the URL to dial, and the name it proves when pinned. */
|
|
20
|
+
export interface DialTarget {
|
|
21
|
+
readonly url: string;
|
|
22
|
+
/** The original `host[:port]`, sent as the `Host` header of a pinned request. */
|
|
23
|
+
readonly host?: string | undefined;
|
|
24
|
+
/** For `https:` — the certificate is verified against the NAME, not the address. */
|
|
25
|
+
readonly serverName?: string | undefined;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const isWildcard = (rule: HostRule): boolean => {
|
|
29
|
+
const cleaned = rule.trim();
|
|
30
|
+
return cleaned === '*' || cleaned.startsWith('*.');
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The connection for `url`, which `interceptVerdict` has ALREADY allowed. Unchanged when an exact
|
|
35
|
+
* rule names the host — the app said that host, internal or not, in a line a reviewer can see —
|
|
36
|
+
* or when the host is an address literal, which core's floor already judged. Otherwise resolved:
|
|
37
|
+
* one address that is not public refuses the request (`X_SCRAPE_HOST_BLOCKED`), and the dial is
|
|
38
|
+
* pinned to the first approved address, so a name that answers public here and private to the
|
|
39
|
+
* socket (DNS rebinding) is never reached. A resolver failure propagates: nothing unscreened is
|
|
40
|
+
* dialled, and the same name would not have resolved for `fetch` either.
|
|
41
|
+
*/
|
|
42
|
+
export async function dialTarget(
|
|
43
|
+
url: string,
|
|
44
|
+
allowHosts: readonly HostRule[],
|
|
45
|
+
resolve: HostResolve,
|
|
46
|
+
): Promise<DialTarget> {
|
|
47
|
+
const parsed = new URL(url);
|
|
48
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return { url };
|
|
49
|
+
const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
|
|
50
|
+
if (classifyAddress(hostname) !== undefined) return { url };
|
|
51
|
+
if (allowHosts.some((rule) => !isWildcard(rule) && hostMatches(hostname, rule))) return { url };
|
|
52
|
+
const addresses = await resolve(hostname);
|
|
53
|
+
const inward = addresses.map((address) => classifyAddress(address) ?? 'unclassifiable');
|
|
54
|
+
const refused =
|
|
55
|
+
addresses.length === 0 ? 'unresolvable' : inward.find((kind) => kind !== 'public');
|
|
56
|
+
if (refused !== undefined) throw hostResolvesInward(url, hostname, refused);
|
|
57
|
+
const pinned = addresses[0] ?? hostname;
|
|
58
|
+
const target = new URL(parsed.href);
|
|
59
|
+
target.hostname = pinned.includes(':') ? `[${pinned}]` : pinned;
|
|
60
|
+
return {
|
|
61
|
+
url: target.href,
|
|
62
|
+
host: parsed.host,
|
|
63
|
+
...(parsed.protocol === 'https:' ? { serverName: hostname } : {}),
|
|
64
|
+
};
|
|
65
|
+
}
|
package/src/rate.ts
CHANGED
|
@@ -9,7 +9,7 @@ export const DEFAULT_NAVIGATION_RATE = 1;
|
|
|
9
9
|
|
|
10
10
|
export type Pacer = (signal?: AbortSignal) => Promise<void>;
|
|
11
11
|
|
|
12
|
-
export function
|
|
12
|
+
export function scrapePacer(rate: number, clock: ScrapeClock): Pacer {
|
|
13
13
|
const intervalMs = 1_000 / rate;
|
|
14
14
|
let nextAt = 0;
|
|
15
15
|
return async (signal?: AbortSignal): Promise<void> => {
|
package/src/recover.ts
CHANGED
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
// Two shapes. A FUNCTION is complete and shipped — an app writes its own fallback, gets the page
|
|
4
4
|
// and the failure, and answers whether the attempt should be retried. `'agent'` is the seam for
|
|
5
5
|
// letting a model re-derive the selector from the page, and it is an honest
|
|
6
|
-
// `X_NOT_IMPLEMENTED` today
|
|
6
|
+
// `X_NOT_IMPLEMENTED` today: correct types
|
|
7
7
|
// so an app can be written against it, one labelled throw, no silent no-op.
|
|
8
8
|
|
|
9
|
-
import {
|
|
9
|
+
import { NotImplementedError } from '@ultimat3/core';
|
|
10
|
+
import { recoverRefused } from './error-throws';
|
|
10
11
|
import type { ScrapePage } from './page';
|
|
11
12
|
|
|
12
13
|
export interface RecoveryAttempt {
|
|
@@ -41,7 +42,7 @@ export async function runRecovery(recovery: Recovery, attempt: RecoveryAttempt):
|
|
|
41
42
|
// The agent path lands with `@ultimat3/ai`'s `llm()` behind it: page HTML in, a selector out,
|
|
42
43
|
// re-run once. Until it does, this throws rather than answering `false` — a recovery that
|
|
43
44
|
// silently declines is indistinguishable from one that was never configured.
|
|
44
|
-
throw
|
|
45
|
+
throw new NotImplementedError({ cause: "recover: 'agent' is not implemented", fix: AGENT_FIX });
|
|
45
46
|
}
|
|
46
47
|
const verdict: unknown = await recovery(attempt);
|
|
47
48
|
if (typeof verdict !== 'boolean') {
|
package/src/rings.ts
CHANGED
|
@@ -107,18 +107,18 @@ export type PageErrorRing = Ring<PageError>;
|
|
|
107
107
|
*
|
|
108
108
|
* Both wrong values fail in opposite directions and neither is survivable. A NEGATIVE capacity
|
|
109
109
|
* makes `while (items.length > capacity) items.shift()` spin forever on an already-empty array
|
|
110
|
-
* (`0 > -1`): `
|
|
110
|
+
* (`0 > -1`): `boundedRing(-1).push(1)` never returns, and it is a synchronous loop on the worker's
|
|
111
111
|
* only thread, inside a `page.on('request')` handler — past `ctx.signal`, past the wedge watchdog
|
|
112
112
|
* and past the job timeout, which is incident #1 in this file's header. `NaN` fails the other way:
|
|
113
113
|
* the comparison is false forever, the ring is UNBOUNDED, and the OOM the bound exists to prevent
|
|
114
114
|
* arrives two hours in. Zero is a ring that discards everything silently, and a fraction is a
|
|
115
115
|
* bound that is not the number anybody asked for.
|
|
116
116
|
*/
|
|
117
|
-
export function
|
|
117
|
+
export function boundedRing<T>(capacity: number = DEFAULT_RING_CAPACITY): Ring<T> {
|
|
118
118
|
assert(
|
|
119
119
|
Number.isSafeInteger(capacity) && capacity > 0,
|
|
120
120
|
`a ring capacity must be a positive integer, got ${String(capacity)}: a negative one spins forever on push() and NaN makes the ring unbounded`,
|
|
121
|
-
`pass a positive capacity —
|
|
121
|
+
`pass a positive capacity — boundedRing(${String(DEFAULT_RING_CAPACITY)}) — or omit it and take DEFAULT_RING_CAPACITY`,
|
|
122
122
|
);
|
|
123
123
|
const items: T[] = [];
|
|
124
124
|
let dropped = 0;
|