@crawlee/playwright 4.0.0-beta.104 → 4.0.0-beta.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { BrowserHook, LoadedRequest, Request, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
|
|
2
2
|
import type { BasicCrawlerOptions } from '@crawlee/basic';
|
|
3
3
|
import { BasicCrawler } from '@crawlee/basic';
|
|
4
|
-
import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
|
|
5
|
-
import {
|
|
4
|
+
import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticsOptions, StatisticState, StorageTransactionView } from '@crawlee/core';
|
|
5
|
+
import { Statistics } from '@crawlee/core';
|
|
6
6
|
import type { Dictionary, Awaitable } from '@crawlee/types';
|
|
7
7
|
import { type CheerioRoot } from '@crawlee/utils';
|
|
8
8
|
import { type Cheerio } from 'cheerio';
|
|
@@ -108,11 +108,12 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
|
|
|
108
108
|
*/
|
|
109
109
|
renderingTypeDetectionRatio?: number;
|
|
110
110
|
/**
|
|
111
|
-
* An optional callback that is called on
|
|
111
|
+
* An optional callback that is called on the storage writes recorded by the request handler in plain
|
|
112
|
+
* HTTP mode (exposed as a read-only {@link StorageTransactionView}).
|
|
112
113
|
* If it returns false, the request is retried in a browser.
|
|
113
|
-
* If no callback is specified, every
|
|
114
|
+
* If no callback is specified, every result is considered valid.
|
|
114
115
|
*/
|
|
115
|
-
resultChecker?: (result:
|
|
116
|
+
resultChecker?: (result: StorageTransactionView) => boolean;
|
|
116
117
|
/**
|
|
117
118
|
* An optional callback that decides whether an error thrown during the plain HTTP request handler
|
|
118
119
|
* should be propagated (instead of falling back to browser navigation).
|
|
@@ -135,18 +136,13 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
|
|
|
135
136
|
*
|
|
136
137
|
* For a stricter, ready-made comparator that also takes enqueued requests and key-value store changes into account, see {@link fullResultComparator}.
|
|
137
138
|
*/
|
|
138
|
-
resultComparator?: (resultA:
|
|
139
|
+
resultComparator?: (resultA: StorageTransactionView, resultB: StorageTransactionView) => boolean | 'equal' | 'different' | 'inconclusive';
|
|
139
140
|
/**
|
|
140
141
|
* A custom rendering type predictor. A predictor passed here is borrowed - the crawler never drives its
|
|
141
142
|
* lifecycle, so set it up yourself (the built-in {@link RenderingTypePredictor} needs `initialize()`).
|
|
142
143
|
* Omit the option and the crawler builds its own from `renderingTypeDetectionRatio` - and initializes it.
|
|
143
144
|
*/
|
|
144
145
|
renderingTypePredictor?: IRenderingTypePredictor;
|
|
145
|
-
/**
|
|
146
|
-
* Prevent direct access to storage in request handlers (only allow using context helpers).
|
|
147
|
-
* Defaults to `true`
|
|
148
|
-
*/
|
|
149
|
-
preventDirectStorageAccess?: boolean;
|
|
150
146
|
}
|
|
151
147
|
/**
|
|
152
148
|
* An extension of {@link PlaywrightCrawler} that uses a more limited request handler interface so that it is able to switch to HTTP-only crawling when it detects it may be possible.
|
|
@@ -182,13 +178,16 @@ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<nev
|
|
|
182
178
|
private resultChecker;
|
|
183
179
|
private shouldPropagateError;
|
|
184
180
|
private resultComparator;
|
|
185
|
-
private preventDirectStorageAccess;
|
|
186
181
|
private staticContextPipeline;
|
|
187
182
|
private browserContextPipeline;
|
|
188
183
|
private individualRequestHandlerTimeoutMillis;
|
|
189
184
|
get stats(): AdaptivePlaywrightCrawlerStatistics;
|
|
190
|
-
private resultObjects;
|
|
191
185
|
private inFlightRenderingTypeDetections;
|
|
186
|
+
/**
|
|
187
|
+
* The write policy of the per-attempt transactions. Defaults the request queue to `deferred`:
|
|
188
|
+
* a discarded attempt's enqueues must never reach the queue.
|
|
189
|
+
*/
|
|
190
|
+
private readonly attemptWritePolicy;
|
|
192
191
|
private teardownHooks;
|
|
193
192
|
constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
|
|
194
193
|
protected _init(): Promise<void>;
|
|
@@ -203,16 +202,14 @@ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<nev
|
|
|
203
202
|
}>;
|
|
204
203
|
private adaptCheerioContext;
|
|
205
204
|
private adaptPlaywrightContext;
|
|
206
|
-
private crawlOne;
|
|
207
|
-
protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
|
|
208
|
-
private commitResult;
|
|
209
|
-
private allowStorageAccess;
|
|
210
205
|
/**
|
|
211
|
-
*
|
|
212
|
-
*
|
|
213
|
-
*
|
|
206
|
+
* Runs one request handler attempt inside its own {@link StorageTransaction}, wrapping the inner
|
|
207
|
+
* (static or browser) context pipeline. The transaction is pushed to `transactions` *at creation
|
|
208
|
+
* time, before the `try`* - the `ok: false` branch of the returned {@link Result} carries no
|
|
209
|
+
* result, and failed attempts are routine here. The caller owns the outcome and disposal.
|
|
214
210
|
*/
|
|
215
|
-
|
|
211
|
+
private crawlOne;
|
|
212
|
+
protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
|
|
216
213
|
private enqueueLinks;
|
|
217
214
|
private createLogProxy;
|
|
218
215
|
teardown(): Promise<void>;
|
|
@@ -243,5 +240,5 @@ export declare function createAdaptivePlaywrightRouter<Context extends AdaptiveP
|
|
|
243
240
|
* });
|
|
244
241
|
* ```
|
|
245
242
|
*/
|
|
246
|
-
export declare function fullResultComparator(resultA:
|
|
243
|
+
export declare function fullResultComparator(resultA: StorageTransactionView, resultB: StorageTransactionView): boolean;
|
|
247
244
|
export {};
|
|
@@ -2,8 +2,9 @@ import { isDeepStrictEqual } from 'node:util';
|
|
|
2
2
|
import { BasicCrawler } from '@crawlee/basic';
|
|
3
3
|
import { extractUrlsFromPage } from '@crawlee/browser';
|
|
4
4
|
import { CheerioCrawler } from '@crawlee/cheerio';
|
|
5
|
-
import { OwnedOrInjected, RequestHandlerError,
|
|
5
|
+
import { createStorageTransaction, OwnedOrInjected, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/core';
|
|
6
6
|
import { extractUrlsFromCheerio } from '@crawlee/utils';
|
|
7
|
+
import ow from 'ow';
|
|
7
8
|
import { addTimeoutToPromise } from '@apify/timeout';
|
|
8
9
|
import { PlaywrightCrawler } from './playwright-crawler.js';
|
|
9
10
|
import { RenderingTypePredictor, } from './utils/rendering-type-prediction.js';
|
|
@@ -86,7 +87,6 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
86
87
|
resultChecker;
|
|
87
88
|
shouldPropagateError;
|
|
88
89
|
resultComparator;
|
|
89
|
-
preventDirectStorageAccess;
|
|
90
90
|
staticContextPipeline;
|
|
91
91
|
browserContextPipeline;
|
|
92
92
|
individualRequestHandlerTimeoutMillis;
|
|
@@ -94,11 +94,25 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
94
94
|
get stats() {
|
|
95
95
|
return super.stats;
|
|
96
96
|
}
|
|
97
|
-
resultObjects = new WeakMap();
|
|
98
97
|
inFlightRenderingTypeDetections = 0;
|
|
98
|
+
/**
|
|
99
|
+
* The write policy of the per-attempt transactions. Defaults the request queue to `deferred`:
|
|
100
|
+
* a discarded attempt's enqueues must never reach the queue.
|
|
101
|
+
*/
|
|
102
|
+
attemptWritePolicy;
|
|
99
103
|
teardownHooks = [];
|
|
100
104
|
constructor(options = {}) {
|
|
101
|
-
const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics,
|
|
105
|
+
const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, transactionalStorage, ...rest } = options;
|
|
106
|
+
// The user's value is replaced by `false` in the `super` call below — validate it separately.
|
|
107
|
+
ow(transactionalStorage, 'transactionalStorage', BasicCrawler.optionsShape.transactionalStorage);
|
|
108
|
+
// Per-attempt buffering is load-bearing here: the handler runs up to twice per request and the
|
|
109
|
+
// losing attempt's writes must be discardable.
|
|
110
|
+
if (transactionalStorage === false) {
|
|
111
|
+
throw new Error('AdaptivePlaywrightCrawler requires transactional storage - it runs the request handler ' +
|
|
112
|
+
'multiple times per request and must be able to discard the storage writes of losing ' +
|
|
113
|
+
'attempts. `transactionalStorage: false` is therefore not supported; a write policy ' +
|
|
114
|
+
'object is accepted and forwarded to the per-attempt transactions.');
|
|
115
|
+
}
|
|
102
116
|
if (statistics !== undefined && !(statistics instanceof AdaptivePlaywrightCrawlerStatistics)) {
|
|
103
117
|
throw new Error('AdaptivePlaywrightCrawler tracks extra fields on its own Statistics subclass and cannot use a ' +
|
|
104
118
|
'plain `statistics` instance. Omit the option to let the crawler build its own.');
|
|
@@ -115,11 +129,19 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
115
129
|
logMessage: `${AdaptivePlaywrightCrawler.name} request statistics:`,
|
|
116
130
|
}),
|
|
117
131
|
contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
|
|
132
|
+
// The base crawler must not wrap requests in a transaction of its own - this crawler opens
|
|
133
|
+
// one per request handler attempt in `crawlOne` instead, forwarding the write policy of the
|
|
134
|
+
// user-facing option (validated above) to those.
|
|
135
|
+
transactionalStorage: false,
|
|
118
136
|
});
|
|
119
137
|
this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
|
|
120
138
|
// `renderingTypeDetectionRatio` only configures the default predictor - an injected one brings its own
|
|
121
139
|
// detection ratio (and its own state), so the option is ignored in that case.
|
|
122
140
|
this.renderingTypePredictor = OwnedOrInjected.resolve(renderingTypePredictor, () => new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }));
|
|
141
|
+
this.attemptWritePolicy = {
|
|
142
|
+
requestQueue: 'deferred',
|
|
143
|
+
...(typeof transactionalStorage === 'object' ? transactionalStorage : {}),
|
|
144
|
+
};
|
|
123
145
|
this.resultChecker = resultChecker ?? (() => true);
|
|
124
146
|
this.shouldPropagateError = shouldPropagateError ?? (() => false);
|
|
125
147
|
if (resultComparator !== undefined) {
|
|
@@ -166,7 +188,6 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
166
188
|
this.browserContextPipeline = browserCrawler.contextPipeline.compose({
|
|
167
189
|
action: this.adaptPlaywrightContext.bind(this),
|
|
168
190
|
});
|
|
169
|
-
this.preventDirectStorageAccess = preventDirectStorageAccess;
|
|
170
191
|
}
|
|
171
192
|
async _init() {
|
|
172
193
|
// Only the predictor we built ourselves is ours to initialize - an injected one is borrowed, so its
|
|
@@ -203,11 +224,6 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
203
224
|
});
|
|
204
225
|
}
|
|
205
226
|
async adaptCheerioContext(cheerioContext) {
|
|
206
|
-
// Capture the original response to avoid infinite recursion when the getter is copied to the context
|
|
207
|
-
const result = this.resultObjects.get(cheerioContext);
|
|
208
|
-
if (result === undefined) {
|
|
209
|
-
throw new Error('Logical error - `this.resultObjects` does not contain the result object');
|
|
210
|
-
}
|
|
211
227
|
return {
|
|
212
228
|
get page() {
|
|
213
229
|
throw new Error('Page object was used in HTTP-only request handler');
|
|
@@ -221,17 +237,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
221
237
|
enqueueLinks: async (options = {}) => {
|
|
222
238
|
const urls = options.urls ??
|
|
223
239
|
extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
|
|
224
|
-
return (await this.enqueueLinks({ ...options, urls }, cheerioContext.request
|
|
240
|
+
return (await this.enqueueLinks({ ...options, urls }, cheerioContext.request));
|
|
225
241
|
},
|
|
226
242
|
response: cheerioContext.response,
|
|
227
243
|
};
|
|
228
244
|
}
|
|
229
245
|
async adaptPlaywrightContext(playwrightContext) {
|
|
246
|
+
// Capture the original response to avoid infinite recursion when the getter is copied to the context
|
|
230
247
|
const originalResponse = playwrightContext.response;
|
|
231
|
-
const result = this.resultObjects.get(playwrightContext);
|
|
232
|
-
if (result === undefined) {
|
|
233
|
-
throw new Error('Logical error - `this.resultObjects` does not contain the result object');
|
|
234
|
-
}
|
|
235
248
|
return {
|
|
236
249
|
response: new Response(Uint8Array.from(await originalResponse.body()), {
|
|
237
250
|
headers: originalResponse.headers(),
|
|
@@ -264,29 +277,35 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
264
277
|
else {
|
|
265
278
|
urls = options.urls;
|
|
266
279
|
}
|
|
267
|
-
return (await this.enqueueLinks({ ...options, urls }, playwrightContext.request
|
|
280
|
+
return (await this.enqueueLinks({ ...options, urls }, playwrightContext.request));
|
|
268
281
|
},
|
|
269
282
|
};
|
|
270
283
|
}
|
|
271
|
-
|
|
272
|
-
|
|
284
|
+
/**
|
|
285
|
+
* Runs one request handler attempt inside its own {@link StorageTransaction}, wrapping the inner
|
|
286
|
+
* (static or browser) context pipeline. The transaction is pushed to `transactions` *at creation
|
|
287
|
+
* time, before the `try`* - the `ok: false` branch of the returned {@link Result} carries no
|
|
288
|
+
* result, and failed attempts are routine here. The caller owns the outcome and disposal.
|
|
289
|
+
*/
|
|
290
|
+
async crawlOne(renderingType, context, useStateFunction, transactions) {
|
|
291
|
+
const transaction = createStorageTransaction({
|
|
292
|
+
policy: this.attemptWritePolicy,
|
|
293
|
+
commitTimeoutMillis: this.internalTimeoutMillis,
|
|
294
|
+
});
|
|
295
|
+
transactions.push(transaction);
|
|
273
296
|
const logs = [];
|
|
274
297
|
const deferredCleanup = [];
|
|
275
|
-
const
|
|
276
|
-
|
|
277
|
-
pushData: result.pushData,
|
|
278
|
-
useState: this.allowStorageAccess(useStateFunction),
|
|
279
|
-
getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
|
|
298
|
+
const attemptBoundContextHelpers = {
|
|
299
|
+
useState: useStateFunction,
|
|
280
300
|
log: this.createLogProxy(context.log, logs),
|
|
281
301
|
registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
|
|
282
302
|
};
|
|
283
303
|
const subCrawlerContext = Object.defineProperties({}, Object.getOwnPropertyDescriptors(context));
|
|
284
|
-
// Mark
|
|
285
|
-
// (which would otherwise override them with the sub-crawler's own versions, losing the
|
|
286
|
-
for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(
|
|
304
|
+
// Mark attempt-bound helpers as non-configurable so they survive the sub-crawler context pipeline
|
|
305
|
+
// (which would otherwise override them with the sub-crawler's own versions, losing the binding).
|
|
306
|
+
for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(attemptBoundContextHelpers))) {
|
|
287
307
|
Object.defineProperty(subCrawlerContext, key, { ...descriptor, configurable: false });
|
|
288
308
|
}
|
|
289
|
-
this.resultObjects.set(subCrawlerContext, result);
|
|
290
309
|
try {
|
|
291
310
|
const callAdaptiveRequestHandler = async () => {
|
|
292
311
|
if (renderingType === 'static') {
|
|
@@ -300,12 +319,8 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
300
319
|
// to resolve any per-route override too - otherwise routes would be silently ignored here
|
|
301
320
|
const routeTimeoutSecs = this.requestHandler.getTimeoutSecs?.(context.request.label);
|
|
302
321
|
const timeoutMillis = routeTimeoutSecs === undefined ? this.individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;
|
|
303
|
-
await addTimeoutToPromise(async () =>
|
|
304
|
-
|
|
305
|
-
throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
|
|
306
|
-
}
|
|
307
|
-
}, callAdaptiveRequestHandler), timeoutMillis, 'Request handler timed out');
|
|
308
|
-
return { result, ok: true, logs };
|
|
322
|
+
await addTimeoutToPromise(async () => transaction.run(callAdaptiveRequestHandler), timeoutMillis, 'Request handler timed out');
|
|
323
|
+
return { result: transaction, ok: true, logs };
|
|
309
324
|
}
|
|
310
325
|
catch (error) {
|
|
311
326
|
return { error, ok: false, logs };
|
|
@@ -323,15 +338,19 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
323
338
|
if (shouldDetectRenderingType) {
|
|
324
339
|
this.inFlightRenderingTypeDetections += 1;
|
|
325
340
|
}
|
|
341
|
+
// Every transaction created for this request - up to two, since the static-then-browser
|
|
342
|
+
// fall-through and the browser-then-detection pair are mutually exclusive. Disposed in the
|
|
343
|
+
// `finally` below, not earlier: the comparators read the journals after `crawlOne` returns.
|
|
344
|
+
const transactions = [];
|
|
326
345
|
try {
|
|
327
346
|
if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
|
|
328
347
|
crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
|
|
329
348
|
this.stats.trackHttpOnlyRequestHandlerRun();
|
|
330
|
-
const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState);
|
|
349
|
+
const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
|
|
331
350
|
if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
|
|
332
351
|
crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
|
|
333
352
|
plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
|
|
334
|
-
await
|
|
353
|
+
await plainHTTPRun.result.commit();
|
|
335
354
|
return;
|
|
336
355
|
}
|
|
337
356
|
// Execution will "fall through" and try running the request handler in a browser
|
|
@@ -371,15 +390,17 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
371
390
|
return this.stateCopy;
|
|
372
391
|
},
|
|
373
392
|
};
|
|
374
|
-
const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker));
|
|
393
|
+
const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker), transactions);
|
|
375
394
|
if (!browserRun.ok) {
|
|
376
395
|
throw browserRun.error;
|
|
377
396
|
}
|
|
378
397
|
browserRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
|
|
379
|
-
await
|
|
398
|
+
await browserRun.result.commit();
|
|
380
399
|
if (shouldDetectRenderingType) {
|
|
381
400
|
crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
|
|
382
|
-
|
|
401
|
+
// The detection attempt's transaction is never committed - its writes exist only for the
|
|
402
|
+
// result comparison.
|
|
403
|
+
const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker), transactions);
|
|
383
404
|
const detectionResult = (() => {
|
|
384
405
|
if (!plainHTTPRun.ok) {
|
|
385
406
|
return 'clientOnly';
|
|
@@ -403,52 +424,23 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
403
424
|
if (shouldDetectRenderingType) {
|
|
404
425
|
this.inFlightRenderingTypeDetections -= 1;
|
|
405
426
|
}
|
|
427
|
+
// A still-open transaction here belongs to a discarded attempt - roll it back, then release.
|
|
428
|
+
for (const transaction of transactions) {
|
|
429
|
+
transaction.rollback();
|
|
430
|
+
transaction.dispose();
|
|
431
|
+
}
|
|
406
432
|
}
|
|
407
433
|
}
|
|
408
|
-
async
|
|
409
|
-
await Promise.all([
|
|
410
|
-
...calls.pushData.map(async (params) => crawlingContext.pushData(...params)),
|
|
411
|
-
...calls.addRequests.map(async (params) => crawlingContext.addRequests(...params)),
|
|
412
|
-
...Object.entries(keyValueStoreChanges).map(async ([storeIdOrName, changes]) => {
|
|
413
|
-
const store = await crawlingContext.getKeyValueStore({ id: storeIdOrName });
|
|
414
|
-
await Promise.all(Object.entries(changes).map(async ([key, { changedValue, options }]) => store.setValue(key, changedValue, options)));
|
|
415
|
-
}),
|
|
416
|
-
]);
|
|
417
|
-
}
|
|
418
|
-
allowStorageAccess(func) {
|
|
419
|
-
return async (...args) => withCheckedStorageAccess(() => { }, async () => func(...args));
|
|
420
|
-
}
|
|
421
|
-
/**
|
|
422
|
-
* Reading the pending request count queries the underlying request manager, which counts as storage access.
|
|
423
|
-
* Since this is internal crawler bookkeeping used to compute the `enqueueLinks` limit (not user-initiated storage
|
|
424
|
-
* access), it must be allowed even while a request handler runs inside the storage-access guard.
|
|
425
|
-
*/
|
|
426
|
-
async getPendingRequestCountApproximation() {
|
|
427
|
-
return this.allowStorageAccess(() => super.getPendingRequestCountApproximation())();
|
|
428
|
-
}
|
|
429
|
-
async enqueueLinks(options, request, result) {
|
|
434
|
+
async enqueueLinks(options, request) {
|
|
430
435
|
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
|
|
431
436
|
enqueueStrategy: options?.strategy,
|
|
432
437
|
finalRequestUrl: request.loadedUrl,
|
|
433
438
|
originalRequestUrl: request.url,
|
|
434
439
|
userProvidedBaseUrl: options?.baseUrl,
|
|
435
440
|
});
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
addedRequests: requests.map(({ uniqueKey, id }) => ({
|
|
440
|
-
uniqueKey,
|
|
441
|
-
requestId: id ?? '',
|
|
442
|
-
wasAlreadyPresent: false,
|
|
443
|
-
wasAlreadyHandled: false,
|
|
444
|
-
})),
|
|
445
|
-
waitForAllRequestsToBeAdded: Promise.resolve([]),
|
|
446
|
-
requestsOverLimit: [],
|
|
447
|
-
};
|
|
448
|
-
};
|
|
449
|
-
// We need to use a mock request queue implementation, in order to add the requests into our result object
|
|
450
|
-
const mockRequestQueue = { addRequestsBatched };
|
|
451
|
-
return await this.enqueueLinksWithCrawlDepth({ ...options, baseUrl }, request, mockRequestQueue);
|
|
441
|
+
// The per-attempt transaction buffers these (the queue policy defaults to `deferred` here),
|
|
442
|
+
// so a discarded attempt's enqueues never reach the queue.
|
|
443
|
+
return await this.enqueueLinksWithCrawlDepth({ ...options, baseUrl }, request, await this.getRequestManager());
|
|
452
444
|
}
|
|
453
445
|
createLogProxy(log, logs) {
|
|
454
446
|
return new Proxy(log, {
|
|
@@ -498,6 +490,5 @@ export function createAdaptivePlaywrightRouter(routesOrSchemas) {
|
|
|
498
490
|
export function fullResultComparator(resultA, resultB) {
|
|
499
491
|
return (isDeepStrictEqual(resultA.datasetItems, resultB.datasetItems) &&
|
|
500
492
|
isDeepStrictEqual(resultA.enqueuedUrls, resultB.enqueuedUrls) &&
|
|
501
|
-
isDeepStrictEqual(resultA.enqueuedUrlLists, resultB.enqueuedUrlLists) &&
|
|
502
493
|
isDeepStrictEqual(resultA.keyValueStoreChanges, resultB.keyValueStoreChanges));
|
|
503
494
|
}
|
|
@@ -215,6 +215,8 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
|
|
|
215
215
|
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
216
216
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
217
217
|
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
218
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
219
|
+
transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/browser").StorageWritePolicy> | undefined>;
|
|
218
220
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
219
221
|
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
220
222
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/playwright",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.105",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,13 +49,13 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
51
|
"@apify/timeout": "^0.4.4",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
54
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
55
|
-
"@crawlee/cheerio": "4.0.0-beta.
|
|
56
|
-
"@crawlee/core": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.105",
|
|
53
|
+
"@crawlee/browser": "4.0.0-beta.105",
|
|
54
|
+
"@crawlee/browser-pool": "4.0.0-beta.105",
|
|
55
|
+
"@crawlee/cheerio": "4.0.0-beta.105",
|
|
56
|
+
"@crawlee/core": "4.0.0-beta.105",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.105",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.105",
|
|
59
59
|
"cheerio": "^1.0.0",
|
|
60
60
|
"idcac-playwright": "^0.1.3",
|
|
61
61
|
"jquery": "^3.7.1",
|
|
@@ -85,5 +85,5 @@
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
},
|
|
88
|
-
"gitHead": "
|
|
88
|
+
"gitHead": "26073a822c7699ac487931383a39192bc3daae7a"
|
|
89
89
|
}
|