@crawlee/puppeteer 4.0.0-beta.63 → 4.0.0-beta.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -8,12 +8,12 @@ import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/clic
|
|
|
8
8
|
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
9
9
|
import type { InterceptHandler } from './utils/puppeteer_request_interception.js';
|
|
10
10
|
import type { BlockRequestsOptions, DirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, PuppeteerContextUtils, SaveSnapshotOptions } from './utils/puppeteer_utils.js';
|
|
11
|
-
export
|
|
11
|
+
export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
|
12
|
+
export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
|
|
12
13
|
}
|
|
13
14
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
14
|
-
export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext
|
|
15
|
+
export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext> {
|
|
15
16
|
}
|
|
16
|
-
export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
|
|
17
17
|
export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerController, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {
|
|
18
18
|
browserPlugins: [PuppeteerPlugin];
|
|
19
19
|
}> {
|
|
@@ -23,25 +23,25 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
|
|
|
23
23
|
launchContext?: PuppeteerLaunchContext;
|
|
24
24
|
/**
|
|
25
25
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
26
|
-
* or browser properties before navigation. The function
|
|
27
|
-
*
|
|
26
|
+
* or browser properties before navigation. The function receives the `crawlingContext`; the options object
|
|
27
|
+
* forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
|
|
28
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context
|
|
29
|
+
* (e.g. to override context members for subsequent hooks and pipeline stages).
|
|
28
30
|
* Example:
|
|
29
31
|
* ```
|
|
30
32
|
* preNavigationHooks: [
|
|
31
|
-
* async (
|
|
32
|
-
* const { page } = crawlingContext;
|
|
33
|
+
* async ({ page, gotoOptions }) => {
|
|
33
34
|
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
|
|
35
|
+
* gotoOptions.timeout = 60_000;
|
|
34
36
|
* },
|
|
35
37
|
* ]
|
|
36
38
|
* ```
|
|
37
|
-
*
|
|
38
|
-
* Modyfing `pageOptions` is supported only in Playwright incognito.
|
|
39
|
-
* See {@link PrePageCreateHook}
|
|
40
39
|
*/
|
|
41
40
|
preNavigationHooks?: PuppeteerHook[];
|
|
42
41
|
/**
|
|
43
42
|
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
44
|
-
* The function accepts `crawlingContext` as the only parameter.
|
|
43
|
+
* The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
|
|
44
|
+
* whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
|
|
45
45
|
* Example:
|
|
46
46
|
* ```
|
|
47
47
|
* postNavigationHooks: [
|
|
@@ -213,7 +213,7 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
213
213
|
*/
|
|
214
214
|
constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext>);
|
|
215
215
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
216
|
-
protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, Dictionary> & {
|
|
216
|
+
protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, Dictionary, Dictionary> & {
|
|
217
217
|
injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
|
|
218
218
|
injectJQuery: () => Promise<void>;
|
|
219
219
|
waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACX,sBAAsB,EACtB,YAAY,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAwB,MAAM,kBAAkB,CAAC;AACxE,OAAO,KAAK,EAAsB,mBAAmB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAEtG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEnE,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,mCAAmC,CAAC;AAC/F,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AAEtE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,2CAA2C,CAAC;AAClF,OAAO,KAAK,EACR,oBAAoB,EACpB,uBAAuB,EACvB,qBAAqB,EACrB,iBAAiB,EACjB,qBAAqB,EACrB,mBAAmB,EACtB,MAAM,4BAA4B,CAAC;AAGpC,MAAM,WAAW,wBAAwB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAC9E,
|
|
1
|
+
{"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACX,sBAAsB,EACtB,YAAY,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAwB,MAAM,kBAAkB,CAAC;AACxE,OAAO,KAAK,EAAsB,mBAAmB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAEtG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEnE,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,mCAAmC,CAAC;AAC/F,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AAEtE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,2CAA2C,CAAC;AAClF,OAAO,KAAK,EACR,oBAAoB,EACpB,uBAAuB,EACvB,qBAAqB,EACrB,iBAAiB,EACjB,qBAAqB,EACrB,mBAAmB,EACtB,MAAM,4BAA4B,CAAC;AAGpC,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE5E,MAAM,WAAW,wBAAwB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAC9E,SACI,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,EAAE,QAAQ,EAAE,oBAAoB,CAAC,EAC/F,qBAAqB;CAAG;AAChC,MAAM,WAAW,aAAc,SAAQ,WAAW,CAAC,wBAAwB,CAAC;CAAG;AAE/E,MAAM,WAAW,uBAAuB,CACpC,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,wBAAwB,GAAG,wBAAwB,GAAG,gBAAgB,CAChG,SAAQ,qBAAqB,CAC3B,IAAI,EACJ,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,gBAAgB,EAChB,eAAe,EACf;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,CACxC;IACG;;OAEG;IACH,aAAa,CAAC,EAAE,sBAAsB,CAAC;IAEvC;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,aAAa,EAAE,CAAC;IAErC;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB,CAAC,EAAE,aAAa,EAAE,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AACH,qBAAa,gBAAgB,CACzB,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,wBAAwB,GAAG,wBAAwB,GAAG,gBAAgB,CAChG,SAAQ,cAAc,CACpB,IAAI,EACJ,YAAY,EACZ,mBAAmB,EACnB;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,EACrC,aAAa,EACb,wBAAwB,EACxB,gBAAgB,EAChB,eAAe,CAClB;IACG,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAGpC;IAEF;;OAEG;gBACS,OAAO,GAAE,uBAAuB,CAAC,gBAAgB,EAAE,eAAe,CAAM;cAqDjE,oBAAoB;+BAUF,MAAM,YAAY,iBAAiB;;oCAL3B,MAAM;sCAkBP,MAAM;kDAQ7B,IAAI,CAAC,qCAAqC,EAAE,MAAM,GAAG,cAAc,CAAC;kCAOjD,oBAAoB;sCAEtB,MAAM,QAAQ,UAAU;8CACV,gBAAgB;iDAEb,gBAAgB;mCAE9B,qBAAqB;iCAEvB,mBAAmB;;;YA3C5C,cAAc;cAiDH,kBAAkB,CACvC,eAAe,EAAE,wBAAwB,EACzC,WAAW,EAAE,uBAAuB;CAI3C;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,qBAAqB,CACjC,OAAO,SAAS,wBAAwB,GAAG,wBAAwB,EACnE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,qDAEzC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAExE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAU5D,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAExE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAU5D,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAgE1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AACH,MAAM,OAAO,gBAGX,SAAQ,cAST;IACa,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,cAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KACzC,CAAC;IAEF;;OAEG;IACH,YAAY,UAAsE,EAAE;QAChF,EAAE,CAAC,OAAO,EAAE,yBAAyB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE5F,MAAM,EACF,aAAa,GAAG,EAAE,EAClB,QAAQ,EACR,kBAAkB,EAClB,sBAAsB,EACtB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,MAAM,kBAAkB,GAAG;YACvB,GAAG,OAAO,CAAC,kBAAkB;SACV,CAAC;QAExB,IAAI,aAAa,CAAC,QAAQ,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACX,oFAAoF;gBAChF,gDAAgD,CACvD,CAAC;QACN,CAAC;QAED,2EAA2E;QAC3E,uFAAuF;QACvF,IAAI,kBAAkB,CAAC,cAAc,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;QAC5G,CAAC;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACpD,aAAa,CAAC,aAAa,CAAC,QAAQ,GAAG,QAAmB,CAAC;QAC/D,CAAC;QAED,MAAM,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAEtF,kBAAkB,CAAC,cAAc,GAAG,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAE9E,KAAK,CAAC;YACF,GAAI,qBAOF;YACF,aAAa;YACb,kBAAkB;YAClB,kBAAkB;YAClB,sBAAsB,EAAE,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;SACxF,CAAC,CAAC;IACP,CAAC;IAEkB,oBAAoB;QACnC,OAAO,KAAK,CAAC,oBAAoB,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5F,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,OAAwE;QACjG,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE;YAClE,MAAM,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QACzE,CAAC,CAAC;QAEF,OAAO;YACH,UAAU,EAAE,KAAK,EAAE,QAAgB,EAAE,OAA2B,EAAE,EAAE,CAChE,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC;YAC9D,YAAY,EAAE,KAAK,IAAI,EAAE;gBACrB,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,KAAK,YAAY,CAAC,UAAU,EAAE,CAAC;oBACpD,OAAO,CAAC,GAAG,CAAC,OAAO,CACf,mIAAmI,CACtI,CAAC;oBACF,MAAM,cAAc,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChD,OAAO;gBACX,CAAC;gBACD,MAAM,cAAc,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,eAAe;YACf,gBAAgB,EAAE,KAAK,EAAE,QAAiB,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE;gBAC7D,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAC/C,CAAC;gBAED,OAAO,cAAc,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YACrG,CAAC;YACD,8BAA8B,EAAE,KAAK,EACjC,OAA6E,EAC/E,EAAE,CACA,cAAc,CAAC,8BAA8B,CAAC;gBAC1C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,YAAY,EAAE,IAAI,CAAC,YAAa;gBAChC,GAAG,OAAO;aACb,CAAC;YACN,aAAa,EAAE,KAAK,EAAE,OAA8B,EAAE,EAAE,CACpD,cAAc,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;YACvD,aAAa,EAAE,CAAC,YAAoB,EAAE,GAAgB,EAAE,EAAE,CAAC,cAAc,CAAC,aAAa,CAAC,YAAY,EAAE,GAAG,CAAC;YAC1G,0BAA0B,EAAE,KAAK,EAAE,OAAyB,EAAE,EAAE,CAC5D,cAAc,CAAC,0BAA0B,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;YACpE,6BAA6B,EAAE,KAAK,EAAE,OAAyB,EAAE,EAAE,CAC/D,cAAc,CAAC,6BAA6B,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;YACvE,cAAc,EAAE,KAAK,EAAE,OAA+B,EAAE,EAAE,CACtD,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;YACxD,YAAY,EAAE,KAAK,EAAE,OAA6B,EAAE,EAAE,CAClD,cAAc,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE,EAAE,CAAC;YACxG,iBAAiB,EAAE,KAAK,IAAI,EAAE,CAAC,cAAc,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC;SAChF,CAAC;IACN,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CACvC,eAAyC,EACzC,WAAoC;QAEpC,OAAO,YAAY,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAGL;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,qBAAqB,CAGnC,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/puppeteer",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.64",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,11 +48,11 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/core": "4.0.0-beta.
|
|
54
|
-
"@crawlee/types": "4.0.0-beta.
|
|
55
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/browser": "4.0.0-beta.64",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.64",
|
|
53
|
+
"@crawlee/core": "4.0.0-beta.64",
|
|
54
|
+
"@crawlee/types": "4.0.0-beta.64",
|
|
55
|
+
"@crawlee/utils": "4.0.0-beta.64",
|
|
56
56
|
"cheerio": "^1.0.0",
|
|
57
57
|
"devtools-protocol": "*",
|
|
58
58
|
"idcac-playwright": "^0.2.0",
|
|
@@ -79,5 +79,5 @@
|
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
81
|
},
|
|
82
|
-
"gitHead": "
|
|
82
|
+
"gitHead": "76ba0fe1101b1e14fb5e1962540107775ad4e513"
|
|
83
83
|
}
|