@crawlee/cheerio 3.10.3-beta.4 → 3.10.3-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -8,6 +8,7 @@ import type { IncomingMessage } from 'http';
|
|
|
8
8
|
import type { EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterRoutes, Configuration, RequestProvider } from '@crawlee/http';
|
|
9
9
|
import { HttpCrawler } from '@crawlee/http';
|
|
10
10
|
import type { Dictionary } from '@crawlee/types';
|
|
11
|
+
import { type CheerioRoot } from '@crawlee/utils';
|
|
11
12
|
import * as cheerio from 'cheerio';
|
|
12
13
|
export type CheerioErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
13
14
|
JSONData extends Dictionary = any> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
|
|
@@ -23,20 +24,34 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
|
|
|
23
24
|
* Cheerio is available only for HTML and XML content types.
|
|
24
25
|
*/
|
|
25
26
|
$: cheerio.CheerioAPI;
|
|
27
|
+
/**
|
|
28
|
+
* Wait for an element matching the selector to appear. Timeout is ignored.
|
|
29
|
+
*
|
|
30
|
+
* **Example usage:**
|
|
31
|
+
* ```ts
|
|
32
|
+
* async requestHandler({ waitForSelector, parseWithCheerio }) {
|
|
33
|
+
* await waitForSelector('article h1');
|
|
34
|
+
* const $ = await parseWithCheerio();
|
|
35
|
+
* const title = $('title').text();
|
|
36
|
+
* });
|
|
37
|
+
* ```
|
|
38
|
+
*/
|
|
39
|
+
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
26
40
|
/**
|
|
27
41
|
* Returns Cheerio handle, this is here to unify the crawler API, so they all have this handy method.
|
|
28
42
|
* It has the same return type as the `$` context property, use it only if you are abstracting your workflow to
|
|
29
43
|
* support different context types in one handler.
|
|
44
|
+
* When provided with the `selector` argument, it will throw if it's not available.
|
|
30
45
|
*
|
|
31
46
|
* **Example usage:**
|
|
32
|
-
* ```
|
|
47
|
+
* ```ts
|
|
33
48
|
* async requestHandler({ parseWithCheerio }) {
|
|
34
49
|
* const $ = await parseWithCheerio();
|
|
35
50
|
* const title = $('title').text();
|
|
36
51
|
* });
|
|
37
52
|
* ```
|
|
38
53
|
*/
|
|
39
|
-
parseWithCheerio(): Promise<
|
|
54
|
+
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
|
40
55
|
}
|
|
41
56
|
export type CheerioRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
42
57
|
JSONData extends Dictionary = any> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";;;AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAE5C,OAAO,KAAK,EACR,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,eAAe,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,WAAW,EAAgE,MAAM,eAAe,CAAC;AAC1G,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"cheerio-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";;;AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAE5C,OAAO,KAAK,EACR,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,eAAe,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,WAAW,EAAgE,MAAM,eAAe,CAAC;AAC1G,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAA0B,MAAM,gBAAgB,CAAC;AAE1E,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,MAAM,MAAM,mBAAmB,CAC3B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE7D,MAAM,WAAW,qBAAqB,CAClC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,kBAAkB,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;CAAG;AAE3E,MAAM,MAAM,WAAW,CACnB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEjE,MAAM,WAAW,sBAAsB,CACnC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,cAAc,CAAC;IACrE;;;OAGG;IACH,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC;IAEtB;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;;;OAaG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,MAAM,qBAAqB,CAC7B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AACH,qBAAa,cAAe,SAAQ,WAAW,CAAC,sBAAsB,CAAC;IACnE;;OAEG;gBAES,OAAO,CAAC,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,aAAa;cAI1C,UAAU,CAC/B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,OAAO,EACd,eAAe,EAAE,sBAAsB;;;;wCAqBG,mBAAmB;;cAYjD,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,OAAO;cAehD,kBAAkB,CAAC,OAAO,EAAE,sBAAsB;CAgB9E;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,eAAe,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAC7C,OAAO,EACP,CAAC,EACD,YAAY,EACZ,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,4DAwB7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,mBAAmB,CAC/B,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,kDAEzC"}
|
|
@@ -132,7 +132,17 @@ class CheerioCrawler extends http_1.HttpCrawler {
|
|
|
132
132
|
});
|
|
133
133
|
}
|
|
134
134
|
async _runRequestHandler(context) {
|
|
135
|
-
context.
|
|
135
|
+
context.waitForSelector = async (selector, _timeoutMs) => {
|
|
136
|
+
if (context.$(selector).get().length === 0) {
|
|
137
|
+
throw new Error(`Selector '${selector}' not found.`);
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
context.parseWithCheerio = async (selector, timeoutMs) => {
|
|
141
|
+
if (selector) {
|
|
142
|
+
await context.waitForSelector(selector, timeoutMs);
|
|
143
|
+
}
|
|
144
|
+
return context.$;
|
|
145
|
+
};
|
|
136
146
|
await super._runRequestHandler(context);
|
|
137
147
|
}
|
|
138
148
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio-crawler.js","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";;;;AAcA,wCAA0G;AAE1G,
|
|
1
|
+
{"version":3,"file":"cheerio-crawler.js","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";;;;AAcA,wCAA0G;AAE1G,0CAA0E;AAE1E,yDAAmC;AACnC,6CAAyC;AACzC,mEAAgE;AA+DhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AACH,MAAa,cAAe,SAAQ,kBAAmC;IACnE;;OAEG;IACH,qEAAqE;IACrE,YAAY,OAA+B,EAAE,MAAsB;QAC/D,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAEkB,KAAK,CAAC,UAAU,CAC/B,QAAyB,EACzB,KAAc,EACd,eAAuC;QAEvC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAExD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAClB,GAAa,EACb;YACI,OAAO,EAAE,KAAK;YACd,yGAAyG;YACzG,gFAAgF;YAChF,2GAA2G;YAC3G,eAAe,EAAE,IAAI;SACN,CACtB,CAAC;QAEF,OAAO;YACH,GAAG;YACH,CAAC;YACD,IAAI,IAAI;gBACJ,OAAO,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,YAAY,EAAE,KAAK,EAAE,cAAoC,EAAE,EAAE;gBACzD,OAAO,0BAA0B,CAAC;oBAC9B,OAAO,EAAE,cAAc;oBACvB,CAAC;oBACD,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;iBACrD,CAAC,CAAC;YACP,CAAC;SACJ,CAAC;IACN,CAAC;IAES,KAAK,CAAC,eAAe,CAAC,QAAyB,EAAE,KAAc;QACrE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,UAAU,GAAG,IAAI,wBAAU,CAC7B,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBACT,IAAI,GAAG;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;oBAChB,OAAO,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC,EACD,EAAE,OAAO,EAAE,KAAK,EAAE,CACrB,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,+BAAc,CAAC,UAAU,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YACxF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;IACP,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,OAA+B;QACvE,OAAO,CAAC,eAAe,GAAG,KAAK,EAAE,QAAiB,EAAE,UAAmB,EAAE,EAAE;YACvE,IAAI,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,OAAO,CAAC,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACvE,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAM,OAAO,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YACvD,CAAC;YAED,OAAO,OAAO,CAAC,CAAC,CAAC;QACrB,CAAC,CAAC;QAEF,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;CACJ;AA5ED,wCA4EC;AAUD,gBAAgB;AACT,KAAK,UAAU,0BAA0B,CAAC,EAC7C,OAAO,EACP,CAAC,EACD,YAAY,EACZ,kBAAkB,EAClB,eAAe,GACW;IAC1B,IAAI,CAAC,CAAC,EAAE,CAAC;QACL,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,OAAO,GAAG,IAAA,6CAAsC,EAAC;QACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAA,8BAAsB,EAC/B,CAAC,EACD,OAAO,EAAE,QAAQ,IAAI,GAAG,EACxB,OAAO,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CAC5D,CAAC;IAEF,OAAO,IAAA,mBAAY,EAAC;QAChB,YAAY;QACZ,IAAI;QACJ,OAAO;QACP,GAAG,OAAO;KACb,CAAC,CAAC;AACP,CAAC;AA9BD,gEA8BC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,mBAAmB,CAGjC,MAAwC;IACtC,OAAO,aAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC;AALD,kDAKC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/cheerio",
|
|
3
|
-
"version": "3.10.3-beta.
|
|
3
|
+
"version": "3.10.3-beta.6",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -53,9 +53,9 @@
|
|
|
53
53
|
"access": "public"
|
|
54
54
|
},
|
|
55
55
|
"dependencies": {
|
|
56
|
-
"@crawlee/http": "3.10.3-beta.
|
|
57
|
-
"@crawlee/types": "3.10.3-beta.
|
|
58
|
-
"@crawlee/utils": "3.10.3-beta.
|
|
56
|
+
"@crawlee/http": "3.10.3-beta.6",
|
|
57
|
+
"@crawlee/types": "3.10.3-beta.6",
|
|
58
|
+
"@crawlee/utils": "3.10.3-beta.6",
|
|
59
59
|
"cheerio": "^1.0.0-rc.12",
|
|
60
60
|
"htmlparser2": "^9.0.0",
|
|
61
61
|
"tslib": "^2.4.0"
|
|
@@ -67,5 +67,5 @@
|
|
|
67
67
|
}
|
|
68
68
|
}
|
|
69
69
|
},
|
|
70
|
-
"gitHead": "
|
|
70
|
+
"gitHead": "e7024b7af7451c76bd20b988ea9866798b202ffe"
|
|
71
71
|
}
|