@crawlee/linkedom 3.10.3-beta.4 → 3.10.3-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -8,7 +8,7 @@ import type { IncomingMessage } from 'http';
|
|
|
8
8
|
import type { HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, ErrorHandler, RequestHandler, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, RequestProvider } from '@crawlee/http';
|
|
9
9
|
import { HttpCrawler } from '@crawlee/http';
|
|
10
10
|
import type { Dictionary } from '@crawlee/types';
|
|
11
|
-
import type
|
|
11
|
+
import { type CheerioRoot } from '@crawlee/utils';
|
|
12
12
|
export type LinkeDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
13
13
|
JSONData extends Dictionary = any> = ErrorHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
|
|
14
14
|
export interface LinkeDOMCrawlerOptions<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
@@ -22,8 +22,23 @@ export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // w
|
|
|
22
22
|
JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData, LinkeDOMCrawler> {
|
|
23
23
|
window: Window;
|
|
24
24
|
document: Document;
|
|
25
|
+
/**
|
|
26
|
+
* Wait for an element matching the selector to appear.
|
|
27
|
+
* Timeout defaults to 5s.
|
|
28
|
+
*
|
|
29
|
+
* **Example usage:**
|
|
30
|
+
* ```ts
|
|
31
|
+
* async requestHandler({ waitForSelector, parseWithCheerio }) {
|
|
32
|
+
* await waitForSelector('article h1');
|
|
33
|
+
* const $ = await parseWithCheerio();
|
|
34
|
+
* const title = $('title').text();
|
|
35
|
+
* });
|
|
36
|
+
* ```
|
|
37
|
+
*/
|
|
38
|
+
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
25
39
|
/**
|
|
26
40
|
* Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.
|
|
41
|
+
* When provided with the `selector` argument, it will first look for the selector with a 5s timeout.
|
|
27
42
|
*
|
|
28
43
|
* **Example usage:**
|
|
29
44
|
* ```javascript
|
|
@@ -33,7 +48,7 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
|
|
|
33
48
|
* });
|
|
34
49
|
* ```
|
|
35
50
|
*/
|
|
36
|
-
parseWithCheerio(): Promise<
|
|
51
|
+
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
|
37
52
|
}
|
|
38
53
|
export type LinkeDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
39
54
|
JSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
|
|
@@ -116,6 +131,7 @@ export declare class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext
|
|
|
116
131
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
117
132
|
enqueueLinks: (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
|
|
118
133
|
}>;
|
|
134
|
+
_runRequestHandler(context: LinkeDOMCrawlingContext): Promise<void>;
|
|
119
135
|
}
|
|
120
136
|
interface EnqueueLinksInternalOptions {
|
|
121
137
|
options?: LinkeDOMCrawlerEnqueueLinksOptions;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"linkedom-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/linkedom-crawler.ts"],"names":[],"mappings":";;;AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAG5C,OAAO,KAAK,EACR,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,YAAY,EACZ,eAAe,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,WAAW,EAKd,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,
|
|
1
|
+
{"version":3,"file":"linkedom-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/linkedom-crawler.ts"],"names":[],"mappings":";;;AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAG5C,OAAO,KAAK,EACR,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,YAAY,EACZ,eAAe,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,WAAW,EAKd,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAS,MAAM,gBAAgB,CAAC;AAKzD,MAAM,MAAM,oBAAoB,CAC5B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,MAAM,WAAW,sBAAsB,CACnC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,kBAAkB,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;CAAG;AAE5E,MAAM,WAAW,kCAAmC,SAAQ,IAAI,CAAC,mBAAmB,EAAE,MAAM,GAAG,cAAc,CAAC;CAAG;AAEjH,MAAM,MAAM,YAAY,CACpB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAElE,MAAM,WAAW,uBAAuB,CACpC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,eAAe,CAAC;IACtE,MAAM,EAAE,MAAM,CAAC;IAMf,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,MAAM,sBAAsB,CAC9B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AAEH,qBAAa,eAAgB,SAAQ,WAAW,CAAC,uBAAuB,CAAC;IACrE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAmB;cAEf,UAAU,CAC/B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,OAAO,EACd,eAAe,EAAE,uBAAuB;;;;wCAeE,kCAAkC;;IAYjE,kBAAkB,CAAC,OAAO,EAAE,uBAAuB;CAyBrE;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,kCAAkC,CAAC;IAC7C,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,eAAe,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,2BAA2B,CAAC,EAC9C,OAAO,EACP,MAAM,EACN,YAAY,EACZ,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,4DAwB7B;AAmBD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,SAAS,uBAAuB,GAAG,uBAAuB,EACjE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,kDAEzC"}
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createLinkeDOMRouter = exports.linkedomCrawlerEnqueueLinks = exports.LinkeDOMCrawler = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
4
5
|
const utilities_1 = require("@apify/utilities");
|
|
5
6
|
const http_1 = require("@crawlee/http");
|
|
7
|
+
const utils_1 = require("@crawlee/utils");
|
|
8
|
+
const cheerio = tslib_1.__importStar(require("cheerio"));
|
|
6
9
|
// @ts-expect-error This throws a compilation error due to TypeScript not inferring the module has CJS versions too
|
|
7
10
|
const cached_1 = require("linkedom/cached");
|
|
8
11
|
/**
|
|
@@ -99,6 +102,26 @@ class LinkeDOMCrawler extends http_1.HttpCrawler {
|
|
|
99
102
|
},
|
|
100
103
|
};
|
|
101
104
|
}
|
|
105
|
+
async _runRequestHandler(context) {
|
|
106
|
+
context.waitForSelector = async (selector, timeoutMs = 5000) => {
|
|
107
|
+
const $ = cheerio.load(context.body);
|
|
108
|
+
if ($(selector).get().length === 0) {
|
|
109
|
+
if (timeoutMs) {
|
|
110
|
+
await (0, utils_1.sleep)(50);
|
|
111
|
+
return context.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
|
|
112
|
+
}
|
|
113
|
+
throw new Error(`Selector '${selector}' not found.`);
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
context.parseWithCheerio = async (selector, _timeoutMs = 5000) => {
|
|
117
|
+
const $ = cheerio.load(context.body);
|
|
118
|
+
if (selector && $(selector).get().length === 0) {
|
|
119
|
+
throw new Error(`Selector '${selector}' not found.`);
|
|
120
|
+
}
|
|
121
|
+
return $;
|
|
122
|
+
};
|
|
123
|
+
await super._runRequestHandler(context);
|
|
124
|
+
}
|
|
102
125
|
}
|
|
103
126
|
exports.LinkeDOMCrawler = LinkeDOMCrawler;
|
|
104
127
|
Object.defineProperty(LinkeDOMCrawler, "parser", {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"linkedom-crawler.js","sourceRoot":"","sources":["../../src/internals/linkedom-crawler.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"linkedom-crawler.js","sourceRoot":"","sources":["../../src/internals/linkedom-crawler.ts"],"names":[],"mappings":";;;;AAEA,gDAAwD;AAYxD,wCAMuB;AAEvB,0CAAyD;AACzD,yDAAmC;AACnC,mHAAmH;AACnH,4CAA4C;AAkE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AAEH,MAAa,eAAgB,SAAQ,kBAAoC;IAGlD,KAAK,CAAC,UAAU,CAC/B,QAAyB,EACzB,KAAc,EACd,eAAwC;QAExC,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;QAElD,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAE3G,OAAO;YACH,MAAM,EAAE,QAAQ,CAAC,WAAW;YAC5B,IAAI,IAAI;gBACJ,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;YAC9C,CAAC;YACD,IAAI,QAAQ;gBACR,iEAAiE;gBACjE,OAAO,QAA+B,CAAC;YAC3C,CAAC;YACD,YAAY,EAAE,KAAK,EAAE,cAAmD,EAAE,EAAE;gBACxE,OAAO,2BAA2B,CAAC;oBAC/B,OAAO,EAAE,cAAc;oBACvB,MAAM,EAAE,QAAQ,CAAC,WAAW;oBAC5B,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;iBACrD,CAAC,CAAC;YACP,CAAC;SACJ,CAAC;IACN,CAAC;IAEQ,KAAK,CAAC,kBAAkB,CAAC,OAAgC;QAC9D,OAAO,CAAC,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,SAAS,GAAG,IAAK,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAErC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,IAAI,SAAS,EAAE,CAAC;oBACZ,MAAM,IAAA,aAAK,EAAC,EAAE,CAAC,CAAC;oBAChB,OAAO,OAAO,CAAC,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC1E,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,OAAO,CAAC,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,UAAU,GAAG,IAAK,EAAE,EAAE;YACvE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAErC,IAAI,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;;AAzDL,0CA0DC;AAzDkB;;;;WAAS,IAAI,kBAAS,EAAE;GAAC;AAmE5C,gBAAgB;AACT,KAAK,UAAU,2BAA2B,CAAC,EAC9C,OAAO,EACP,MAAM,EACN,YAAY,EACZ,kBAAkB,EAClB,eAAe,GACW;IAC1B,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,OAAO,GAAG,IAAA,6CAAsC,EAAC;QACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,qBAAqB,CAC9B,MAAM,EACN,OAAO,EAAE,QAAQ,IAAI,GAAG,EACxB,OAAO,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CAC5D,CAAC;IAEF,OAAO,IAAA,mBAAY,EAAC;QAChB,YAAY;QACZ,IAAI;QACJ,OAAO;QACP,GAAG,OAAO;KACb,CAAC,CAAC;AACP,CAAC;AA9BD,kEA8BC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,MAAc,EAAE,QAAgB,EAAE,OAAe;IAC5E,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;SACxD,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;SACnD,GAAG,CAAC,CAAC,IAAwB,EAAE,EAAE;QAC9B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,OAAO,IAAA,qBAAc,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAa,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,oBAAoB,CAGlC,MAAwC;IACtC,OAAO,aAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC;AALD,oDAKC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/linkedom",
|
|
3
|
-
"version": "3.10.3-beta.
|
|
3
|
+
"version": "3.10.3-beta.6",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -55,8 +55,8 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@apify/timeout": "^0.3.0",
|
|
57
57
|
"@apify/utilities": "^2.7.10",
|
|
58
|
-
"@crawlee/http": "3.10.3-beta.
|
|
59
|
-
"@crawlee/types": "3.10.3-beta.
|
|
58
|
+
"@crawlee/http": "3.10.3-beta.6",
|
|
59
|
+
"@crawlee/types": "3.10.3-beta.6",
|
|
60
60
|
"linkedom": "^0.18.0",
|
|
61
61
|
"ow": "^0.28.2",
|
|
62
62
|
"tslib": "^2.4.0"
|
|
@@ -68,5 +68,5 @@
|
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
},
|
|
71
|
-
"gitHead": "
|
|
71
|
+
"gitHead": "e7024b7af7451c76bd20b988ea9866798b202ffe"
|
|
72
72
|
}
|