@crawlee/linkedom 3.10.3-beta.12 → 3.10.3-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -8,7 +8,7 @@ import type { IncomingMessage } from 'http';
|
|
|
8
8
|
import type { HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, ErrorHandler, RequestHandler, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, RequestProvider } from '@crawlee/http';
|
|
9
9
|
import { HttpCrawler } from '@crawlee/http';
|
|
10
10
|
import type { Dictionary } from '@crawlee/types';
|
|
11
|
-
import
|
|
11
|
+
import type * as cheerio from 'cheerio';
|
|
12
12
|
export type LinkeDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
13
13
|
JSONData extends Dictionary = any> = ErrorHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
|
|
14
14
|
export interface LinkeDOMCrawlerOptions<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
@@ -22,23 +22,8 @@ export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // w
|
|
|
22
22
|
JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData, LinkeDOMCrawler> {
|
|
23
23
|
window: Window;
|
|
24
24
|
document: Document;
|
|
25
|
-
/**
|
|
26
|
-
* Wait for an element matching the selector to appear.
|
|
27
|
-
* Timeout defaults to 5s.
|
|
28
|
-
*
|
|
29
|
-
* **Example usage:**
|
|
30
|
-
* ```ts
|
|
31
|
-
* async requestHandler({ waitForSelector, parseWithCheerio }) {
|
|
32
|
-
* await waitForSelector('article h1');
|
|
33
|
-
* const $ = await parseWithCheerio();
|
|
34
|
-
* const title = $('title').text();
|
|
35
|
-
* });
|
|
36
|
-
* ```
|
|
37
|
-
*/
|
|
38
|
-
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
39
25
|
/**
|
|
40
26
|
* Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.
|
|
41
|
-
* When provided with the `selector` argument, it will first look for the selector with a 5s timeout.
|
|
42
27
|
*
|
|
43
28
|
* **Example usage:**
|
|
44
29
|
* ```javascript
|
|
@@ -48,7 +33,7 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
|
|
|
48
33
|
* });
|
|
49
34
|
* ```
|
|
50
35
|
*/
|
|
51
|
-
parseWithCheerio(
|
|
36
|
+
parseWithCheerio(): Promise<cheerio.CheerioAPI>;
|
|
52
37
|
}
|
|
53
38
|
export type LinkeDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
54
39
|
JSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
|
|
@@ -131,7 +116,6 @@ export declare class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext
|
|
|
131
116
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
132
117
|
enqueueLinks: (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
|
|
133
118
|
}>;
|
|
134
|
-
_runRequestHandler(context: LinkeDOMCrawlingContext): Promise<void>;
|
|
135
119
|
}
|
|
136
120
|
interface EnqueueLinksInternalOptions {
|
|
137
121
|
options?: LinkeDOMCrawlerEnqueueLinksOptions;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"linkedom-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/linkedom-crawler.ts"],"names":[],"mappings":";;;AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAG5C,OAAO,KAAK,EACR,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,YAAY,EACZ,eAAe,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,WAAW,EAKd,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,
|
|
1
|
+
{"version":3,"file":"linkedom-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/linkedom-crawler.ts"],"names":[],"mappings":";;;AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAG5C,OAAO,KAAK,EACR,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,YAAY,EACZ,eAAe,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,WAAW,EAKd,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,KAAK,OAAO,MAAM,SAAS,CAAC;AAIxC,MAAM,MAAM,oBAAoB,CAC5B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,MAAM,WAAW,sBAAsB,CACnC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,kBAAkB,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;CAAG;AAE5E,MAAM,WAAW,kCAAmC,SAAQ,IAAI,CAAC,mBAAmB,EAAE,MAAM,GAAG,cAAc,CAAC;CAAG;AAEjH,MAAM,MAAM,YAAY,CACpB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAElE,MAAM,WAAW,uBAAuB,CACpC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,eAAe,CAAC;IACtE,MAAM,EAAE,MAAM,CAAC;IAMf,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;;;;;;;;;OAUG;IACH,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;CACnD;AAED,MAAM,MAAM,sBAAsB,CAC9B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AAEH,qBAAa,eAAgB,SAAQ,WAAW,CAAC,uBAAuB,CAAC;IACrE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAmB;cAEf,UAAU,CAC/B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,OAAO,EACd,eAAe,EAAE,uBAAuB;;;;wCAeE,kCAAkC;;CAWnF;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,kCAAkC,CAAC;IAC7C,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,eAAe,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,2BAA2B,CAAC,EAC9C,OAAO,EACP,MAAM,EACN,YAAY,EACZ,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,4DAwB7B;AAmBD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,SAAS,uBAAuB,GAAG,uBAAuB,EACjE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,kDAEzC"}
|
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createLinkeDOMRouter = exports.linkedomCrawlerEnqueueLinks = exports.LinkeDOMCrawler = void 0;
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
4
|
const utilities_1 = require("@apify/utilities");
|
|
6
5
|
const http_1 = require("@crawlee/http");
|
|
7
|
-
const utils_1 = require("@crawlee/utils");
|
|
8
|
-
const cheerio = tslib_1.__importStar(require("cheerio"));
|
|
9
6
|
// @ts-expect-error This throws a compilation error due to TypeScript not inferring the module has CJS versions too
|
|
10
7
|
const cached_1 = require("linkedom/cached");
|
|
11
8
|
/**
|
|
@@ -102,26 +99,6 @@ class LinkeDOMCrawler extends http_1.HttpCrawler {
|
|
|
102
99
|
},
|
|
103
100
|
};
|
|
104
101
|
}
|
|
105
|
-
async _runRequestHandler(context) {
|
|
106
|
-
context.waitForSelector = async (selector, timeoutMs = 5000) => {
|
|
107
|
-
const $ = cheerio.load(context.body);
|
|
108
|
-
if ($(selector).get().length === 0) {
|
|
109
|
-
if (timeoutMs) {
|
|
110
|
-
await (0, utils_1.sleep)(50);
|
|
111
|
-
return context.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
|
|
112
|
-
}
|
|
113
|
-
throw new Error(`Selector '${selector}' not found.`);
|
|
114
|
-
}
|
|
115
|
-
};
|
|
116
|
-
context.parseWithCheerio = async (selector, _timeoutMs = 5000) => {
|
|
117
|
-
const $ = cheerio.load(context.body);
|
|
118
|
-
if (selector && $(selector).get().length === 0) {
|
|
119
|
-
throw new Error(`Selector '${selector}' not found.`);
|
|
120
|
-
}
|
|
121
|
-
return $;
|
|
122
|
-
};
|
|
123
|
-
await super._runRequestHandler(context);
|
|
124
|
-
}
|
|
125
102
|
}
|
|
126
103
|
exports.LinkeDOMCrawler = LinkeDOMCrawler;
|
|
127
104
|
Object.defineProperty(LinkeDOMCrawler, "parser", {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"linkedom-crawler.js","sourceRoot":"","sources":["../../src/internals/linkedom-crawler.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"linkedom-crawler.js","sourceRoot":"","sources":["../../src/internals/linkedom-crawler.ts"],"names":[],"mappings":";;;AAEA,gDAAwD;AAYxD,wCAMuB;AAGvB,mHAAmH;AACnH,4CAA4C;AAkD5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AAEH,MAAa,eAAgB,SAAQ,kBAAoC;IAGlD,KAAK,CAAC,UAAU,CAC/B,QAAyB,EACzB,KAAc,EACd,eAAwC;QAExC,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;QAElD,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAE3G,OAAO;YACH,MAAM,EAAE,QAAQ,CAAC,WAAW;YAC5B,IAAI,IAAI;gBACJ,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;YAC9C,CAAC;YACD,IAAI,QAAQ;gBACR,iEAAiE;gBACjE,OAAO,QAA+B,CAAC;YAC3C,CAAC;YACD,YAAY,EAAE,KAAK,EAAE,cAAmD,EAAE,EAAE;gBACxE,OAAO,2BAA2B,CAAC;oBAC/B,OAAO,EAAE,cAAc;oBACvB,MAAM,EAAE,QAAQ,CAAC,WAAW;oBAC5B,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;iBACrD,CAAC,CAAC;YACP,CAAC;SACJ,CAAC;IACN,CAAC;;AA/BL,0CAgCC;AA/BkB;;;;WAAS,IAAI,kBAAS,EAAE;GAAC;AAyC5C,gBAAgB;AACT,KAAK,UAAU,2BAA2B,CAAC,EAC9C,OAAO,EACP,MAAM,EACN,YAAY,EACZ,kBAAkB,EAClB,eAAe,GACW;IAC1B,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,OAAO,GAAG,IAAA,6CAAsC,EAAC;QACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,qBAAqB,CAC9B,MAAM,EACN,OAAO,EAAE,QAAQ,IAAI,GAAG,EACxB,OAAO,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CAC5D,CAAC;IAEF,OAAO,IAAA,mBAAY,EAAC;QAChB,YAAY;QACZ,IAAI;QACJ,OAAO;QACP,GAAG,OAAO;KACb,CAAC,CAAC;AACP,CAAC;AA9BD,kEA8BC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,MAAc,EAAE,QAAgB,EAAE,OAAe;IAC5E,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;SACxD,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;SACnD,GAAG,CAAC,CAAC,IAAwB,EAAE,EAAE;QAC9B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,OAAO,IAAA,qBAAc,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAa,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,oBAAoB,CAGlC,MAAwC;IACtC,OAAO,aAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC;AALD,oDAKC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/linkedom",
|
|
3
|
-
"version": "3.10.3-beta.
|
|
3
|
+
"version": "3.10.3-beta.2",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -55,8 +55,8 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@apify/timeout": "^0.3.0",
|
|
57
57
|
"@apify/utilities": "^2.7.10",
|
|
58
|
-
"@crawlee/http": "3.10.3-beta.
|
|
59
|
-
"@crawlee/types": "3.10.3-beta.
|
|
58
|
+
"@crawlee/http": "3.10.3-beta.2",
|
|
59
|
+
"@crawlee/types": "3.10.3-beta.2",
|
|
60
60
|
"linkedom": "^0.18.0",
|
|
61
61
|
"ow": "^0.28.2",
|
|
62
62
|
"tslib": "^2.4.0"
|
|
@@ -68,5 +68,5 @@
|
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
},
|
|
71
|
-
"gitHead": "
|
|
71
|
+
"gitHead": "6de56e7b41a79aa00990d0b2d7450c8b522a68f8"
|
|
72
72
|
}
|