@crawlee/utils 4.0.0-beta.26 → 4.0.0-beta.260
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +10 -17
- package/index.js +8 -16
- package/internal.d.ts +12 -0
- package/internal.js +11 -0
- package/internals/blocked.d.ts +0 -1
- package/internals/blocked.js +0 -1
- package/internals/cheerio.d.ts +1 -3
- package/internals/cheerio.js +25 -16
- package/internals/extract-microdata.d.ts +24 -0
- package/internals/extract-microdata.js +118 -0
- package/internals/extract-urls.d.ts +1 -2
- package/internals/extract-urls.js +18 -17
- package/internals/general.d.ts +0 -25
- package/internals/general.js +2 -110
- package/internals/http-request.d.ts +5 -0
- package/internals/http-request.js +36 -0
- package/internals/iterables.d.ts +0 -80
- package/internals/iterables.js +0 -126
- package/internals/mutation-journal.d.ts +29 -0
- package/internals/mutation-journal.js +173 -0
- package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
- package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
- package/internals/robots.d.ts +35 -15
- package/internals/robots.js +66 -30
- package/internals/schemas.d.ts +114 -0
- package/internals/schemas.js +115 -0
- package/internals/sitemap.d.ts +44 -3
- package/internals/sitemap.js +173 -97
- package/internals/social.d.ts +4 -3
- package/internals/social.js +36 -11
- package/internals/url.d.ts +67 -1
- package/internals/url.js +118 -2
- package/internals/validation.d.ts +9 -0
- package/internals/validation.js +16 -0
- package/package.json +16 -15
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/blocked.d.ts.map +0 -1
- package/internals/blocked.js.map +0 -1
- package/internals/cheerio.d.ts.map +0 -1
- package/internals/cheerio.js.map +0 -1
- package/internals/chunk.d.ts +0 -2
- package/internals/chunk.d.ts.map +0 -1
- package/internals/chunk.js +0 -40
- package/internals/chunk.js.map +0 -1
- package/internals/debug.d.ts +0 -32
- package/internals/debug.d.ts.map +0 -1
- package/internals/debug.js +0 -41
- package/internals/debug.js.map +0 -1
- package/internals/extract-urls.d.ts.map +0 -1
- package/internals/extract-urls.js.map +0 -1
- package/internals/general.d.ts.map +0 -1
- package/internals/general.js.map +0 -1
- package/internals/iterables.d.ts.map +0 -1
- package/internals/iterables.js.map +0 -1
- package/internals/open_graph_parser.d.ts.map +0 -1
- package/internals/open_graph_parser.js.map +0 -1
- package/internals/robots.d.ts.map +0 -1
- package/internals/robots.js.map +0 -1
- package/internals/sitemap.d.ts.map +0 -1
- package/internals/sitemap.js.map +0 -1
- package/internals/social.d.ts.map +0 -1
- package/internals/social.js.map +0 -1
- package/internals/system-info/cpu-info.d.ts +0 -64
- package/internals/system-info/cpu-info.d.ts.map +0 -1
- package/internals/system-info/cpu-info.js +0 -211
- package/internals/system-info/cpu-info.js.map +0 -1
- package/internals/system-info/memory-info.d.ts +0 -28
- package/internals/system-info/memory-info.d.ts.map +0 -1
- package/internals/system-info/memory-info.js +0 -118
- package/internals/system-info/memory-info.js.map +0 -1
- package/internals/system-info/ps-tree.d.ts +0 -18
- package/internals/system-info/ps-tree.d.ts.map +0 -1
- package/internals/system-info/ps-tree.js +0 -145
- package/internals/system-info/ps-tree.js.map +0 -1
- package/internals/typedefs.d.ts +0 -5
- package/internals/typedefs.d.ts.map +0 -1
- package/internals/typedefs.js +0 -9
- package/internals/typedefs.js.map +0 -1
- package/internals/url.d.ts.map +0 -1
- package/internals/url.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift"
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
<p align=center>
|
|
17
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg"
|
|
18
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg"
|
|
19
|
-
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord"
|
|
20
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status"
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
21
21
|
</p>
|
|
22
22
|
|
|
23
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
|
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
34
34
|
|
|
35
35
|
We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
|
|
36
36
|
|
|
37
|
-
> Crawlee requires **Node.js
|
|
37
|
+
> Crawlee requires **Node.js 22.13 or higher**.
|
|
38
38
|
|
|
39
39
|
### With Crawlee CLI
|
|
40
40
|
|
|
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
89
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
98
98
|
{
|
|
99
99
|
"overrides": {
|
|
100
100
|
"apify": {
|
|
101
|
-
"@crawlee/core": "
|
|
102
|
-
"@crawlee/types": "
|
|
103
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
}
|
package/index.d.ts
CHANGED
|
@@ -1,19 +1,12 @@
|
|
|
1
|
-
export
|
|
2
|
-
export
|
|
3
|
-
export
|
|
4
|
-
export
|
|
5
|
-
export
|
|
6
|
-
export * from './internals/debug.js';
|
|
1
|
+
export { htmlToText } from './internals/cheerio.js';
|
|
2
|
+
export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
|
|
3
|
+
export { EnqueueStrategy } from './internals/url.js';
|
|
4
|
+
export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/extract-urls.js';
|
|
5
|
+
export { sleep } from './internals/general.js';
|
|
7
6
|
export * as social from './internals/social.js';
|
|
8
|
-
export * from './internals/
|
|
9
|
-
export * from './internals/
|
|
7
|
+
export * from './internals/extract-microdata.js';
|
|
8
|
+
export * from './internals/open-graph-parser.js';
|
|
10
9
|
export * from './internals/robots.js';
|
|
11
|
-
export
|
|
12
|
-
export
|
|
13
|
-
export
|
|
14
|
-
export * from './internals/sitemap.js';
|
|
15
|
-
export * from './internals/url.js';
|
|
16
|
-
export { getCurrentCpuTicksV2, CpuSample } from './internals/system-info/cpu-info.js';
|
|
17
|
-
export { getMemoryInfo, MemoryInfo } from './internals/system-info/memory-info.js';
|
|
18
|
-
export { Dictionary, Awaitable, Constructor } from '@crawlee/types';
|
|
19
|
-
//# sourceMappingURL=index.d.ts.map
|
|
10
|
+
export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
|
|
11
|
+
export type { ParseSitemapOptions } from './internals/sitemap.js';
|
|
12
|
+
export { ArgumentValidationError } from './internals/validation.js';
|
package/index.js
CHANGED
|
@@ -1,18 +1,10 @@
|
|
|
1
|
-
export
|
|
2
|
-
export
|
|
3
|
-
export
|
|
4
|
-
export
|
|
5
|
-
export * from './internals/general.js';
|
|
6
|
-
export * from './internals/debug.js';
|
|
1
|
+
export { htmlToText } from './internals/cheerio.js';
|
|
2
|
+
export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
|
|
3
|
+
export { EnqueueStrategy } from './internals/url.js';
|
|
4
|
+
export { sleep } from './internals/general.js';
|
|
7
5
|
export * as social from './internals/social.js';
|
|
8
|
-
export * from './internals/
|
|
9
|
-
export * from './internals/
|
|
6
|
+
export * from './internals/extract-microdata.js';
|
|
7
|
+
export * from './internals/open-graph-parser.js';
|
|
10
8
|
export * from './internals/robots.js';
|
|
11
|
-
export
|
|
12
|
-
export
|
|
13
|
-
export * from './internals/robots.js';
|
|
14
|
-
export * from './internals/sitemap.js';
|
|
15
|
-
export * from './internals/url.js';
|
|
16
|
-
export { getCurrentCpuTicksV2 } from './internals/system-info/cpu-info.js';
|
|
17
|
-
export { getMemoryInfo } from './internals/system-info/memory-info.js';
|
|
18
|
-
//# sourceMappingURL=index.js.map
|
|
9
|
+
export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
|
|
10
|
+
export { ArgumentValidationError } from './internals/validation.js';
|
package/internal.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export * from './internals/blocked.js';
|
|
2
|
+
export type { CheerioAPI, Cheerio, Element } from './internals/cheerio.js';
|
|
3
|
+
export { extractUrlsFromCheerio } from './internals/cheerio.js';
|
|
4
|
+
export { tryAbsoluteURL } from './internals/extract-urls.js';
|
|
5
|
+
export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
|
|
6
|
+
export { processHttpRequestOptions } from './internals/http-request.js';
|
|
7
|
+
export * from './internals/iterables.js';
|
|
8
|
+
export { MutationJournal } from './internals/mutation-journal.js';
|
|
9
|
+
export * from './internals/url.js';
|
|
10
|
+
export * from './internals/validation.js';
|
|
11
|
+
export * as schemas from './internals/schemas.js';
|
|
12
|
+
export { parseSitemap, type SitemapUrl } from './internals/sitemap.js';
|
package/internal.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export * from './internals/blocked.js';
|
|
2
|
+
export { extractUrlsFromCheerio } from './internals/cheerio.js';
|
|
3
|
+
export { tryAbsoluteURL } from './internals/extract-urls.js';
|
|
4
|
+
export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
|
|
5
|
+
export { processHttpRequestOptions } from './internals/http-request.js';
|
|
6
|
+
export * from './internals/iterables.js';
|
|
7
|
+
export { MutationJournal } from './internals/mutation-journal.js';
|
|
8
|
+
export * from './internals/url.js';
|
|
9
|
+
export * from './internals/validation.js';
|
|
10
|
+
export * as schemas from './internals/schemas.js';
|
|
11
|
+
export { parseSitemap } from './internals/sitemap.js';
|
package/internals/blocked.d.ts
CHANGED
package/internals/blocked.js
CHANGED
package/internals/cheerio.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { CheerioAPI } from 'cheerio';
|
|
2
|
-
export type CheerioRoot = CheerioAPI;
|
|
3
2
|
export type { CheerioAPI, Cheerio } from 'cheerio';
|
|
4
3
|
export type { Element } from 'domhandler';
|
|
5
4
|
/**
|
|
@@ -29,7 +28,7 @@ export type { Element } from 'domhandler';
|
|
|
29
28
|
* @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
|
|
30
29
|
* @return Plain text
|
|
31
30
|
*/
|
|
32
|
-
export declare function htmlToText(htmlOrCheerioElement: string |
|
|
31
|
+
export declare function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string>;
|
|
33
32
|
/**
|
|
34
33
|
* Extracts URLs from a given Cheerio object.
|
|
35
34
|
*
|
|
@@ -40,4 +39,3 @@ export declare function htmlToText(htmlOrCheerioElement: string | CheerioRoot):
|
|
|
40
39
|
* @return An array of absolute URLs
|
|
41
40
|
*/
|
|
42
41
|
export declare function extractUrlsFromCheerio($: CheerioAPI, selector?: string, baseUrl?: string): string[];
|
|
43
|
-
//# sourceMappingURL=cheerio.d.ts.map
|
package/internals/cheerio.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio';
|
|
2
1
|
import { tryAbsoluteURL } from './extract-urls.js';
|
|
3
2
|
// NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.
|
|
4
3
|
const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;
|
|
@@ -30,11 +29,20 @@ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|
|
|
|
30
29
|
* @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
|
|
31
30
|
* @return Plain text
|
|
32
31
|
*/
|
|
33
|
-
export function htmlToText(htmlOrCheerioElement) {
|
|
32
|
+
export async function htmlToText(htmlOrCheerioElement) {
|
|
33
|
+
const { load } = await import('cheerio/slim');
|
|
34
34
|
if (!htmlOrCheerioElement)
|
|
35
35
|
return '';
|
|
36
|
-
const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement :
|
|
37
|
-
|
|
36
|
+
const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
|
|
37
|
+
// Never read the accumulated text mid-loop: each read flattens V8's cons-string, making the traversal quadratic
|
|
38
|
+
const parts = [];
|
|
39
|
+
let last = '';
|
|
40
|
+
const push = (part) => {
|
|
41
|
+
if (!part)
|
|
42
|
+
return;
|
|
43
|
+
parts.push(part);
|
|
44
|
+
last = part[part.length - 1];
|
|
45
|
+
};
|
|
38
46
|
const process = (elems) => {
|
|
39
47
|
const len = elems ? elems.length : 0;
|
|
40
48
|
for (let i = 0; i < len; i++) {
|
|
@@ -42,40 +50,42 @@ export function htmlToText(htmlOrCheerioElement) {
|
|
|
42
50
|
if (elem.type === 'text') {
|
|
43
51
|
// Compress spaces, unless we're inside <pre> element
|
|
44
52
|
let compr;
|
|
45
|
-
if (elem.parent
|
|
46
|
-
|
|
53
|
+
if (elem.parent?.tagName === 'pre') {
|
|
54
|
+
// Strip single newline right after `<pre>` that htmlparser2 keeps
|
|
55
|
+
compr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;
|
|
56
|
+
}
|
|
47
57
|
else
|
|
48
58
|
compr = elem.data.replace(/\s+/g, ' ');
|
|
49
59
|
// If text is empty or ends with a whitespace, don't add the leading whitespace
|
|
50
|
-
if (compr.startsWith(' ') &&
|
|
60
|
+
if (compr.startsWith(' ') && (last === '' || /\s/.test(last)))
|
|
51
61
|
compr = compr.substring(1);
|
|
52
|
-
|
|
62
|
+
push(compr);
|
|
53
63
|
}
|
|
54
64
|
else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {
|
|
55
65
|
// Skip comments and special elements
|
|
56
66
|
}
|
|
57
67
|
else if (elem.tagName === 'br') {
|
|
58
|
-
|
|
68
|
+
push('\n');
|
|
59
69
|
}
|
|
60
70
|
else if (elem.tagName === 'td') {
|
|
61
71
|
process(elem.children);
|
|
62
|
-
|
|
72
|
+
push('\t');
|
|
63
73
|
}
|
|
64
74
|
else {
|
|
65
75
|
// Block elements must be surrounded by newlines (unless beginning of text)
|
|
66
76
|
const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
|
|
67
|
-
if (isBlockTag &&
|
|
68
|
-
|
|
77
|
+
if (isBlockTag && last !== '' && last !== '\n')
|
|
78
|
+
push('\n');
|
|
69
79
|
process(elem.children);
|
|
70
|
-
if (isBlockTag &&
|
|
71
|
-
|
|
80
|
+
if (isBlockTag && last !== '\n')
|
|
81
|
+
push('\n');
|
|
72
82
|
}
|
|
73
83
|
}
|
|
74
84
|
};
|
|
75
85
|
// If HTML document has body, only convert that, otherwise convert the entire HTML
|
|
76
86
|
const $body = $('body');
|
|
77
87
|
process($body.length > 0 ? $body : $.root());
|
|
78
|
-
return
|
|
88
|
+
return parts.join('').trim();
|
|
79
89
|
}
|
|
80
90
|
/**
|
|
81
91
|
* Extracts URLs from a given Cheerio object.
|
|
@@ -107,4 +117,3 @@ export function extractUrlsFromCheerio($, selector = 'a', baseUrl = '') {
|
|
|
107
117
|
})
|
|
108
118
|
.filter(Boolean);
|
|
109
119
|
}
|
|
110
|
-
//# sourceMappingURL=cheerio.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { CheerioAPI } from 'cheerio';
|
|
2
|
+
/** The value of a microdata property: either text or a nested item. */
|
|
3
|
+
export type MicrodataValue = string | MicrodataItem;
|
|
4
|
+
/** A single schema.org item extracted from a document. */
|
|
5
|
+
export interface MicrodataItem {
|
|
6
|
+
/** Tokens of the item's `itemtype` attribute. */
|
|
7
|
+
type?: string[];
|
|
8
|
+
/** The item's `itemid` attribute. */
|
|
9
|
+
id?: string;
|
|
10
|
+
/** Values keyed by `itemprop` name, an array where the property repeats. */
|
|
11
|
+
properties: Record<string, MicrodataValue | MicrodataValue[]>;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Easily parse all schema.org microdata from a page with just a `CheerioAPI` object or raw HTML,
|
|
15
|
+
* following the [microdata processing model](https://html.spec.whatwg.org/multipage/microdata.html#microdata).
|
|
16
|
+
*
|
|
17
|
+
* Text values are trimmed and their inner whitespace collapsed. URL-valued attributes are returned
|
|
18
|
+
* verbatim rather than resolved against the document's base URL.
|
|
19
|
+
*
|
|
20
|
+
* @param htmlOrCheerioElement A `CheerioAPI` object, or a string of raw HTML.
|
|
21
|
+
* @returns The document's top-level items. Nested items are the property values of their parent.
|
|
22
|
+
*/
|
|
23
|
+
export declare function extractMicrodata(raw: string): Promise<MicrodataItem[]>;
|
|
24
|
+
export declare function extractMicrodata($: CheerioAPI): Promise<MicrodataItem[]>;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { isTag } from 'domhandler';
|
|
2
|
+
export async function extractMicrodata(htmlOrCheerioElement) {
|
|
3
|
+
// Dynamic so that importing `@crawlee/utils` does not pull in cheerio - see #3836.
|
|
4
|
+
const { load } = await import('cheerio/slim');
|
|
5
|
+
const $ = typeof htmlOrCheerioElement === 'string' ? load(htmlOrCheerioElement) : htmlOrCheerioElement;
|
|
6
|
+
const context = { $ };
|
|
7
|
+
return $('[itemscope]')
|
|
8
|
+
.toArray()
|
|
9
|
+
.filter((element) => !('itemprop' in element.attribs))
|
|
10
|
+
.map((element) => parseItem(context, element, new Set()));
|
|
11
|
+
}
|
|
12
|
+
function parseItem(context, element, ancestors) {
|
|
13
|
+
const item = { properties: {} };
|
|
14
|
+
const type = uniqueTokens(element.attribs.itemtype);
|
|
15
|
+
const id = element.attribs.itemid;
|
|
16
|
+
if (type.length > 0) {
|
|
17
|
+
item.type = type;
|
|
18
|
+
}
|
|
19
|
+
if (id) {
|
|
20
|
+
item.id = id.trim();
|
|
21
|
+
}
|
|
22
|
+
ancestors.add(element);
|
|
23
|
+
for (const propertyElement of collectPropertyElements(context, element)) {
|
|
24
|
+
const value = getPropertyValue(context, propertyElement, ancestors);
|
|
25
|
+
for (const name of uniqueTokens(propertyElement.attribs.itemprop)) {
|
|
26
|
+
addProperty(item.properties, name, value);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
ancestors.delete(element);
|
|
30
|
+
return item;
|
|
31
|
+
}
|
|
32
|
+
function collectPropertyElements(context, scope) {
|
|
33
|
+
const elements = [];
|
|
34
|
+
collectFromNodes(scope.children, elements);
|
|
35
|
+
for (const id of uniqueTokens(scope.attribs.itemref)) {
|
|
36
|
+
const referenced = (context.idIndex ??= indexIds(context.$)).get(id);
|
|
37
|
+
if (referenced) {
|
|
38
|
+
collectFromNodes([referenced], elements);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return elements;
|
|
42
|
+
}
|
|
43
|
+
function collectFromNodes(nodes, elements) {
|
|
44
|
+
for (const node of nodes) {
|
|
45
|
+
if (!isTag(node)) {
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
if ('itemprop' in node.attribs) {
|
|
49
|
+
elements.push(node);
|
|
50
|
+
}
|
|
51
|
+
// A nested item owns everything below it, so its subtree is not part of the enclosing item.
|
|
52
|
+
if (!('itemscope' in node.attribs)) {
|
|
53
|
+
collectFromNodes(node.children, elements);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
function indexIds($) {
|
|
58
|
+
const index = new Map();
|
|
59
|
+
for (const element of $('[id]').toArray()) {
|
|
60
|
+
// Duplicate ids are invalid HTML; `getElementById` resolves them to the first element.
|
|
61
|
+
if (!index.has(element.attribs.id)) {
|
|
62
|
+
index.set(element.attribs.id, element);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return index;
|
|
66
|
+
}
|
|
67
|
+
function getPropertyValue(context, element, ancestors) {
|
|
68
|
+
if ('itemscope' in element.attribs) {
|
|
69
|
+
// `itemref` can point back at an enclosing item, which the spec treats as an error.
|
|
70
|
+
return ancestors.has(element) ? { properties: {} } : parseItem(context, element, ancestors);
|
|
71
|
+
}
|
|
72
|
+
const { attribs } = element;
|
|
73
|
+
switch (element.tagName.toLowerCase()) {
|
|
74
|
+
case 'meta':
|
|
75
|
+
return attribs.content ?? '';
|
|
76
|
+
case 'audio':
|
|
77
|
+
case 'embed':
|
|
78
|
+
case 'iframe':
|
|
79
|
+
case 'img':
|
|
80
|
+
case 'source':
|
|
81
|
+
case 'track':
|
|
82
|
+
case 'video':
|
|
83
|
+
return attribs.src ?? '';
|
|
84
|
+
case 'a':
|
|
85
|
+
case 'area':
|
|
86
|
+
case 'link':
|
|
87
|
+
return attribs.href ?? '';
|
|
88
|
+
case 'object':
|
|
89
|
+
return attribs.data ?? '';
|
|
90
|
+
case 'data':
|
|
91
|
+
case 'meter':
|
|
92
|
+
return attribs.value ?? '';
|
|
93
|
+
case 'time':
|
|
94
|
+
return attribs.datetime ?? context.$(element).text().replace(/\s+/g, ' ').trim();
|
|
95
|
+
default:
|
|
96
|
+
return context.$(element).text().replace(/\s+/g, ' ').trim();
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
function addProperty(properties, name, value) {
|
|
100
|
+
const existing = properties[name];
|
|
101
|
+
if (existing === undefined) {
|
|
102
|
+
properties[name] = value;
|
|
103
|
+
}
|
|
104
|
+
else if (Array.isArray(existing)) {
|
|
105
|
+
existing.push(value);
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
properties[name] = [existing, value];
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/** `itemtype`, `itemprop` and `itemref` are all unordered sets of unique space-separated tokens. */
|
|
112
|
+
function uniqueTokens(value) {
|
|
113
|
+
if (!value) {
|
|
114
|
+
return [];
|
|
115
|
+
}
|
|
116
|
+
const tokens = value.split(/\s+/).filter(Boolean);
|
|
117
|
+
return tokens.length > 1 ? [...new Set(tokens)] : tokens;
|
|
118
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BaseHttpClient } from '@crawlee/
|
|
1
|
+
import type { BaseHttpClient } from '@crawlee/http-client';
|
|
2
2
|
export interface DownloadListOfUrlsOptions {
|
|
3
3
|
/**
|
|
4
4
|
* URL to the file
|
|
@@ -46,4 +46,3 @@ export declare function extractUrls(options: ExtractUrlsOptions): string[];
|
|
|
46
46
|
* Helper function used to validate URLs used when extracting URLs from a page
|
|
47
47
|
*/
|
|
48
48
|
export declare function tryAbsoluteURL(href: string, baseUrl: string): string | undefined;
|
|
49
|
-
//# sourceMappingURL=extract-urls.d.ts.map
|
|
@@ -1,21 +1,27 @@
|
|
|
1
1
|
import { FetchHttpClient } from '@crawlee/http-client';
|
|
2
|
-
import
|
|
2
|
+
import { z } from 'zod';
|
|
3
3
|
import { URL_NO_COMMAS_REGEX } from './general.js';
|
|
4
|
+
import { httpClient as httpClientSchema } from './schemas.js';
|
|
5
|
+
import { parseArgument } from './validation.js';
|
|
6
|
+
const downloadListOfUrlsOptionsSchema = z.strictObject({
|
|
7
|
+
url: z.url(),
|
|
8
|
+
encoding: z.string().default('utf8'),
|
|
9
|
+
urlRegExp: z.instanceof(RegExp).default(URL_NO_COMMAS_REGEX),
|
|
10
|
+
proxyUrl: z.string().optional(),
|
|
11
|
+
httpClient: httpClientSchema.default(() => new FetchHttpClient()),
|
|
12
|
+
});
|
|
13
|
+
const extractUrlsOptionsSchema = z.strictObject({
|
|
14
|
+
string: z.string(),
|
|
15
|
+
urlRegExp: z.instanceof(RegExp).default(URL_NO_COMMAS_REGEX),
|
|
16
|
+
});
|
|
4
17
|
/**
|
|
5
18
|
* Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
|
|
6
19
|
* Optionally, custom regular expression and encoding may be provided.
|
|
7
20
|
*/
|
|
8
21
|
export async function downloadListOfUrls(options) {
|
|
9
|
-
|
|
10
|
-
url: ow.string.url,
|
|
11
|
-
encoding: ow.optional.string,
|
|
12
|
-
urlRegExp: ow.optional.regExp,
|
|
13
|
-
proxyUrl: ow.optional.string,
|
|
14
|
-
httpClient: ow.optional.object,
|
|
15
|
-
}));
|
|
16
|
-
const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl, httpClient = new FetchHttpClient(), } = options;
|
|
22
|
+
const { url, encoding, urlRegExp, proxyUrl, httpClient } = parseArgument(options, downloadListOfUrlsOptionsSchema);
|
|
17
23
|
// Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
|
|
18
|
-
const match =
|
|
24
|
+
const match = /^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/.exec(url);
|
|
19
25
|
let fixedUrl = url;
|
|
20
26
|
if (match) {
|
|
21
27
|
fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`;
|
|
@@ -30,13 +36,9 @@ export async function downloadListOfUrls(options) {
|
|
|
30
36
|
* Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
|
|
31
37
|
*/
|
|
32
38
|
export function extractUrls(options) {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
urlRegExp: ow.optional.regExp,
|
|
36
|
-
}));
|
|
37
|
-
const lines = options.string.split('\n');
|
|
39
|
+
const { string, urlRegExp } = parseArgument(options, extractUrlsOptionsSchema);
|
|
40
|
+
const lines = string.split('\n');
|
|
38
41
|
const result = [];
|
|
39
|
-
const urlRegExp = options.urlRegExp ?? URL_NO_COMMAS_REGEX;
|
|
40
42
|
for (const line of lines) {
|
|
41
43
|
result.push(...(line.match(urlRegExp) ?? []));
|
|
42
44
|
}
|
|
@@ -53,4 +55,3 @@ export function tryAbsoluteURL(href, baseUrl) {
|
|
|
53
55
|
return undefined;
|
|
54
56
|
}
|
|
55
57
|
}
|
|
56
|
-
//# sourceMappingURL=extract-urls.js.map
|
package/internals/general.d.ts
CHANGED
|
@@ -8,25 +8,6 @@ export declare const URL_NO_COMMAS_REGEX: RegExp;
|
|
|
8
8
|
* Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
|
|
9
9
|
*/
|
|
10
10
|
export declare const URL_WITH_COMMAS_REGEX: RegExp;
|
|
11
|
-
/**
|
|
12
|
-
* Returns a `Promise` that resolves to true if the code is running in a Docker container.
|
|
13
|
-
*/
|
|
14
|
-
export declare function isDocker(forceReset?: boolean): Promise<boolean>;
|
|
15
|
-
/**
|
|
16
|
-
* Detects if crawlee is running in a containerized environment.
|
|
17
|
-
*/
|
|
18
|
-
export declare function isContainerized(): Promise<boolean>;
|
|
19
|
-
export declare function isLambda(): boolean;
|
|
20
|
-
/**
|
|
21
|
-
* gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
|
|
22
|
-
* @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
|
|
23
|
-
*/
|
|
24
|
-
export declare function getCgroupsVersion(forceReset?: boolean): Promise<"V1" | "V2" | null>;
|
|
25
|
-
/**
|
|
26
|
-
* Computes a weighted average of an array of numbers, complemented by an array of weights.
|
|
27
|
-
* @ignore
|
|
28
|
-
*/
|
|
29
|
-
export declare function weightedAvg(arrValues: number[], arrWeights: number[]): number;
|
|
30
11
|
/**
|
|
31
12
|
* Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
|
|
32
13
|
* in your code, e.g. to prevent overloading of target website or to avoid bot detection.
|
|
@@ -44,14 +25,8 @@ export declare function weightedAvg(arrValues: number[], arrWeights: number[]):
|
|
|
44
25
|
* @param millis Period of time to sleep, in milliseconds. If not a positive number, the returned promise resolves immediately.
|
|
45
26
|
*/
|
|
46
27
|
export declare function sleep(millis?: number): Promise<void>;
|
|
47
|
-
/**
|
|
48
|
-
* Converts SNAKE_CASE to camelCase.
|
|
49
|
-
* @ignore
|
|
50
|
-
*/
|
|
51
|
-
export declare function snakeCaseToCamelCase(snakeCaseStr: string): string;
|
|
52
28
|
/**
|
|
53
29
|
* Traverses DOM and expands shadow-root elements (created by custom components).
|
|
54
30
|
* @ignore
|
|
55
31
|
*/
|
|
56
32
|
export declare function expandShadowRoots(document: Document): string;
|
|
57
|
-
//# sourceMappingURL=general.d.ts.map
|