@crawlee/basic 4.0.0-beta.268 → 4.0.0-beta.269
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -34,13 +34,13 @@ export declare function createSkippedRequestArgs(source: string | Source, reason
|
|
|
34
34
|
/**
|
|
35
35
|
* @ignore
|
|
36
36
|
*/
|
|
37
|
-
export declare function updateEnqueueLinksPatternCache(item:
|
|
37
|
+
export declare function updateEnqueueLinksPatternCache(item: UrlPatternInput, pattern: UrlPatternObject): void;
|
|
38
38
|
/**
|
|
39
39
|
* Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
|
|
40
40
|
* to construct Glob objects from Glob pattern strings.
|
|
41
41
|
* @ignore
|
|
42
42
|
*/
|
|
43
|
-
export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
|
|
43
|
+
export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): (GlobObject & UrlPatternObject)[];
|
|
44
44
|
/**
|
|
45
45
|
* @internal
|
|
46
46
|
*/
|
|
@@ -50,7 +50,7 @@ export declare function validateGlobPattern(glob: string): string;
|
|
|
50
50
|
* to check RegExps input and return valid RegExps.
|
|
51
51
|
* @ignore
|
|
52
52
|
*/
|
|
53
|
-
export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
|
|
53
|
+
export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): (RegExpObject & UrlPatternObject)[];
|
|
54
54
|
/**
|
|
55
55
|
* The compiled form of a {@link UrlPatternInput}, produced by {@link constructUrlPatternObjects}. Deliberately
|
|
56
56
|
* not re-exported from the package entry point — it is an implementation detail of the pattern matching helpers.
|
|
@@ -59,6 +59,7 @@ export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegE
|
|
|
59
59
|
export interface UrlPatternObject {
|
|
60
60
|
glob?: string;
|
|
61
61
|
regexp?: RegExp;
|
|
62
|
+
match: (url: string) => boolean;
|
|
62
63
|
}
|
|
63
64
|
/**
|
|
64
65
|
* Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
|
|
@@ -45,8 +45,7 @@ export function createSkippedRequestArgs(source, reason) {
|
|
|
45
45
|
export function updateEnqueueLinksPatternCache(item, pattern) {
|
|
46
46
|
enqueueLinksPatternCache.set(item, pattern);
|
|
47
47
|
if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
|
|
48
|
-
|
|
49
|
-
enqueueLinksPatternCache.delete(key);
|
|
48
|
+
enqueueLinksPatternCache.delete(enqueueLinksPatternCache.keys().next().value);
|
|
50
49
|
}
|
|
51
50
|
}
|
|
52
51
|
/**
|
|
@@ -70,16 +69,12 @@ export function constructGlobObjectsFromGlobs(globs) {
|
|
|
70
69
|
return false;
|
|
71
70
|
})
|
|
72
71
|
.map((item) => {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
}
|
|
80
|
-
else {
|
|
81
|
-
globObject = { glob: validateGlobPattern(item.glob) };
|
|
82
|
-
}
|
|
72
|
+
const cached = enqueueLinksPatternCache.get(item);
|
|
73
|
+
if (cached)
|
|
74
|
+
return cached;
|
|
75
|
+
const glob = validateGlobPattern(typeof item === 'string' ? item : item.glob);
|
|
76
|
+
const minimatch = new Minimatch(glob, { nocase: true });
|
|
77
|
+
const globObject = { glob, match: (url) => minimatch.match(url) };
|
|
83
78
|
updateEnqueueLinksPatternCache(item, globObject);
|
|
84
79
|
return globObject;
|
|
85
80
|
});
|
|
@@ -100,16 +95,11 @@ export function validateGlobPattern(glob) {
|
|
|
100
95
|
*/
|
|
101
96
|
export function constructRegExpObjectsFromRegExps(regexps) {
|
|
102
97
|
return regexps.map((item) => {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
regexpObject = { regexp: item };
|
|
109
|
-
}
|
|
110
|
-
else {
|
|
111
|
-
regexpObject = { regexp: item.regexp };
|
|
112
|
-
}
|
|
98
|
+
const cached = enqueueLinksPatternCache.get(item);
|
|
99
|
+
if (cached)
|
|
100
|
+
return cached;
|
|
101
|
+
const regexp = item instanceof RegExp ? item : item.regexp;
|
|
102
|
+
const regexpObject = { regexp, match: (url) => regexp.test(url) };
|
|
113
103
|
updateEnqueueLinksPatternCache(item, regexpObject);
|
|
114
104
|
return regexpObject;
|
|
115
105
|
});
|
|
@@ -138,21 +128,19 @@ export function constructUrlPatternObjects(patterns) {
|
|
|
138
128
|
* @ignore
|
|
139
129
|
*/
|
|
140
130
|
export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedRequestOptions) {
|
|
141
|
-
const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
|
|
142
|
-
const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
|
|
143
131
|
return requestOptions
|
|
144
132
|
.filter((opts) => {
|
|
145
|
-
const matchesExclude =
|
|
133
|
+
const matchesExclude = excludePatterns.some(({ match }) => match(opts.url));
|
|
146
134
|
if (matchesExclude) {
|
|
147
135
|
onSkippedRequestOptions?.(opts);
|
|
148
136
|
}
|
|
149
137
|
return !matchesExclude;
|
|
150
138
|
})
|
|
151
139
|
.map((opts) => {
|
|
152
|
-
if (!
|
|
140
|
+
if (!includePatterns?.length) {
|
|
153
141
|
return { ...opts, enqueueStrategy: strategy };
|
|
154
142
|
}
|
|
155
|
-
for (const { match } of
|
|
143
|
+
for (const { match } of includePatterns) {
|
|
156
144
|
if (match(opts.url)) {
|
|
157
145
|
return { ...opts, enqueueStrategy: strategy };
|
|
158
146
|
}
|
|
@@ -212,24 +200,6 @@ export function createRequestOptions(sources, options = {}) {
|
|
|
212
200
|
return requestOptions;
|
|
213
201
|
});
|
|
214
202
|
}
|
|
215
|
-
/**
|
|
216
|
-
* @ignore
|
|
217
|
-
*/
|
|
218
|
-
function createPatternObjectMatcher(urlPatternObject) {
|
|
219
|
-
const { regexp, glob } = urlPatternObject;
|
|
220
|
-
let match;
|
|
221
|
-
if (regexp) {
|
|
222
|
-
match = (url) => regexp.test(url);
|
|
223
|
-
}
|
|
224
|
-
else if (glob) {
|
|
225
|
-
const m = new Minimatch(glob, { nocase: true });
|
|
226
|
-
match = (url) => m.match(url);
|
|
227
|
-
}
|
|
228
|
-
else {
|
|
229
|
-
match = () => false;
|
|
230
|
-
}
|
|
231
|
-
return { match };
|
|
232
|
-
}
|
|
233
203
|
/**
|
|
234
204
|
* Applies a {@link RequestTransform} function to a list of request options.
|
|
235
205
|
* Options for which the transform returns a falsy value are removed from the list.
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Transform } from 'node:stream';
|
|
2
2
|
import { EnqueueStrategy } from '@crawlee/utils';
|
|
3
3
|
import { parseArgument, parseSitemap, schemas } from '@crawlee/utils/internal';
|
|
4
|
-
import { minimatch } from 'minimatch';
|
|
5
4
|
import { z } from 'zod';
|
|
6
5
|
import { EventType, KeyValueStore, purgeDefaultStorages, Request, RequestManagerTandem, RequestQueue, serviceLocator, } from '@crawlee/core';
|
|
7
6
|
import { constructUrlPatternObjects, urlPatternSchema } from './enqueue-links/index.js';
|
|
@@ -123,27 +122,14 @@ export class SitemapRequestLoader {
|
|
|
123
122
|
highWaterMark,
|
|
124
123
|
}).pause();
|
|
125
124
|
}
|
|
126
|
-
/**
|
|
127
|
-
* Returns a function that checks whether the provided pattern matches the closure URL.
|
|
128
|
-
* @param url URL to be checked.
|
|
129
|
-
* @returns A matcher function that checks whether the pattern matches the closure URL.
|
|
130
|
-
*/
|
|
131
|
-
#matchesUrl(url) {
|
|
132
|
-
return (patternObject) => {
|
|
133
|
-
const { regexp, glob } = patternObject;
|
|
134
|
-
const matchesRegex = (regexp && url.match(regexp)) || false;
|
|
135
|
-
const matchesGlob = (glob && minimatch(url, glob, { nocase: true })) || false;
|
|
136
|
-
return Boolean(matchesRegex || matchesGlob);
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
125
|
/**
|
|
140
126
|
* Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
|
|
141
127
|
* @param url URL to be checked.
|
|
142
128
|
* @returns `true` if the URL matches the patterns, `false` otherwise.
|
|
143
129
|
*/
|
|
144
130
|
#isUrlMatchingPatterns(url) {
|
|
145
|
-
return (!this.#urlExcludePatternObjects.some(
|
|
146
|
-
(this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(
|
|
131
|
+
return (!this.#urlExcludePatternObjects.some(({ match }) => match(url)) &&
|
|
132
|
+
(this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(({ match }) => match(url))));
|
|
147
133
|
}
|
|
148
134
|
/**
|
|
149
135
|
* Adds a URL to the queue of parsed URLs.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.269",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.269",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.269",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.269",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.269",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.269"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "ee60f6c7d6aefb5a275a199a118244f75f4675e6"
|
|
70
70
|
}
|