@crawlee/basic 4.0.0-beta.259 → 4.0.0-beta.260
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,9 +2,8 @@ import { mkdir, writeFile } from 'node:fs/promises';
|
|
|
2
2
|
import { dirname } from 'node:path';
|
|
3
3
|
import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
|
|
4
4
|
import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
|
|
5
|
-
import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
|
|
5
|
+
import { filterUrl, isAsyncIterable, isIterable, matchesEnqueueStrategy, parseArgument, ROTATE_PROXY_ERRORS, schemas, } from '@crawlee/utils/internal';
|
|
6
6
|
import { RobotsTxtFile } from '@crawlee/utils';
|
|
7
|
-
import { getDomain } from 'tldts';
|
|
8
7
|
import { z } from 'zod';
|
|
9
8
|
import { LruCache } from '@apify/datastructures';
|
|
10
9
|
import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
@@ -15,7 +14,7 @@ import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
|
15
14
|
import { CrawlingRequest, RequestState } from './crawling-request.js';
|
|
16
15
|
import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
|
|
17
16
|
import { Statistics } from './crawlers/statistics.js';
|
|
18
|
-
import { applyRequestTransform,
|
|
17
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
|
|
19
18
|
import { parseRetryAfterHeader } from './http.js';
|
|
20
19
|
import { CrawlerRun } from './crawler-run.js';
|
|
21
20
|
import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
|
|
@@ -1116,7 +1115,7 @@ export class BasicCrawler {
|
|
|
1116
1115
|
const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
|
|
1117
1116
|
// The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
|
|
1118
1117
|
// (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
|
|
1119
|
-
const
|
|
1118
|
+
const strategyBaseUrl = options.baseUrl ? new URL(options.baseUrl) : undefined;
|
|
1120
1119
|
const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
|
|
1121
1120
|
const maxCrawlDepth = this.#maxCrawlDepth;
|
|
1122
1121
|
const validateRequestUserData = this.#validateRequestUserData.bind(this);
|
|
@@ -1167,7 +1166,14 @@ export class BasicCrawler {
|
|
|
1167
1166
|
// Filter by user patterns first (with exclude)...
|
|
1168
1167
|
let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
|
|
1169
1168
|
// ...then filter by the enqueue strategy (making this an AND check)
|
|
1170
|
-
|
|
1169
|
+
if (strategyBaseUrl) {
|
|
1170
|
+
filtered = filtered.filter((opts) => {
|
|
1171
|
+
const { allowed } = filterUrl(opts.url, strategyBaseUrl, strategy);
|
|
1172
|
+
if (!allowed)
|
|
1173
|
+
onSkippedByFilter(opts);
|
|
1174
|
+
return allowed;
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1171
1177
|
if (filtered.length === 0) {
|
|
1172
1178
|
continue;
|
|
1173
1179
|
}
|
|
@@ -1879,30 +1885,7 @@ export class BasicCrawler {
|
|
|
1879
1885
|
if (!loadedUrl) {
|
|
1880
1886
|
return true;
|
|
1881
1887
|
}
|
|
1882
|
-
|
|
1883
|
-
const loadedBaseUrl = new URL(loadedUrl);
|
|
1884
|
-
switch (strategy) {
|
|
1885
|
-
case EnqueueStrategy.SameHostname: {
|
|
1886
|
-
return baseUrl.hostname === loadedBaseUrl.hostname;
|
|
1887
|
-
}
|
|
1888
|
-
case EnqueueStrategy.SameDomain: {
|
|
1889
|
-
const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
|
|
1890
|
-
if (baseUrlHostname) {
|
|
1891
|
-
const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });
|
|
1892
|
-
return baseUrlHostname === loadedBaseUrlHostname;
|
|
1893
|
-
}
|
|
1894
|
-
// Can happen for IPs, we just check like same origin
|
|
1895
|
-
return baseUrl.origin === loadedBaseUrl.origin;
|
|
1896
|
-
}
|
|
1897
|
-
case EnqueueStrategy.SameOrigin: {
|
|
1898
|
-
// Same as hostname, but also checks protocol
|
|
1899
|
-
return baseUrl.origin === loadedBaseUrl.origin;
|
|
1900
|
-
}
|
|
1901
|
-
case EnqueueStrategy.All:
|
|
1902
|
-
default: {
|
|
1903
|
-
return baseUrl.protocol === 'http:' || baseUrl.protocol === 'https:';
|
|
1904
|
-
}
|
|
1905
|
-
}
|
|
1888
|
+
return matchesEnqueueStrategy(strategy, new URL(loadedUrl), new URL(url));
|
|
1906
1889
|
}
|
|
1907
1890
|
}
|
|
1908
1891
|
/** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Dictionary } from '@crawlee/types';
|
|
2
2
|
import type { EnqueueStrategyOption, RequestQueueOperationOptions } from '@crawlee/core';
|
|
3
|
-
import type { RequestTransform, SkippedRequestCallback, UrlPatternInput
|
|
3
|
+
import type { RequestTransform, SkippedRequestCallback, UrlPatternInput } from './shared.js';
|
|
4
4
|
/**
|
|
5
5
|
* Options shared by the `extractLinks()` context helper across crawler types.
|
|
6
6
|
*/
|
|
@@ -145,8 +145,3 @@ export interface ResolveBaseUrl {
|
|
|
145
145
|
originalRequestUrl: string;
|
|
146
146
|
finalRequestUrl?: string;
|
|
147
147
|
}
|
|
148
|
-
/**
|
|
149
|
-
* @internal
|
|
150
|
-
* Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
|
|
151
|
-
*/
|
|
152
|
-
export declare function buildEnqueueStrategyPatterns(baseUrl: string, strategy: EnqueueStrategyOption): UrlPatternObject[];
|
|
@@ -33,44 +33,3 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
|
|
|
33
33
|
// before actually finding the urls
|
|
34
34
|
return originalUrlOrigin;
|
|
35
35
|
}
|
|
36
|
-
/**
|
|
37
|
-
* @internal
|
|
38
|
-
* Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
|
|
39
|
-
*/
|
|
40
|
-
export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
|
|
41
|
-
const url = new URL(baseUrl);
|
|
42
|
-
switch (strategy) {
|
|
43
|
-
case EnqueueStrategy.SameHostname:
|
|
44
|
-
// We need to get the origin of the passed in domain in the event someone sets baseUrl
|
|
45
|
-
// to an url like https://example.com/deep/default/path and one of the found urls is an
|
|
46
|
-
// absolute relative path (/path/to/page)
|
|
47
|
-
return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
|
|
48
|
-
case EnqueueStrategy.SameDomain: {
|
|
49
|
-
// Get the actual hostname from the base url
|
|
50
|
-
const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
|
|
51
|
-
if (baseUrlHostname) {
|
|
52
|
-
// We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
|
|
53
|
-
url.hostname = baseUrlHostname;
|
|
54
|
-
return [
|
|
55
|
-
{ glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
|
|
56
|
-
{ glob: ignoreHttpSchema(`${url.origin}/**`) },
|
|
57
|
-
];
|
|
58
|
-
}
|
|
59
|
-
// We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
|
|
60
|
-
// as SameDomainAndSubdomain
|
|
61
|
-
return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
|
|
62
|
-
}
|
|
63
|
-
case EnqueueStrategy.SameOrigin:
|
|
64
|
-
// The same behavior as SameHostname, but respecting the protocol of the URL
|
|
65
|
-
return [{ glob: `${url.origin}/**` }];
|
|
66
|
-
case EnqueueStrategy.All:
|
|
67
|
-
default:
|
|
68
|
-
return [{ glob: `http{s,}://**` }];
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Internal function that changes the enqueue glob patterns to match both http and https
|
|
73
|
-
*/
|
|
74
|
-
function ignoreHttpSchema(pattern) {
|
|
75
|
-
return pattern.replace(/^(https?):\/\//, 'http{s,}://');
|
|
76
|
-
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.260",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.260",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.260",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.260",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.260",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.260"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "c96c87d09eb5df1bcdf6365563ea99739f612a35"
|
|
70
70
|
}
|