@crawlee/basic 4.0.0-rc.1 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +6 -6
- package/index.js +6 -6
- package/internals/autoscaling/{autoscaled_pool.d.ts → autoscaled-pool.d.ts} +6 -6
- package/internals/autoscaling/{autoscaled_pool.js → autoscaled-pool.js} +3 -1
- package/internals/autoscaling/{concurrency_system.js → concurrency-system.js} +1 -1
- package/internals/autoscaling/{cpu_load_signal.d.ts → cpu-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{cpu_load_signal.js → cpu-load-signal.js} +1 -1
- package/internals/autoscaling/{event_loop_load_signal.d.ts → event-loop-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{event_loop_load_signal.js → event-loop-load-signal.js} +1 -1
- package/internals/autoscaling/index.d.ts +8 -8
- package/internals/autoscaling/index.js +8 -8
- package/internals/autoscaling/{load_signal.js → load-signal.js} +1 -1
- package/internals/autoscaling/{memory_load_signal.d.ts → memory-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{memory_load_signal.js → memory-load-signal.js} +1 -1
- package/internals/autoscaling/snapshotter.d.ts +5 -5
- package/internals/autoscaling/snapshotter.js +4 -4
- package/internals/autoscaling/{storage_backend_load_signal.d.ts → storage-backend-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{storage_backend_load_signal.js → storage-backend-load-signal.js} +1 -1
- package/internals/autoscaling/{system_status.d.ts → system-status.d.ts} +1 -1
- package/internals/autoscaling/{system_status.js → system-status.js} +1 -1
- package/internals/basic-crawler.d.ts +16 -11
- package/internals/basic-crawler.js +61 -66
- package/internals/{cookie_utils.js → cookie-utils.js} +1 -1
- package/internals/crawler-run.d.ts +2 -2
- package/internals/crawler-run.js +3 -1
- package/internals/crawlers/{crawler_commons.d.ts → crawler-commons.d.ts} +2 -2
- package/internals/crawlers/{error_snapshotter.d.ts → error-snapshotter.d.ts} +2 -2
- package/internals/crawlers/{error_tracker.d.ts → error-tracker.d.ts} +1 -1
- package/internals/crawlers/{error_tracker.js → error-tracker.js} +1 -1
- package/internals/crawlers/index.d.ts +3 -3
- package/internals/crawlers/index.js +2 -2
- package/internals/crawlers/statistics.d.ts +11 -1
- package/internals/crawlers/statistics.js +39 -5
- package/internals/{crawling_request.js → crawling-request.js} +1 -1
- package/internals/{enqueue_links/enqueue_links.d.ts → enqueue-links/enqueue-links.d.ts} +1 -6
- package/internals/{enqueue_links/enqueue_links.js → enqueue-links/enqueue-links.js} +0 -41
- package/internals/{enqueue_links → enqueue-links}/index.d.ts +1 -1
- package/internals/{enqueue_links → enqueue-links}/index.js +1 -1
- package/internals/{enqueue_links → enqueue-links}/shared.d.ts +5 -4
- package/internals/{enqueue_links → enqueue-links}/shared.js +15 -45
- package/internals/errors.d.ts +6 -1
- package/internals/errors.js +25 -0
- package/internals/router.d.ts +2 -2
- package/internals/send-request.js +17 -8
- package/internals/{session_pool → session-pool}/index.d.ts +1 -1
- package/internals/{session_pool → session-pool}/index.js +1 -1
- package/internals/{sitemap_request_loader.d.ts → sitemap-request-loader.d.ts} +8 -1
- package/internals/{sitemap_request_loader.js → sitemap-request-loader.js} +116 -56
- package/internals/{throttling_request_manager.d.ts → throttling-request-manager.d.ts} +7 -7
- package/internals/{throttling_request_manager.js → throttling-request-manager.js} +61 -44
- package/package.json +15 -15
- /package/internals/autoscaling/{concurrency_system.d.ts → concurrency-system.d.ts} +0 -0
- /package/internals/autoscaling/{load_signal.d.ts → load-signal.d.ts} +0 -0
- /package/internals/autoscaling/{weighted_avg.d.ts → weighted-avg.d.ts} +0 -0
- /package/internals/autoscaling/{weighted_avg.js → weighted-avg.js} +0 -0
- /package/internals/{cookie_utils.d.ts → cookie-utils.d.ts} +0 -0
- /package/internals/crawlers/{context_pipeline.d.ts → context-pipeline.d.ts} +0 -0
- /package/internals/crawlers/{context_pipeline.js → context-pipeline.js} +0 -0
- /package/internals/crawlers/{crawler_commons.js → crawler-commons.js} +0 -0
- /package/internals/crawlers/{error_snapshotter.js → error-snapshotter.js} +0 -0
- /package/internals/{crawling_request.d.ts → crawling-request.d.ts} +0 -0
- /package/internals/{session_pool → session-pool}/consts.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/consts.js +0 -0
- /package/internals/{session_pool → session-pool}/errors.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/errors.js +0 -0
- /package/internals/{session_pool → session-pool}/fingerprint.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/fingerprint.js +0 -0
- /package/internals/{session_pool/session_pool.d.ts → session-pool/session-pool.d.ts} +0 -0
- /package/internals/{session_pool/session_pool.js → session-pool/session-pool.js} +0 -0
- /package/internals/{session_pool → session-pool}/session.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/session.js +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CrawleeLogger } from '@crawlee/core';
|
|
2
|
-
import type { AutoscaledPoolOptions } from './autoscaling/
|
|
3
|
-
import type { IConcurrencySystem } from './autoscaling/
|
|
2
|
+
import type { AutoscaledPoolOptions } from './autoscaling/autoscaled-pool.js';
|
|
3
|
+
import type { IConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
4
4
|
/** Everything a run needs to exist. Whatever a crawler resolves per run is resolved before the run is built. */
|
|
5
5
|
export interface CrawlerRunSetup {
|
|
6
6
|
log: CrawleeLogger;
|
package/internals/crawler-run.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AutoscaledPool } from './autoscaling/
|
|
1
|
+
import { AutoscaledPool } from './autoscaling/autoscaled-pool.js';
|
|
2
2
|
/**
|
|
3
3
|
* Everything that belongs to a single {@link BasicCrawler.run|`crawler.run()`} — the task loop dispatching its
|
|
4
4
|
* requests, the concurrency governor those requests are booked against, and the bookkeeping that must not survive
|
|
@@ -38,6 +38,8 @@ export class CrawlerRun {
|
|
|
38
38
|
* running by now — the task loop refuses to book anything against one that is not.
|
|
39
39
|
*/
|
|
40
40
|
async dispatchRequests() {
|
|
41
|
+
if (this.#ended)
|
|
42
|
+
return;
|
|
41
43
|
this.#dispatching = true;
|
|
42
44
|
try {
|
|
43
45
|
await this.#autoscaledPool.run();
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { Awaitable, Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
|
|
2
2
|
import type { ReadonlyDeep } from 'type-fest';
|
|
3
3
|
import type { AddRequestsBatchedResult, CrawleeLogger, Dataset, KeyValueStore, Request, RequestOptions, Source, StorageIdentifier } from '@crawlee/core';
|
|
4
|
-
import type { EnqueueUrlsOptions } from '../
|
|
5
|
-
import type { CrawlingRequest } from '../
|
|
4
|
+
import type { EnqueueUrlsOptions } from '../enqueue-links/index.js';
|
|
5
|
+
import type { CrawlingRequest } from '../crawling-request.js';
|
|
6
6
|
/** @internal */
|
|
7
7
|
export type IsAny<T> = 0 extends 1 & T ? true : false;
|
|
8
8
|
/**
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { CrawlingContext } from '
|
|
2
|
-
import type { ErrnoException } from './
|
|
1
|
+
import type { CrawlingContext } from './crawler-commons.js';
|
|
2
|
+
import type { ErrnoException } from './error-tracker.js';
|
|
3
3
|
import type { SnapshottableProperties } from './internals/types.js';
|
|
4
4
|
interface ErrorSnapshot {
|
|
5
5
|
screenshotFileName?: string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { inspect } from 'node:util';
|
|
2
|
-
import { ErrorSnapshotter } from './
|
|
2
|
+
import { ErrorSnapshotter } from './error-snapshotter.js';
|
|
3
3
|
const extractPathFromStackTraceLine = (line) => {
|
|
4
4
|
const lastStartingRoundBracketIndex = line.lastIndexOf('(');
|
|
5
5
|
if (lastStartingRoundBracketIndex !== -1) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export type * from './
|
|
1
|
+
export * from './context-pipeline.js';
|
|
2
|
+
export type * from './crawler-commons.js';
|
|
3
3
|
export * from './statistics.js';
|
|
4
|
-
export * from './
|
|
4
|
+
export * from './error-tracker.js';
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './context-pipeline.js';
|
|
2
2
|
export * from './statistics.js';
|
|
3
|
-
export * from './
|
|
3
|
+
export * from './error-tracker.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CrawleeLogger, SyncStateConversion } from '@crawlee/core';
|
|
2
2
|
import { KeyValueStore } from '@crawlee/core';
|
|
3
|
-
import { ErrorTracker } from './
|
|
3
|
+
import { ErrorTracker } from './error-tracker.js';
|
|
4
4
|
/**
|
|
5
5
|
* Persistence-related options to control how and when crawler's data gets persisted.
|
|
6
6
|
*/
|
|
@@ -50,6 +50,11 @@ export interface IStatistics<StateExtension extends object = {}> {
|
|
|
50
50
|
startCapturing(): Promise<void>;
|
|
51
51
|
/** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
|
|
52
52
|
stopCapturing(): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Registers a retry attempt in real-time when a request temporarily fails.
|
|
55
|
+
* Optional - crawlers call this when a request is reclaimed back to the queue or list.
|
|
56
|
+
*/
|
|
57
|
+
registerRetry?(retryCount: number): void;
|
|
53
58
|
/**
|
|
54
59
|
* Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
|
|
55
60
|
* with no persistence of its own can omit it.
|
|
@@ -155,6 +160,11 @@ export declare class Statistics<StateExtension extends object = {}, PersistedSta
|
|
|
155
160
|
* Stops logging and remove event listeners, then persist
|
|
156
161
|
*/
|
|
157
162
|
stopCapturing(): Promise<void>;
|
|
163
|
+
/**
|
|
164
|
+
* Registers a retry attempt in real-time when a request temporarily fails.
|
|
165
|
+
* Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
|
|
166
|
+
*/
|
|
167
|
+
registerRetry(retryCount: number): void;
|
|
158
168
|
/**
|
|
159
169
|
* Persist internal state to the key value store.
|
|
160
170
|
*
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { convertStateSync, KeyValueStore, RecoverableState, serviceLocator, validators } from '@crawlee/core';
|
|
2
2
|
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import { ErrorTracker } from './
|
|
4
|
+
import { ErrorTracker } from './error-tracker.js';
|
|
5
5
|
/**
|
|
6
6
|
* @ignore
|
|
7
7
|
*/
|
|
@@ -38,7 +38,7 @@ const errorTrackerConfig = {
|
|
|
38
38
|
* order here *is* the record's field order (guarded by a test).
|
|
39
39
|
*
|
|
40
40
|
* JSON has no infinity, so the three fields that are `Infinity` until the first request settles are written as
|
|
41
|
-
* `null`. Both
|
|
41
|
+
* `null`. Both the private serialize and deserialize steps of `Statistics` run through this,
|
|
42
42
|
* which is what keeps them describing the same record.
|
|
43
43
|
*
|
|
44
44
|
* Nothing is optional on purpose: the record has always carried every field, so one missing a field is not one of
|
|
@@ -410,11 +410,45 @@ export class Statistics {
|
|
|
410
410
|
this.state.crawlerFinishedAt = new Date();
|
|
411
411
|
await this.#recoverableState.teardown();
|
|
412
412
|
}
|
|
413
|
-
|
|
414
|
-
|
|
413
|
+
/**
|
|
414
|
+
* Registers a retry attempt in real-time when a request temporarily fails.
|
|
415
|
+
* Includes safeguards against NaN, negative values, sparse arrays, and migration state mismatch.
|
|
416
|
+
*/
|
|
417
|
+
registerRetry(retryCount) {
|
|
418
|
+
if (!Number.isInteger(retryCount) || retryCount < 0)
|
|
419
|
+
return;
|
|
420
|
+
if (retryCount === 1) {
|
|
415
421
|
this.state.requestsRetries++;
|
|
416
|
-
|
|
422
|
+
}
|
|
423
|
+
// Fill intermediate indices with 0 to prevent sparse arrays and JSON serialization null values
|
|
424
|
+
while (this.requestRetryHistogram.length <= retryCount) {
|
|
425
|
+
this.requestRetryHistogram.push(0);
|
|
426
|
+
}
|
|
417
427
|
this.requestRetryHistogram[retryCount]++;
|
|
428
|
+
const previousRetryCount = retryCount - 1;
|
|
429
|
+
if (previousRetryCount > 0) {
|
|
430
|
+
while (this.requestRetryHistogram.length <= previousRetryCount) {
|
|
431
|
+
this.requestRetryHistogram.push(0);
|
|
432
|
+
}
|
|
433
|
+
// Decrement previous count only if it is strictly greater than 0
|
|
434
|
+
if (this.requestRetryHistogram[previousRetryCount] > 0) {
|
|
435
|
+
this.requestRetryHistogram[previousRetryCount]--;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
#saveRetryCountForRequest(retryCount) {
|
|
440
|
+
if (!Number.isInteger(retryCount) || retryCount < 0)
|
|
441
|
+
return;
|
|
442
|
+
while (this.requestRetryHistogram.length <= retryCount) {
|
|
443
|
+
this.requestRetryHistogram.push(0);
|
|
444
|
+
}
|
|
445
|
+
if (retryCount === 0) {
|
|
446
|
+
this.requestRetryHistogram[0]++;
|
|
447
|
+
}
|
|
448
|
+
else if (this.requestRetryHistogram[retryCount] === 0) {
|
|
449
|
+
this.requestRetryHistogram[retryCount]++;
|
|
450
|
+
this.state.requestsRetries += retryCount;
|
|
451
|
+
}
|
|
418
452
|
}
|
|
419
453
|
/**
|
|
420
454
|
* Persist internal state to the key value store.
|
|
@@ -99,7 +99,7 @@ export class CrawlingRequest extends Request {
|
|
|
99
99
|
message = omitStack
|
|
100
100
|
? errorOrMessage.message
|
|
101
101
|
: // .stack includes the message
|
|
102
|
-
errorOrMessage.stack;
|
|
102
|
+
(errorOrMessage.stack ?? errorOrMessage.message);
|
|
103
103
|
}
|
|
104
104
|
else if (Reflect.has(Object(errorOrMessage), 'message')) {
|
|
105
105
|
message = Reflect.get(Object(errorOrMessage), 'message');
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Dictionary } from '@crawlee/types';
|
|
2
2
|
import type { EnqueueStrategyOption, RequestQueueOperationOptions } from '@crawlee/core';
|
|
3
|
-
import type { RequestTransform, SkippedRequestCallback, UrlPatternInput
|
|
3
|
+
import type { RequestTransform, SkippedRequestCallback, UrlPatternInput } from './shared.js';
|
|
4
4
|
/**
|
|
5
5
|
* Options shared by the `extractLinks()` context helper across crawler types.
|
|
6
6
|
*/
|
|
@@ -145,8 +145,3 @@ export interface ResolveBaseUrl {
|
|
|
145
145
|
originalRequestUrl: string;
|
|
146
146
|
finalRequestUrl?: string;
|
|
147
147
|
}
|
|
148
|
-
/**
|
|
149
|
-
* @internal
|
|
150
|
-
* Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
|
|
151
|
-
*/
|
|
152
|
-
export declare function buildEnqueueStrategyPatterns(baseUrl: string, strategy: EnqueueStrategyOption): UrlPatternObject[];
|
|
@@ -33,44 +33,3 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
|
|
|
33
33
|
// before actually finding the urls
|
|
34
34
|
return originalUrlOrigin;
|
|
35
35
|
}
|
|
36
|
-
/**
|
|
37
|
-
* @internal
|
|
38
|
-
* Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
|
|
39
|
-
*/
|
|
40
|
-
export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
|
|
41
|
-
const url = new URL(baseUrl);
|
|
42
|
-
switch (strategy) {
|
|
43
|
-
case EnqueueStrategy.SameHostname:
|
|
44
|
-
// We need to get the origin of the passed in domain in the event someone sets baseUrl
|
|
45
|
-
// to an url like https://example.com/deep/default/path and one of the found urls is an
|
|
46
|
-
// absolute relative path (/path/to/page)
|
|
47
|
-
return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
|
|
48
|
-
case EnqueueStrategy.SameDomain: {
|
|
49
|
-
// Get the actual hostname from the base url
|
|
50
|
-
const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
|
|
51
|
-
if (baseUrlHostname) {
|
|
52
|
-
// We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
|
|
53
|
-
url.hostname = baseUrlHostname;
|
|
54
|
-
return [
|
|
55
|
-
{ glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
|
|
56
|
-
{ glob: ignoreHttpSchema(`${url.origin}/**`) },
|
|
57
|
-
];
|
|
58
|
-
}
|
|
59
|
-
// We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
|
|
60
|
-
// as SameDomainAndSubdomain
|
|
61
|
-
return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
|
|
62
|
-
}
|
|
63
|
-
case EnqueueStrategy.SameOrigin:
|
|
64
|
-
// The same behavior as SameHostname, but respecting the protocol of the URL
|
|
65
|
-
return [{ glob: `${url.origin}/**` }];
|
|
66
|
-
case EnqueueStrategy.All:
|
|
67
|
-
default:
|
|
68
|
-
return [{ glob: `http{s,}://**` }];
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Internal function that changes the enqueue glob patterns to match both http and https
|
|
73
|
-
*/
|
|
74
|
-
function ignoreHttpSchema(pattern) {
|
|
75
|
-
return pattern.replace(/^(https?):\/\//, 'http{s,}://');
|
|
76
|
-
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './enqueue-links.js';
|
|
2
2
|
export { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, updateEnqueueLinksPatternCache, urlPatternSchema, validateGlobPattern, } from './shared.js';
|
|
3
3
|
export type { GlobInput, GlobObject, RegExpInput, RegExpObject, RequestTransform, SkippedRequestCallback, UrlPatternInput, } from './shared.js';
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './enqueue-links.js';
|
|
2
2
|
// Not `export *`: keeps the internal `UrlPatternObject` off the public surface.
|
|
3
3
|
export { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, updateEnqueueLinksPatternCache, urlPatternSchema, validateGlobPattern, } from './shared.js';
|
|
@@ -2,7 +2,7 @@ import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
import type { EnqueueStrategyOption, RequestOptions, Source } from '@crawlee/core';
|
|
4
4
|
import { Request } from '@crawlee/core';
|
|
5
|
-
import type { SkippedRequestReason } from '../
|
|
5
|
+
import type { SkippedRequestReason } from '../crawling-request.js';
|
|
6
6
|
export interface GlobObject {
|
|
7
7
|
glob: string;
|
|
8
8
|
}
|
|
@@ -34,13 +34,13 @@ export declare function createSkippedRequestArgs(source: string | Source, reason
|
|
|
34
34
|
/**
|
|
35
35
|
* @ignore
|
|
36
36
|
*/
|
|
37
|
-
export declare function updateEnqueueLinksPatternCache(item:
|
|
37
|
+
export declare function updateEnqueueLinksPatternCache(item: UrlPatternInput, pattern: UrlPatternObject): void;
|
|
38
38
|
/**
|
|
39
39
|
* Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
|
|
40
40
|
* to construct Glob objects from Glob pattern strings.
|
|
41
41
|
* @ignore
|
|
42
42
|
*/
|
|
43
|
-
export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
|
|
43
|
+
export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): (GlobObject & UrlPatternObject)[];
|
|
44
44
|
/**
|
|
45
45
|
* @internal
|
|
46
46
|
*/
|
|
@@ -50,7 +50,7 @@ export declare function validateGlobPattern(glob: string): string;
|
|
|
50
50
|
* to check RegExps input and return valid RegExps.
|
|
51
51
|
* @ignore
|
|
52
52
|
*/
|
|
53
|
-
export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
|
|
53
|
+
export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): (RegExpObject & UrlPatternObject)[];
|
|
54
54
|
/**
|
|
55
55
|
* The compiled form of a {@link UrlPatternInput}, produced by {@link constructUrlPatternObjects}. Deliberately
|
|
56
56
|
* not re-exported from the package entry point — it is an implementation detail of the pattern matching helpers.
|
|
@@ -59,6 +59,7 @@ export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegE
|
|
|
59
59
|
export interface UrlPatternObject {
|
|
60
60
|
glob?: string;
|
|
61
61
|
regexp?: RegExp;
|
|
62
|
+
match: (url: string) => boolean;
|
|
62
63
|
}
|
|
63
64
|
/**
|
|
64
65
|
* Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
|
|
@@ -45,8 +45,7 @@ export function createSkippedRequestArgs(source, reason) {
|
|
|
45
45
|
export function updateEnqueueLinksPatternCache(item, pattern) {
|
|
46
46
|
enqueueLinksPatternCache.set(item, pattern);
|
|
47
47
|
if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
|
|
48
|
-
|
|
49
|
-
enqueueLinksPatternCache.delete(key);
|
|
48
|
+
enqueueLinksPatternCache.delete(enqueueLinksPatternCache.keys().next().value);
|
|
50
49
|
}
|
|
51
50
|
}
|
|
52
51
|
/**
|
|
@@ -70,16 +69,12 @@ export function constructGlobObjectsFromGlobs(globs) {
|
|
|
70
69
|
return false;
|
|
71
70
|
})
|
|
72
71
|
.map((item) => {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
}
|
|
80
|
-
else {
|
|
81
|
-
globObject = { glob: validateGlobPattern(item.glob) };
|
|
82
|
-
}
|
|
72
|
+
const cached = enqueueLinksPatternCache.get(item);
|
|
73
|
+
if (cached)
|
|
74
|
+
return cached;
|
|
75
|
+
const glob = validateGlobPattern(typeof item === 'string' ? item : item.glob);
|
|
76
|
+
const minimatch = new Minimatch(glob, { nocase: true });
|
|
77
|
+
const globObject = { glob, match: (url) => minimatch.match(url) };
|
|
83
78
|
updateEnqueueLinksPatternCache(item, globObject);
|
|
84
79
|
return globObject;
|
|
85
80
|
});
|
|
@@ -100,16 +95,11 @@ export function validateGlobPattern(glob) {
|
|
|
100
95
|
*/
|
|
101
96
|
export function constructRegExpObjectsFromRegExps(regexps) {
|
|
102
97
|
return regexps.map((item) => {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
regexpObject = { regexp: item };
|
|
109
|
-
}
|
|
110
|
-
else {
|
|
111
|
-
regexpObject = { regexp: item.regexp };
|
|
112
|
-
}
|
|
98
|
+
const cached = enqueueLinksPatternCache.get(item);
|
|
99
|
+
if (cached)
|
|
100
|
+
return cached;
|
|
101
|
+
const regexp = item instanceof RegExp ? item : item.regexp;
|
|
102
|
+
const regexpObject = { regexp, match: (url) => regexp.test(url) };
|
|
113
103
|
updateEnqueueLinksPatternCache(item, regexpObject);
|
|
114
104
|
return regexpObject;
|
|
115
105
|
});
|
|
@@ -138,21 +128,19 @@ export function constructUrlPatternObjects(patterns) {
|
|
|
138
128
|
* @ignore
|
|
139
129
|
*/
|
|
140
130
|
export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedRequestOptions) {
|
|
141
|
-
const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
|
|
142
|
-
const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
|
|
143
131
|
return requestOptions
|
|
144
132
|
.filter((opts) => {
|
|
145
|
-
const matchesExclude =
|
|
133
|
+
const matchesExclude = excludePatterns.some(({ match }) => match(opts.url));
|
|
146
134
|
if (matchesExclude) {
|
|
147
135
|
onSkippedRequestOptions?.(opts);
|
|
148
136
|
}
|
|
149
137
|
return !matchesExclude;
|
|
150
138
|
})
|
|
151
139
|
.map((opts) => {
|
|
152
|
-
if (!
|
|
140
|
+
if (!includePatterns?.length) {
|
|
153
141
|
return { ...opts, enqueueStrategy: strategy };
|
|
154
142
|
}
|
|
155
|
-
for (const { match } of
|
|
143
|
+
for (const { match } of includePatterns) {
|
|
156
144
|
if (match(opts.url)) {
|
|
157
145
|
return { ...opts, enqueueStrategy: strategy };
|
|
158
146
|
}
|
|
@@ -212,24 +200,6 @@ export function createRequestOptions(sources, options = {}) {
|
|
|
212
200
|
return requestOptions;
|
|
213
201
|
});
|
|
214
202
|
}
|
|
215
|
-
/**
|
|
216
|
-
* @ignore
|
|
217
|
-
*/
|
|
218
|
-
function createPatternObjectMatcher(urlPatternObject) {
|
|
219
|
-
const { regexp, glob } = urlPatternObject;
|
|
220
|
-
let match;
|
|
221
|
-
if (regexp) {
|
|
222
|
-
match = (url) => regexp.test(url);
|
|
223
|
-
}
|
|
224
|
-
else if (glob) {
|
|
225
|
-
const m = new Minimatch(glob, { nocase: true });
|
|
226
|
-
match = (url) => m.match(url);
|
|
227
|
-
}
|
|
228
|
-
else {
|
|
229
|
-
match = () => false;
|
|
230
|
-
}
|
|
231
|
-
return { match };
|
|
232
|
-
}
|
|
233
203
|
/**
|
|
234
204
|
* Applies a {@link RequestTransform} function to a list of request options.
|
|
235
205
|
* Options for which the transform returns a falsy value are removed from the list.
|
package/internals/errors.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CriticalError, NonRetryableError } from '@crawlee/core';
|
|
2
|
-
import type { SkippedRequestReason } from './
|
|
2
|
+
import type { SkippedRequestReason } from './crawling-request.js';
|
|
3
3
|
/**
|
|
4
4
|
* @ignore
|
|
5
5
|
*/
|
|
@@ -66,3 +66,8 @@ export declare class RequestHandlerError extends Error {
|
|
|
66
66
|
*/
|
|
67
67
|
export declare class NavigationSkippedError extends NonRetryableError {
|
|
68
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
|
|
71
|
+
* @internal
|
|
72
|
+
*/
|
|
73
|
+
export declare function skippedNavigationRequest<R extends object>(request: R): R;
|
package/internals/errors.js
CHANGED
|
@@ -81,3 +81,28 @@ export class RequestHandlerError extends Error {
|
|
|
81
81
|
*/
|
|
82
82
|
export class NavigationSkippedError extends NonRetryableError {
|
|
83
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Wraps a `skipNavigation` request so that reading `loadedUrl` throws a {@link NavigationSkippedError}.
|
|
86
|
+
* @internal
|
|
87
|
+
*/
|
|
88
|
+
export function skippedNavigationRequest(request) {
|
|
89
|
+
return new Proxy(request, {
|
|
90
|
+
get(target, propertyName) {
|
|
91
|
+
if (propertyName === 'loadedUrl') {
|
|
92
|
+
throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
93
|
+
}
|
|
94
|
+
// `target` as receiver and bound methods, or `#` members throw on the proxy
|
|
95
|
+
const value = Reflect.get(target, propertyName, target);
|
|
96
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
97
|
+
},
|
|
98
|
+
// Hide `loadedUrl` from enumeration so bookkeeping (spread/JSON/zod) skips it; direct reads still throw above.
|
|
99
|
+
getOwnPropertyDescriptor(target, propertyName) {
|
|
100
|
+
const descriptor = Reflect.getOwnPropertyDescriptor(target, propertyName);
|
|
101
|
+
if (propertyName === 'loadedUrl' && descriptor) {
|
|
102
|
+
return { ...descriptor, enumerable: false };
|
|
103
|
+
}
|
|
104
|
+
return descriptor;
|
|
105
|
+
},
|
|
106
|
+
set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
|
|
107
|
+
});
|
|
108
|
+
}
|
package/internals/router.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { CrawlingRequest } from './
|
|
1
|
+
import type { CrawlingRequest } from './crawling-request.js';
|
|
2
2
|
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { StandardSchemaV1 } from '@standard-schema/spec';
|
|
4
|
-
import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/
|
|
4
|
+
import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler-commons.js';
|
|
5
5
|
/**
|
|
6
6
|
* The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
|
|
7
7
|
* Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { processHttpRequestOptions } from '@crawlee/utils/internal';
|
|
1
3
|
/**
|
|
2
4
|
* Prepares a function to be used as the `sendRequest` context helper.
|
|
3
5
|
*
|
|
@@ -9,24 +11,31 @@
|
|
|
9
11
|
export function createSendRequest(httpClient, originRequest, session) {
|
|
10
12
|
return async (overrideRequest = {}, overrideOptions = {}) => {
|
|
11
13
|
const baseRequest = originRequest.intoFetchAPIRequest();
|
|
12
|
-
const mergedUrl = overrideRequest.url ?? baseRequest.url;
|
|
13
|
-
const mergedMethod = overrideRequest.method ?? baseRequest.method;
|
|
14
14
|
const mergedHeaders = new Headers(baseRequest.headers);
|
|
15
15
|
if (overrideRequest.headers) {
|
|
16
16
|
overrideRequest.headers.forEach((value, key) => {
|
|
17
17
|
mergedHeaders.set(key, value);
|
|
18
18
|
});
|
|
19
19
|
}
|
|
20
|
-
const
|
|
21
|
-
|
|
20
|
+
const opts = processHttpRequestOptions({
|
|
21
|
+
...overrideRequest,
|
|
22
|
+
url: overrideRequest.url ?? baseRequest.url,
|
|
23
|
+
method: overrideRequest.method ?? originRequest.method,
|
|
22
24
|
headers: mergedHeaders,
|
|
23
|
-
|
|
25
|
+
});
|
|
26
|
+
const request = new Request(opts.url, {
|
|
27
|
+
method: opts.method,
|
|
28
|
+
headers: opts.headers,
|
|
29
|
+
body: opts.body ? Readable.toWeb(opts.body) : baseRequest.body,
|
|
30
|
+
// Node-specific option to make the request body work with streams
|
|
31
|
+
duplex: 'half',
|
|
24
32
|
});
|
|
25
33
|
return httpClient.sendRequest(request, {
|
|
26
34
|
session,
|
|
27
|
-
cookieJar: overrideOptions
|
|
28
|
-
|
|
29
|
-
|
|
35
|
+
cookieJar: overrideOptions.cookieJar ?? opts.cookieJar ?? session.cookieJar,
|
|
36
|
+
proxyUrl: overrideOptions.proxyUrl ?? opts.proxyUrl,
|
|
37
|
+
timeoutMillis: overrideOptions.timeoutMillis ?? opts.timeout,
|
|
38
|
+
signal: overrideOptions.signal ?? opts.signal,
|
|
30
39
|
ignoreTlsErrors: overrideOptions.ignoreTlsErrors,
|
|
31
40
|
});
|
|
32
41
|
};
|
|
@@ -3,7 +3,7 @@ import type { ParseSitemapOptions } from '@crawlee/utils';
|
|
|
3
3
|
import { EnqueueStrategy } from '@crawlee/utils';
|
|
4
4
|
import type { IRequestLoader, IRequestManager, RequestLoaderStatus } from '@crawlee/core';
|
|
5
5
|
import { Request } from '@crawlee/core';
|
|
6
|
-
import type { UrlPatternInput } from './
|
|
6
|
+
import type { UrlPatternInput } from './enqueue-links/index.js';
|
|
7
7
|
interface UrlConstraints {
|
|
8
8
|
/**
|
|
9
9
|
* An array of URL patterns that URLs must match to be included.
|
|
@@ -80,6 +80,13 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {
|
|
|
80
80
|
* Custom HTTP client to be used for sitemap loading.
|
|
81
81
|
*/
|
|
82
82
|
httpClient?: BaseHttpClient;
|
|
83
|
+
/**
|
|
84
|
+
* Maximum number of sitemaps to load in parallel.
|
|
85
|
+
* This includes both the initial sitemaps and any nested sitemaps discovered during parsing.
|
|
86
|
+
*
|
|
87
|
+
* @default 1
|
|
88
|
+
*/
|
|
89
|
+
concurrency?: number;
|
|
83
90
|
}
|
|
84
91
|
/**
|
|
85
92
|
* A list of URLs to crawl parsed from a sitemap.
|