@crawlee/basic 4.0.0-rc.1 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +6 -6
- package/index.js +6 -6
- package/internals/autoscaling/{autoscaled_pool.d.ts → autoscaled-pool.d.ts} +6 -6
- package/internals/autoscaling/{autoscaled_pool.js → autoscaled-pool.js} +3 -1
- package/internals/autoscaling/{concurrency_system.js → concurrency-system.js} +1 -1
- package/internals/autoscaling/{cpu_load_signal.d.ts → cpu-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{cpu_load_signal.js → cpu-load-signal.js} +1 -1
- package/internals/autoscaling/{event_loop_load_signal.d.ts → event-loop-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{event_loop_load_signal.js → event-loop-load-signal.js} +1 -1
- package/internals/autoscaling/index.d.ts +8 -8
- package/internals/autoscaling/index.js +8 -8
- package/internals/autoscaling/{load_signal.js → load-signal.js} +1 -1
- package/internals/autoscaling/{memory_load_signal.d.ts → memory-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{memory_load_signal.js → memory-load-signal.js} +1 -1
- package/internals/autoscaling/snapshotter.d.ts +5 -5
- package/internals/autoscaling/snapshotter.js +4 -4
- package/internals/autoscaling/{storage_backend_load_signal.d.ts → storage-backend-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{storage_backend_load_signal.js → storage-backend-load-signal.js} +1 -1
- package/internals/autoscaling/{system_status.d.ts → system-status.d.ts} +1 -1
- package/internals/autoscaling/{system_status.js → system-status.js} +1 -1
- package/internals/basic-crawler.d.ts +16 -11
- package/internals/basic-crawler.js +61 -66
- package/internals/{cookie_utils.js → cookie-utils.js} +1 -1
- package/internals/crawler-run.d.ts +2 -2
- package/internals/crawler-run.js +3 -1
- package/internals/crawlers/{crawler_commons.d.ts → crawler-commons.d.ts} +2 -2
- package/internals/crawlers/{error_snapshotter.d.ts → error-snapshotter.d.ts} +2 -2
- package/internals/crawlers/{error_tracker.d.ts → error-tracker.d.ts} +1 -1
- package/internals/crawlers/{error_tracker.js → error-tracker.js} +1 -1
- package/internals/crawlers/index.d.ts +3 -3
- package/internals/crawlers/index.js +2 -2
- package/internals/crawlers/statistics.d.ts +11 -1
- package/internals/crawlers/statistics.js +39 -5
- package/internals/{crawling_request.js → crawling-request.js} +1 -1
- package/internals/{enqueue_links/enqueue_links.d.ts → enqueue-links/enqueue-links.d.ts} +1 -6
- package/internals/{enqueue_links/enqueue_links.js → enqueue-links/enqueue-links.js} +0 -41
- package/internals/{enqueue_links → enqueue-links}/index.d.ts +1 -1
- package/internals/{enqueue_links → enqueue-links}/index.js +1 -1
- package/internals/{enqueue_links → enqueue-links}/shared.d.ts +5 -4
- package/internals/{enqueue_links → enqueue-links}/shared.js +15 -45
- package/internals/errors.d.ts +6 -1
- package/internals/errors.js +25 -0
- package/internals/router.d.ts +2 -2
- package/internals/send-request.js +17 -8
- package/internals/{session_pool → session-pool}/index.d.ts +1 -1
- package/internals/{session_pool → session-pool}/index.js +1 -1
- package/internals/{sitemap_request_loader.d.ts → sitemap-request-loader.d.ts} +8 -1
- package/internals/{sitemap_request_loader.js → sitemap-request-loader.js} +116 -56
- package/internals/{throttling_request_manager.d.ts → throttling-request-manager.d.ts} +7 -7
- package/internals/{throttling_request_manager.js → throttling-request-manager.js} +61 -44
- package/package.json +15 -15
- /package/internals/autoscaling/{concurrency_system.d.ts → concurrency-system.d.ts} +0 -0
- /package/internals/autoscaling/{load_signal.d.ts → load-signal.d.ts} +0 -0
- /package/internals/autoscaling/{weighted_avg.d.ts → weighted-avg.d.ts} +0 -0
- /package/internals/autoscaling/{weighted_avg.js → weighted-avg.js} +0 -0
- /package/internals/{cookie_utils.d.ts → cookie-utils.d.ts} +0 -0
- /package/internals/crawlers/{context_pipeline.d.ts → context-pipeline.d.ts} +0 -0
- /package/internals/crawlers/{context_pipeline.js → context-pipeline.js} +0 -0
- /package/internals/crawlers/{crawler_commons.js → crawler-commons.js} +0 -0
- /package/internals/crawlers/{error_snapshotter.js → error-snapshotter.js} +0 -0
- /package/internals/{crawling_request.d.ts → crawling-request.d.ts} +0 -0
- /package/internals/{session_pool → session-pool}/consts.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/consts.js +0 -0
- /package/internals/{session_pool → session-pool}/errors.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/errors.js +0 -0
- /package/internals/{session_pool → session-pool}/fingerprint.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/fingerprint.js +0 -0
- /package/internals/{session_pool/session_pool.d.ts → session-pool/session-pool.d.ts} +0 -0
- /package/internals/{session_pool/session_pool.js → session-pool/session-pool.js} +0 -0
- /package/internals/{session_pool → session-pool}/session.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/session.js +0 -0
package/index.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
export * from '@crawlee/core';
|
|
2
2
|
export * from './internals/autoscaling/index.js';
|
|
3
3
|
export * from './internals/basic-crawler.js';
|
|
4
|
-
export * from './internals/
|
|
4
|
+
export * from './internals/cookie-utils.js';
|
|
5
5
|
export * from './internals/crawlers/index.js';
|
|
6
|
-
export * from './internals/
|
|
7
|
-
export * from './internals/
|
|
6
|
+
export * from './internals/crawling-request.js';
|
|
7
|
+
export * from './internals/enqueue-links/index.js';
|
|
8
8
|
export * from './internals/errors.js';
|
|
9
9
|
export * from './internals/http.js';
|
|
10
10
|
export * from './internals/router.js';
|
|
11
|
-
export * from './internals/
|
|
12
|
-
export * from './internals/
|
|
13
|
-
export * from './internals/
|
|
11
|
+
export * from './internals/session-pool/index.js';
|
|
12
|
+
export * from './internals/sitemap-request-loader.js';
|
|
13
|
+
export * from './internals/throttling-request-manager.js';
|
package/index.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
export * from '@crawlee/core';
|
|
2
2
|
export * from './internals/autoscaling/index.js';
|
|
3
3
|
export * from './internals/basic-crawler.js';
|
|
4
|
-
export * from './internals/
|
|
4
|
+
export * from './internals/cookie-utils.js';
|
|
5
5
|
export * from './internals/crawlers/index.js';
|
|
6
|
-
export * from './internals/
|
|
7
|
-
export * from './internals/
|
|
6
|
+
export * from './internals/crawling-request.js';
|
|
7
|
+
export * from './internals/enqueue-links/index.js';
|
|
8
8
|
export * from './internals/errors.js';
|
|
9
9
|
export * from './internals/http.js';
|
|
10
10
|
export * from './internals/router.js';
|
|
11
|
-
export * from './internals/
|
|
12
|
-
export * from './internals/
|
|
13
|
-
export * from './internals/
|
|
11
|
+
export * from './internals/session-pool/index.js';
|
|
12
|
+
export * from './internals/sitemap-request-loader.js';
|
|
13
|
+
export * from './internals/throttling-request-manager.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CrawleeLogger } from '@crawlee/core';
|
|
2
|
-
import type { ConcurrencyConsumer, IConcurrencySystem } from './
|
|
2
|
+
import type { ConcurrencyConsumer, IConcurrencySystem } from './concurrency-system.js';
|
|
3
3
|
/**
|
|
4
4
|
* The two predicates that steer a task loop: *is there work ready?* and *are we done?* These are the parts of the loop
|
|
5
5
|
* a caller legitimately overrides, as opposed to the task itself (`runTaskFunction`), which the loop's driver owns.
|
|
@@ -20,15 +20,13 @@ export interface TaskLoopPredicates {
|
|
|
20
20
|
*/
|
|
21
21
|
isFinishedFunction?: () => Promise<boolean>;
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/** @internal */
|
|
24
|
+
export interface AutoscaledPoolOptions extends TaskLoopPredicates {
|
|
24
25
|
/**
|
|
25
26
|
* How often the pool should check if a new task is ready, in seconds.
|
|
26
27
|
* @default 0.5
|
|
27
28
|
*/
|
|
28
29
|
maybeRunIntervalSecs?: number;
|
|
29
|
-
}
|
|
30
|
-
/** @internal */
|
|
31
|
-
export interface AutoscaledPoolOptions extends TaskLoopOptions {
|
|
32
30
|
/**
|
|
33
31
|
* The governor that decides whether there is free compute for one more task. Typically a
|
|
34
32
|
* {@link ConcurrencySystem}, but any {@link IConcurrencySystem} works. Share a single instance across
|
|
@@ -60,7 +58,9 @@ export interface AutoscaledPoolOptions extends TaskLoopOptions {
|
|
|
60
58
|
/**
|
|
61
59
|
* Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
|
|
62
60
|
* The pool only starts new tasks while its {@link IConcurrencySystem|concurrency system} reports free capacity —
|
|
63
|
-
* that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget.
|
|
61
|
+
* that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget. The default
|
|
62
|
+
* {@link ConcurrencySystem} keeps `minConcurrency` tasks running even while the system is overloaded, and only
|
|
63
|
+
* starts additional ones when there is enough free CPU and memory.
|
|
64
64
|
*
|
|
65
65
|
* Before running the pool, you need to implement the following three functions:
|
|
66
66
|
* {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
|
|
@@ -23,7 +23,9 @@ const autoscaledPoolOptionsSchema = z.strictObject({
|
|
|
23
23
|
/**
|
|
24
24
|
* Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
|
|
25
25
|
* The pool only starts new tasks while its {@link IConcurrencySystem|concurrency system} reports free capacity —
|
|
26
|
-
* that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget.
|
|
26
|
+
* that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget. The default
|
|
27
|
+
* {@link ConcurrencySystem} keeps `minConcurrency` tasks running even while the system is overloaded, and only
|
|
28
|
+
* starts additional ones when there is enough free CPU and memory.
|
|
27
29
|
*
|
|
28
30
|
* Before running the pool, you need to implement the following three functions:
|
|
29
31
|
* {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
|
|
@@ -3,7 +3,7 @@ import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
5
5
|
import { Snapshotter } from './snapshotter.js';
|
|
6
|
-
import { SystemStatus } from './
|
|
6
|
+
import { SystemStatus } from './system-status.js';
|
|
7
7
|
const concurrencySchema = z.number().int().gte(1).optional();
|
|
8
8
|
// `schemas.anyObject` passes values through by reference, so the load signal instances inside
|
|
9
9
|
// `loadSignals` and class instances like loggers keep their prototype.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SystemInfo } from '@crawlee/core';
|
|
2
|
-
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './
|
|
2
|
+
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
|
|
3
3
|
/**
|
|
4
4
|
* A snapshot produced by the built-in CPU signal.
|
|
5
5
|
* @internal
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { EventType, serviceLocator } from '@crawlee/core';
|
|
2
|
-
import { SnapshotStore } from './
|
|
2
|
+
import { SnapshotStore } from './load-signal.js';
|
|
3
3
|
/**
|
|
4
4
|
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
|
|
5
5
|
* CPU is overloaded.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
2
|
-
import { SnapshotStore } from './
|
|
2
|
+
import { SnapshotStore } from './load-signal.js';
|
|
3
3
|
/**
|
|
4
4
|
* Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
|
|
5
5
|
*
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './
|
|
4
|
-
export * from './
|
|
5
|
-
export * from './
|
|
6
|
-
export * from './
|
|
1
|
+
export * from './autoscaled-pool.js';
|
|
2
|
+
export * from './concurrency-system.js';
|
|
3
|
+
export * from './cpu-load-signal.js';
|
|
4
|
+
export * from './event-loop-load-signal.js';
|
|
5
|
+
export * from './load-signal.js';
|
|
6
|
+
export * from './memory-load-signal.js';
|
|
7
7
|
export * from './snapshotter.js';
|
|
8
|
-
export * from './
|
|
9
|
-
export * from './
|
|
8
|
+
export * from './storage-backend-load-signal.js';
|
|
9
|
+
export * from './system-status.js';
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './
|
|
4
|
-
export * from './
|
|
5
|
-
export * from './
|
|
6
|
-
export * from './
|
|
1
|
+
export * from './autoscaled-pool.js';
|
|
2
|
+
export * from './concurrency-system.js';
|
|
3
|
+
export * from './cpu-load-signal.js';
|
|
4
|
+
export * from './event-loop-load-signal.js';
|
|
5
|
+
export * from './load-signal.js';
|
|
6
|
+
export * from './memory-load-signal.js';
|
|
7
7
|
export * from './snapshotter.js';
|
|
8
|
-
export * from './
|
|
9
|
-
export * from './
|
|
8
|
+
export * from './storage-backend-load-signal.js';
|
|
9
|
+
export * from './system-status.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { weightedAvg } from './
|
|
1
|
+
import { weightedAvg } from './weighted-avg.js';
|
|
2
2
|
/**
|
|
3
3
|
* A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
|
|
4
4
|
* and so can yours — it is the only part of their machinery worth reusing.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SystemInfo } from '@crawlee/core';
|
|
2
|
-
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './
|
|
2
|
+
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
|
|
3
3
|
/**
|
|
4
4
|
* A snapshot produced by the built-in memory signal.
|
|
5
5
|
* @internal
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { EventType, serviceLocator } from '@crawlee/core';
|
|
2
2
|
import { getMemoryInfo, isContainerized } from '@crawlee/core/internal';
|
|
3
|
-
import { SnapshotStore } from './
|
|
3
|
+
import { SnapshotStore } from './load-signal.js';
|
|
4
4
|
const RESERVE_MEMORY_RATIO = 0.5;
|
|
5
5
|
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
|
|
6
6
|
/**
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type { CpuLoadSignalOptions } from './
|
|
2
|
-
import type { EventLoopLoadSignalOptions } from './
|
|
3
|
-
import type { LoadSignal, LoadSignalStartContext } from './
|
|
4
|
-
import type { MemoryLoadSignalOptions } from './
|
|
5
|
-
import type { StorageBackendLoadSignalOptions } from './
|
|
1
|
+
import type { CpuLoadSignalOptions } from './cpu-load-signal.js';
|
|
2
|
+
import type { EventLoopLoadSignalOptions } from './event-loop-load-signal.js';
|
|
3
|
+
import type { LoadSignal, LoadSignalStartContext } from './load-signal.js';
|
|
4
|
+
import type { MemoryLoadSignalOptions } from './memory-load-signal.js';
|
|
5
|
+
import type { StorageBackendLoadSignalOptions } from './storage-backend-load-signal.js';
|
|
6
6
|
/**
|
|
7
7
|
* The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
|
|
8
8
|
*
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { CpuLoadSignal } from './
|
|
2
|
-
import { EventLoopLoadSignal } from './
|
|
3
|
-
import { MemoryLoadSignal } from './
|
|
4
|
-
import { StorageBackendLoadSignal } from './
|
|
1
|
+
import { CpuLoadSignal } from './cpu-load-signal.js';
|
|
2
|
+
import { EventLoopLoadSignal } from './event-loop-load-signal.js';
|
|
3
|
+
import { MemoryLoadSignal } from './memory-load-signal.js';
|
|
4
|
+
import { StorageBackendLoadSignal } from './storage-backend-load-signal.js';
|
|
5
5
|
/**
|
|
6
6
|
* Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
|
|
7
7
|
* {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link StorageBackendLoadSignal} — constructing
|
package/internals/autoscaling/{storage_backend_load_signal.d.ts → storage-backend-load-signal.d.ts}
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './
|
|
1
|
+
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
|
|
2
2
|
/**
|
|
3
3
|
* A snapshot produced by the built-in storage backend (rate-limit) signal.
|
|
4
4
|
* @internal
|
package/internals/autoscaling/{storage_backend_load_signal.js → storage-backend-load-signal.js}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { serviceLocator } from '@crawlee/core';
|
|
2
2
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
3
|
-
import { SnapshotStore } from './
|
|
3
|
+
import { SnapshotStore } from './load-signal.js';
|
|
4
4
|
const RATE_LIMIT_ERROR_RETRY_COUNT = 2;
|
|
5
5
|
/**
|
|
6
6
|
* Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SystemInfo } from '@crawlee/core';
|
|
2
|
-
import type { LoadSignal } from './
|
|
2
|
+
import type { LoadSignal } from './load-signal.js';
|
|
3
3
|
import type { Snapshotter } from './snapshotter.js';
|
|
4
4
|
/**
|
|
5
5
|
* How far back the *current* system status looks by default — the window that gates task dispatch.
|
|
@@ -5,14 +5,15 @@ import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, Stor
|
|
|
5
5
|
import { RobotsTxtFile } from '@crawlee/utils';
|
|
6
6
|
import type { ReadonlyDeep } from 'type-fest';
|
|
7
7
|
import { z } from 'zod';
|
|
8
|
-
import type {
|
|
9
|
-
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/
|
|
10
|
-
import { ConcurrencySystem } from './autoscaling/
|
|
11
|
-
import type { FinalStatistics } from './autoscaling/
|
|
12
|
-
import {
|
|
13
|
-
import
|
|
8
|
+
import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
|
|
9
|
+
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
10
|
+
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
11
|
+
import type { FinalStatistics } from './autoscaling/system-status.js';
|
|
12
|
+
import type { ContextMiddleware } from './crawlers/context-pipeline.js';
|
|
13
|
+
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
14
|
+
import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
|
|
14
15
|
import type { IStatistics, StatisticState } from './crawlers/statistics.js';
|
|
15
|
-
import type { EnqueueUrlsOptions, SkippedRequestCallback } from './
|
|
16
|
+
import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue-links/index.js';
|
|
16
17
|
import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
|
|
17
18
|
export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
18
19
|
}
|
|
@@ -65,6 +66,10 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
65
66
|
* **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
|
|
66
67
|
* those, use a `postNavigationHook` or the `requestHandler` instead.
|
|
67
68
|
*
|
|
69
|
+
* The second argument is the same {@link CleanupRegistrar|`onCleanup`} registrar used by
|
|
70
|
+
* {@link ContextPipeline.compose}. Use it to register teardown callbacks that run after the
|
|
71
|
+
* request handler or a pipeline failure, in reverse registration order.
|
|
72
|
+
*
|
|
68
73
|
* **Example usage:**
|
|
69
74
|
*
|
|
70
75
|
* ```javascript
|
|
@@ -83,7 +88,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
83
88
|
* });
|
|
84
89
|
* ```
|
|
85
90
|
*/
|
|
86
|
-
extendContext?:
|
|
91
|
+
extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
|
|
87
92
|
/**
|
|
88
93
|
* *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
|
|
89
94
|
*
|
|
@@ -174,7 +179,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
174
179
|
* Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
|
|
175
180
|
* shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
|
|
176
181
|
*/
|
|
177
|
-
taskLoopOptions?:
|
|
182
|
+
taskLoopOptions?: TaskLoopPredicates;
|
|
178
183
|
/**
|
|
179
184
|
* A pre-configured concurrency governor — the component that decides whether there is free compute for one more
|
|
180
185
|
* task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
|
|
@@ -278,7 +283,9 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
278
283
|
*
|
|
279
284
|
* Currently supports:
|
|
280
285
|
* - [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)
|
|
286
|
+
* - [**AWS WAF** Challenge and CAPTCHA](https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge.html)
|
|
281
287
|
* - [**Google Search** Rate Limiting](https://www.google.com/sorry/)
|
|
288
|
+
* - **Imperva Incapsula**
|
|
282
289
|
*/
|
|
283
290
|
retryOnBlocked?: boolean;
|
|
284
291
|
/**
|
|
@@ -654,8 +661,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
|
|
|
654
661
|
protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
|
|
655
662
|
/**
|
|
656
663
|
* Handles errors thrown by user provided requestHandler()
|
|
657
|
-
*
|
|
658
|
-
* @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
|
|
659
664
|
*/
|
|
660
665
|
private requestFunctionErrorHandler;
|
|
661
666
|
private handleFailedRequestHandler;
|
|
@@ -2,29 +2,28 @@ import { mkdir, writeFile } from 'node:fs/promises';
|
|
|
2
2
|
import { dirname } from 'node:path';
|
|
3
3
|
import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
|
|
4
4
|
import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
|
|
5
|
-
import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
|
|
5
|
+
import { filterUrl, isAsyncIterable, isIterable, matchesEnqueueStrategy, parseArgument, ROTATE_PROXY_ERRORS, schemas, } from '@crawlee/utils/internal';
|
|
6
6
|
import { RobotsTxtFile } from '@crawlee/utils';
|
|
7
|
-
import { getDomain } from 'tldts';
|
|
8
7
|
import { z } from 'zod';
|
|
9
8
|
import { LruCache } from '@apify/datastructures';
|
|
10
9
|
import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
11
10
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
12
|
-
import { ConcurrencySystem } from './autoscaling/
|
|
13
|
-
import { mergeCookies } from './
|
|
14
|
-
import { ContextPipeline } from './crawlers/
|
|
15
|
-
import { CrawlingRequest, RequestState } from './
|
|
11
|
+
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
12
|
+
import { mergeCookies } from './cookie-utils.js';
|
|
13
|
+
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
14
|
+
import { CrawlingRequest, RequestState } from './crawling-request.js';
|
|
16
15
|
import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
|
|
17
16
|
import { Statistics } from './crawlers/statistics.js';
|
|
18
|
-
import { applyRequestTransform,
|
|
17
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
|
|
19
18
|
import { parseRetryAfterHeader } from './http.js';
|
|
20
19
|
import { CrawlerRun } from './crawler-run.js';
|
|
21
20
|
import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
|
|
22
21
|
import { createSendRequest } from './send-request.js';
|
|
23
22
|
import { Router, validateUserData } from './router.js';
|
|
24
|
-
import { BLOCKED_STATUS_CODES } from './
|
|
25
|
-
import { Session } from './
|
|
26
|
-
import { SessionPool } from './
|
|
27
|
-
import { ThrottlingRequestManager } from './
|
|
23
|
+
import { BLOCKED_STATUS_CODES } from './session-pool/consts.js';
|
|
24
|
+
import { Session } from './session-pool/session.js';
|
|
25
|
+
import { SessionPool } from './session-pool/session-pool.js';
|
|
26
|
+
import { ThrottlingRequestManager } from './throttling-request-manager.js';
|
|
28
27
|
class LazyDefaultHttpClient extends BaseHttpClient {
|
|
29
28
|
#delegatePromise;
|
|
30
29
|
constructor(options) {
|
|
@@ -471,14 +470,14 @@ export class BasicCrawler {
|
|
|
471
470
|
// Navigation, the navigation hooks and the request handler are timed individually, but the
|
|
472
471
|
// phases between them are not, so a request could still get stuck indefinitely. This is the
|
|
473
472
|
// catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
|
|
474
|
-
await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source
|
|
473
|
+
await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source), (error) => this.#handleRequestFailure(error, crawlingContext, source))));
|
|
475
474
|
}
|
|
476
475
|
catch (error) {
|
|
477
476
|
// The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
|
|
478
477
|
// take the normal error flow. Anything else is fatal.
|
|
479
478
|
if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
|
|
480
479
|
const unwrappedError = this.#unwrapError(error);
|
|
481
|
-
await this.requestFunctionErrorHandler(unwrappedError, crawlingContext,
|
|
480
|
+
await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, this.requestManager);
|
|
482
481
|
if (!this.#errorAbsolvesSession(unwrappedError)) {
|
|
483
482
|
crawlingContext.session?.markBad();
|
|
484
483
|
}
|
|
@@ -511,7 +510,7 @@ export class BasicCrawler {
|
|
|
511
510
|
this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
|
|
512
511
|
return true;
|
|
513
512
|
}
|
|
514
|
-
// `maybeFinish()` calls this only once nothing is in flight (`
|
|
513
|
+
// `maybeFinish()` calls this only once nothing is in flight (`autoscaled-pool.ts`) - the point
|
|
515
514
|
// where a crawl that cannot progress becomes distinguishable from one that is merely waiting,
|
|
516
515
|
// and the only place where throwing does not abandon requests mid-processing.
|
|
517
516
|
const state = await this.requestManager?.checkReadiness();
|
|
@@ -625,7 +624,7 @@ export class BasicCrawler {
|
|
|
625
624
|
context[navigationDeadlineKey] += extraMillis;
|
|
626
625
|
}
|
|
627
626
|
// Extension failure must not fail the request that asked for more time.
|
|
628
|
-
this.requestManager?.
|
|
627
|
+
this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {
|
|
629
628
|
this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
|
|
630
629
|
});
|
|
631
630
|
},
|
|
@@ -686,7 +685,7 @@ export class BasicCrawler {
|
|
|
686
685
|
let contextPipeline;
|
|
687
686
|
if (extendContext !== undefined) {
|
|
688
687
|
contextPipeline = ContextPipeline.create()
|
|
689
|
-
.compose(
|
|
688
|
+
.compose(extendContext)
|
|
690
689
|
.chain(subclassPipeline);
|
|
691
690
|
}
|
|
692
691
|
else {
|
|
@@ -795,18 +794,6 @@ export class BasicCrawler {
|
|
|
795
794
|
if (this.running) {
|
|
796
795
|
throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
|
|
797
796
|
}
|
|
798
|
-
if (this.#hasFinishedBefore) {
|
|
799
|
-
// A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
|
|
800
|
-
await this.#statisticsDep.ifOwned(async (stats) => {
|
|
801
|
-
stats.reset();
|
|
802
|
-
await stats.resetStore();
|
|
803
|
-
});
|
|
804
|
-
// A session pool the crawler built follows the same blank-slate rule as its statistics.
|
|
805
|
-
await this.#sessionPoolDep.ifOwned(async (pool) => {
|
|
806
|
-
pool.reset();
|
|
807
|
-
await pool.resetStore();
|
|
808
|
-
});
|
|
809
|
-
}
|
|
810
797
|
// An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
|
|
811
798
|
// resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
|
|
812
799
|
// run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
|
|
@@ -819,6 +806,18 @@ export class BasicCrawler {
|
|
|
819
806
|
});
|
|
820
807
|
this.#run = run;
|
|
821
808
|
try {
|
|
809
|
+
if (this.#hasFinishedBefore) {
|
|
810
|
+
// A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
|
|
811
|
+
await this.#statisticsDep.ifOwned(async (stats) => {
|
|
812
|
+
stats.reset();
|
|
813
|
+
await stats.resetStore();
|
|
814
|
+
});
|
|
815
|
+
// A session pool the crawler built follows the same blank-slate rule as its statistics.
|
|
816
|
+
await this.#sessionPoolDep.ifOwned(async (pool) => {
|
|
817
|
+
pool.reset();
|
|
818
|
+
await pool.resetStore();
|
|
819
|
+
});
|
|
820
|
+
}
|
|
822
821
|
await purgeDefaultStorages({
|
|
823
822
|
onlyPurgeOnce: true,
|
|
824
823
|
storageBackend: serviceLocator.getStorageBackend(),
|
|
@@ -970,7 +969,7 @@ export class BasicCrawler {
|
|
|
970
969
|
this.#requestManager = await this.#openOwnedRequestQueue();
|
|
971
970
|
}
|
|
972
971
|
// Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
|
|
973
|
-
// now that `
|
|
972
|
+
// now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,
|
|
974
973
|
// but guard so we do not re-issue it on every call.
|
|
975
974
|
if (!this.#requestManagerTimeoutsApplied) {
|
|
976
975
|
this.#requestManagerTimeoutsApplied = true;
|
|
@@ -1007,7 +1006,7 @@ export class BasicCrawler {
|
|
|
1007
1006
|
// raise-only, so erring high here is safe.
|
|
1008
1007
|
const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
|
|
1009
1008
|
const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
|
|
1010
|
-
await requestManager.
|
|
1009
|
+
await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));
|
|
1011
1010
|
}
|
|
1012
1011
|
/**
|
|
1013
1012
|
* Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
|
|
@@ -1116,7 +1115,7 @@ export class BasicCrawler {
|
|
|
1116
1115
|
const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
|
|
1117
1116
|
// The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
|
|
1118
1117
|
// (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
|
|
1119
|
-
const
|
|
1118
|
+
const strategyBaseUrl = options.baseUrl ? new URL(options.baseUrl) : undefined;
|
|
1120
1119
|
const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
|
|
1121
1120
|
const maxCrawlDepth = this.#maxCrawlDepth;
|
|
1122
1121
|
const validateRequestUserData = this.#validateRequestUserData.bind(this);
|
|
@@ -1167,7 +1166,14 @@ export class BasicCrawler {
|
|
|
1167
1166
|
// Filter by user patterns first (with exclude)...
|
|
1168
1167
|
let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
|
|
1169
1168
|
// ...then filter by the enqueue strategy (making this an AND check)
|
|
1170
|
-
|
|
1169
|
+
if (strategyBaseUrl) {
|
|
1170
|
+
filtered = filtered.filter((opts) => {
|
|
1171
|
+
const { allowed } = filterUrl(opts.url, strategyBaseUrl, strategy);
|
|
1172
|
+
if (!allowed)
|
|
1173
|
+
onSkippedByFilter(opts);
|
|
1174
|
+
return allowed;
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1171
1177
|
if (filtered.length === 0) {
|
|
1172
1178
|
continue;
|
|
1173
1179
|
}
|
|
@@ -1185,7 +1191,7 @@ export class BasicCrawler {
|
|
|
1185
1191
|
}
|
|
1186
1192
|
const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
|
|
1187
1193
|
forefront: options.forefront,
|
|
1188
|
-
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
|
|
1194
|
+
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,
|
|
1189
1195
|
batchSize: options.batchSize,
|
|
1190
1196
|
waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
|
|
1191
1197
|
maxNewRequests: requestLimit,
|
|
@@ -1202,6 +1208,7 @@ export class BasicCrawler {
|
|
|
1202
1208
|
});
|
|
1203
1209
|
// Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
|
|
1204
1210
|
// when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
|
|
1211
|
+
// Background failures are already logged by `drainRequestBatches`.
|
|
1205
1212
|
void waitForAllRequestsToBeAdded.catch(() => { });
|
|
1206
1213
|
return { ...result, waitForAllRequestsToBeAdded };
|
|
1207
1214
|
}
|
|
@@ -1455,7 +1462,15 @@ export class BasicCrawler {
|
|
|
1455
1462
|
if (cachedRobotsTxtFile) {
|
|
1456
1463
|
return cachedRobotsTxtFile;
|
|
1457
1464
|
}
|
|
1458
|
-
|
|
1465
|
+
// Fetch robots.txt the way pages are fetched: through the crawler's HTTP client, with a session's browser
|
|
1466
|
+
// fingerprint and proxy. Sites with bot protection answer a bare request with 4xx or 5xx, and that status
|
|
1467
|
+
// alone would then decide whether everything is allowed or disallowed.
|
|
1468
|
+
const robotsTxtFile = await RobotsTxtFile.find(url, {
|
|
1469
|
+
httpClient: this.httpClient,
|
|
1470
|
+
session: await this.sessionPool.getSession(),
|
|
1471
|
+
timeoutMillis: 30_000,
|
|
1472
|
+
logger: this.log,
|
|
1473
|
+
});
|
|
1459
1474
|
this.#robotsTxtFileCache.add(origin, robotsTxtFile);
|
|
1460
1475
|
return robotsTxtFile;
|
|
1461
1476
|
}
|
|
@@ -1492,13 +1507,14 @@ export class BasicCrawler {
|
|
|
1492
1507
|
}
|
|
1493
1508
|
/** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
|
|
1494
1509
|
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1495
|
-
async handleRequest(crawlingContext, requestSource
|
|
1510
|
+
async handleRequest(crawlingContext, requestSource) {
|
|
1496
1511
|
// An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
|
|
1497
1512
|
// which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
|
|
1498
1513
|
// execute (and re-report) on top of a request the crawler has already moved past.
|
|
1499
1514
|
if (crawlingContext[timeoutExpiredKey]?.()) {
|
|
1500
1515
|
return;
|
|
1501
1516
|
}
|
|
1517
|
+
const { request } = crawlingContext;
|
|
1502
1518
|
request.state = RequestState.REQUEST_HANDLER;
|
|
1503
1519
|
await this.runRequestHandler(crawlingContext);
|
|
1504
1520
|
// Commit *before* marking the request as handled, so a commit failure fails the request and
|
|
@@ -1516,7 +1532,8 @@ export class BasicCrawler {
|
|
|
1516
1532
|
* or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
|
|
1517
1533
|
* error handler has seen the context.
|
|
1518
1534
|
*/
|
|
1519
|
-
async #handleRequestFailure(rawError, crawlingContext, requestSource
|
|
1535
|
+
async #handleRequestFailure(rawError, crawlingContext, requestSource) {
|
|
1536
|
+
const { request } = crawlingContext;
|
|
1520
1537
|
// Roll back *before* any error handler runs - error handlers write to real storage precisely
|
|
1521
1538
|
// because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
|
|
1522
1539
|
currentStorageTransaction()?.rollback();
|
|
@@ -1528,18 +1545,19 @@ export class BasicCrawler {
|
|
|
1528
1545
|
? await this.#trySkipRequest(unwrappedError, request, requestSource)
|
|
1529
1546
|
: unwrappedError;
|
|
1530
1547
|
if (err) {
|
|
1531
|
-
await this.#runErrorHandler(err, crawlingContext, requestSource
|
|
1548
|
+
await this.#runErrorHandler(err, crawlingContext, requestSource);
|
|
1532
1549
|
}
|
|
1533
1550
|
}
|
|
1534
1551
|
finally {
|
|
1535
1552
|
resumeRequestTimeout?.();
|
|
1536
1553
|
}
|
|
1537
1554
|
}
|
|
1538
|
-
async #runErrorHandler(err, context, requestSource
|
|
1555
|
+
async #runErrorHandler(err, context, requestSource) {
|
|
1556
|
+
const { request } = context;
|
|
1539
1557
|
const retryCountBefore = request.retryCount;
|
|
1540
1558
|
try {
|
|
1541
1559
|
request.state = RequestState.ERROR_HANDLER;
|
|
1542
|
-
await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context,
|
|
1560
|
+
await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1543
1561
|
// The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
|
|
1544
1562
|
if (request.state !== RequestState.SKIPPED) {
|
|
1545
1563
|
request.state = RequestState.DONE;
|
|
@@ -1648,11 +1666,10 @@ export class BasicCrawler {
|
|
|
1648
1666
|
}
|
|
1649
1667
|
/**
|
|
1650
1668
|
* Handles errors thrown by user provided requestHandler()
|
|
1651
|
-
*
|
|
1652
|
-
* @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
|
|
1653
1669
|
*/
|
|
1654
1670
|
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1655
|
-
async requestFunctionErrorHandler(error, crawlingContext,
|
|
1671
|
+
async requestFunctionErrorHandler(error, crawlingContext, source) {
|
|
1672
|
+
const { request } = crawlingContext;
|
|
1656
1673
|
if (error instanceof RequestThrottledError) {
|
|
1657
1674
|
// The domain told us to come back later, so the request was never really attempted. Put it back where
|
|
1658
1675
|
// it was, without recording a failure - it costs neither a retry nor session reputation.
|
|
@@ -1695,6 +1712,7 @@ export class BasicCrawler {
|
|
|
1695
1712
|
}
|
|
1696
1713
|
if (!request.noRetry) {
|
|
1697
1714
|
request.retryCount++;
|
|
1715
|
+
this.statistics.registerRetry?.(request.retryCount);
|
|
1698
1716
|
const { url, retryCount, id } = request;
|
|
1699
1717
|
// We don't want to see the stack trace in the logs by default, when we are going to retry the request.
|
|
1700
1718
|
// Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
|
|
@@ -1867,30 +1885,7 @@ export class BasicCrawler {
|
|
|
1867
1885
|
if (!loadedUrl) {
|
|
1868
1886
|
return true;
|
|
1869
1887
|
}
|
|
1870
|
-
|
|
1871
|
-
const loadedBaseUrl = new URL(loadedUrl);
|
|
1872
|
-
switch (strategy) {
|
|
1873
|
-
case EnqueueStrategy.SameHostname: {
|
|
1874
|
-
return baseUrl.hostname === loadedBaseUrl.hostname;
|
|
1875
|
-
}
|
|
1876
|
-
case EnqueueStrategy.SameDomain: {
|
|
1877
|
-
const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
|
|
1878
|
-
if (baseUrlHostname) {
|
|
1879
|
-
const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });
|
|
1880
|
-
return baseUrlHostname === loadedBaseUrlHostname;
|
|
1881
|
-
}
|
|
1882
|
-
// Can happen for IPs, we just check like same origin
|
|
1883
|
-
return baseUrl.origin === loadedBaseUrl.origin;
|
|
1884
|
-
}
|
|
1885
|
-
case EnqueueStrategy.SameOrigin: {
|
|
1886
|
-
// Same as hostname, but also checks protocol
|
|
1887
|
-
return baseUrl.origin === loadedBaseUrl.origin;
|
|
1888
|
-
}
|
|
1889
|
-
case EnqueueStrategy.All:
|
|
1890
|
-
default: {
|
|
1891
|
-
return baseUrl.protocol === 'http:' || baseUrl.protocol === 'https:';
|
|
1892
|
-
}
|
|
1893
|
-
}
|
|
1888
|
+
return matchesEnqueueStrategy(strategy, new URL(loadedUrl), new URL(url));
|
|
1894
1889
|
}
|
|
1895
1890
|
}
|
|
1896
1891
|
/** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
|