@crawlee/basic 4.0.0-beta.223 → 4.0.0-beta.225
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +6 -6
- package/index.js +6 -6
- package/internals/autoscaling/{autoscaled_pool.d.ts → autoscaled-pool.d.ts} +1 -1
- package/internals/autoscaling/{concurrency_system.js → concurrency-system.js} +1 -1
- package/internals/autoscaling/{cpu_load_signal.d.ts → cpu-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{cpu_load_signal.js → cpu-load-signal.js} +1 -1
- package/internals/autoscaling/{event_loop_load_signal.d.ts → event-loop-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{event_loop_load_signal.js → event-loop-load-signal.js} +1 -1
- package/internals/autoscaling/index.d.ts +8 -8
- package/internals/autoscaling/index.js +8 -8
- package/internals/autoscaling/{load_signal.js → load-signal.js} +1 -1
- package/internals/autoscaling/{memory_load_signal.d.ts → memory-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{memory_load_signal.js → memory-load-signal.js} +1 -1
- package/internals/autoscaling/snapshotter.d.ts +5 -5
- package/internals/autoscaling/snapshotter.js +4 -4
- package/internals/autoscaling/{storage_backend_load_signal.d.ts → storage-backend-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{storage_backend_load_signal.js → storage-backend-load-signal.js} +1 -1
- package/internals/autoscaling/{system_status.d.ts → system-status.d.ts} +1 -1
- package/internals/autoscaling/{system_status.js → system-status.js} +1 -1
- package/internals/basic-crawler.d.ts +7 -7
- package/internals/basic-crawler.js +10 -10
- package/internals/{cookie_utils.js → cookie-utils.js} +1 -1
- package/internals/crawler-run.d.ts +2 -2
- package/internals/crawler-run.js +1 -1
- package/internals/crawlers/{crawler_commons.d.ts → crawler-commons.d.ts} +2 -2
- package/internals/crawlers/{error_snapshotter.d.ts → error-snapshotter.d.ts} +2 -2
- package/internals/crawlers/{error_tracker.d.ts → error-tracker.d.ts} +1 -1
- package/internals/crawlers/{error_tracker.js → error-tracker.js} +1 -1
- package/internals/crawlers/index.d.ts +3 -3
- package/internals/crawlers/index.js +2 -2
- package/internals/crawlers/statistics.d.ts +1 -1
- package/internals/crawlers/statistics.js +1 -1
- package/internals/{enqueue_links → enqueue-links}/index.d.ts +1 -1
- package/internals/{enqueue_links → enqueue-links}/index.js +1 -1
- package/internals/{enqueue_links → enqueue-links}/shared.d.ts +1 -1
- package/internals/errors.d.ts +1 -1
- package/internals/router.d.ts +2 -2
- package/internals/{session_pool → session-pool}/index.d.ts +1 -1
- package/internals/{session_pool → session-pool}/index.js +1 -1
- package/internals/{sitemap_request_loader.d.ts → sitemap-request-loader.d.ts} +1 -1
- package/internals/{sitemap_request_loader.js → sitemap-request-loader.js} +1 -1
- package/package.json +7 -7
- /package/internals/autoscaling/{autoscaled_pool.js → autoscaled-pool.js} +0 -0
- /package/internals/autoscaling/{concurrency_system.d.ts → concurrency-system.d.ts} +0 -0
- /package/internals/autoscaling/{load_signal.d.ts → load-signal.d.ts} +0 -0
- /package/internals/autoscaling/{weighted_avg.d.ts → weighted-avg.d.ts} +0 -0
- /package/internals/autoscaling/{weighted_avg.js → weighted-avg.js} +0 -0
- /package/internals/{cookie_utils.d.ts → cookie-utils.d.ts} +0 -0
- /package/internals/crawlers/{context_pipeline.d.ts → context-pipeline.d.ts} +0 -0
- /package/internals/crawlers/{context_pipeline.js → context-pipeline.js} +0 -0
- /package/internals/crawlers/{crawler_commons.js → crawler-commons.js} +0 -0
- /package/internals/crawlers/{error_snapshotter.js → error-snapshotter.js} +0 -0
- /package/internals/{crawling_request.d.ts → crawling-request.d.ts} +0 -0
- /package/internals/{crawling_request.js → crawling-request.js} +0 -0
- /package/internals/{enqueue_links/enqueue_links.d.ts → enqueue-links/enqueue-links.d.ts} +0 -0
- /package/internals/{enqueue_links/enqueue_links.js → enqueue-links/enqueue-links.js} +0 -0
- /package/internals/{enqueue_links → enqueue-links}/shared.js +0 -0
- /package/internals/{session_pool → session-pool}/consts.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/consts.js +0 -0
- /package/internals/{session_pool → session-pool}/errors.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/errors.js +0 -0
- /package/internals/{session_pool → session-pool}/fingerprint.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/fingerprint.js +0 -0
- /package/internals/{session_pool/session_pool.d.ts → session-pool/session-pool.d.ts} +0 -0
- /package/internals/{session_pool/session_pool.js → session-pool/session-pool.js} +0 -0
- /package/internals/{session_pool → session-pool}/session.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/session.js +0 -0
- /package/internals/{throttling_request_manager.d.ts → throttling-request-manager.d.ts} +0 -0
- /package/internals/{throttling_request_manager.js → throttling-request-manager.js} +0 -0
package/index.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
export * from '@crawlee/core';
|
|
2
2
|
export * from './internals/autoscaling/index.js';
|
|
3
3
|
export * from './internals/basic-crawler.js';
|
|
4
|
-
export * from './internals/
|
|
4
|
+
export * from './internals/cookie-utils.js';
|
|
5
5
|
export * from './internals/crawlers/index.js';
|
|
6
|
-
export * from './internals/
|
|
7
|
-
export * from './internals/
|
|
6
|
+
export * from './internals/crawling-request.js';
|
|
7
|
+
export * from './internals/enqueue-links/index.js';
|
|
8
8
|
export * from './internals/errors.js';
|
|
9
9
|
export * from './internals/http.js';
|
|
10
10
|
export * from './internals/router.js';
|
|
11
|
-
export * from './internals/
|
|
12
|
-
export * from './internals/
|
|
13
|
-
export * from './internals/
|
|
11
|
+
export * from './internals/session-pool/index.js';
|
|
12
|
+
export * from './internals/sitemap-request-loader.js';
|
|
13
|
+
export * from './internals/throttling-request-manager.js';
|
package/index.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
export * from '@crawlee/core';
|
|
2
2
|
export * from './internals/autoscaling/index.js';
|
|
3
3
|
export * from './internals/basic-crawler.js';
|
|
4
|
-
export * from './internals/
|
|
4
|
+
export * from './internals/cookie-utils.js';
|
|
5
5
|
export * from './internals/crawlers/index.js';
|
|
6
|
-
export * from './internals/
|
|
7
|
-
export * from './internals/
|
|
6
|
+
export * from './internals/crawling-request.js';
|
|
7
|
+
export * from './internals/enqueue-links/index.js';
|
|
8
8
|
export * from './internals/errors.js';
|
|
9
9
|
export * from './internals/http.js';
|
|
10
10
|
export * from './internals/router.js';
|
|
11
|
-
export * from './internals/
|
|
12
|
-
export * from './internals/
|
|
13
|
-
export * from './internals/
|
|
11
|
+
export * from './internals/session-pool/index.js';
|
|
12
|
+
export * from './internals/sitemap-request-loader.js';
|
|
13
|
+
export * from './internals/throttling-request-manager.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CrawleeLogger } from '@crawlee/core';
|
|
2
|
-
import type { ConcurrencyConsumer, IConcurrencySystem } from './
|
|
2
|
+
import type { ConcurrencyConsumer, IConcurrencySystem } from './concurrency-system.js';
|
|
3
3
|
/**
|
|
4
4
|
* The two predicates that steer a task loop: *is there work ready?* and *are we done?* These are the parts of the loop
|
|
5
5
|
* a caller legitimately overrides, as opposed to the task itself (`runTaskFunction`), which the loop's driver owns.
|
|
@@ -3,7 +3,7 @@ import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
5
5
|
import { Snapshotter } from './snapshotter.js';
|
|
6
|
-
import { SystemStatus } from './
|
|
6
|
+
import { SystemStatus } from './system-status.js';
|
|
7
7
|
const concurrencySchema = z.number().int().gte(1).optional();
|
|
8
8
|
// `schemas.anyObject` passes values through by reference, so the load signal instances inside
|
|
9
9
|
// `loadSignals` and class instances like loggers keep their prototype.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SystemInfo } from '@crawlee/core';
|
|
2
|
-
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './
|
|
2
|
+
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
|
|
3
3
|
/**
|
|
4
4
|
* A snapshot produced by the built-in CPU signal.
|
|
5
5
|
* @internal
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { EventType, serviceLocator } from '@crawlee/core';
|
|
2
|
-
import { SnapshotStore } from './
|
|
2
|
+
import { SnapshotStore } from './load-signal.js';
|
|
3
3
|
/**
|
|
4
4
|
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
|
|
5
5
|
* CPU is overloaded.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
2
|
-
import { SnapshotStore } from './
|
|
2
|
+
import { SnapshotStore } from './load-signal.js';
|
|
3
3
|
/**
|
|
4
4
|
* Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
|
|
5
5
|
*
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './
|
|
4
|
-
export * from './
|
|
5
|
-
export * from './
|
|
6
|
-
export * from './
|
|
1
|
+
export * from './autoscaled-pool.js';
|
|
2
|
+
export * from './concurrency-system.js';
|
|
3
|
+
export * from './cpu-load-signal.js';
|
|
4
|
+
export * from './event-loop-load-signal.js';
|
|
5
|
+
export * from './load-signal.js';
|
|
6
|
+
export * from './memory-load-signal.js';
|
|
7
7
|
export * from './snapshotter.js';
|
|
8
|
-
export * from './
|
|
9
|
-
export * from './
|
|
8
|
+
export * from './storage-backend-load-signal.js';
|
|
9
|
+
export * from './system-status.js';
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './
|
|
4
|
-
export * from './
|
|
5
|
-
export * from './
|
|
6
|
-
export * from './
|
|
1
|
+
export * from './autoscaled-pool.js';
|
|
2
|
+
export * from './concurrency-system.js';
|
|
3
|
+
export * from './cpu-load-signal.js';
|
|
4
|
+
export * from './event-loop-load-signal.js';
|
|
5
|
+
export * from './load-signal.js';
|
|
6
|
+
export * from './memory-load-signal.js';
|
|
7
7
|
export * from './snapshotter.js';
|
|
8
|
-
export * from './
|
|
9
|
-
export * from './
|
|
8
|
+
export * from './storage-backend-load-signal.js';
|
|
9
|
+
export * from './system-status.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { weightedAvg } from './
|
|
1
|
+
import { weightedAvg } from './weighted-avg.js';
|
|
2
2
|
/**
|
|
3
3
|
* A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
|
|
4
4
|
* and so can yours — it is the only part of their machinery worth reusing.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SystemInfo } from '@crawlee/core';
|
|
2
|
-
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './
|
|
2
|
+
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
|
|
3
3
|
/**
|
|
4
4
|
* A snapshot produced by the built-in memory signal.
|
|
5
5
|
* @internal
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { EventType, serviceLocator } from '@crawlee/core';
|
|
2
2
|
import { getMemoryInfo, isContainerized } from '@crawlee/core/internal';
|
|
3
|
-
import { SnapshotStore } from './
|
|
3
|
+
import { SnapshotStore } from './load-signal.js';
|
|
4
4
|
const RESERVE_MEMORY_RATIO = 0.5;
|
|
5
5
|
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
|
|
6
6
|
/**
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type { CpuLoadSignalOptions } from './
|
|
2
|
-
import type { EventLoopLoadSignalOptions } from './
|
|
3
|
-
import type { LoadSignal, LoadSignalStartContext } from './
|
|
4
|
-
import type { MemoryLoadSignalOptions } from './
|
|
5
|
-
import type { StorageBackendLoadSignalOptions } from './
|
|
1
|
+
import type { CpuLoadSignalOptions } from './cpu-load-signal.js';
|
|
2
|
+
import type { EventLoopLoadSignalOptions } from './event-loop-load-signal.js';
|
|
3
|
+
import type { LoadSignal, LoadSignalStartContext } from './load-signal.js';
|
|
4
|
+
import type { MemoryLoadSignalOptions } from './memory-load-signal.js';
|
|
5
|
+
import type { StorageBackendLoadSignalOptions } from './storage-backend-load-signal.js';
|
|
6
6
|
/**
|
|
7
7
|
* The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
|
|
8
8
|
*
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { CpuLoadSignal } from './
|
|
2
|
-
import { EventLoopLoadSignal } from './
|
|
3
|
-
import { MemoryLoadSignal } from './
|
|
4
|
-
import { StorageBackendLoadSignal } from './
|
|
1
|
+
import { CpuLoadSignal } from './cpu-load-signal.js';
|
|
2
|
+
import { EventLoopLoadSignal } from './event-loop-load-signal.js';
|
|
3
|
+
import { MemoryLoadSignal } from './memory-load-signal.js';
|
|
4
|
+
import { StorageBackendLoadSignal } from './storage-backend-load-signal.js';
|
|
5
5
|
/**
|
|
6
6
|
* Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
|
|
7
7
|
* {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link StorageBackendLoadSignal} — constructing
|
package/internals/autoscaling/{storage_backend_load_signal.d.ts → storage-backend-load-signal.d.ts}
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './
|
|
1
|
+
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load-signal.js';
|
|
2
2
|
/**
|
|
3
3
|
* A snapshot produced by the built-in storage backend (rate-limit) signal.
|
|
4
4
|
* @internal
|
package/internals/autoscaling/{storage_backend_load_signal.js → storage-backend-load-signal.js}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { serviceLocator } from '@crawlee/core';
|
|
2
2
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
3
|
-
import { SnapshotStore } from './
|
|
3
|
+
import { SnapshotStore } from './load-signal.js';
|
|
4
4
|
const RATE_LIMIT_ERROR_RETRY_COUNT = 2;
|
|
5
5
|
/**
|
|
6
6
|
* Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SystemInfo } from '@crawlee/core';
|
|
2
|
-
import type { LoadSignal } from './
|
|
2
|
+
import type { LoadSignal } from './load-signal.js';
|
|
3
3
|
import type { Snapshotter } from './snapshotter.js';
|
|
4
4
|
/**
|
|
5
5
|
* How far back the *current* system status looks by default — the window that gates task dispatch.
|
|
@@ -5,14 +5,14 @@ import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, Stor
|
|
|
5
5
|
import { RobotsTxtFile } from '@crawlee/utils';
|
|
6
6
|
import type { ReadonlyDeep } from 'type-fest';
|
|
7
7
|
import { z } from 'zod';
|
|
8
|
-
import type { TaskLoopOptions } from './autoscaling/
|
|
9
|
-
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/
|
|
10
|
-
import { ConcurrencySystem } from './autoscaling/
|
|
11
|
-
import type { FinalStatistics } from './autoscaling/
|
|
12
|
-
import { ContextPipeline } from './crawlers/
|
|
13
|
-
import type { CrawlingContext, TypedRequestsLike } from './crawlers/
|
|
8
|
+
import type { TaskLoopOptions } from './autoscaling/autoscaled-pool.js';
|
|
9
|
+
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
10
|
+
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
11
|
+
import type { FinalStatistics } from './autoscaling/system-status.js';
|
|
12
|
+
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
13
|
+
import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
|
|
14
14
|
import type { IStatistics, StatisticState } from './crawlers/statistics.js';
|
|
15
|
-
import type { EnqueueUrlsOptions, SkippedRequestCallback } from './
|
|
15
|
+
import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue-links/index.js';
|
|
16
16
|
import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
|
|
17
17
|
export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
18
18
|
}
|
|
@@ -9,22 +9,22 @@ import { z } from 'zod';
|
|
|
9
9
|
import { LruCache } from '@apify/datastructures';
|
|
10
10
|
import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
11
11
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
12
|
-
import { ConcurrencySystem } from './autoscaling/
|
|
13
|
-
import { mergeCookies } from './
|
|
14
|
-
import { ContextPipeline } from './crawlers/
|
|
15
|
-
import { CrawlingRequest, RequestState } from './
|
|
12
|
+
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
13
|
+
import { mergeCookies } from './cookie-utils.js';
|
|
14
|
+
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
15
|
+
import { CrawlingRequest, RequestState } from './crawling-request.js';
|
|
16
16
|
import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
|
|
17
17
|
import { Statistics } from './crawlers/statistics.js';
|
|
18
|
-
import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './
|
|
18
|
+
import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
|
|
19
19
|
import { parseRetryAfterHeader } from './http.js';
|
|
20
20
|
import { CrawlerRun } from './crawler-run.js';
|
|
21
21
|
import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
|
|
22
22
|
import { createSendRequest } from './send-request.js';
|
|
23
23
|
import { Router, validateUserData } from './router.js';
|
|
24
|
-
import { BLOCKED_STATUS_CODES } from './
|
|
25
|
-
import { Session } from './
|
|
26
|
-
import { SessionPool } from './
|
|
27
|
-
import { ThrottlingRequestManager } from './
|
|
24
|
+
import { BLOCKED_STATUS_CODES } from './session-pool/consts.js';
|
|
25
|
+
import { Session } from './session-pool/session.js';
|
|
26
|
+
import { SessionPool } from './session-pool/session-pool.js';
|
|
27
|
+
import { ThrottlingRequestManager } from './throttling-request-manager.js';
|
|
28
28
|
class LazyDefaultHttpClient extends BaseHttpClient {
|
|
29
29
|
#delegatePromise;
|
|
30
30
|
constructor(options) {
|
|
@@ -511,7 +511,7 @@ export class BasicCrawler {
|
|
|
511
511
|
this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
|
|
512
512
|
return true;
|
|
513
513
|
}
|
|
514
|
-
// `maybeFinish()` calls this only once nothing is in flight (`
|
|
514
|
+
// `maybeFinish()` calls this only once nothing is in flight (`autoscaled-pool.ts`) - the point
|
|
515
515
|
// where a crawl that cannot progress becomes distinguishable from one that is merely waiting,
|
|
516
516
|
// and the only place where throwing does not abandon requests mid-processing.
|
|
517
517
|
const state = await this.requestManager?.checkReadiness();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CrawleeLogger } from '@crawlee/core';
|
|
2
|
-
import type { AutoscaledPoolOptions } from './autoscaling/
|
|
3
|
-
import type { IConcurrencySystem } from './autoscaling/
|
|
2
|
+
import type { AutoscaledPoolOptions } from './autoscaling/autoscaled-pool.js';
|
|
3
|
+
import type { IConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
4
4
|
/** Everything a run needs to exist. Whatever a crawler resolves per run is resolved before the run is built. */
|
|
5
5
|
export interface CrawlerRunSetup {
|
|
6
6
|
log: CrawleeLogger;
|
package/internals/crawler-run.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AutoscaledPool } from './autoscaling/
|
|
1
|
+
import { AutoscaledPool } from './autoscaling/autoscaled-pool.js';
|
|
2
2
|
/**
|
|
3
3
|
* Everything that belongs to a single {@link BasicCrawler.run|`crawler.run()`} — the task loop dispatching its
|
|
4
4
|
* requests, the concurrency governor those requests are booked against, and the bookkeeping that must not survive
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { Awaitable, Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
|
|
2
2
|
import type { ReadonlyDeep } from 'type-fest';
|
|
3
3
|
import type { AddRequestsBatchedResult, CrawleeLogger, Dataset, KeyValueStore, Request, RequestOptions, Source, StorageIdentifier } from '@crawlee/core';
|
|
4
|
-
import type { EnqueueUrlsOptions } from '../
|
|
5
|
-
import type { CrawlingRequest } from '../
|
|
4
|
+
import type { EnqueueUrlsOptions } from '../enqueue-links/index.js';
|
|
5
|
+
import type { CrawlingRequest } from '../crawling-request.js';
|
|
6
6
|
/** @internal */
|
|
7
7
|
export type IsAny<T> = 0 extends 1 & T ? true : false;
|
|
8
8
|
/**
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { CrawlingContext } from '
|
|
2
|
-
import type { ErrnoException } from './
|
|
1
|
+
import type { CrawlingContext } from './crawler-commons.js';
|
|
2
|
+
import type { ErrnoException } from './error-tracker.js';
|
|
3
3
|
import type { SnapshottableProperties } from './internals/types.js';
|
|
4
4
|
interface ErrorSnapshot {
|
|
5
5
|
screenshotFileName?: string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { inspect } from 'node:util';
|
|
2
|
-
import { ErrorSnapshotter } from './
|
|
2
|
+
import { ErrorSnapshotter } from './error-snapshotter.js';
|
|
3
3
|
const extractPathFromStackTraceLine = (line) => {
|
|
4
4
|
const lastStartingRoundBracketIndex = line.lastIndexOf('(');
|
|
5
5
|
if (lastStartingRoundBracketIndex !== -1) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export type * from './
|
|
1
|
+
export * from './context-pipeline.js';
|
|
2
|
+
export type * from './crawler-commons.js';
|
|
3
3
|
export * from './statistics.js';
|
|
4
|
-
export * from './
|
|
4
|
+
export * from './error-tracker.js';
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './context-pipeline.js';
|
|
2
2
|
export * from './statistics.js';
|
|
3
|
-
export * from './
|
|
3
|
+
export * from './error-tracker.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CrawleeLogger, SyncStateConversion } from '@crawlee/core';
|
|
2
2
|
import { KeyValueStore } from '@crawlee/core';
|
|
3
|
-
import { ErrorTracker } from './
|
|
3
|
+
import { ErrorTracker } from './error-tracker.js';
|
|
4
4
|
/**
|
|
5
5
|
* Persistence-related options to control how and when crawler's data gets persisted.
|
|
6
6
|
*/
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { convertStateSync, KeyValueStore, RecoverableState, serviceLocator, validators } from '@crawlee/core';
|
|
2
2
|
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import { ErrorTracker } from './
|
|
4
|
+
import { ErrorTracker } from './error-tracker.js';
|
|
5
5
|
/**
|
|
6
6
|
* @ignore
|
|
7
7
|
*/
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './enqueue-links.js';
|
|
2
2
|
export { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, updateEnqueueLinksPatternCache, urlPatternSchema, validateGlobPattern, } from './shared.js';
|
|
3
3
|
export type { GlobInput, GlobObject, RegExpInput, RegExpObject, RequestTransform, SkippedRequestCallback, UrlPatternInput, } from './shared.js';
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './enqueue-links.js';
|
|
2
2
|
// Not `export *`: keeps the internal `UrlPatternObject` off the public surface.
|
|
3
3
|
export { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, updateEnqueueLinksPatternCache, urlPatternSchema, validateGlobPattern, } from './shared.js';
|
|
@@ -2,7 +2,7 @@ import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
import type { EnqueueStrategyOption, RequestOptions, Source } from '@crawlee/core';
|
|
4
4
|
import { Request } from '@crawlee/core';
|
|
5
|
-
import type { SkippedRequestReason } from '../
|
|
5
|
+
import type { SkippedRequestReason } from '../crawling-request.js';
|
|
6
6
|
export interface GlobObject {
|
|
7
7
|
glob: string;
|
|
8
8
|
}
|
package/internals/errors.d.ts
CHANGED
package/internals/router.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { CrawlingRequest } from './
|
|
1
|
+
import type { CrawlingRequest } from './crawling-request.js';
|
|
2
2
|
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { StandardSchemaV1 } from '@standard-schema/spec';
|
|
4
|
-
import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/
|
|
4
|
+
import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler-commons.js';
|
|
5
5
|
/**
|
|
6
6
|
* The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
|
|
7
7
|
* Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
|
|
@@ -3,7 +3,7 @@ import type { ParseSitemapOptions } from '@crawlee/utils';
|
|
|
3
3
|
import { EnqueueStrategy } from '@crawlee/utils';
|
|
4
4
|
import type { IRequestLoader, IRequestManager, RequestLoaderStatus } from '@crawlee/core';
|
|
5
5
|
import { Request } from '@crawlee/core';
|
|
6
|
-
import type { UrlPatternInput } from './
|
|
6
|
+
import type { UrlPatternInput } from './enqueue-links/index.js';
|
|
7
7
|
interface UrlConstraints {
|
|
8
8
|
/**
|
|
9
9
|
* An array of URL patterns that URLs must match to be included.
|
|
@@ -4,7 +4,7 @@ import { parseArgument, parseSitemap, schemas } from '@crawlee/utils/internal';
|
|
|
4
4
|
import { minimatch } from 'minimatch';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
import { EventType, KeyValueStore, purgeDefaultStorages, Request, RequestManagerTandem, RequestQueue, serviceLocator, } from '@crawlee/core';
|
|
7
|
-
import { constructUrlPatternObjects, urlPatternSchema } from './
|
|
7
|
+
import { constructUrlPatternObjects, urlPatternSchema } from './enqueue-links/index.js';
|
|
8
8
|
const sitemapRequestLoaderOptionsSchema = z.strictObject({
|
|
9
9
|
sitemapUrls: schemas.arrayOf(z.string(), 'strings'),
|
|
10
10
|
proxyUrl: z.string().optional(),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.225",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.0.1",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.225",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.225",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.225",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.225",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.0.0",
|
|
51
51
|
"csv-stringify": "^6.5.2",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.5.4"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.225"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "5d71fa0f785d7dc5736d95bdf5f4ec907e5a1c42"
|
|
70
70
|
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|