@crawlee/core 4.0.0-beta.1 → 4.0.0-beta.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +69 -156
- package/autoscaling/autoscaled_pool.js +97 -252
- package/autoscaling/client_load_signal.d.ts +59 -0
- package/autoscaling/client_load_signal.js +73 -0
- package/autoscaling/concurrency_system.d.ts +283 -0
- package/autoscaling/concurrency_system.js +350 -0
- package/autoscaling/cpu_load_signal.d.ts +44 -0
- package/autoscaling/cpu_load_signal.js +46 -0
- package/autoscaling/event_loop_load_signal.d.ts +54 -0
- package/autoscaling/event_loop_load_signal.js +60 -0
- package/autoscaling/index.d.ts +6 -1
- package/autoscaling/index.js +6 -1
- package/autoscaling/load_signal.d.ts +99 -0
- package/autoscaling/load_signal.js +103 -0
- package/autoscaling/memory_load_signal.d.ts +56 -0
- package/autoscaling/memory_load_signal.js +106 -0
- package/autoscaling/snapshotter.d.ts +61 -163
- package/autoscaling/snapshotter.js +45 -270
- package/autoscaling/system_status.d.ts +63 -83
- package/autoscaling/system_status.js +90 -120
- package/autoscaling/weighted_avg.d.ts +5 -0
- package/autoscaling/weighted_avg.js +14 -0
- package/byte_utils.d.ts +17 -0
- package/byte_utils.js +42 -0
- package/configuration.d.ts +96 -230
- package/configuration.js +170 -226
- package/cookie_utils.d.ts +4 -3
- package/cookie_utils.js +22 -13
- package/crawlers/context_pipeline.d.ts +70 -0
- package/crawlers/context_pipeline.js +122 -0
- package/crawlers/crawler_commons.d.ts +96 -40
- package/crawlers/crawler_commons.js +15 -24
- package/crawlers/error_snapshotter.d.ts +3 -3
- package/crawlers/error_snapshotter.js +2 -3
- package/crawlers/error_tracker.d.ts +2 -2
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +1 -3
- package/crawlers/index.js +1 -3
- package/crawlers/internals/types.d.ts +7 -0
- package/crawlers/internals/types.js +1 -0
- package/crawlers/statistics.d.ts +28 -23
- package/crawlers/statistics.js +38 -34
- package/debug.d.ts +36 -0
- package/debug.js +70 -0
- package/enqueue_links/enqueue_links.d.ts +59 -68
- package/enqueue_links/enqueue_links.js +57 -62
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +39 -26
- package/enqueue_links/shared.js +89 -67
- package/errors.d.ts +53 -4
- package/errors.js +70 -5
- package/events/event_manager.d.ts +34 -8
- package/events/event_manager.js +8 -10
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -5
- package/events/local_event_manager.js +37 -40
- package/index.d.ts +6 -4
- package/index.js +5 -3
- package/iterables.d.ts +79 -0
- package/iterables.js +134 -0
- package/log.d.ts +82 -3
- package/log.js +102 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +46 -0
- package/memory-storage/memory-storage.js +136 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +113 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +203 -0
- package/memory-storage/resource-clients/request-queue.d.ts +96 -0
- package/memory-storage/resource-clients/request-queue.js +421 -0
- package/memory-storage/utils.d.ts +16 -0
- package/memory-storage/utils.js +41 -0
- package/owned_or_injected.d.ts +60 -0
- package/owned_or_injected.js +98 -0
- package/package.json +13 -12
- package/proxy_configuration.d.ts +29 -152
- package/proxy_configuration.js +21 -173
- package/recoverable_state.d.ts +120 -0
- package/recoverable_state.js +143 -0
- package/request.d.ts +85 -17
- package/request.js +107 -36
- package/router.d.ts +194 -19
- package/router.js +177 -32
- package/serialization.d.ts +0 -1
- package/serialization.js +1 -2
- package/service_locator.d.ts +156 -0
- package/service_locator.js +244 -0
- package/session_pool/consts.d.ts +1 -2
- package/session_pool/consts.js +1 -2
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +9 -0
- package/session_pool/fingerprint.js +30 -0
- package/session_pool/index.d.ts +0 -2
- package/session_pool/index.js +0 -2
- package/session_pool/session.d.ts +37 -75
- package/session_pool/session.js +49 -102
- package/session_pool/session_pool.d.ts +85 -90
- package/session_pool/session_pool.js +131 -120
- package/storages/access_checking.d.ts +1 -2
- package/storages/access_checking.js +5 -2
- package/storages/dataset.d.ts +103 -54
- package/storages/dataset.js +174 -132
- package/storages/index.d.ts +8 -7
- package/storages/index.js +6 -7
- package/storages/key_value_store.d.ts +167 -39
- package/storages/key_value_store.js +274 -127
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +23 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +54 -98
- package/storages/request_list.js +99 -75
- package/storages/request_loader.d.ts +96 -0
- package/storages/request_loader.js +1 -0
- package/storages/request_manager.d.ts +33 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +106 -0
- package/storages/request_manager_tandem.js +197 -0
- package/storages/request_queue.d.ts +287 -47
- package/storages/request_queue.js +620 -215
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +38 -46
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +58 -65
- package/storages/storage_instance_manager.d.ts +88 -0
- package/storages/storage_instance_manager.js +256 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/utils.d.ts +54 -9
- package/storages/utils.js +64 -13
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +0 -6
- package/typedefs.js +0 -1
- package/validators.d.ts +8 -1
- package/validators.js +10 -3
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js +0 -12
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_provider.d.ts +0 -307
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
package/errors.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { inspectValue } from './debug.js';
|
|
1
2
|
/**
|
|
2
3
|
* Errors of `NonRetryableError` type will never be retried by the crawler.
|
|
3
4
|
*/
|
|
@@ -14,6 +15,28 @@ export class CriticalError extends NonRetryableError {
|
|
|
14
15
|
*/
|
|
15
16
|
export class MissingRouteError extends CriticalError {
|
|
16
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
|
|
20
|
+
*
|
|
21
|
+
* As the `userData` does not change between attempts, this error is non-retryable.
|
|
22
|
+
*/
|
|
23
|
+
export class RequestValidationError extends NonRetryableError {
|
|
24
|
+
label;
|
|
25
|
+
issues;
|
|
26
|
+
constructor(label, issues) {
|
|
27
|
+
const details = issues
|
|
28
|
+
.map((issue) => {
|
|
29
|
+
const path = (issue.path ?? [])
|
|
30
|
+
.map((segment) => (typeof segment === 'object' ? segment.key : segment))
|
|
31
|
+
.join('.');
|
|
32
|
+
return `- ${path ? `${path}: ` : ''}${issue.message}`;
|
|
33
|
+
})
|
|
34
|
+
.join('\n');
|
|
35
|
+
super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);
|
|
36
|
+
this.label = label;
|
|
37
|
+
this.issues = issues;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
17
40
|
/**
|
|
18
41
|
* Errors of `RetryRequestError` type will always be retried by the crawler.
|
|
19
42
|
*
|
|
@@ -25,13 +48,55 @@ export class RetryRequestError extends Error {
|
|
|
25
48
|
}
|
|
26
49
|
}
|
|
27
50
|
/**
|
|
28
|
-
* Errors of `SessionError` type
|
|
51
|
+
* Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
|
|
29
52
|
*
|
|
30
|
-
*
|
|
53
|
+
* The retry counts towards the `maxRequestRetries` limit, just like any other error.
|
|
31
54
|
*/
|
|
32
|
-
export class SessionError extends
|
|
55
|
+
export class SessionError extends Error {
|
|
33
56
|
constructor(message) {
|
|
34
|
-
super(`Detected a session error,
|
|
57
|
+
super(`Detected a session error, retiring session... ${message ? `\n${message}` : ''}`);
|
|
35
58
|
}
|
|
36
59
|
}
|
|
37
|
-
|
|
60
|
+
/**
|
|
61
|
+
* Thrown when a requested session is not found in the referenced SessionPool.
|
|
62
|
+
*/
|
|
63
|
+
export class MissingSessionError extends Error {
|
|
64
|
+
constructor(sessionId) {
|
|
65
|
+
super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
export class ContextPipelineInterruptedError extends Error {
|
|
69
|
+
constructor(message) {
|
|
70
|
+
super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
export class ContextPipelineInitializationError extends Error {
|
|
74
|
+
constructor(error, options) {
|
|
75
|
+
super(undefined, { cause: error, ...options });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
export class ContextPipelineCleanupError extends CriticalError {
|
|
79
|
+
constructor(error, options) {
|
|
80
|
+
super(undefined, { cause: error, ...options });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
export class RequestHandlerError extends Error {
|
|
84
|
+
constructor(error, options) {
|
|
85
|
+
super(undefined, { cause: error, ...options });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Thrown when attempting to set a different service instance after one has already been retrieved.
|
|
90
|
+
*/
|
|
91
|
+
export class ServiceConflictError extends Error {
|
|
92
|
+
constructor(serviceName, newValue, existingValue) {
|
|
93
|
+
super(`Service ${serviceName} is already in use. ` +
|
|
94
|
+
`Existing value: ${inspectValue(existingValue)}, attempted new value: ${inspectValue(newValue)}.`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Thrown by crawlers when `skipNavigation` is used on a request.
|
|
99
|
+
* Subclasses can catch this error to skip their own navigation-dependent logic.
|
|
100
|
+
*/
|
|
101
|
+
export class NavigationSkippedError extends NonRetryableError {
|
|
102
|
+
}
|
|
@@ -1,28 +1,55 @@
|
|
|
1
1
|
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
2
2
|
import type { BetterIntervalID } from '@apify/utilities';
|
|
3
|
-
|
|
3
|
+
export interface EventManagerOptions {
|
|
4
|
+
/** Interval between emitted `persistState` events in milliseconds. */
|
|
5
|
+
persistStateIntervalMillis: number;
|
|
6
|
+
}
|
|
4
7
|
export declare const enum EventType {
|
|
5
8
|
PERSIST_STATE = "persistState",
|
|
6
9
|
SYSTEM_INFO = "systemInfo",
|
|
7
10
|
MIGRATING = "migrating",
|
|
8
11
|
ABORTING = "aborting",
|
|
9
|
-
EXIT = "exit"
|
|
12
|
+
EXIT = "exit",
|
|
13
|
+
STATUS_MESSAGE = "statusMessage"
|
|
14
|
+
}
|
|
15
|
+
export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';
|
|
16
|
+
/**
|
|
17
|
+
* Payload emitted with the {@link EventType.STATUS_MESSAGE|`statusMessage`} event.
|
|
18
|
+
*
|
|
19
|
+
* The crawler broadcasts these whenever it wants to report its progress (e.g. periodically, or on
|
|
20
|
+
* start/finish). Consumers such as the Apify SDK can listen for the event and propagate the message
|
|
21
|
+
* to the platform. This keeps the crawler decoupled from any specific status-reporting backend.
|
|
22
|
+
*/
|
|
23
|
+
export interface EventStatusMessageData {
|
|
24
|
+
/**
|
|
25
|
+
* Identifies the crawler that emitted the message.
|
|
26
|
+
*
|
|
27
|
+
* Either the user-provided `id` from the crawler options, or a randomly generated one.
|
|
28
|
+
* Since a single event manager may be shared by multiple crawlers, consumers can use this
|
|
29
|
+
* to attribute the message to a specific crawler instance.
|
|
30
|
+
*/
|
|
31
|
+
crawlerId: string;
|
|
32
|
+
/** The human-readable status message. */
|
|
33
|
+
message: string;
|
|
34
|
+
/** Whether this is the final status message of the run. */
|
|
35
|
+
isStatusMessageTerminal?: boolean;
|
|
36
|
+
/** The log level the message was logged with. */
|
|
37
|
+
level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
|
|
10
38
|
}
|
|
11
|
-
export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';
|
|
12
39
|
interface Intervals {
|
|
13
40
|
persistState?: BetterIntervalID;
|
|
14
41
|
systemInfo?: BetterIntervalID;
|
|
15
42
|
}
|
|
16
43
|
export declare abstract class EventManager {
|
|
17
|
-
readonly config: Configuration;
|
|
18
44
|
protected events: AsyncEventEmitter<{}>;
|
|
19
45
|
protected initialized: boolean;
|
|
20
46
|
protected intervals: Intervals;
|
|
21
47
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
22
|
-
protected log: import("@
|
|
23
|
-
|
|
48
|
+
protected log: import("@crawlee/types").CrawleeLogger;
|
|
49
|
+
private persistStateIntervalMillis;
|
|
50
|
+
constructor(options: EventManagerOptions);
|
|
24
51
|
/**
|
|
25
|
-
* Initializes the event manager by
|
|
52
|
+
* Initializes the event manager by starting the `persistState` event interval.
|
|
26
53
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
27
54
|
*/
|
|
28
55
|
init(): Promise<void>;
|
|
@@ -49,4 +76,3 @@ export declare abstract class EventManager {
|
|
|
49
76
|
waitForAllListenersToComplete(): Promise<boolean>;
|
|
50
77
|
}
|
|
51
78
|
export {};
|
|
52
|
-
//# sourceMappingURL=event_manager.d.ts.map
|
package/events/event_manager.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
2
|
-
import log from '@apify/log';
|
|
3
2
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
4
|
-
import {
|
|
3
|
+
import { serviceLocator } from '../service_locator.js';
|
|
5
4
|
export var EventType;
|
|
6
5
|
(function (EventType) {
|
|
7
6
|
EventType["PERSIST_STATE"] = "persistState";
|
|
@@ -9,30 +8,30 @@ export var EventType;
|
|
|
9
8
|
EventType["MIGRATING"] = "migrating";
|
|
10
9
|
EventType["ABORTING"] = "aborting";
|
|
11
10
|
EventType["EXIT"] = "exit";
|
|
11
|
+
EventType["STATUS_MESSAGE"] = "statusMessage";
|
|
12
12
|
})(EventType || (EventType = {}));
|
|
13
13
|
export class EventManager {
|
|
14
|
-
config;
|
|
15
14
|
events = new AsyncEventEmitter();
|
|
16
15
|
initialized = false;
|
|
17
16
|
intervals = {};
|
|
18
|
-
log =
|
|
19
|
-
|
|
20
|
-
|
|
17
|
+
log = serviceLocator.getLogger().child({ prefix: 'Events' });
|
|
18
|
+
persistStateIntervalMillis;
|
|
19
|
+
constructor(options) {
|
|
20
|
+
this.persistStateIntervalMillis = options.persistStateIntervalMillis;
|
|
21
21
|
this.events.setMaxListeners(50);
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
|
-
* Initializes the event manager by
|
|
24
|
+
* Initializes the event manager by starting the `persistState` event interval.
|
|
25
25
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
26
26
|
*/
|
|
27
27
|
async init() {
|
|
28
28
|
if (this.initialized) {
|
|
29
29
|
return;
|
|
30
30
|
}
|
|
31
|
-
const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
|
|
32
31
|
this.intervals.persistState = betterSetInterval((intervalCallback) => {
|
|
33
32
|
this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
|
|
34
33
|
intervalCallback();
|
|
35
|
-
}, persistStateIntervalMillis);
|
|
34
|
+
}, this.persistStateIntervalMillis);
|
|
36
35
|
this.initialized = true;
|
|
37
36
|
}
|
|
38
37
|
/**
|
|
@@ -86,4 +85,3 @@ export class EventManager {
|
|
|
86
85
|
return this.events.waitForAllListenersToComplete();
|
|
87
86
|
}
|
|
88
87
|
}
|
|
89
|
-
//# sourceMappingURL=event_manager.js.map
|
package/events/index.d.ts
CHANGED
package/events/index.js
CHANGED
|
@@ -1,8 +1,19 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Configuration } from '../configuration.js';
|
|
2
|
+
import { EventManager, type EventManagerOptions } from './event_manager.js';
|
|
3
|
+
export interface LocalEventManagerOptions extends EventManagerOptions {
|
|
4
|
+
/** Interval between emitted `systemInfo` events in milliseconds. */
|
|
5
|
+
systemInfoIntervalMillis: number;
|
|
6
|
+
}
|
|
2
7
|
export declare class LocalEventManager extends EventManager {
|
|
3
|
-
private
|
|
8
|
+
private systemInfoIntervalMillis;
|
|
9
|
+
constructor(options: LocalEventManagerOptions);
|
|
10
|
+
/**
|
|
11
|
+
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
12
|
+
* Uses the global configuration from the service locator if none is provided.
|
|
13
|
+
*/
|
|
14
|
+
static fromConfiguration(configuration?: Configuration): LocalEventManager;
|
|
4
15
|
/**
|
|
5
|
-
* Initializes the EventManager and sets up periodic `systemInfo`
|
|
16
|
+
* Initializes the EventManager and sets up periodic `systemInfo` events.
|
|
6
17
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
7
18
|
*/
|
|
8
19
|
init(): Promise<void>;
|
|
@@ -18,12 +29,11 @@ export declare class LocalEventManager extends EventManager {
|
|
|
18
29
|
* @internal
|
|
19
30
|
*/
|
|
20
31
|
isContainerizedWrapper(): Promise<boolean>;
|
|
21
|
-
private getCurrentCpuTicks;
|
|
22
32
|
/**
|
|
23
33
|
* Creates a SystemInfo object based on local metrics.
|
|
24
34
|
*/
|
|
25
35
|
private createSystemInfo;
|
|
26
36
|
private createCpuInfo;
|
|
27
37
|
private createMemoryInfo;
|
|
38
|
+
private getMemoryInfo;
|
|
28
39
|
}
|
|
29
|
-
//# sourceMappingURL=local_event_manager.d.ts.map
|
|
@@ -1,12 +1,25 @@
|
|
|
1
|
-
import os from 'node:os';
|
|
2
|
-
import { getCurrentCpuTicksV2, getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
|
|
3
|
-
import log from '@apify/log';
|
|
4
1
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
2
|
+
import { serviceLocator } from '../service_locator.js';
|
|
5
3
|
import { EventManager } from './event_manager.js';
|
|
6
4
|
export class LocalEventManager extends EventManager {
|
|
7
|
-
|
|
5
|
+
systemInfoIntervalMillis;
|
|
6
|
+
constructor(options) {
|
|
7
|
+
super(options);
|
|
8
|
+
this.systemInfoIntervalMillis = options.systemInfoIntervalMillis;
|
|
9
|
+
}
|
|
8
10
|
/**
|
|
9
|
-
*
|
|
11
|
+
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
12
|
+
* Uses the global configuration from the service locator if none is provided.
|
|
13
|
+
*/
|
|
14
|
+
static fromConfiguration(configuration) {
|
|
15
|
+
const resolvedConfiguration = configuration ?? serviceLocator.getConfiguration();
|
|
16
|
+
return new LocalEventManager({
|
|
17
|
+
persistStateIntervalMillis: resolvedConfiguration.persistStateIntervalMillis,
|
|
18
|
+
systemInfoIntervalMillis: resolvedConfiguration.systemInfoIntervalMillis,
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Initializes the EventManager and sets up periodic `systemInfo` events.
|
|
10
23
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
11
24
|
*/
|
|
12
25
|
async init() {
|
|
@@ -14,9 +27,8 @@ export class LocalEventManager extends EventManager {
|
|
|
14
27
|
return;
|
|
15
28
|
}
|
|
16
29
|
await super.init();
|
|
17
|
-
const systemInfoIntervalMillis = this.config.get('systemInfoIntervalMillis');
|
|
18
30
|
this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
|
|
19
|
-
this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
|
|
31
|
+
this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.systemInfoIntervalMillis);
|
|
20
32
|
}
|
|
21
33
|
/**
|
|
22
34
|
* @inheritDoc
|
|
@@ -33,7 +45,7 @@ export class LocalEventManager extends EventManager {
|
|
|
33
45
|
*/
|
|
34
46
|
async emitSystemInfoEvent(intervalCallback) {
|
|
35
47
|
const info = await this.createSystemInfo({
|
|
36
|
-
maxUsedCpuRatio:
|
|
48
|
+
maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,
|
|
37
49
|
});
|
|
38
50
|
this.events.emit("systemInfo" /* EventType.SYSTEM_INFO */, info);
|
|
39
51
|
intervalCallback();
|
|
@@ -42,17 +54,8 @@ export class LocalEventManager extends EventManager {
|
|
|
42
54
|
* @internal
|
|
43
55
|
*/
|
|
44
56
|
async isContainerizedWrapper() {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
getCurrentCpuTicks() {
|
|
48
|
-
const cpus = os.cpus();
|
|
49
|
-
return cpus.reduce((acc, cpu) => {
|
|
50
|
-
const cpuTimes = Object.values(cpu.times);
|
|
51
|
-
return {
|
|
52
|
-
idle: acc.idle + cpu.times.idle,
|
|
53
|
-
total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
|
|
54
|
-
};
|
|
55
|
-
}, { idle: 0, total: 0 });
|
|
57
|
+
const { isContainerized } = await import('../system-info/runtime.js');
|
|
58
|
+
return serviceLocator.getConfiguration().containerized ?? (await isContainerized());
|
|
56
59
|
}
|
|
57
60
|
/**
|
|
58
61
|
* Creates a SystemInfo object based on local metrics.
|
|
@@ -65,18 +68,11 @@ export class LocalEventManager extends EventManager {
|
|
|
65
68
|
};
|
|
66
69
|
}
|
|
67
70
|
async createCpuInfo(options) {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
const ticks = this.getCurrentCpuTicks();
|
|
76
|
-
const idleTicksDelta = ticks.idle - this.previousTicks.idle;
|
|
77
|
-
const totalTicksDelta = ticks.total - this.previousTicks.total;
|
|
78
|
-
const usedCpuRatio = totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
|
|
79
|
-
Object.assign(this.previousTicks, ticks);
|
|
71
|
+
const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');
|
|
72
|
+
const usedCpuRatio = await getCurrentCpuTicksV2({
|
|
73
|
+
containerized: await this.isContainerizedWrapper(),
|
|
74
|
+
logger: serviceLocator.getLogger(),
|
|
75
|
+
});
|
|
80
76
|
return {
|
|
81
77
|
cpuCurrentUsage: usedCpuRatio * 100,
|
|
82
78
|
isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
|
|
@@ -84,21 +80,22 @@ export class LocalEventManager extends EventManager {
|
|
|
84
80
|
}
|
|
85
81
|
async createMemoryInfo() {
|
|
86
82
|
try {
|
|
87
|
-
|
|
88
|
-
const memInfo = await getMemoryInfoV2(await this.isContainerizedWrapper());
|
|
89
|
-
return {
|
|
90
|
-
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
|
|
91
|
-
};
|
|
92
|
-
}
|
|
93
|
-
const memInfo = await getMemoryInfo();
|
|
83
|
+
const memInfo = await this.getMemoryInfo();
|
|
94
84
|
return {
|
|
85
|
+
memTotalBytes: memInfo.totalBytes,
|
|
95
86
|
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
|
|
96
87
|
};
|
|
97
88
|
}
|
|
98
89
|
catch (err) {
|
|
99
|
-
log.exception(err, 'Memory snapshot failed.');
|
|
90
|
+
this.log.exception(err, 'Memory snapshot failed.');
|
|
100
91
|
return {};
|
|
101
92
|
}
|
|
102
93
|
}
|
|
94
|
+
async getMemoryInfo() {
|
|
95
|
+
const { getMemoryInfo } = await import('../system-info/memory-info.js');
|
|
96
|
+
return getMemoryInfo({
|
|
97
|
+
containerized: await this.isContainerizedWrapper(),
|
|
98
|
+
logger: serviceLocator.getLogger(),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
103
101
|
}
|
|
104
|
-
//# sourceMappingURL=local_event_manager.js.map
|
package/index.d.ts
CHANGED
|
@@ -1,19 +1,21 @@
|
|
|
1
|
+
export * from './debug.js';
|
|
1
2
|
export * from './errors.js';
|
|
2
3
|
export * from './autoscaling/index.js';
|
|
3
4
|
export * from './configuration.js';
|
|
5
|
+
export * from './service_locator.js';
|
|
4
6
|
export * from './crawlers/index.js';
|
|
5
7
|
export * from './enqueue_links/index.js';
|
|
6
8
|
export * from './events/index.js';
|
|
7
|
-
export * from './http_clients/index.js';
|
|
8
9
|
export * from './log.js';
|
|
10
|
+
export * from './owned_or_injected.js';
|
|
9
11
|
export * from './proxy_configuration.js';
|
|
10
12
|
export * from './request.js';
|
|
11
13
|
export * from './router.js';
|
|
12
14
|
export * from './serialization.js';
|
|
13
15
|
export * from './session_pool/index.js';
|
|
14
16
|
export * from './storages/index.js';
|
|
17
|
+
export * from './memory-storage/index.js';
|
|
15
18
|
export * from './validators.js';
|
|
16
19
|
export * from './cookie_utils.js';
|
|
17
|
-
export
|
|
18
|
-
export {
|
|
19
|
-
//# sourceMappingURL=index.d.ts.map
|
|
20
|
+
export * from './recoverable_state.js';
|
|
21
|
+
export type { StorageBackend } from '@crawlee/types';
|
package/index.js
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
|
+
export * from './debug.js';
|
|
1
2
|
export * from './errors.js';
|
|
2
3
|
export * from './autoscaling/index.js';
|
|
3
4
|
export * from './configuration.js';
|
|
5
|
+
export * from './service_locator.js';
|
|
4
6
|
export * from './crawlers/index.js';
|
|
5
7
|
export * from './enqueue_links/index.js';
|
|
6
8
|
export * from './events/index.js';
|
|
7
|
-
export * from './http_clients/index.js';
|
|
8
9
|
export * from './log.js';
|
|
10
|
+
export * from './owned_or_injected.js';
|
|
9
11
|
export * from './proxy_configuration.js';
|
|
10
12
|
export * from './request.js';
|
|
11
13
|
export * from './router.js';
|
|
12
14
|
export * from './serialization.js';
|
|
13
15
|
export * from './session_pool/index.js';
|
|
14
16
|
export * from './storages/index.js';
|
|
17
|
+
export * from './memory-storage/index.js';
|
|
15
18
|
export * from './validators.js';
|
|
16
19
|
export * from './cookie_utils.js';
|
|
17
|
-
export
|
|
18
|
-
//# sourceMappingURL=index.js.map
|
|
20
|
+
export * from './recoverable_state.js';
|
package/iterables.d.ts
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Converts any iterable or async iterable to an async iterable.
|
|
3
|
+
* @internal
|
|
4
|
+
*
|
|
5
|
+
* @yields Each item from the input iterable
|
|
6
|
+
*
|
|
7
|
+
* **Example usage:**
|
|
8
|
+
* ```ts
|
|
9
|
+
* const syncArray = [1, 2, 3];
|
|
10
|
+
* for await (const item of asyncifyIterable(syncArray)) {
|
|
11
|
+
* console.log(item); // 1, 2, 3
|
|
12
|
+
* }
|
|
13
|
+
* ```
|
|
14
|
+
*/
|
|
15
|
+
export declare function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
|
|
16
|
+
/**
|
|
17
|
+
* Lazily splits the input async iterable into chunks of specified size.
|
|
18
|
+
* The last chunk may contain fewer items if the total number of items
|
|
19
|
+
* is not evenly divisible by the chunk size.
|
|
20
|
+
* @internal
|
|
21
|
+
*
|
|
22
|
+
* @yields Arrays of items, each containing up to chunkSize items
|
|
23
|
+
*
|
|
24
|
+
* **Example usage:**
|
|
25
|
+
* ```ts
|
|
26
|
+
* const numbers = async function* () {
|
|
27
|
+
* for (let i = 1; i <= 10; i++) yield i;
|
|
28
|
+
* };
|
|
29
|
+
*
|
|
30
|
+
* for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
|
|
31
|
+
* console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
|
|
32
|
+
* }
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number | (() => number)): AsyncIterable<T[]>;
|
|
36
|
+
/**
|
|
37
|
+
* An async iterator that also supports peeking at the next value without consuming it.
|
|
38
|
+
* Extends both AsyncIterator and AsyncIterable interfaces.
|
|
39
|
+
* @internal
|
|
40
|
+
*/
|
|
41
|
+
export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
|
|
42
|
+
/**
|
|
43
|
+
* Peeks at the next value without consuming it from the iterator.
|
|
44
|
+
* Subsequent calls to peek() will return the same value until next() is called.
|
|
45
|
+
*
|
|
46
|
+
* @returns Promise that resolves to the next value, or undefined if the iterator is exhausted
|
|
47
|
+
*/
|
|
48
|
+
peek(): Promise<T | undefined>;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* An async iterable that yields peekable async iterators.
|
|
52
|
+
* @internal
|
|
53
|
+
*/
|
|
54
|
+
export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
|
|
55
|
+
[Symbol.asyncIterator](): PeekableAsyncIterator<T>;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Wraps an async iterable to provide peek functionality, allowing you to look at
|
|
59
|
+
* the next value without consuming it from the iterator.
|
|
60
|
+
* @internal
|
|
61
|
+
*
|
|
62
|
+
* @param iterable - The async iterable to make peekable
|
|
63
|
+
*
|
|
64
|
+
* **Example usage:**
|
|
65
|
+
* ```ts
|
|
66
|
+
* const numbers = async function* () {
|
|
67
|
+
* yield 1; yield 2; yield 3;
|
|
68
|
+
* };
|
|
69
|
+
*
|
|
70
|
+
* const peekable = peekableAsyncIterable(numbers());
|
|
71
|
+
* const iterator = peekable[Symbol.asyncIterator]();
|
|
72
|
+
*
|
|
73
|
+
* console.log(await iterator.peek()); // 1 (doesn't consume)
|
|
74
|
+
* console.log(await iterator.peek()); // 1 (still doesn't consume)
|
|
75
|
+
* console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
|
|
76
|
+
* console.log(await iterator.peek()); // 2 (next value)
|
|
77
|
+
* ```
|
|
78
|
+
*/
|
|
79
|
+
export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;
|
package/iterables.js
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { inspect } from 'node:util';
|
|
2
|
+
/**
|
|
3
|
+
* Converts any iterable or async iterable to an async iterable.
|
|
4
|
+
* @internal
|
|
5
|
+
*
|
|
6
|
+
* @yields Each item from the input iterable
|
|
7
|
+
*
|
|
8
|
+
* **Example usage:**
|
|
9
|
+
* ```ts
|
|
10
|
+
* const syncArray = [1, 2, 3];
|
|
11
|
+
* for await (const item of asyncifyIterable(syncArray)) {
|
|
12
|
+
* console.log(item); // 1, 2, 3
|
|
13
|
+
* }
|
|
14
|
+
* ```
|
|
15
|
+
*/
|
|
16
|
+
export async function* asyncifyIterable(iterable) {
|
|
17
|
+
yield* iterable;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Lazily splits the input async iterable into chunks of specified size.
|
|
21
|
+
* The last chunk may contain fewer items if the total number of items
|
|
22
|
+
* is not evenly divisible by the chunk size.
|
|
23
|
+
* @internal
|
|
24
|
+
*
|
|
25
|
+
* @yields Arrays of items, each containing up to chunkSize items
|
|
26
|
+
*
|
|
27
|
+
* **Example usage:**
|
|
28
|
+
* ```ts
|
|
29
|
+
* const numbers = async function* () {
|
|
30
|
+
* for (let i = 1; i <= 10; i++) yield i;
|
|
31
|
+
* };
|
|
32
|
+
*
|
|
33
|
+
* for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
|
|
34
|
+
* console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
|
|
35
|
+
* }
|
|
36
|
+
* ```
|
|
37
|
+
*/
|
|
38
|
+
export async function* chunkedAsyncIterable(iterable, chunkSize) {
|
|
39
|
+
const getChunkSize = typeof chunkSize === 'function' ? chunkSize : () => chunkSize;
|
|
40
|
+
if (typeof chunkSize === 'number' && chunkSize < 1) {
|
|
41
|
+
throw new Error(`Chunk size must be a positive number (${inspect(chunkSize)}) received`);
|
|
42
|
+
}
|
|
43
|
+
const iterator = Symbol.asyncIterator in iterable
|
|
44
|
+
? iterable[Symbol.asyncIterator]()
|
|
45
|
+
: iterable[Symbol.iterator]();
|
|
46
|
+
while (true) {
|
|
47
|
+
const currentSize = getChunkSize();
|
|
48
|
+
if (currentSize < 1)
|
|
49
|
+
break;
|
|
50
|
+
const chunk = [];
|
|
51
|
+
for (let i = 0; i < currentSize; i++) {
|
|
52
|
+
const next = await iterator.next();
|
|
53
|
+
if (next.done) {
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
chunk.push(next.value);
|
|
57
|
+
}
|
|
58
|
+
if (chunk.length === 0)
|
|
59
|
+
break;
|
|
60
|
+
yield chunk;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Wraps an async iterable to provide peek functionality, allowing you to look at
|
|
65
|
+
* the next value without consuming it from the iterator.
|
|
66
|
+
* @internal
|
|
67
|
+
*
|
|
68
|
+
* @param iterable - The async iterable to make peekable
|
|
69
|
+
*
|
|
70
|
+
* **Example usage:**
|
|
71
|
+
* ```ts
|
|
72
|
+
* const numbers = async function* () {
|
|
73
|
+
* yield 1; yield 2; yield 3;
|
|
74
|
+
* };
|
|
75
|
+
*
|
|
76
|
+
* const peekable = peekableAsyncIterable(numbers());
|
|
77
|
+
* const iterator = peekable[Symbol.asyncIterator]();
|
|
78
|
+
*
|
|
79
|
+
* console.log(await iterator.peek()); // 1 (doesn't consume)
|
|
80
|
+
* console.log(await iterator.peek()); // 1 (still doesn't consume)
|
|
81
|
+
* console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
|
|
82
|
+
* console.log(await iterator.peek()); // 2 (next value)
|
|
83
|
+
* ```
|
|
84
|
+
*/
|
|
85
|
+
export function peekableAsyncIterable(iterable) {
|
|
86
|
+
const iterator = asyncifyIterable(iterable)[Symbol.asyncIterator]();
|
|
87
|
+
let peekedValue;
|
|
88
|
+
let isExhausted = false;
|
|
89
|
+
const peekableIterator = {
|
|
90
|
+
async next() {
|
|
91
|
+
// If we have peeked a value, return it and clear the peek
|
|
92
|
+
if (peekedValue !== undefined) {
|
|
93
|
+
const result = peekedValue;
|
|
94
|
+
peekedValue = undefined;
|
|
95
|
+
if (result.done) {
|
|
96
|
+
isExhausted = true;
|
|
97
|
+
return { done: true, value: undefined };
|
|
98
|
+
}
|
|
99
|
+
return { done: false, value: result.value };
|
|
100
|
+
}
|
|
101
|
+
if (isExhausted) {
|
|
102
|
+
return { done: true, value: undefined };
|
|
103
|
+
}
|
|
104
|
+
const result = await iterator.next();
|
|
105
|
+
if (result.done) {
|
|
106
|
+
isExhausted = true;
|
|
107
|
+
}
|
|
108
|
+
return result;
|
|
109
|
+
},
|
|
110
|
+
async peek() {
|
|
111
|
+
if (peekedValue !== undefined) {
|
|
112
|
+
return peekedValue.done ? undefined : peekedValue.value;
|
|
113
|
+
}
|
|
114
|
+
if (isExhausted) {
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
const result = await iterator.next();
|
|
118
|
+
peekedValue = { done: result.done ?? false, value: result.value };
|
|
119
|
+
if (result.done) {
|
|
120
|
+
isExhausted = true;
|
|
121
|
+
return undefined;
|
|
122
|
+
}
|
|
123
|
+
return result.value;
|
|
124
|
+
},
|
|
125
|
+
[Symbol.asyncIterator]() {
|
|
126
|
+
return this;
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
return {
|
|
130
|
+
[Symbol.asyncIterator]() {
|
|
131
|
+
return peekableIterator;
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|