@crawlee/core 4.0.0-beta.11 → 4.0.0-beta.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +67 -172
- package/autoscaling/autoscaled_pool.js +165 -320
- package/autoscaling/client_load_signal.d.ts +55 -0
- package/autoscaling/client_load_signal.js +73 -0
- package/autoscaling/concurrency_system.d.ts +268 -0
- package/autoscaling/concurrency_system.js +351 -0
- package/autoscaling/cpu_load_signal.d.ts +43 -0
- package/autoscaling/cpu_load_signal.js +46 -0
- package/autoscaling/event_loop_load_signal.d.ts +51 -0
- package/autoscaling/event_loop_load_signal.js +60 -0
- package/autoscaling/index.d.ts +6 -1
- package/autoscaling/index.js +6 -1
- package/autoscaling/load_signal.d.ts +99 -0
- package/autoscaling/load_signal.js +104 -0
- package/autoscaling/memory_load_signal.d.ts +47 -0
- package/autoscaling/memory_load_signal.js +105 -0
- package/autoscaling/snapshotter.d.ts +58 -163
- package/autoscaling/snapshotter.js +45 -263
- package/autoscaling/system_status.d.ts +62 -84
- package/autoscaling/system_status.js +92 -122
- package/autoscaling/weighted_avg.d.ts +5 -0
- package/autoscaling/weighted_avg.js +14 -0
- package/byte_utils.d.ts +17 -0
- package/byte_utils.js +42 -0
- package/configuration.d.ts +96 -223
- package/configuration.js +170 -222
- package/cookie_utils.d.ts +4 -3
- package/cookie_utils.js +22 -13
- package/crawlers/context_pipeline.d.ts +10 -1
- package/crawlers/context_pipeline.js +31 -8
- package/crawlers/crawler_commons.d.ts +90 -83
- package/crawlers/crawler_commons.js +1 -116
- package/crawlers/error_snapshotter.d.ts +1 -4
- package/crawlers/error_snapshotter.js +3 -6
- package/crawlers/error_tracker.d.ts +0 -1
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +1 -3
- package/crawlers/index.js +0 -3
- package/crawlers/internals/types.d.ts +0 -1
- package/crawlers/internals/types.js +0 -1
- package/crawlers/statistics.d.ts +91 -41
- package/crawlers/statistics.js +83 -72
- package/debug.d.ts +36 -0
- package/debug.js +70 -0
- package/enqueue_links/enqueue_links.d.ts +59 -68
- package/enqueue_links/enqueue_links.js +57 -62
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +40 -27
- package/enqueue_links/shared.js +90 -68
- package/errors.d.ts +41 -4
- package/errors.js +50 -5
- package/events/event_manager.d.ts +34 -8
- package/events/event_manager.js +8 -10
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -3
- package/events/local_event_manager.js +37 -11
- package/index.d.ts +6 -4
- package/index.js +5 -3
- package/iterables.d.ts +79 -0
- package/iterables.js +134 -0
- package/log.d.ts +82 -3
- package/log.js +106 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +42 -0
- package/memory-storage/memory-storage.js +136 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +114 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +204 -0
- package/memory-storage/resource-clients/request-queue.d.ts +77 -0
- package/memory-storage/resource-clients/request-queue.js +422 -0
- package/memory-storage/utils.d.ts +16 -0
- package/memory-storage/utils.js +41 -0
- package/owned_or_injected.d.ts +58 -0
- package/owned_or_injected.js +98 -0
- package/package.json +13 -12
- package/proxy_configuration.d.ts +27 -152
- package/proxy_configuration.js +27 -179
- package/recoverable_state.d.ts +111 -0
- package/recoverable_state.js +143 -0
- package/request.d.ts +86 -17
- package/request.js +117 -41
- package/router.d.ts +193 -21
- package/router.js +188 -43
- package/serialization.d.ts +0 -1
- package/serialization.js +9 -11
- package/service_locator.d.ts +147 -0
- package/service_locator.js +244 -0
- package/session_pool/consts.d.ts +1 -2
- package/session_pool/consts.js +1 -2
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +9 -0
- package/session_pool/fingerprint.js +30 -0
- package/session_pool/index.d.ts +0 -2
- package/session_pool/index.js +0 -2
- package/session_pool/session.d.ts +37 -86
- package/session_pool/session.js +85 -138
- package/session_pool/session_pool.d.ts +77 -91
- package/session_pool/session_pool.js +159 -147
- package/storages/dataset.d.ts +114 -54
- package/storages/dataset.js +285 -144
- package/storages/index.d.ts +9 -8
- package/storages/index.js +7 -8
- package/storages/key_value_store.d.ts +185 -42
- package/storages/key_value_store.js +424 -151
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +22 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +52 -116
- package/storages/request_list.js +158 -133
- package/storages/request_loader.d.ts +96 -0
- package/storages/request_loader.js +1 -0
- package/storages/request_manager.d.ts +33 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +97 -0
- package/storages/request_manager_tandem.js +197 -0
- package/storages/request_queue.d.ts +290 -47
- package/storages/request_queue.js +825 -216
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +37 -88
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +136 -143
- package/storages/storage_instance_manager.d.ts +87 -0
- package/storages/storage_instance_manager.js +256 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/transaction.d.ts +252 -0
- package/storages/transaction.js +251 -0
- package/storages/utils.d.ts +54 -9
- package/storages/utils.js +64 -13
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +0 -6
- package/typedefs.js +0 -1
- package/validators.d.ts +8 -1
- package/validators.js +10 -3
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js +0 -12
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts +0 -13
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js +0 -14
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_provider.d.ts +0 -308
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
package/crawlers/statistics.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import { Configuration } from '../configuration.js';
|
|
1
|
+
import type { CrawleeLogger } from '../log.js';
|
|
3
2
|
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
4
3
|
import { ErrorTracker } from './error_tracker.js';
|
|
5
4
|
/**
|
|
@@ -12,30 +11,90 @@ export interface PersistenceOptions {
|
|
|
12
11
|
*/
|
|
13
12
|
enable?: boolean;
|
|
14
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* The statistics surface a crawler depends on: recording per-request outcomes, tracking errors, and driving the
|
|
16
|
+
* capture lifecycle for a run. Injected via the crawler's `statistics` option, so a custom implementation (or a
|
|
17
|
+
* {@link Statistics} subclass tracking extra fields) can be plugged in without subclassing the crawler.
|
|
18
|
+
*
|
|
19
|
+
* The owned-only mutators the crawler uses to *own* a default it built - `reset()`/`resetStore()` - are deliberately
|
|
20
|
+
* absent: an injected instance is borrowed, and the crawler never wipes it. Those live on the concrete
|
|
21
|
+
* {@link Statistics} only.
|
|
22
|
+
*
|
|
23
|
+
* @category Crawlers
|
|
24
|
+
*/
|
|
25
|
+
export interface IStatistics {
|
|
26
|
+
/** Tracker for errors on the final retry of a request. */
|
|
27
|
+
readonly errorTracker: ErrorTracker;
|
|
28
|
+
/** Tracker for errors on retries prior to the final one. */
|
|
29
|
+
readonly errorTrackerRetry: ErrorTracker;
|
|
30
|
+
/** The live statistics state the crawler reads for status messages and the final summary. */
|
|
31
|
+
readonly state: StatisticState;
|
|
32
|
+
/** Retries histogram - index `i` holds the number of requests that finished after `i` retries. */
|
|
33
|
+
readonly requestRetryHistogram: number[];
|
|
34
|
+
/** Marks a request as started, so its duration can be measured on finish/fail. */
|
|
35
|
+
startJob(id: number | string): void;
|
|
36
|
+
/** Marks a started request as finished, updating the finished counters and durations. */
|
|
37
|
+
finishJob(id: number | string, retryCount: number): void;
|
|
38
|
+
/** Marks a started request as failed, updating the failed counters and durations. */
|
|
39
|
+
failJob(id: number | string, retryCount: number): void;
|
|
40
|
+
/** Drops a started request without counting it as finished or failed (e.g. skipped by robots.txt). */
|
|
41
|
+
discardJob(id: number | string): void;
|
|
42
|
+
/** Increments the counter for the given HTTP status code. */
|
|
43
|
+
registerStatusCode(code: number): void;
|
|
44
|
+
/** Computes the derived aggregates (averages, per-minute rates, totals) from the current state. */
|
|
45
|
+
calculate(): CalculatedStatistics;
|
|
46
|
+
/** Begins a capture window: loads any persisted state, subscribes to persistence events, starts periodic logging. */
|
|
47
|
+
startCapturing(): Promise<void>;
|
|
48
|
+
/** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
|
|
49
|
+
stopCapturing(): Promise<void>;
|
|
50
|
+
/**
|
|
51
|
+
* Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
|
|
52
|
+
* with no persistence of its own can omit it.
|
|
53
|
+
*/
|
|
54
|
+
persistState?(options?: PersistenceOptions): Promise<void>;
|
|
55
|
+
}
|
|
56
|
+
/** The derived aggregates computed by {@link IStatistics.calculate} from the current {@link StatisticState}. */
|
|
57
|
+
export interface CalculatedStatistics {
|
|
58
|
+
/** Mean duration of a failed request, in milliseconds; `Infinity` when nothing has failed. */
|
|
59
|
+
requestAvgFailedDurationMillis: number;
|
|
60
|
+
/** Mean duration of a finished request, in milliseconds; `Infinity` when nothing has finished. */
|
|
61
|
+
requestAvgFinishedDurationMillis: number;
|
|
62
|
+
/** Requests finished per minute over the run so far. */
|
|
63
|
+
requestsFinishedPerMinute: number;
|
|
64
|
+
/** Requests failed per minute over the run so far. */
|
|
65
|
+
requestsFailedPerMinute: number;
|
|
66
|
+
/** Combined duration of all finished and failed requests, in milliseconds. */
|
|
67
|
+
requestTotalDurationMillis: number;
|
|
68
|
+
/** Total number of settled requests (finished plus failed). */
|
|
69
|
+
requestsTotal: number;
|
|
70
|
+
/** Wall-clock runtime since capturing started, in milliseconds. */
|
|
71
|
+
crawlerRuntimeMillis: number;
|
|
72
|
+
}
|
|
15
73
|
/**
|
|
16
74
|
* The statistics class provides an interface to collecting and logging run
|
|
17
75
|
* statistics for requests.
|
|
18
76
|
*
|
|
19
77
|
* All statistic information is saved on key value store
|
|
20
|
-
* under the key `
|
|
78
|
+
* under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
|
|
21
79
|
* migrations and abort/resurrect
|
|
22
80
|
*
|
|
23
81
|
* @category Crawlers
|
|
24
82
|
*/
|
|
25
|
-
export declare class Statistics {
|
|
83
|
+
export declare class Statistics implements IStatistics {
|
|
84
|
+
#private;
|
|
26
85
|
private static id;
|
|
27
86
|
/**
|
|
28
87
|
* An error tracker for final retry errors.
|
|
29
88
|
*/
|
|
30
|
-
errorTracker: ErrorTracker;
|
|
89
|
+
readonly errorTracker: ErrorTracker;
|
|
31
90
|
/**
|
|
32
91
|
* An error tracker for retry errors prior to the final retry.
|
|
33
92
|
*/
|
|
34
|
-
errorTrackerRetry: ErrorTracker;
|
|
93
|
+
readonly errorTrackerRetry: ErrorTracker;
|
|
35
94
|
/**
|
|
36
95
|
* Statistic instance id.
|
|
37
96
|
*/
|
|
38
|
-
readonly id:
|
|
97
|
+
readonly id: string;
|
|
39
98
|
/**
|
|
40
99
|
* Current statistic state used for doing calculations on {@link Statistics.calculate} calls
|
|
41
100
|
*/
|
|
@@ -44,23 +103,13 @@ export declare class Statistics {
|
|
|
44
103
|
* Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
|
|
45
104
|
*/
|
|
46
105
|
readonly requestRetryHistogram: number[];
|
|
47
|
-
/**
|
|
48
|
-
* Contains the associated Configuration instance
|
|
49
|
-
*/
|
|
50
|
-
private readonly config;
|
|
51
106
|
protected keyValueStore?: KeyValueStore;
|
|
52
|
-
protected persistStateKey: string;
|
|
53
|
-
private logIntervalMillis;
|
|
54
|
-
private logMessage;
|
|
55
|
-
private listener;
|
|
56
|
-
private requestsInProgress;
|
|
107
|
+
protected readonly persistStateKey: string;
|
|
57
108
|
private readonly log;
|
|
58
|
-
private
|
|
59
|
-
private logInterval;
|
|
60
|
-
private events;
|
|
61
|
-
private persistenceOptions;
|
|
109
|
+
private get events();
|
|
62
110
|
/**
|
|
63
|
-
*
|
|
111
|
+
* Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
|
|
112
|
+
* persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
|
|
64
113
|
*/
|
|
65
114
|
constructor(options?: StatisticsOptions);
|
|
66
115
|
/**
|
|
@@ -90,18 +139,16 @@ export declare class Statistics {
|
|
|
90
139
|
* @ignore
|
|
91
140
|
*/
|
|
92
141
|
failJob(id: number | string, retryCount: number): void;
|
|
142
|
+
/**
|
|
143
|
+
* Discards a started job without affecting the finished/failed counters, e.g. when a request
|
|
144
|
+
* turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
|
|
145
|
+
* @ignore
|
|
146
|
+
*/
|
|
147
|
+
discardJob(id: number | string): void;
|
|
93
148
|
/**
|
|
94
149
|
* Calculate the current statistics
|
|
95
150
|
*/
|
|
96
|
-
calculate():
|
|
97
|
-
requestAvgFailedDurationMillis: number;
|
|
98
|
-
requestAvgFinishedDurationMillis: number;
|
|
99
|
-
requestsFinishedPerMinute: number;
|
|
100
|
-
requestsFailedPerMinute: number;
|
|
101
|
-
requestTotalDurationMillis: number;
|
|
102
|
-
requestsTotal: number;
|
|
103
|
-
crawlerRuntimeMillis: number;
|
|
104
|
-
};
|
|
151
|
+
calculate(): CalculatedStatistics;
|
|
105
152
|
/**
|
|
106
153
|
* Initializes the key value store for persisting the statistics,
|
|
107
154
|
* displaying the current state in predefined intervals
|
|
@@ -111,7 +158,7 @@ export declare class Statistics {
|
|
|
111
158
|
* Stops logging and remove event listeners, then persist
|
|
112
159
|
*/
|
|
113
160
|
stopCapturing(): Promise<void>;
|
|
114
|
-
|
|
161
|
+
private saveRetryCountForJob;
|
|
115
162
|
/**
|
|
116
163
|
* Persist internal state to the key value store
|
|
117
164
|
* @param options - Override the persistence options provided in the constructor
|
|
@@ -120,8 +167,8 @@ export declare class Statistics {
|
|
|
120
167
|
/**
|
|
121
168
|
* Loads the current statistic from the key value store if any
|
|
122
169
|
*/
|
|
123
|
-
protected
|
|
124
|
-
|
|
170
|
+
protected maybeLoadStatistics(): Promise<void>;
|
|
171
|
+
private teardown;
|
|
125
172
|
/**
|
|
126
173
|
* Make this class serializable when called with `JSON.stringify(statsInstance)` directly
|
|
127
174
|
* or through `keyValueStore.setValue('KEY', statsInstance)`
|
|
@@ -146,17 +193,12 @@ export interface StatisticsOptions {
|
|
|
146
193
|
* Parent logger instance, the statistics will create a child logger from this.
|
|
147
194
|
* @default crawler.log
|
|
148
195
|
*/
|
|
149
|
-
log?:
|
|
196
|
+
log?: CrawleeLogger;
|
|
150
197
|
/**
|
|
151
198
|
* Key value store instance to persist the statistics.
|
|
152
199
|
* If not provided, the default one will be used when capturing starts
|
|
153
200
|
*/
|
|
154
201
|
keyValueStore?: KeyValueStore;
|
|
155
|
-
/**
|
|
156
|
-
* Configuration instance to use
|
|
157
|
-
* @default Configuration.getGlobalConfig()
|
|
158
|
-
*/
|
|
159
|
-
config?: Configuration;
|
|
160
202
|
/**
|
|
161
203
|
* Control how and when to persist the statistics.
|
|
162
204
|
*/
|
|
@@ -166,13 +208,22 @@ export interface StatisticsOptions {
|
|
|
166
208
|
* @default false
|
|
167
209
|
*/
|
|
168
210
|
saveErrorSnapshots?: boolean;
|
|
211
|
+
/**
|
|
212
|
+
* A unique identifier for this statistics instance. This ID is used for persistence
|
|
213
|
+
* to the key value store, ensuring the same statistics can be loaded after script restarts.
|
|
214
|
+
*
|
|
215
|
+
* If not provided, an auto-incremented ID will be used for backward compatibility.
|
|
216
|
+
* This means statistics may not persist correctly across script restarts
|
|
217
|
+
* if crawler creation order changes.
|
|
218
|
+
*/
|
|
219
|
+
id?: string;
|
|
169
220
|
}
|
|
170
221
|
/**
|
|
171
222
|
* Format of the persisted stats
|
|
172
223
|
*/
|
|
173
224
|
export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
|
|
174
225
|
requestRetryHistogram: number[];
|
|
175
|
-
statsId:
|
|
226
|
+
statsId: string;
|
|
176
227
|
requestAvgFailedDurationMillis: number;
|
|
177
228
|
requestAvgFinishedDurationMillis: number;
|
|
178
229
|
requestTotalDurationMillis: number;
|
|
@@ -201,4 +252,3 @@ export interface StatisticState {
|
|
|
201
252
|
retryErrors: Record<string, unknown>;
|
|
202
253
|
requestsWithStatusCode: Record<string, number>;
|
|
203
254
|
}
|
|
204
|
-
//# sourceMappingURL=statistics.d.ts.map
|
package/crawlers/statistics.js
CHANGED
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
import ow from 'ow';
|
|
2
|
-
import {
|
|
3
|
-
import { log as defaultLog } from '../log.js';
|
|
2
|
+
import { serviceLocator } from '../service_locator.js';
|
|
4
3
|
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
5
4
|
import { ErrorTracker } from './error_tracker.js';
|
|
6
5
|
/**
|
|
7
6
|
* @ignore
|
|
8
7
|
*/
|
|
9
8
|
class Job {
|
|
10
|
-
lastRunAt = null;
|
|
11
|
-
durationMillis;
|
|
9
|
+
#lastRunAt = null;
|
|
10
|
+
#durationMillis;
|
|
12
11
|
run() {
|
|
13
|
-
this
|
|
12
|
+
this.#lastRunAt = Date.now();
|
|
14
13
|
}
|
|
15
14
|
finish() {
|
|
16
|
-
this
|
|
17
|
-
return this
|
|
15
|
+
this.#durationMillis = Date.now() - this.#lastRunAt;
|
|
16
|
+
return this.#durationMillis;
|
|
18
17
|
}
|
|
19
18
|
}
|
|
20
19
|
const errorTrackerConfig = {
|
|
@@ -30,12 +29,13 @@ const errorTrackerConfig = {
|
|
|
30
29
|
* statistics for requests.
|
|
31
30
|
*
|
|
32
31
|
* All statistic information is saved on key value store
|
|
33
|
-
* under the key `
|
|
32
|
+
* under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
|
|
34
33
|
* migrations and abort/resurrect
|
|
35
34
|
*
|
|
36
35
|
* @category Crawlers
|
|
37
36
|
*/
|
|
38
37
|
export class Statistics {
|
|
38
|
+
// kept as TS-private: statistics tests read the static counter directly
|
|
39
39
|
static id = 0;
|
|
40
40
|
/**
|
|
41
41
|
* An error tracker for final retry errors.
|
|
@@ -48,7 +48,7 @@ export class Statistics {
|
|
|
48
48
|
/**
|
|
49
49
|
* Statistic instance id.
|
|
50
50
|
*/
|
|
51
|
-
id
|
|
51
|
+
id;
|
|
52
52
|
/**
|
|
53
53
|
* Current statistic state used for doing calculations on {@link Statistics.calculate} calls
|
|
54
54
|
*/
|
|
@@ -57,23 +57,26 @@ export class Statistics {
|
|
|
57
57
|
* Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
|
|
58
58
|
*/
|
|
59
59
|
requestRetryHistogram = [];
|
|
60
|
-
/**
|
|
61
|
-
* Contains the associated Configuration instance
|
|
62
|
-
*/
|
|
63
|
-
config;
|
|
64
60
|
keyValueStore = undefined;
|
|
65
|
-
persistStateKey
|
|
66
|
-
logIntervalMillis;
|
|
67
|
-
logMessage;
|
|
68
|
-
listener;
|
|
69
|
-
requestsInProgress = new Map();
|
|
61
|
+
persistStateKey;
|
|
62
|
+
#logIntervalMillis;
|
|
63
|
+
#logMessage;
|
|
64
|
+
#listener;
|
|
65
|
+
#requestsInProgress = new Map();
|
|
70
66
|
log;
|
|
71
|
-
instanceStart;
|
|
72
|
-
logInterval;
|
|
73
|
-
events;
|
|
74
|
-
persistenceOptions;
|
|
67
|
+
#instanceStart;
|
|
68
|
+
#logInterval;
|
|
69
|
+
#events;
|
|
70
|
+
#persistenceOptions;
|
|
71
|
+
get events() {
|
|
72
|
+
if (!this.#events) {
|
|
73
|
+
this.#events = serviceLocator.getEventManager();
|
|
74
|
+
}
|
|
75
|
+
return this.#events;
|
|
76
|
+
}
|
|
75
77
|
/**
|
|
76
|
-
*
|
|
78
|
+
* Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
|
|
79
|
+
* persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
|
|
77
80
|
*/
|
|
78
81
|
constructor(options = {}) {
|
|
79
82
|
ow(options, ow.object.exactShape({
|
|
@@ -81,23 +84,23 @@ export class Statistics {
|
|
|
81
84
|
logMessage: ow.optional.string,
|
|
82
85
|
log: ow.optional.object,
|
|
83
86
|
keyValueStore: ow.optional.object,
|
|
84
|
-
config: ow.optional.object,
|
|
85
87
|
persistenceOptions: ow.optional.object,
|
|
86
88
|
saveErrorSnapshots: ow.optional.boolean,
|
|
89
|
+
id: ow.optional.any(ow.number, ow.string),
|
|
87
90
|
}));
|
|
88
|
-
const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore,
|
|
91
|
+
const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, persistenceOptions = {
|
|
89
92
|
enable: true,
|
|
90
|
-
}, saveErrorSnapshots = false, } = options;
|
|
91
|
-
this.
|
|
93
|
+
}, saveErrorSnapshots = false, id, } = options;
|
|
94
|
+
this.id = id ?? String(Statistics.id++);
|
|
95
|
+
this.persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
|
|
96
|
+
this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
|
|
92
97
|
this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
|
|
93
98
|
this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
|
|
94
|
-
this
|
|
95
|
-
this
|
|
99
|
+
this.#logIntervalMillis = logIntervalSecs * 1000;
|
|
100
|
+
this.#logMessage = logMessage;
|
|
96
101
|
this.keyValueStore = keyValueStore;
|
|
97
|
-
this
|
|
98
|
-
this
|
|
99
|
-
this.config = config;
|
|
100
|
-
this.persistenceOptions = persistenceOptions;
|
|
102
|
+
this.#listener = this.persistState.bind(this);
|
|
103
|
+
this.#persistenceOptions = persistenceOptions;
|
|
101
104
|
// initialize by "resetting"
|
|
102
105
|
this.reset();
|
|
103
106
|
}
|
|
@@ -126,15 +129,15 @@ export class Statistics {
|
|
|
126
129
|
retryErrors: this.errorTrackerRetry.result,
|
|
127
130
|
};
|
|
128
131
|
this.requestRetryHistogram.length = 0;
|
|
129
|
-
this
|
|
130
|
-
this
|
|
131
|
-
this.
|
|
132
|
+
this.#requestsInProgress.clear();
|
|
133
|
+
this.#instanceStart = Date.now();
|
|
134
|
+
this.teardown();
|
|
132
135
|
}
|
|
133
136
|
/**
|
|
134
137
|
* @param options - Override the persistence options provided in the constructor
|
|
135
138
|
*/
|
|
136
139
|
async resetStore(options) {
|
|
137
|
-
if (!this
|
|
140
|
+
if (!this.#persistenceOptions.enable && !options?.enable) {
|
|
138
141
|
return;
|
|
139
142
|
}
|
|
140
143
|
if (!this.keyValueStore) {
|
|
@@ -157,49 +160,57 @@ export class Statistics {
|
|
|
157
160
|
* @ignore
|
|
158
161
|
*/
|
|
159
162
|
startJob(id) {
|
|
160
|
-
let job = this
|
|
163
|
+
let job = this.#requestsInProgress.get(id);
|
|
161
164
|
if (!job)
|
|
162
165
|
job = new Job();
|
|
163
166
|
job.run();
|
|
164
|
-
this
|
|
167
|
+
this.#requestsInProgress.set(id, job);
|
|
165
168
|
}
|
|
166
169
|
/**
|
|
167
170
|
* Mark job as finished and sets the state
|
|
168
171
|
* @ignore
|
|
169
172
|
*/
|
|
170
173
|
finishJob(id, retryCount) {
|
|
171
|
-
const job = this
|
|
174
|
+
const job = this.#requestsInProgress.get(id);
|
|
172
175
|
if (!job)
|
|
173
176
|
return;
|
|
174
177
|
const jobDurationMillis = job.finish();
|
|
175
178
|
this.state.requestsFinished++;
|
|
176
179
|
this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
|
|
177
|
-
this.
|
|
180
|
+
this.saveRetryCountForJob(retryCount);
|
|
178
181
|
if (jobDurationMillis < this.state.requestMinDurationMillis)
|
|
179
182
|
this.state.requestMinDurationMillis = jobDurationMillis;
|
|
180
183
|
if (jobDurationMillis > this.state.requestMaxDurationMillis)
|
|
181
184
|
this.state.requestMaxDurationMillis = jobDurationMillis;
|
|
182
|
-
this
|
|
185
|
+
this.#requestsInProgress.delete(id);
|
|
183
186
|
}
|
|
184
187
|
/**
|
|
185
188
|
* Mark job as failed and sets the state
|
|
186
189
|
* @ignore
|
|
187
190
|
*/
|
|
188
191
|
failJob(id, retryCount) {
|
|
189
|
-
const job = this
|
|
192
|
+
const job = this.#requestsInProgress.get(id);
|
|
190
193
|
if (!job)
|
|
191
194
|
return;
|
|
192
195
|
this.state.requestTotalFailedDurationMillis += job.finish();
|
|
193
196
|
this.state.requestsFailed++;
|
|
194
|
-
this.
|
|
195
|
-
this
|
|
197
|
+
this.saveRetryCountForJob(retryCount);
|
|
198
|
+
this.#requestsInProgress.delete(id);
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Discards a started job without affecting the finished/failed counters, e.g. when a request
|
|
202
|
+
* turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
|
|
203
|
+
* @ignore
|
|
204
|
+
*/
|
|
205
|
+
discardJob(id) {
|
|
206
|
+
this.#requestsInProgress.delete(id);
|
|
196
207
|
}
|
|
197
208
|
/**
|
|
198
209
|
* Calculate the current statistics
|
|
199
210
|
*/
|
|
200
211
|
calculate() {
|
|
201
212
|
const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
|
|
202
|
-
const totalMillis = Date.now() - this
|
|
213
|
+
const totalMillis = Date.now() - this.#instanceStart;
|
|
203
214
|
const totalMinutes = totalMillis / 1000 / 60;
|
|
204
215
|
return {
|
|
205
216
|
requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
|
|
@@ -216,30 +227,35 @@ export class Statistics {
|
|
|
216
227
|
* displaying the current state in predefined intervals
|
|
217
228
|
*/
|
|
218
229
|
async startCapturing() {
|
|
219
|
-
|
|
230
|
+
// A single instance drives one logging interval and one PERSIST_STATE listener, so a second concurrent
|
|
231
|
+
// capture (e.g. sharing one instance across crawlers running at once) would orphan the first. Fail loudly.
|
|
232
|
+
if (this.#logInterval) {
|
|
233
|
+
throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');
|
|
234
|
+
}
|
|
235
|
+
this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
|
|
220
236
|
if (this.state.crawlerStartedAt === null) {
|
|
221
237
|
this.state.crawlerStartedAt = new Date();
|
|
222
238
|
}
|
|
223
|
-
if (this
|
|
224
|
-
await this.
|
|
225
|
-
this.events.on("persistState" /* EventType.PERSIST_STATE */, this
|
|
239
|
+
if (this.#persistenceOptions.enable) {
|
|
240
|
+
await this.maybeLoadStatistics();
|
|
241
|
+
this.events.on("persistState" /* EventType.PERSIST_STATE */, this.#listener);
|
|
226
242
|
}
|
|
227
|
-
this
|
|
228
|
-
this.log.info(this
|
|
243
|
+
this.#logInterval = setInterval(() => {
|
|
244
|
+
this.log.info(this.#logMessage, {
|
|
229
245
|
...this.calculate(),
|
|
230
246
|
retryHistogram: this.requestRetryHistogram,
|
|
231
247
|
});
|
|
232
|
-
}, this
|
|
248
|
+
}, this.#logIntervalMillis);
|
|
233
249
|
}
|
|
234
250
|
/**
|
|
235
251
|
* Stops logging and remove event listeners, then persist
|
|
236
252
|
*/
|
|
237
253
|
async stopCapturing() {
|
|
238
|
-
this.
|
|
254
|
+
this.teardown();
|
|
239
255
|
this.state.crawlerFinishedAt = new Date();
|
|
240
256
|
await this.persistState();
|
|
241
257
|
}
|
|
242
|
-
|
|
258
|
+
saveRetryCountForJob(retryCount) {
|
|
243
259
|
if (retryCount > 0)
|
|
244
260
|
this.state.requestsRetries++;
|
|
245
261
|
this.requestRetryHistogram[retryCount] ??= 0;
|
|
@@ -250,7 +266,7 @@ export class Statistics {
|
|
|
250
266
|
* @param options - Override the persistence options provided in the constructor
|
|
251
267
|
*/
|
|
252
268
|
async persistState(options) {
|
|
253
|
-
if (!this
|
|
269
|
+
if (!this.#persistenceOptions.enable && !options?.enable) {
|
|
254
270
|
return;
|
|
255
271
|
}
|
|
256
272
|
// this might be called before startCapturing was called without using await, should not crash
|
|
@@ -258,20 +274,14 @@ export class Statistics {
|
|
|
258
274
|
return;
|
|
259
275
|
}
|
|
260
276
|
this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
|
|
261
|
-
// use half the interval of `persistState` to avoid race conditions
|
|
262
|
-
const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
|
|
263
|
-
const timeoutSecs = persistStateIntervalMillis / 2_000;
|
|
264
277
|
await this.keyValueStore
|
|
265
|
-
.setValue(this.persistStateKey, this.toJSON()
|
|
266
|
-
timeoutSecs,
|
|
267
|
-
doNotRetryTimeouts: true,
|
|
268
|
-
})
|
|
278
|
+
.setValue(this.persistStateKey, this.toJSON())
|
|
269
279
|
.catch((error) => this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }));
|
|
270
280
|
}
|
|
271
281
|
/**
|
|
272
282
|
* Loads the current statistic from the key value store if any
|
|
273
283
|
*/
|
|
274
|
-
async
|
|
284
|
+
async maybeLoadStatistics() {
|
|
275
285
|
// this might be called before startCapturing was called without using await, should not crash
|
|
276
286
|
if (!this.keyValueStore) {
|
|
277
287
|
return;
|
|
@@ -303,15 +313,17 @@ export class Statistics {
|
|
|
303
313
|
this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
|
|
304
314
|
this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
|
|
305
315
|
this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
|
|
306
|
-
this
|
|
316
|
+
this.#instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
|
|
307
317
|
this.log.debug('Loaded from KeyValueStore');
|
|
308
318
|
}
|
|
309
|
-
|
|
319
|
+
teardown() {
|
|
310
320
|
// this can be called before a call to startCapturing happens (or in a 'finally' block)
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
321
|
+
// Only unsubscribe if event manager was already resolved — avoid eagerly resolving it
|
|
322
|
+
// (e.g. during the constructor's reset() call, which would capture the wrong context)
|
|
323
|
+
this.#events?.off("persistState" /* EventType.PERSIST_STATE */, this.#listener);
|
|
324
|
+
if (this.#logInterval) {
|
|
325
|
+
clearInterval(this.#logInterval);
|
|
326
|
+
this.#logInterval = null;
|
|
315
327
|
}
|
|
316
328
|
}
|
|
317
329
|
/**
|
|
@@ -324,7 +336,7 @@ export class Statistics {
|
|
|
324
336
|
// omit duplicated information
|
|
325
337
|
const result = {
|
|
326
338
|
...this.state,
|
|
327
|
-
crawlerLastStartTimestamp: this
|
|
339
|
+
crawlerLastStartTimestamp: this.#instanceStart,
|
|
328
340
|
crawlerFinishedAt: this.state.crawlerFinishedAt
|
|
329
341
|
? new Date(this.state.crawlerFinishedAt).toISOString()
|
|
330
342
|
: null,
|
|
@@ -343,4 +355,3 @@ export class Statistics {
|
|
|
343
355
|
return result;
|
|
344
356
|
}
|
|
345
357
|
}
|
|
346
|
-
//# sourceMappingURL=statistics.js.map
|
package/debug.d.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { IncomingMessage } from 'node:http';
|
|
2
|
+
import type { Dictionary } from '@crawlee/types';
|
|
3
|
+
import type { Request } from './request.js';
|
|
4
|
+
interface BrowserResponseLike {
|
|
5
|
+
status(): number;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
|
|
9
|
+
*
|
|
10
|
+
* @param request [Request](https://sdk.apify.com/docs/api/request) object.
|
|
11
|
+
* @param [response]
|
|
12
|
+
* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
|
|
13
|
+
* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
|
|
14
|
+
* @param [additionalFields] Object containing additional fields to be added.
|
|
15
|
+
*
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
|
|
19
|
+
/**
|
|
20
|
+
* Returns a human-readable label for an unknown value,
|
|
21
|
+
* suitable for embedding in error messages and log output.
|
|
22
|
+
*
|
|
23
|
+
* Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
|
|
24
|
+
* otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
|
|
25
|
+
*
|
|
26
|
+
* @internal
|
|
27
|
+
*/
|
|
28
|
+
export declare function inspectValue(value: unknown): string;
|
|
29
|
+
/**
|
|
30
|
+
* Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
|
|
31
|
+
* by their constructor name. Used for building validation error messages.
|
|
32
|
+
*
|
|
33
|
+
* @internal
|
|
34
|
+
*/
|
|
35
|
+
export declare function getObjectType(value: unknown): string;
|
|
36
|
+
export {};
|
package/debug.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { inspect } from 'node:util';
|
|
2
|
+
import ow from 'ow';
|
|
3
|
+
/**
|
|
4
|
+
* Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
|
|
5
|
+
*
|
|
6
|
+
* @param request [Request](https://sdk.apify.com/docs/api/request) object.
|
|
7
|
+
* @param [response]
|
|
8
|
+
* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
|
|
9
|
+
* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
|
|
10
|
+
* @param [additionalFields] Object containing additional fields to be added.
|
|
11
|
+
*
|
|
12
|
+
* @internal
|
|
13
|
+
*/
|
|
14
|
+
export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
|
|
15
|
+
ow(request, ow.object);
|
|
16
|
+
ow(response, ow.object);
|
|
17
|
+
ow(additionalFields, ow.object);
|
|
18
|
+
return {
|
|
19
|
+
requestId: request.id,
|
|
20
|
+
url: request.url,
|
|
21
|
+
loadedUrl: request.loadedUrl,
|
|
22
|
+
method: request.method,
|
|
23
|
+
retryCount: request.retryCount,
|
|
24
|
+
errorMessages: request.errorMessages,
|
|
25
|
+
// Puppeteer response has .status() function and NodeJS response, statusCode property.
|
|
26
|
+
statusCode: 'status' in response && response.status instanceof Function
|
|
27
|
+
? response.status()
|
|
28
|
+
: response.statusCode,
|
|
29
|
+
...additionalFields,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Returns a human-readable label for an unknown value,
|
|
34
|
+
* suitable for embedding in error messages and log output.
|
|
35
|
+
*
|
|
36
|
+
* Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
|
|
37
|
+
* otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
|
|
38
|
+
*
|
|
39
|
+
* @internal
|
|
40
|
+
*/
|
|
41
|
+
export function inspectValue(value) {
|
|
42
|
+
if (typeof value === 'object' && value !== null && value.constructor?.name) {
|
|
43
|
+
return value.constructor.name;
|
|
44
|
+
}
|
|
45
|
+
return inspect(value, {
|
|
46
|
+
depth: 0,
|
|
47
|
+
compact: true,
|
|
48
|
+
maxStringLength: 64,
|
|
49
|
+
breakLength: Infinity,
|
|
50
|
+
colors: false,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
|
|
55
|
+
* by their constructor name. Used for building validation error messages.
|
|
56
|
+
*
|
|
57
|
+
* @internal
|
|
58
|
+
*/
|
|
59
|
+
export function getObjectType(value) {
|
|
60
|
+
const simple = typeof value;
|
|
61
|
+
if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {
|
|
62
|
+
return simple;
|
|
63
|
+
}
|
|
64
|
+
const objectType = Object.prototype.toString.call(value);
|
|
65
|
+
const type = /\[object (\w+)]/.exec(objectType)[1];
|
|
66
|
+
if (type === 'Uint8Array') {
|
|
67
|
+
return 'Buffer';
|
|
68
|
+
}
|
|
69
|
+
return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();
|
|
70
|
+
}
|