@crawlee/core 4.0.0-beta.18 → 4.0.0-beta.181
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/byte_utils.d.ts +17 -0
- package/byte_utils.js +42 -0
- package/configuration.d.ts +90 -226
- package/configuration.js +164 -225
- package/debug.d.ts +36 -0
- package/debug.js +70 -0
- package/errors.d.ts +40 -19
- package/errors.js +53 -28
- package/events/event_manager.d.ts +35 -9
- package/events/event_manager.js +12 -13
- package/events/index.d.ts +1 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -3
- package/events/local_event_manager.js +39 -13
- package/events/system_info.d.ts +38 -0
- package/events/system_info.js +1 -0
- package/index.d.ts +7 -11
- package/index.js +8 -10
- package/internal.d.ts +8 -0
- package/internal.js +9 -0
- package/iterables.d.ts +79 -0
- package/iterables.js +134 -0
- package/log.d.ts +77 -1
- package/log.js +134 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +44 -0
- package/memory-storage/memory-storage.js +160 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +106 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +199 -0
- package/memory-storage/resource-clients/request-queue.d.ts +77 -0
- package/memory-storage/resource-clients/request-queue.js +414 -0
- package/memory-storage/utils.d.ts +16 -0
- package/memory-storage/utils.js +41 -0
- package/owned_or_injected.d.ts +58 -0
- package/owned_or_injected.js +98 -0
- package/package.json +18 -19
- package/proxy_configuration.d.ts +22 -128
- package/proxy_configuration.js +32 -144
- package/recoverable_state.d.ts +85 -51
- package/recoverable_state.js +178 -72
- package/request.d.ts +60 -17
- package/request.js +130 -69
- package/serialization.d.ts +0 -1
- package/serialization.js +15 -15
- package/service_locator.d.ts +165 -0
- package/service_locator.js +254 -0
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +114 -56
- package/storages/dataset.js +292 -149
- package/storages/index.d.ts +7 -9
- package/storages/index.js +5 -9
- package/storages/key_value_store.d.ts +187 -50
- package/storages/key_value_store.js +453 -172
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +22 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +54 -129
- package/storages/request_list.js +272 -245
- package/storages/request_loader.d.ts +127 -0
- package/storages/request_loader.js +36 -0
- package/storages/request_manager.d.ts +109 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +50 -23
- package/storages/request_manager_tandem.js +132 -57
- package/storages/request_queue.d.ts +286 -48
- package/storages/request_queue.js +757 -218
- package/storages/storage_instance_manager.d.ts +87 -0
- package/storages/storage_instance_manager.js +256 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/transaction.d.ts +260 -0
- package/storages/transaction.js +287 -0
- package/storages/utils.d.ts +59 -11
- package/storages/utils.js +75 -15
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +0 -6
- package/typedefs.js +0 -1
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +23 -18
- package/validators.js +14 -18
- package/autoscaling/autoscaled_pool.d.ts +0 -300
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js +0 -533
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts +0 -4
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js +0 -4
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts +0 -189
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js +0 -285
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts +0 -181
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js +0 -169
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts +0 -43
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js +0 -111
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts +0 -61
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js +0 -99
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts +0 -195
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js +0 -108
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js +0 -12
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts +0 -60
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js +0 -118
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts +0 -55
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js +0 -309
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts +0 -7
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js +0 -7
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts +0 -8
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js +0 -2
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts +0 -204
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js +0 -346
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts +0 -249
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js +0 -269
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts +0 -3
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js +0 -3
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts +0 -73
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js +0 -212
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -91
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -44
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -20
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -102
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/recoverable_state.d.ts.map +0 -1
- package/recoverable_state.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts +0 -134
- package/router.d.ts.map +0 -1
- package/router.js +0 -164
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts +0 -4
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js +0 -4
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts +0 -8
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js +0 -12
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts +0 -6
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js +0 -6
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts +0 -203
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js +0 -278
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts +0 -258
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js +0 -387
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts +0 -13
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js +0 -14
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_list_adapter.d.ts +0 -58
- package/storages/request_list_adapter.d.ts.map +0 -1
- package/storages/request_list_adapter.js +0 -81
- package/storages/request_list_adapter.js.map +0 -1
- package/storages/request_manager_tandem.d.ts.map +0 -1
- package/storages/request_manager_tandem.js.map +0 -1
- package/storages/request_provider.d.ts +0 -371
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -585
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts +0 -245
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js +0 -430
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
package/configuration.js
CHANGED
|
@@ -1,15 +1,80 @@
|
|
|
1
|
-
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
1
|
import { EventEmitter } from 'node:events';
|
|
3
2
|
import { readFileSync } from 'node:fs';
|
|
4
3
|
import { join } from 'node:path';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import
|
|
8
|
-
|
|
9
|
-
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { log, LogLevel } from './log.js';
|
|
6
|
+
import { serviceLocator } from './service_locator.js';
|
|
7
|
+
// Crawlee attaches many listeners to shared EventEmitters (one per crawler/session/autoscaled pool),
|
|
8
|
+
// which can exceed Node's default limit of 10 and trigger spurious MaxListenersExceededWarning logs.
|
|
9
|
+
// Raising the global default avoids false positives; real leaks will still manifest as unbounded growth.
|
|
10
|
+
// TODO: tracked in https://github.com/apify/crawlee/issues/3615 — find a less side-effecting place for this.
|
|
11
|
+
EventEmitter.defaultMaxListeners = 50;
|
|
12
|
+
export function field(schema, envVar) {
|
|
13
|
+
return { schema, envVar };
|
|
14
|
+
}
|
|
15
|
+
// --- Zod preprocessors ---
|
|
16
|
+
/** Zod preprocessor treating `'0'` and `'false'` as falsy. */
|
|
17
|
+
export const coerceBoolean = z.preprocess((val) => {
|
|
18
|
+
if (typeof val === 'string') {
|
|
19
|
+
return !['0', 'false'].includes(val.toLowerCase());
|
|
20
|
+
}
|
|
21
|
+
return val;
|
|
22
|
+
}, z.boolean());
|
|
23
|
+
export const coerceNumber = z.preprocess((val) => {
|
|
24
|
+
if (typeof val === 'string')
|
|
25
|
+
return Number(val);
|
|
26
|
+
return val;
|
|
27
|
+
}, z.number());
|
|
28
|
+
/** Zod schema accepting both LogLevel enum values and string names (case-insensitive). */
|
|
29
|
+
const logLevelSchema = z.preprocess((val) => {
|
|
30
|
+
if (val == null)
|
|
31
|
+
return val;
|
|
32
|
+
const s = String(val);
|
|
33
|
+
if (Number.isFinite(+s))
|
|
34
|
+
return +s;
|
|
35
|
+
const key = s.toUpperCase();
|
|
36
|
+
if (key in LogLevel)
|
|
37
|
+
return LogLevel[key];
|
|
38
|
+
return val;
|
|
39
|
+
}, z.enum(LogLevel));
|
|
40
|
+
// --- Crawlee config field definitions ---
|
|
41
|
+
export const crawleeConfigFields = {
|
|
42
|
+
/** @default true */
|
|
43
|
+
purgeOnStart: field(coerceBoolean.default(true), 'CRAWLEE_PURGE_ON_START'),
|
|
44
|
+
/** @default 0.95 */
|
|
45
|
+
maxUsedCpuRatio: field(coerceNumber.default(0.95)),
|
|
46
|
+
/** @default 0.25 */
|
|
47
|
+
availableMemoryRatio: field(coerceNumber.default(0.25), 'CRAWLEE_AVAILABLE_MEMORY_RATIO'),
|
|
48
|
+
memoryMbytes: field(coerceNumber.optional(), 'CRAWLEE_MEMORY_MBYTES'),
|
|
49
|
+
/** @default 60_000 */
|
|
50
|
+
persistStateIntervalMillis: field(coerceNumber.default(60_000), 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS'),
|
|
51
|
+
/**
|
|
52
|
+
* Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
|
|
53
|
+
* the request handler timeout (twice it, and never below 5 minutes).
|
|
54
|
+
*/
|
|
55
|
+
internalTimeoutMillis: field(coerceNumber.optional(), 'CRAWLEE_INTERNAL_TIMEOUT'),
|
|
56
|
+
/** @default 1_000 */
|
|
57
|
+
systemInfoIntervalMillis: field(coerceNumber.default(1_000)),
|
|
58
|
+
/** @default 'INPUT' */
|
|
59
|
+
inputKey: field(z.string().default('INPUT'), 'CRAWLEE_INPUT_KEY'),
|
|
60
|
+
/** @default true */
|
|
61
|
+
headless: field(coerceBoolean.default(true), 'CRAWLEE_HEADLESS'),
|
|
62
|
+
/** @default false */
|
|
63
|
+
xvfb: field(coerceBoolean.default(false), 'CRAWLEE_XVFB'),
|
|
64
|
+
chromeExecutablePath: field(z.string().optional(), 'CRAWLEE_CHROME_EXECUTABLE_PATH'),
|
|
65
|
+
defaultBrowserPath: field(z.string().optional(), 'CRAWLEE_DEFAULT_BROWSER_PATH'),
|
|
66
|
+
/** @default false */
|
|
67
|
+
disableBrowserSandbox: field(coerceBoolean.default(false), 'CRAWLEE_DISABLE_BROWSER_SANDBOX'),
|
|
68
|
+
logLevel: field(logLevelSchema.optional(), 'CRAWLEE_LOG_LEVEL'),
|
|
69
|
+
/** @default true */
|
|
70
|
+
persistStorage: field(coerceBoolean.default(true), 'CRAWLEE_PERSIST_STORAGE'),
|
|
71
|
+
/** @default './storage' */
|
|
72
|
+
storageDir: field(z.string().default('./storage'), 'CRAWLEE_STORAGE_DIR'),
|
|
73
|
+
containerized: field(coerceBoolean.optional(), 'CRAWLEE_CONTAINERIZED'),
|
|
74
|
+
};
|
|
10
75
|
/**
|
|
11
76
|
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
|
|
12
|
-
* global singleton instance of this class available via `Configuration.
|
|
77
|
+
* global singleton instance of this class available via `Configuration.getGlobalConfiguration()`.
|
|
13
78
|
* Places that depend on a configurable behaviour depend on this class, as they have the global
|
|
14
79
|
* instance as the default value.
|
|
15
80
|
*
|
|
@@ -18,14 +83,10 @@ import { entries } from './typedefs.js';
|
|
|
18
83
|
* import { BasicCrawler, Configuration } from 'crawlee';
|
|
19
84
|
*
|
|
20
85
|
* // Get the global configuration
|
|
21
|
-
* const config = Configuration.
|
|
22
|
-
* //
|
|
23
|
-
*
|
|
24
|
-
* config.
|
|
25
|
-
*
|
|
26
|
-
* // No need to pass the configuration to the crawler,
|
|
27
|
-
* // as it's using the global configuration by default
|
|
28
|
-
* const crawler = new BasicCrawler();
|
|
86
|
+
* const config = Configuration.getGlobalConfiguration();
|
|
87
|
+
* // Access configuration values directly as properties
|
|
88
|
+
* console.log(config.headless);
|
|
89
|
+
* console.log(config.persistStateIntervalMillis);
|
|
29
90
|
* ```
|
|
30
91
|
*
|
|
31
92
|
* *Using custom configuration:*
|
|
@@ -35,15 +96,14 @@ import { entries } from './typedefs.js';
|
|
|
35
96
|
* // Create a new configuration
|
|
36
97
|
* const config = new Configuration({ persistStateIntervalMillis: 30_000 });
|
|
37
98
|
* // Pass the configuration to the crawler
|
|
38
|
-
* const crawler = new BasicCrawler({
|
|
99
|
+
* const crawler = new BasicCrawler({ configuration: config });
|
|
39
100
|
* ```
|
|
40
101
|
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* so the options provided in constructor will override those. In other words, the precedence is:
|
|
102
|
+
* Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
|
|
103
|
+
* The priority order for resolving values is (highest to lowest):
|
|
44
104
|
*
|
|
45
105
|
* ```text
|
|
46
|
-
*
|
|
106
|
+
* constructor options > environment variables > crawlee.json > schema defaults
|
|
47
107
|
* ```
|
|
48
108
|
*
|
|
49
109
|
* ## Supported Configuration Options
|
|
@@ -53,250 +113,129 @@ import { entries } from './typedefs.js';
|
|
|
53
113
|
* `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
|
|
54
114
|
* `logLevel` | `CRAWLEE_LOG_LEVEL` | -
|
|
55
115
|
* `headless` | `CRAWLEE_HEADLESS` | `true`
|
|
56
|
-
* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
|
|
57
|
-
* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
|
|
58
|
-
* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
|
|
59
116
|
* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
|
|
117
|
+
* `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
|
|
60
118
|
* `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
|
|
61
119
|
* `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
|
|
120
|
+
* `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`
|
|
62
121
|
*
|
|
63
122
|
* ## Advanced Configuration Options
|
|
64
123
|
*
|
|
65
124
|
* Key | Environment Variable | Default Value
|
|
66
125
|
* ---|---|---
|
|
67
126
|
* `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
|
|
68
|
-
* `xvfb` | `CRAWLEE_XVFB` |
|
|
127
|
+
* `xvfb` | `CRAWLEE_XVFB` | `false`
|
|
69
128
|
* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
|
|
70
129
|
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
71
130
|
* `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
|
|
72
131
|
* `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
|
|
73
|
-
* `containerized | `CRAWLEE_CONTAINERIZED | -
|
|
132
|
+
* `containerized` | `CRAWLEE_CONTAINERIZED` | -
|
|
74
133
|
*/
|
|
134
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
|
|
75
135
|
export class Configuration {
|
|
76
136
|
/**
|
|
77
|
-
*
|
|
78
|
-
|
|
79
|
-
static ENV_MAP = {
|
|
80
|
-
CRAWLEE_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
|
|
81
|
-
CRAWLEE_PURGE_ON_START: 'purgeOnStart',
|
|
82
|
-
CRAWLEE_MEMORY_MBYTES: 'memoryMbytes',
|
|
83
|
-
CRAWLEE_DEFAULT_DATASET_ID: 'defaultDatasetId',
|
|
84
|
-
CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
|
|
85
|
-
CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
|
|
86
|
-
CRAWLEE_INPUT_KEY: 'inputKey',
|
|
87
|
-
CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
|
|
88
|
-
CRAWLEE_HEADLESS: 'headless',
|
|
89
|
-
CRAWLEE_XVFB: 'xvfb',
|
|
90
|
-
CRAWLEE_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
|
|
91
|
-
CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
|
|
92
|
-
CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
|
|
93
|
-
CRAWLEE_LOG_LEVEL: 'logLevel',
|
|
94
|
-
CRAWLEE_PERSIST_STORAGE: 'persistStorage',
|
|
95
|
-
CRAWLEE_CONTAINERIZED: 'containerized',
|
|
96
|
-
};
|
|
97
|
-
static BOOLEAN_VARS = [
|
|
98
|
-
'purgeOnStart',
|
|
99
|
-
'headless',
|
|
100
|
-
'xvfb',
|
|
101
|
-
'disableBrowserSandbox',
|
|
102
|
-
'persistStorage',
|
|
103
|
-
'containerized',
|
|
104
|
-
];
|
|
105
|
-
static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis'];
|
|
106
|
-
static COMMA_SEPARATED_LIST_VARS = [];
|
|
107
|
-
static DEFAULTS = {
|
|
108
|
-
defaultKeyValueStoreId: 'default',
|
|
109
|
-
defaultDatasetId: 'default',
|
|
110
|
-
defaultRequestQueueId: 'default',
|
|
111
|
-
inputKey: 'INPUT',
|
|
112
|
-
maxUsedCpuRatio: 0.95,
|
|
113
|
-
availableMemoryRatio: 0.25,
|
|
114
|
-
storageClientOptions: {},
|
|
115
|
-
purgeOnStart: true,
|
|
116
|
-
headless: true,
|
|
117
|
-
persistStateIntervalMillis: 60_000,
|
|
118
|
-
systemInfoIntervalMillis: 1_000,
|
|
119
|
-
persistStorage: true,
|
|
120
|
-
};
|
|
121
|
-
/**
|
|
122
|
-
* Provides access to the current-instance-scoped Configuration without passing it around in parameters.
|
|
123
|
-
* @internal
|
|
137
|
+
* Field definitions for this configuration class.
|
|
138
|
+
* Subclasses override this to register additional fields.
|
|
124
139
|
*/
|
|
125
|
-
static
|
|
126
|
-
|
|
127
|
-
services = new Map();
|
|
128
|
-
/** @internal */
|
|
129
|
-
static globalConfig;
|
|
130
|
-
storageManagers = new Map();
|
|
140
|
+
static fields = crawleeConfigFields;
|
|
141
|
+
#resolvedValues;
|
|
131
142
|
/**
|
|
132
|
-
* Creates new `Configuration` instance with provided options.
|
|
143
|
+
* Creates new `Configuration` instance with provided options.
|
|
144
|
+
* Constructor options take precedence over environment variables, which take precedence
|
|
145
|
+
* over crawlee.json values, which take precedence over schema defaults.
|
|
133
146
|
*/
|
|
134
147
|
constructor(options = {}) {
|
|
135
|
-
this.
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
: LogLevel[String(logLevel).toUpperCase()];
|
|
144
|
-
log.setLevel(level);
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
/**
|
|
148
|
-
* Returns configured value. First checks the environment variables, then provided configuration,
|
|
149
|
-
* fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
|
|
150
|
-
* in the above section.
|
|
151
|
-
*/
|
|
152
|
-
get(key, defaultValue) {
|
|
153
|
-
// prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option
|
|
154
|
-
let envValue;
|
|
155
|
-
for (const [k, v] of entries(Configuration.ENV_MAP)) {
|
|
156
|
-
if (key === v) {
|
|
157
|
-
envValue = process.env[k];
|
|
158
|
-
if (envValue) {
|
|
159
|
-
break;
|
|
160
|
-
}
|
|
161
|
-
}
|
|
148
|
+
const fields = this.constructor.fields;
|
|
149
|
+
const fileOptions = Configuration.loadFileOptions();
|
|
150
|
+
this.#resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
|
|
151
|
+
this.registerAccessors();
|
|
152
|
+
// Set the log level
|
|
153
|
+
const logLevel = this.logLevel;
|
|
154
|
+
if (logLevel != null) {
|
|
155
|
+
log.setLevel(logLevel);
|
|
162
156
|
}
|
|
163
|
-
if (envValue != null) {
|
|
164
|
-
return this._castEnvValue(key, envValue);
|
|
165
|
-
}
|
|
166
|
-
// check instance level options
|
|
167
|
-
if (this.options.has(key)) {
|
|
168
|
-
return this.options.get(key);
|
|
169
|
-
}
|
|
170
|
-
// fallback to defaults
|
|
171
|
-
return (defaultValue ?? Configuration.DEFAULTS[key] ?? envValue);
|
|
172
|
-
}
|
|
173
|
-
_castEnvValue(key, value) {
|
|
174
|
-
if (Configuration.INTEGER_VARS.includes(key)) {
|
|
175
|
-
return +value;
|
|
176
|
-
}
|
|
177
|
-
if (Configuration.BOOLEAN_VARS.includes(key)) {
|
|
178
|
-
// 0, false and empty string are considered falsy values
|
|
179
|
-
return !['0', 'false', ''].includes(String(value).toLowerCase());
|
|
180
|
-
}
|
|
181
|
-
if (Configuration.COMMA_SEPARATED_LIST_VARS.includes(key)) {
|
|
182
|
-
if (!value)
|
|
183
|
-
return [];
|
|
184
|
-
return String(value)
|
|
185
|
-
.split(',')
|
|
186
|
-
.map((v) => v.trim());
|
|
187
|
-
}
|
|
188
|
-
return value;
|
|
189
157
|
}
|
|
190
158
|
/**
|
|
191
|
-
*
|
|
192
|
-
* To reset a value, we can omit the `value` argument or pass `undefined` there.
|
|
193
|
-
*/
|
|
194
|
-
set(key, value) {
|
|
195
|
-
this.options.set(key, value);
|
|
196
|
-
}
|
|
197
|
-
/**
|
|
198
|
-
* Sets value for given option. Only affects the global `Configuration` instance, the value will not be propagated down to the env var.
|
|
199
|
-
* To reset a value, we can omit the `value` argument or pass `undefined` there.
|
|
200
|
-
*/
|
|
201
|
-
static set(key, value) {
|
|
202
|
-
this.getGlobalConfig().set(key, value);
|
|
203
|
-
}
|
|
204
|
-
/**
|
|
205
|
-
* Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
|
|
206
|
-
* this {@link Configuration} instance. Only first call of this method will create the client, following calls will
|
|
207
|
-
* return the same client instance.
|
|
159
|
+
* Returns the global configuration instance. It will respect the environment variables.
|
|
208
160
|
*
|
|
209
|
-
*
|
|
210
|
-
* multiple instances, one for each variant of the options.
|
|
211
|
-
* @internal
|
|
161
|
+
* Delegates to the global ServiceLocator, making it the single source of truth for service management.
|
|
212
162
|
*/
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
return this.options.get('storageClient');
|
|
216
|
-
}
|
|
217
|
-
const options = this.options.get('storageClientOptions');
|
|
218
|
-
return this.createMemoryStorage(options);
|
|
219
|
-
}
|
|
220
|
-
getEventManager() {
|
|
221
|
-
if (this.options.has('eventManager')) {
|
|
222
|
-
return this.options.get('eventManager');
|
|
223
|
-
}
|
|
224
|
-
if (this.services.has('eventManager')) {
|
|
225
|
-
return this.services.get('eventManager');
|
|
226
|
-
}
|
|
227
|
-
const eventManager = new LocalEventManager(this);
|
|
228
|
-
this.services.set('eventManager', eventManager);
|
|
229
|
-
return eventManager;
|
|
163
|
+
static getGlobalConfiguration() {
|
|
164
|
+
return serviceLocator.getConfiguration();
|
|
230
165
|
}
|
|
231
166
|
/**
|
|
232
|
-
*
|
|
233
|
-
*
|
|
167
|
+
* Resolves all field values once using the priority chain:
|
|
168
|
+
* constructor options > env vars > crawlee.json > schema defaults.
|
|
234
169
|
*/
|
|
235
|
-
|
|
236
|
-
const
|
|
237
|
-
|
|
238
|
-
|
|
170
|
+
static resolveAll(fields, userOptions, fileOptions) {
|
|
171
|
+
const values = {};
|
|
172
|
+
for (const [key, fieldDef] of Object.entries(fields)) {
|
|
173
|
+
// 1. Constructor options (highest priority)
|
|
174
|
+
if (key in userOptions && userOptions[key] !== undefined) {
|
|
175
|
+
values[key] = fieldDef.schema.parse(userOptions[key]);
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
// 2. Environment variables
|
|
179
|
+
const envValue = Configuration.readEnvVar(fieldDef);
|
|
180
|
+
if (envValue != null) {
|
|
181
|
+
values[key] = fieldDef.schema.parse(envValue);
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
// 3. crawlee.json file options
|
|
185
|
+
if (key in fileOptions && fileOptions[key] !== undefined) {
|
|
186
|
+
values[key] = fieldDef.schema.parse(fileOptions[key]);
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
// 4. Schema default (by parsing undefined through the schema)
|
|
190
|
+
const parsed = fieldDef.schema.safeParse(undefined);
|
|
191
|
+
values[key] = parsed.success ? parsed.data : undefined;
|
|
239
192
|
}
|
|
240
|
-
|
|
241
|
-
persistStorage: this.get('persistStorage'),
|
|
242
|
-
// Override persistStorage if user provides it via storageClientOptions
|
|
243
|
-
...options,
|
|
244
|
-
});
|
|
245
|
-
this.services.set(cacheKey, storage);
|
|
246
|
-
return storage;
|
|
247
|
-
}
|
|
248
|
-
useStorageClient(client) {
|
|
249
|
-
this.options.set('storageClient', client);
|
|
250
|
-
}
|
|
251
|
-
static useStorageClient(client) {
|
|
252
|
-
this.getGlobalConfig().useStorageClient(client);
|
|
253
|
-
}
|
|
254
|
-
useEventManager(events) {
|
|
255
|
-
this.options.set('eventManager', events);
|
|
193
|
+
return values;
|
|
256
194
|
}
|
|
257
195
|
/**
|
|
258
|
-
*
|
|
196
|
+
* Registers getters (and throwing setters) on the instance for each field.
|
|
259
197
|
*/
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
198
|
+
registerAccessors() {
|
|
199
|
+
const fields = this.constructor.fields;
|
|
200
|
+
const descriptors = {};
|
|
201
|
+
for (const key of Object.keys(fields)) {
|
|
202
|
+
descriptors[key] = {
|
|
203
|
+
get: () => this.#resolvedValues[key],
|
|
204
|
+
set() {
|
|
205
|
+
throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
|
|
206
|
+
},
|
|
207
|
+
enumerable: true,
|
|
208
|
+
configurable: false,
|
|
209
|
+
};
|
|
263
210
|
}
|
|
264
|
-
|
|
265
|
-
return Configuration.globalConfig;
|
|
266
|
-
}
|
|
267
|
-
/**
|
|
268
|
-
* Gets default {@link StorageClient} instance.
|
|
269
|
-
*/
|
|
270
|
-
static getStorageClient() {
|
|
271
|
-
return this.getGlobalConfig().getStorageClient();
|
|
211
|
+
Object.defineProperties(this, descriptors);
|
|
272
212
|
}
|
|
273
213
|
/**
|
|
274
|
-
*
|
|
214
|
+
* Reads the first defined env var value for a field definition.
|
|
215
|
+
* Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
|
|
216
|
+
* (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
|
|
275
217
|
*/
|
|
276
|
-
static
|
|
277
|
-
|
|
218
|
+
static readEnvVar(fieldDef) {
|
|
219
|
+
if (!fieldDef.envVar)
|
|
220
|
+
return undefined;
|
|
221
|
+
const envVars = Array.isArray(fieldDef.envVar) ? fieldDef.envVar : [fieldDef.envVar];
|
|
222
|
+
for (const envVar of envVars) {
|
|
223
|
+
const value = process.env[envVar];
|
|
224
|
+
if (value != null && value !== '')
|
|
225
|
+
return value;
|
|
226
|
+
}
|
|
227
|
+
return undefined;
|
|
278
228
|
}
|
|
279
229
|
/**
|
|
280
|
-
*
|
|
281
|
-
* if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
|
|
230
|
+
* Loads config options from crawlee.json in the current working directory.
|
|
282
231
|
*/
|
|
283
|
-
static
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
try {
|
|
291
|
-
const file = readFileSync(path);
|
|
292
|
-
const optionsFromFileConfig = JSON.parse(file.toString());
|
|
293
|
-
Object.assign(options, optionsFromFileConfig);
|
|
294
|
-
}
|
|
295
|
-
catch {
|
|
296
|
-
// ignore
|
|
297
|
-
}
|
|
232
|
+
static loadFileOptions() {
|
|
233
|
+
try {
|
|
234
|
+
const file = readFileSync(join(process.cwd(), 'crawlee.json'));
|
|
235
|
+
return JSON.parse(file.toString());
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
return {};
|
|
298
239
|
}
|
|
299
|
-
this.options = new Map(entries(options));
|
|
300
240
|
}
|
|
301
241
|
}
|
|
302
|
-
//# sourceMappingURL=configuration.js.map
|
package/debug.d.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { IncomingMessage } from 'node:http';
|
|
2
|
+
import type { Dictionary } from '@crawlee/types';
|
|
3
|
+
import type { Request } from './request.js';
|
|
4
|
+
interface BrowserResponseLike {
|
|
5
|
+
status(): number;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
|
|
9
|
+
*
|
|
10
|
+
* @param request [Request](https://sdk.apify.com/docs/api/request) object.
|
|
11
|
+
* @param [response]
|
|
12
|
+
* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
|
|
13
|
+
* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
|
|
14
|
+
* @param [additionalFields] Object containing additional fields to be added.
|
|
15
|
+
*
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
|
|
19
|
+
/**
|
|
20
|
+
* Returns a human-readable label for an unknown value,
|
|
21
|
+
* suitable for embedding in error messages and log output.
|
|
22
|
+
*
|
|
23
|
+
* Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
|
|
24
|
+
* otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
|
|
25
|
+
*
|
|
26
|
+
* @internal
|
|
27
|
+
*/
|
|
28
|
+
export declare function inspectValue(value: unknown): string;
|
|
29
|
+
/**
|
|
30
|
+
* Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
|
|
31
|
+
* by their constructor name. Used for building validation error messages.
|
|
32
|
+
*
|
|
33
|
+
* @internal
|
|
34
|
+
*/
|
|
35
|
+
export declare function getObjectType(value: unknown): string;
|
|
36
|
+
export {};
|
package/debug.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { inspect } from 'node:util';
|
|
2
|
+
import { parseArgument, schemas } from './validators.js';
|
|
3
|
+
/**
|
|
4
|
+
* Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
|
|
5
|
+
*
|
|
6
|
+
* @param request [Request](https://sdk.apify.com/docs/api/request) object.
|
|
7
|
+
* @param [response]
|
|
8
|
+
* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
|
|
9
|
+
* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
|
|
10
|
+
* @param [additionalFields] Object containing additional fields to be added.
|
|
11
|
+
*
|
|
12
|
+
* @internal
|
|
13
|
+
*/
|
|
14
|
+
export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
|
|
15
|
+
parseArgument(request, schemas.anyObject);
|
|
16
|
+
parseArgument(response, schemas.anyObject);
|
|
17
|
+
parseArgument(additionalFields, schemas.anyObject);
|
|
18
|
+
return {
|
|
19
|
+
requestId: request.id,
|
|
20
|
+
url: request.url,
|
|
21
|
+
loadedUrl: request.loadedUrl,
|
|
22
|
+
method: request.method,
|
|
23
|
+
retryCount: request.retryCount,
|
|
24
|
+
errorMessages: request.errorMessages,
|
|
25
|
+
// Puppeteer response has .status() function and NodeJS response, statusCode property.
|
|
26
|
+
statusCode: 'status' in response && response.status instanceof Function
|
|
27
|
+
? response.status()
|
|
28
|
+
: response.statusCode,
|
|
29
|
+
...additionalFields,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Returns a human-readable label for an unknown value,
|
|
34
|
+
* suitable for embedding in error messages and log output.
|
|
35
|
+
*
|
|
36
|
+
* Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
|
|
37
|
+
* otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
|
|
38
|
+
*
|
|
39
|
+
* @internal
|
|
40
|
+
*/
|
|
41
|
+
export function inspectValue(value) {
|
|
42
|
+
if (typeof value === 'object' && value !== null && value.constructor?.name) {
|
|
43
|
+
return value.constructor.name;
|
|
44
|
+
}
|
|
45
|
+
return inspect(value, {
|
|
46
|
+
depth: 0,
|
|
47
|
+
compact: true,
|
|
48
|
+
maxStringLength: 64,
|
|
49
|
+
breakLength: Infinity,
|
|
50
|
+
colors: false,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
|
|
55
|
+
* by their constructor name. Used for building validation error messages.
|
|
56
|
+
*
|
|
57
|
+
* @internal
|
|
58
|
+
*/
|
|
59
|
+
export function getObjectType(value) {
|
|
60
|
+
const simple = typeof value;
|
|
61
|
+
if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {
|
|
62
|
+
return simple;
|
|
63
|
+
}
|
|
64
|
+
const objectType = Object.prototype.toString.call(value);
|
|
65
|
+
const type = /\[object (\w+)]/.exec(objectType)[1];
|
|
66
|
+
if (type === 'Uint8Array') {
|
|
67
|
+
return 'Buffer';
|
|
68
|
+
}
|
|
69
|
+
return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();
|
|
70
|
+
}
|
package/errors.d.ts
CHANGED
|
@@ -10,36 +10,57 @@ export declare class NonRetryableError extends Error {
|
|
|
10
10
|
export declare class CriticalError extends NonRetryableError {
|
|
11
11
|
}
|
|
12
12
|
/**
|
|
13
|
-
*
|
|
13
|
+
* A schema validation issue, structurally compatible with `StandardSchemaV1.Issue`. Declared here so that
|
|
14
|
+
* error types do not have to depend on `@standard-schema/spec`.
|
|
14
15
|
*/
|
|
15
|
-
export
|
|
16
|
+
export interface SchemaIssue {
|
|
17
|
+
readonly message: string;
|
|
18
|
+
readonly path?: readonly (PropertyKey | {
|
|
19
|
+
key: PropertyKey;
|
|
20
|
+
})[];
|
|
16
21
|
}
|
|
17
22
|
/**
|
|
18
|
-
*
|
|
23
|
+
* Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
|
|
19
24
|
*
|
|
20
|
-
*
|
|
25
|
+
* As the `userData` does not change between attempts, this error is non-retryable.
|
|
21
26
|
*/
|
|
22
|
-
export declare class
|
|
23
|
-
|
|
27
|
+
export declare class RequestValidationError extends NonRetryableError {
|
|
28
|
+
readonly label: string | symbol;
|
|
29
|
+
readonly issues: readonly SchemaIssue[];
|
|
30
|
+
constructor(label: string | symbol, issues: readonly SchemaIssue[]);
|
|
24
31
|
}
|
|
25
32
|
/**
|
|
26
|
-
*
|
|
33
|
+
* Thrown by {@link RecoverableState} when a persisted state record does not match its `stateSchema`.
|
|
27
34
|
*
|
|
28
|
-
*
|
|
35
|
+
* Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the
|
|
36
|
+
* state is for, so {@link RecoverableState.initialize} always throws and leaves the choice to the caller.
|
|
29
37
|
*/
|
|
30
|
-
export declare class
|
|
31
|
-
|
|
38
|
+
export declare class StateValidationError extends Error {
|
|
39
|
+
readonly persistStateKey: string;
|
|
40
|
+
readonly issues: readonly SchemaIssue[];
|
|
41
|
+
constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
|
|
32
42
|
}
|
|
33
|
-
|
|
43
|
+
/**
|
|
44
|
+
* Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
|
|
45
|
+
*
|
|
46
|
+
* The retry counts towards the `maxRequestRetries` limit, just like any other error.
|
|
47
|
+
*/
|
|
48
|
+
export declare class SessionError extends Error {
|
|
34
49
|
constructor(message?: string);
|
|
35
50
|
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
51
|
+
/**
|
|
52
|
+
* Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
|
|
53
|
+
* that ran after the request's writes had already been committed.
|
|
54
|
+
*
|
|
55
|
+
* Non-retryable by nature: the writes are durable, so re-running the request handler would duplicate
|
|
56
|
+
* them. A callback that throws a `NonRetryableError` of its own is left alone.
|
|
57
|
+
*/
|
|
58
|
+
export declare class AfterCommitError extends NonRetryableError {
|
|
59
|
+
constructor(cause: unknown);
|
|
41
60
|
}
|
|
42
|
-
|
|
43
|
-
|
|
61
|
+
/**
|
|
62
|
+
* Thrown when attempting to set a different service instance after one has already been retrieved.
|
|
63
|
+
*/
|
|
64
|
+
export declare class ServiceConflictError extends Error {
|
|
65
|
+
constructor(serviceName: string, newValue: unknown, existingValue: unknown);
|
|
44
66
|
}
|
|
45
|
-
//# sourceMappingURL=errors.d.ts.map
|