@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +18 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +88 -227
- package/configuration.d.ts.map +1 -1
- package/configuration.js +162 -223
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +4 -2
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +18 -12
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +71 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +123 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +20 -29
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -20
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_utils.d.ts +2 -2
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -1
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +16 -16
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +22 -25
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +36 -21
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +47 -26
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +25 -8
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +69 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +33 -3
- package/errors.d.ts.map +1 -1
- package/errors.js +48 -4
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +34 -7
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +8 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +33 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +4 -2
- package/index.d.ts.map +1 -1
- package/index.js +3 -1
- package/index.js.map +1 -1
- package/log.d.ts +82 -2
- package/log.d.ts.map +1 -1
- package/log.js +102 -0
- package/log.js.map +1 -1
- package/memory-storage/consts.d.ts +5 -0
- package/memory-storage/consts.d.ts.map +1 -0
- package/memory-storage/consts.js +5 -0
- package/memory-storage/consts.js.map +1 -0
- package/memory-storage/index.d.ts +2 -0
- package/memory-storage/index.d.ts.map +1 -0
- package/memory-storage/index.js +2 -0
- package/memory-storage/index.js.map +1 -0
- package/memory-storage/memory-storage.d.ts +47 -0
- package/memory-storage/memory-storage.d.ts.map +1 -0
- package/memory-storage/memory-storage.js +137 -0
- package/memory-storage/memory-storage.js.map +1 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +5 -0
- package/memory-storage/resource-clients/common/base-client.d.ts.map +1 -0
- package/memory-storage/resource-clients/common/base-client.js +7 -0
- package/memory-storage/resource-clients/common/base-client.js.map +1 -0
- package/memory-storage/resource-clients/dataset.d.ts +41 -0
- package/memory-storage/resource-clients/dataset.d.ts.map +1 -0
- package/memory-storage/resource-clients/dataset.js +114 -0
- package/memory-storage/resource-clients/dataset.js.map +1 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +64 -0
- package/memory-storage/resource-clients/key-value-store.d.ts.map +1 -0
- package/memory-storage/resource-clients/key-value-store.js +204 -0
- package/memory-storage/resource-clients/key-value-store.js.map +1 -0
- package/memory-storage/resource-clients/request-queue.d.ts +97 -0
- package/memory-storage/resource-clients/request-queue.d.ts.map +1 -0
- package/memory-storage/resource-clients/request-queue.js +422 -0
- package/memory-storage/resource-clients/request-queue.js.map +1 -0
- package/memory-storage/utils.d.ts +12 -0
- package/memory-storage/utils.d.ts.map +1 -0
- package/memory-storage/utils.js +28 -0
- package/memory-storage/utils.js.map +1 -0
- package/package.json +12 -10
- package/proxy_configuration.d.ts +14 -148
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +19 -167
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +142 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +84 -12
- package/request.d.ts.map +1 -1
- package/request.js +107 -27
- package/request.js.map +1 -1
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/serialization.js +1 -1
- package/serialization.js.map +1 -1
- package/service_locator.d.ts +157 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +239 -0
- package/service_locator.js.map +1 -0
- package/session_pool/consts.d.ts +1 -1
- package/session_pool/consts.d.ts.map +1 -1
- package/session_pool/consts.js +1 -1
- package/session_pool/consts.js.map +1 -1
- package/session_pool/fingerprint.d.ts +10 -0
- package/session_pool/fingerprint.d.ts.map +1 -0
- package/session_pool/fingerprint.js +31 -0
- package/session_pool/fingerprint.js.map +1 -0
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +35 -72
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +46 -98
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +71 -71
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +111 -99
- package/session_pool/session_pool.js.map +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js +5 -1
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts +98 -47
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +166 -123
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +8 -6
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +6 -6
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +163 -33
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +267 -119
- package/storages/key_value_store.js.map +1 -1
- package/storages/key_value_store_codec.d.ts +33 -0
- package/storages/key_value_store_codec.d.ts.map +1 -0
- package/storages/key_value_store_codec.js +114 -0
- package/storages/key_value_store_codec.js.map +1 -0
- package/storages/request_list.d.ts +32 -81
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +49 -39
- package/storages/request_list.js.map +1 -1
- package/storages/request_loader.d.ts +97 -0
- package/storages/request_loader.d.ts.map +1 -0
- package/storages/request_loader.js +2 -0
- package/storages/request_loader.js.map +1 -0
- package/storages/request_manager.d.ts +34 -0
- package/storages/request_manager.d.ts.map +1 -0
- package/storages/request_manager.js +2 -0
- package/storages/request_manager.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +107 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +198 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_queue.d.ts +282 -45
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +592 -213
- package/storages/request_queue.js.map +1 -1
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +29 -24
- package/storages/sitemap_request_loader.d.ts.map +1 -0
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +50 -46
- package/storages/sitemap_request_loader.js.map +1 -0
- package/storages/storage_instance_manager.d.ts +92 -0
- package/storages/storage_instance_manager.d.ts.map +1 -0
- package/storages/storage_instance_manager.js +261 -0
- package/storages/storage_instance_manager.js.map +1 -0
- package/storages/storage_stats.d.ts +49 -0
- package/storages/storage_stats.d.ts.map +1 -0
- package/storages/storage_stats.js +30 -0
- package/storages/storage_stats.js.map +1 -0
- package/storages/utils.d.ts +51 -5
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +60 -8
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/validators.d.ts +8 -0
- package/validators.d.ts.map +1 -1
- package/validators.js +8 -0
- package/validators.js.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/storages/request_provider.d.ts +0 -307
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -1,42 +1,15 @@
|
|
|
1
|
-
import { readFile } from 'node:fs/promises';
|
|
2
|
-
import { join } from 'node:path';
|
|
3
|
-
import JSON5 from 'json5';
|
|
4
1
|
import ow, { ArgumentError } from 'ow';
|
|
5
2
|
import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
|
|
6
|
-
import log from '@apify/log';
|
|
7
|
-
import { jsonStringifyExtended } from '@apify/utilities';
|
|
8
3
|
import { Configuration } from '../configuration.js';
|
|
4
|
+
import { serviceLocator } from '../service_locator.js';
|
|
9
5
|
import { checkStorageAccess } from './access_checking.js';
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
export const maybeStringify = (value, options) => {
|
|
18
|
-
// If contentType is missing, value will be stringified to JSON
|
|
19
|
-
if (options.contentType === null || options.contentType === undefined) {
|
|
20
|
-
options.contentType = 'application/json; charset=utf-8';
|
|
21
|
-
try {
|
|
22
|
-
// Format JSON to simplify debugging, the overheads with compression is negligible
|
|
23
|
-
value = jsonStringifyExtended(value, null, 2);
|
|
24
|
-
}
|
|
25
|
-
catch (e) {
|
|
26
|
-
const error = e;
|
|
27
|
-
// Give more meaningful error message
|
|
28
|
-
if (error.message?.includes('Invalid string length')) {
|
|
29
|
-
error.message = 'Object is too large';
|
|
30
|
-
}
|
|
31
|
-
throw new Error(`The "value" parameter cannot be stringified to JSON: ${error.message}`);
|
|
32
|
-
}
|
|
33
|
-
if (value === undefined) {
|
|
34
|
-
throw new Error('The "value" parameter was stringified to JSON and returned undefined. ' +
|
|
35
|
-
"Make sure you're not trying to stringify an undefined value.");
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
return value;
|
|
39
|
-
};
|
|
6
|
+
import { parseValue, serializeValue } from './key_value_store_codec.js';
|
|
7
|
+
import { StorageStatsTracker } from './storage_stats.js';
|
|
8
|
+
import { resolveStorageIdentifier } from './storage_instance_manager.js';
|
|
9
|
+
import { createDualIterable, purgeDefaultStorages } from './utils.js';
|
|
10
|
+
import { isBuffer, isStream } from '@crawlee/utils';
|
|
11
|
+
/** @internal */
|
|
12
|
+
const KVS_KEYS_DEFAULT_LIMIT = 1000;
|
|
40
13
|
/**
|
|
41
14
|
* The `KeyValueStore` class represents a key-value store, a simple data storage that is used
|
|
42
15
|
* for saving and reading data records or files. Each data record is
|
|
@@ -97,11 +70,16 @@ export class KeyValueStore {
|
|
|
97
70
|
config;
|
|
98
71
|
id;
|
|
99
72
|
name;
|
|
100
|
-
|
|
101
|
-
client;
|
|
73
|
+
backend;
|
|
102
74
|
persistStateEventStarted = false;
|
|
103
75
|
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
|
|
104
76
|
cache = new Map();
|
|
77
|
+
statsTracker = new StorageStatsTracker({
|
|
78
|
+
readCount: 0,
|
|
79
|
+
writeCount: 0,
|
|
80
|
+
deleteCount: 0,
|
|
81
|
+
listCount: 0,
|
|
82
|
+
});
|
|
105
83
|
/**
|
|
106
84
|
* @internal
|
|
107
85
|
*/
|
|
@@ -109,8 +87,14 @@ export class KeyValueStore {
|
|
|
109
87
|
this.config = config;
|
|
110
88
|
this.id = options.id;
|
|
111
89
|
this.name = options.name;
|
|
112
|
-
this.
|
|
113
|
-
|
|
90
|
+
this.backend = options.backend;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Backend-independent usage counters tracked for this key-value store (read / write / delete /
|
|
94
|
+
* list operations issued to the underlying storage backend). Counted per backend call.
|
|
95
|
+
*/
|
|
96
|
+
get stats() {
|
|
97
|
+
return this.statsTracker.current;
|
|
114
98
|
}
|
|
115
99
|
/**
|
|
116
100
|
* Gets a value from the key-value store.
|
|
@@ -147,8 +131,53 @@ export class KeyValueStore {
|
|
|
147
131
|
async getValue(key, defaultValue) {
|
|
148
132
|
checkStorageAccess();
|
|
149
133
|
ow(key, ow.string.nonEmpty);
|
|
150
|
-
|
|
151
|
-
|
|
134
|
+
this.statsTracker.add('readCount');
|
|
135
|
+
const record = await this.backend.getValue(key);
|
|
136
|
+
// A missing record falls back to the default; a record that parses to a falsy value (including
|
|
137
|
+
// a stored literal `null`) is returned verbatim, so callers can tell "stored null" from "absent".
|
|
138
|
+
if (!record) {
|
|
139
|
+
return defaultValue ?? null;
|
|
140
|
+
}
|
|
141
|
+
// Storage backends are byte transports — the value is raw bytes; the frontend parses it here.
|
|
142
|
+
return parseValue(record.value, record.contentType ?? null);
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Reads a record from the key-value store without parsing the value.
|
|
146
|
+
*
|
|
147
|
+
* Use this when you need the raw bytes and the content type — for example, to run your own
|
|
148
|
+
* parser (`simdjson`, a custom XML library, etc.) or to forward the bytes verbatim.
|
|
149
|
+
*
|
|
150
|
+
* There is no symmetric `setRecord` method, because {@link KeyValueStore.setValue} already
|
|
151
|
+
* passes a `Buffer` (or `string` / `Stream`) through unchanged when an explicit `contentType`
|
|
152
|
+
* is provided. To write pre-serialized bytes, call
|
|
153
|
+
* `setValue(key, buffer, { contentType: 'application/json; charset=utf-8' })`.
|
|
154
|
+
*
|
|
155
|
+
* Returns `null` if the record does not exist.
|
|
156
|
+
*
|
|
157
|
+
* **Example usage:**
|
|
158
|
+
* ```javascript
|
|
159
|
+
* const store = await KeyValueStore.open();
|
|
160
|
+
* const record = await store.getRecord('huge.json');
|
|
161
|
+
* if (record) {
|
|
162
|
+
* const data = simdjson.parse(record.value);
|
|
163
|
+
* }
|
|
164
|
+
* ```
|
|
165
|
+
*
|
|
166
|
+
* @param key
|
|
167
|
+
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
168
|
+
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
|
|
169
|
+
*/
|
|
170
|
+
async getRecord(key) {
|
|
171
|
+
checkStorageAccess();
|
|
172
|
+
ow(key, ow.string.nonEmpty);
|
|
173
|
+
this.statsTracker.add('readCount');
|
|
174
|
+
const record = await this.backend.getValue(key);
|
|
175
|
+
if (!record)
|
|
176
|
+
return null;
|
|
177
|
+
return {
|
|
178
|
+
value: record.value,
|
|
179
|
+
contentType: record.contentType ?? null,
|
|
180
|
+
};
|
|
152
181
|
}
|
|
153
182
|
/**
|
|
154
183
|
* Tests whether a record with the given key exists in the key-value store without retrieving its value.
|
|
@@ -159,7 +188,7 @@ export class KeyValueStore {
|
|
|
159
188
|
async recordExists(key) {
|
|
160
189
|
checkStorageAccess();
|
|
161
190
|
ow(key, ow.string.nonEmpty);
|
|
162
|
-
return this.
|
|
191
|
+
return this.backend.recordExists(key);
|
|
163
192
|
}
|
|
164
193
|
async getAutoSavedValue(key, defaultValue = {}) {
|
|
165
194
|
checkStorageAccess();
|
|
@@ -182,21 +211,44 @@ export class KeyValueStore {
|
|
|
182
211
|
if (this.persistStateEventStarted) {
|
|
183
212
|
return;
|
|
184
213
|
}
|
|
185
|
-
|
|
186
|
-
const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
|
|
187
|
-
const timeoutSecs = persistStateIntervalMillis / 2_000;
|
|
188
|
-
this.config.getEventManager().on('persistState', async () => {
|
|
214
|
+
serviceLocator.getEventManager().on('persistState', async () => {
|
|
189
215
|
const promises = [];
|
|
190
216
|
for (const [key, value] of this.cache) {
|
|
191
|
-
promises.push(this.setValue(key, value
|
|
192
|
-
timeoutSecs,
|
|
193
|
-
doNotRetryTimeouts: true,
|
|
194
|
-
}).catch((error) => log.warning(`Failed to persist the state value to ${key}`, { error })));
|
|
217
|
+
promises.push(this.setValue(key, value).catch((error) => serviceLocator.getLogger().warning(`Failed to persist the state value to ${key}`, { error })));
|
|
195
218
|
}
|
|
196
219
|
await Promise.all(promises);
|
|
197
220
|
});
|
|
198
221
|
this.persistStateEventStarted = true;
|
|
199
222
|
}
|
|
223
|
+
async *fetchKeyValuePages(options, mapRecord) {
|
|
224
|
+
for await (const page of this.fetchKeyPages(options)) {
|
|
225
|
+
const results = [];
|
|
226
|
+
for (const item of page) {
|
|
227
|
+
this.statsTracker.add('readCount');
|
|
228
|
+
const record = await this.backend.getValue(item.key);
|
|
229
|
+
if (record) {
|
|
230
|
+
const parsed = parseValue(record.value, record.contentType ?? null);
|
|
231
|
+
results.push(mapRecord(item.key, parsed));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
yield results;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
async *fetchKeyPages(options, limit = KVS_KEYS_DEFAULT_LIMIT) {
|
|
238
|
+
let exclusiveStartKey;
|
|
239
|
+
while (true) {
|
|
240
|
+
this.statsTracker.add('listCount');
|
|
241
|
+
const { items, isTruncated, nextExclusiveStartKey } = await this.backend.listKeys({
|
|
242
|
+
...options,
|
|
243
|
+
exclusiveStartKey,
|
|
244
|
+
limit,
|
|
245
|
+
});
|
|
246
|
+
yield items;
|
|
247
|
+
if (!isTruncated)
|
|
248
|
+
break;
|
|
249
|
+
exclusiveStartKey = nextExclusiveStartKey;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
200
252
|
/**
|
|
201
253
|
* Saves or deletes a record in the key-value store.
|
|
202
254
|
* The function returns a promise that resolves once the record has been saved or deleted.
|
|
@@ -247,15 +299,11 @@ export class KeyValueStore {
|
|
|
247
299
|
validator: ow.isValid(k, ow.string.matches(KEY_VALUE_STORE_KEY_REGEX)),
|
|
248
300
|
message: `The "key" argument "${key}" must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.'()`,
|
|
249
301
|
})));
|
|
250
|
-
if (options.contentType &&
|
|
251
|
-
|
|
252
|
-
(ow.isValid(value, ow.object) && typeof value.pipe === 'function'))) {
|
|
253
|
-
throw new ArgumentError('The "value" parameter must be a String, Buffer or Stream when "options.contentType" is specified.', this.setValue);
|
|
302
|
+
if (options.contentType && !(typeof value === 'string' || isBuffer(value) || isStream(value))) {
|
|
303
|
+
throw new ArgumentError('The "value" parameter must be a String, Buffer, ArrayBuffer, TypedArray, or Stream when "options.contentType" is specified.', this.setValue);
|
|
254
304
|
}
|
|
255
305
|
ow(options, ow.object.exactShape({
|
|
256
306
|
contentType: ow.optional.string.nonEmpty,
|
|
257
|
-
timeoutSecs: ow.optional.number,
|
|
258
|
-
doNotRetryTimeouts: ow.optional.boolean,
|
|
259
307
|
}));
|
|
260
308
|
// Make copy of options, don't update what user passed.
|
|
261
309
|
const optionsCopy = { ...options };
|
|
@@ -276,16 +324,16 @@ export class KeyValueStore {
|
|
|
276
324
|
}
|
|
277
325
|
}
|
|
278
326
|
// In this case delete the record.
|
|
279
|
-
if (value === null)
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
327
|
+
if (value === null) {
|
|
328
|
+
this.statsTracker.add('deleteCount');
|
|
329
|
+
return this.backend.deleteValue(key);
|
|
330
|
+
}
|
|
331
|
+
const serialized = serializeValue(value, optionsCopy.contentType);
|
|
332
|
+
this.statsTracker.add('writeCount');
|
|
333
|
+
return this.backend.setValue({
|
|
283
334
|
key,
|
|
284
|
-
value,
|
|
285
|
-
contentType:
|
|
286
|
-
}, {
|
|
287
|
-
timeoutSecs: optionsCopy.timeoutSecs,
|
|
288
|
-
doNotRetryTimeouts: optionsCopy.doNotRetryTimeouts,
|
|
335
|
+
value: serialized.value,
|
|
336
|
+
contentType: serialized.contentType,
|
|
289
337
|
});
|
|
290
338
|
}
|
|
291
339
|
/**
|
|
@@ -294,9 +342,8 @@ export class KeyValueStore {
|
|
|
294
342
|
*/
|
|
295
343
|
async drop() {
|
|
296
344
|
checkStorageAccess();
|
|
297
|
-
await this.
|
|
298
|
-
|
|
299
|
-
manager.closeStorage(this);
|
|
345
|
+
await this.backend.drop();
|
|
346
|
+
serviceLocator.getStorageInstanceManager().removeFromCache(this);
|
|
300
347
|
}
|
|
301
348
|
/** @internal */
|
|
302
349
|
clearCache() {
|
|
@@ -307,7 +354,7 @@ export class KeyValueStore {
|
|
|
307
354
|
* Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
|
|
308
355
|
* Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
|
|
309
356
|
* is the record key, `index` is a zero-based index of the key in the current iteration
|
|
310
|
-
*
|
|
357
|
+
* and `info` is an object that contains a single property `size`
|
|
311
358
|
* indicating size of the record in bytes.
|
|
312
359
|
*
|
|
313
360
|
* If the `iteratee` function returns a Promise then it is awaited before the next call.
|
|
@@ -326,29 +373,131 @@ export class KeyValueStore {
|
|
|
326
373
|
*/
|
|
327
374
|
async forEachKey(iteratee, options = {}) {
|
|
328
375
|
checkStorageAccess();
|
|
329
|
-
return this._forEachKey(iteratee, options);
|
|
330
|
-
}
|
|
331
|
-
async _forEachKey(iteratee, options = {}, index = 0) {
|
|
332
|
-
const { exclusiveStartKey } = options;
|
|
333
376
|
ow(iteratee, ow.function);
|
|
334
377
|
ow(options, ow.object.exactShape({
|
|
335
|
-
|
|
378
|
+
prefix: ow.optional.string,
|
|
336
379
|
}));
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
380
|
+
let index = 0;
|
|
381
|
+
for await (const page of this.fetchKeyPages(options)) {
|
|
382
|
+
for (const item of page) {
|
|
383
|
+
await iteratee(item.key, index++, { size: item.size });
|
|
384
|
+
}
|
|
341
385
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* Returns key-value store keys.
|
|
389
|
+
*
|
|
390
|
+
* When awaited (`await store.keys()`), returns all keys as a flat `string[]` array.
|
|
391
|
+
* When used as an async iterable (`for await...of`), iterates over all keys across pages
|
|
392
|
+
* without loading everything into memory at once.
|
|
393
|
+
*
|
|
394
|
+
* **Example usage:**
|
|
395
|
+
* ```javascript
|
|
396
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
397
|
+
*
|
|
398
|
+
* // Iterate over all keys (memory-efficient for large stores)
|
|
399
|
+
* for await (const key of keyValueStore.keys()) {
|
|
400
|
+
* console.log(key);
|
|
401
|
+
* }
|
|
402
|
+
*
|
|
403
|
+
* // Or fetch all keys at once
|
|
404
|
+
* const allKeys = await keyValueStore.keys();
|
|
405
|
+
* console.log(allKeys);
|
|
406
|
+
* ```
|
|
407
|
+
*
|
|
408
|
+
* @param options Options for the iteration.
|
|
409
|
+
*/
|
|
410
|
+
keys(options = {}) {
|
|
411
|
+
checkStorageAccess();
|
|
412
|
+
return createDualIterable({
|
|
413
|
+
createPages: () => this.fetchKeyPages(options),
|
|
414
|
+
extractItems: (page) => page.map((item) => item.key),
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Returns key-value store values.
|
|
419
|
+
*
|
|
420
|
+
* When awaited (`await store.values()`), returns all values as a flat `T[]` array.
|
|
421
|
+
* When used as an async iterable (`for await...of`), iterates over all values across pages
|
|
422
|
+
* without loading everything into memory at once.
|
|
423
|
+
*
|
|
424
|
+
* **Example usage:**
|
|
425
|
+
* ```javascript
|
|
426
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
427
|
+
*
|
|
428
|
+
* // Iterate over all values (memory-efficient for large stores)
|
|
429
|
+
* for await (const value of keyValueStore.values()) {
|
|
430
|
+
* console.log(value);
|
|
431
|
+
* }
|
|
432
|
+
*
|
|
433
|
+
* // Or fetch all values at once
|
|
434
|
+
* const allValues = await keyValueStore.values();
|
|
435
|
+
* console.log(allValues);
|
|
436
|
+
* ```
|
|
437
|
+
*
|
|
438
|
+
* @param options Options for the iteration.
|
|
439
|
+
*/
|
|
440
|
+
values(options = {}) {
|
|
441
|
+
checkStorageAccess();
|
|
442
|
+
return createDualIterable({
|
|
443
|
+
createPages: () => this.fetchKeyValuePages(options, (_key, value) => value),
|
|
444
|
+
extractItems: (page) => page,
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Returns key-value store entries (key-value pairs).
|
|
449
|
+
*
|
|
450
|
+
* When awaited (`await store.entries()`), returns all entries as a flat `[key, value][]` array.
|
|
451
|
+
* When used as an async iterable (`for await...of`), iterates over all entries across pages
|
|
452
|
+
* without loading everything into memory at once.
|
|
453
|
+
*
|
|
454
|
+
* **Example usage:**
|
|
455
|
+
* ```javascript
|
|
456
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
457
|
+
*
|
|
458
|
+
* // Iterate over all entries (memory-efficient for large stores)
|
|
459
|
+
* for await (const [key, value] of keyValueStore.entries()) {
|
|
460
|
+
* console.log(`${key}: ${value}`);
|
|
461
|
+
* }
|
|
462
|
+
*
|
|
463
|
+
* // Or fetch all entries at once
|
|
464
|
+
* const allEntries = await keyValueStore.entries();
|
|
465
|
+
* console.log(allEntries);
|
|
466
|
+
* ```
|
|
467
|
+
*
|
|
468
|
+
* @param options Options for the iteration.
|
|
469
|
+
*/
|
|
470
|
+
entries(options = {}) {
|
|
471
|
+
checkStorageAccess();
|
|
472
|
+
return createDualIterable({
|
|
473
|
+
createPages: () => this.fetchKeyValuePages(options, (key, value) => [key, value]),
|
|
474
|
+
extractItems: (page) => page,
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* Default async iterator for the key-value store, iterating over entries (key-value pairs).
|
|
479
|
+
* Allows using the store directly in a `for await...of` loop.
|
|
480
|
+
*
|
|
481
|
+
* **Example usage:**
|
|
482
|
+
* ```javascript
|
|
483
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
484
|
+
* for await (const [key, value] of keyValueStore) {
|
|
485
|
+
* console.log(`${key}: ${value}`);
|
|
486
|
+
* }
|
|
487
|
+
* ```
|
|
488
|
+
*/
|
|
489
|
+
async *[Symbol.asyncIterator]() {
|
|
490
|
+
yield* this.entries();
|
|
345
491
|
}
|
|
346
492
|
/**
|
|
347
493
|
* Returns a file URL for the given key.
|
|
494
|
+
*
|
|
495
|
+
* If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
|
|
496
|
+
*
|
|
497
|
+
* @param key The key of the record to generate the public URL for.
|
|
348
498
|
*/
|
|
349
|
-
getPublicUrl(key) {
|
|
350
|
-
|
|
351
|
-
return `file://${process.cwd()}/storage/key_value_stores/${name}/${key}`;
|
|
499
|
+
async getPublicUrl(key) {
|
|
500
|
+
return this.backend.getPublicUrl(key);
|
|
352
501
|
}
|
|
353
502
|
/**
|
|
354
503
|
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
@@ -359,23 +508,27 @@ export class KeyValueStore {
|
|
|
359
508
|
*
|
|
360
509
|
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
361
510
|
*
|
|
362
|
-
* @param [
|
|
363
|
-
* ID or name of the key-value store to be opened. If
|
|
364
|
-
*
|
|
511
|
+
* @param [identifier]
|
|
512
|
+
* ID or name of the key-value store to be opened. If a string is provided, it will first be
|
|
513
|
+
* looked up as an ID; if no such storage exists, it will be treated as a name.
|
|
514
|
+
* If `null` or `undefined`, the function returns the default key-value store associated with the crawler run.
|
|
365
515
|
* @param [options] Storage manager options.
|
|
366
516
|
*/
|
|
367
|
-
static async open(
|
|
517
|
+
static async open(identifier, options = {}) {
|
|
368
518
|
checkStorageAccess();
|
|
369
|
-
ow(storeIdOrName, ow.optional.any(ow.string, ow.null));
|
|
370
519
|
ow(options, ow.object.exactShape({
|
|
371
520
|
config: ow.optional.object.instanceOf(Configuration),
|
|
372
|
-
|
|
521
|
+
storageBackend: ow.optional.object,
|
|
373
522
|
}));
|
|
374
523
|
options.config ??= Configuration.getGlobalConfig();
|
|
375
|
-
options.
|
|
376
|
-
await purgeDefaultStorages({ onlyPurgeOnce: true,
|
|
377
|
-
const
|
|
378
|
-
return
|
|
524
|
+
const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
|
|
525
|
+
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, config: options.config });
|
|
526
|
+
const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'KeyValueStore');
|
|
527
|
+
return serviceLocator.getStorageInstanceManager().openStorage(this, {
|
|
528
|
+
...resolved,
|
|
529
|
+
backendOpener: () => storageBackend.createKeyValueStoreBackend(resolved),
|
|
530
|
+
backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name,
|
|
531
|
+
});
|
|
379
532
|
}
|
|
380
533
|
/**
|
|
381
534
|
* Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
@@ -410,6 +563,22 @@ export class KeyValueStore {
|
|
|
410
563
|
const store = await this.open();
|
|
411
564
|
return store.getValue(key, defaultValue);
|
|
412
565
|
}
|
|
566
|
+
/**
|
|
567
|
+
* Reads a record from the default {@link KeyValueStore} associated with the current crawler run
|
|
568
|
+
* without parsing the value.
|
|
569
|
+
*
|
|
570
|
+
* This is just a convenient shortcut for {@link KeyValueStore.getRecord}. Returns `null` if the
|
|
571
|
+
* record does not exist.
|
|
572
|
+
*
|
|
573
|
+
* @param key
|
|
574
|
+
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
575
|
+
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
|
|
576
|
+
* @ignore
|
|
577
|
+
*/
|
|
578
|
+
static async getRecord(key) {
|
|
579
|
+
const store = await this.open();
|
|
580
|
+
return store.getRecord(key);
|
|
581
|
+
}
|
|
413
582
|
/**
|
|
414
583
|
* Tests whether a record with the given key exists in the default {@link KeyValueStore} associated with the current crawler run.
|
|
415
584
|
* @param key The queried record key.
|
|
@@ -460,8 +629,9 @@ export class KeyValueStore {
|
|
|
460
629
|
}
|
|
461
630
|
/**
|
|
462
631
|
* Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
463
|
-
*
|
|
464
|
-
*
|
|
632
|
+
*
|
|
633
|
+
* The input is read from the default {@link KeyValueStore} under the configured input key
|
|
634
|
+
* (`CRAWLEE_INPUT_KEY`, default `INPUT`).
|
|
465
635
|
*
|
|
466
636
|
* Note that the `getInput()` function does not cache the value read from the key-value store.
|
|
467
637
|
* If you need to use the input multiple times in your crawler,
|
|
@@ -479,29 +649,7 @@ export class KeyValueStore {
|
|
|
479
649
|
*/
|
|
480
650
|
static async getInput() {
|
|
481
651
|
const store = await this.open();
|
|
482
|
-
|
|
483
|
-
const cwd = process.cwd();
|
|
484
|
-
const possibleExtensions = ['', '.json', '.txt'];
|
|
485
|
-
// Attempt to read input from root file instead of key-value store
|
|
486
|
-
for (const extension of possibleExtensions) {
|
|
487
|
-
const inputFile = join(cwd, `${inputKey}${extension}`);
|
|
488
|
-
let input;
|
|
489
|
-
// Try getting the file from the file system
|
|
490
|
-
try {
|
|
491
|
-
input = await readFile(inputFile);
|
|
492
|
-
}
|
|
493
|
-
catch {
|
|
494
|
-
continue;
|
|
495
|
-
}
|
|
496
|
-
// Attempt to parse as JSON, or return the input as is otherwise
|
|
497
|
-
try {
|
|
498
|
-
return JSON5.parse(input.toString());
|
|
499
|
-
}
|
|
500
|
-
catch {
|
|
501
|
-
return input;
|
|
502
|
-
}
|
|
503
|
-
}
|
|
504
|
-
return store.getValue(inputKey);
|
|
652
|
+
return store.getValue(store.config.inputKey);
|
|
505
653
|
}
|
|
506
654
|
}
|
|
507
655
|
//# sourceMappingURL=key_value_store.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEvC,OAAO,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,GAAG,MAAM,YAAY,CAAC;AAC7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACpE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI,CAAC;YACD,kFAAkF;YAClF,KAAK,GAAG,qBAAqB,CAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;QAChF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBACnD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;YAC1C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,CAAC;QAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CACX,wEAAwE;gBACpE,8DAA8D,CACrE,CAAC;QACN,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAM,OAAO,aAAa;IAeT;IAdJ,EAAE,CAAS;IACX,IAAI,CAAU;IACd,aAAa,CAA2B;IAChC,MAAM,CAAsB;IACrC,wBAAwB,GAAG,KAAK,CAAC;IAEzC,wHAAwH;IACvG,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEvD;;OAEG;IACH,YACI,OAA6B,EACpB,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAkED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAQ,MAAM,EAAE,KAAW,IAAI,YAAY,IAAI,IAAI,CAAC;IACxD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACpC,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QAExD,0HAA0H;QAC1H,wHAAwH;QACxH,qEAAqE;QACrE,kIAAkI;QAClI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACpC,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,OAAO;QACX,CAAC;QAED,mEAAmE;QACnE,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,MAAM,WAAW,GAAG,0BAA0B,GAAG,KAAK,CAAC;QAEvD,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,QAAQ,GAAoB,EAAE,CAAC;YAErC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACpC,QAAQ,CAAC,IAAI,CACT,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE;oBACtB,WAAW;oBACX,kBAAkB,EAAE,IAAI;iBAC3B,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,wCAAwC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAC7F,CAAC;YACN,CAAC;YAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,EAAE,CACE,GAAG,EACH,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,uBAAuB,GAAG,mGAAmG;SACzI,CAAC,CAAC,CACN,CAAC;QACF,IACI,OAAO,CAAC,WAAW;YACnB,CAAC,CACG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;gBACnD,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CACrF,EACH,CAAC;YACC,MAAM,IAAI,aAAa,CACnB,mGAAmG,EACnG,IAAI,CAAC,QAAQ,CAChB,CAAC;QACN,CAAC;QACD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;YACxC,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SAC1C,CAAC,CACL,CAAC;QAEF,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,WAAW,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YACvC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACjB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACnC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAK,KAAoB,CAAC,CAAC;qBAC5C,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CACxB;YACI,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,EACD;YACI,WAAW,EAAE,WAAW,CAAC,WAAW;YACpC,kBAAkB,EAAE,WAAW,CAAC,kBAAkB;SACrD,CACJ,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CACrB,QAAqB,EACrB,UAAwC,EAAE,EAC1C,KAAK,GAAG,CAAC;QAET,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CACL,CAAC;QAEF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,GAAW;QACpB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACpE,OAAO,UAAU,OAAO,CAAC,GAAG,EAAE,6BAA6B,IAAI,IAAI,GAAG,EAAE,CAAC;IAC7E,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAEhE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACrE,CAAC;IA4DD;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAiB,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC;QAE/C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,kBAAkB,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAEjD,kEAAkE;QAClE,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,SAAS,EAAE,CAAC,CAAC;YACvD,IAAI,KAAa,CAAC;YAElB,4CAA4C;YAC5C,IAAI,CAAC;gBACD,KAAK,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;YAED,gEAAgE;YAChE,IAAI,CAAC;gBACD,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAM,CAAC;YAC9C,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO,KAAqB,CAAC;YACjC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC,QAAQ,CAAI,QAAQ,CAAC,CAAC;IACvC,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEvC,OAAO,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAE1D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAExE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGzD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAEpD,gBAAgB;AAChB,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAM,OAAO,aAAa;IAqBT;IApBJ,EAAE,CAAS;IACX,IAAI,CAAU;IACN,OAAO,CAAuB;IACvC,wBAAwB,GAAG,KAAK,CAAC;IAEzC,wHAAwH;IACvG,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEtC,YAAY,GAAG,IAAI,mBAAmB,CAAqB;QACxE,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;QACd,SAAS,EAAE,CAAC;KACf,CAAC,CAAC;IAEH;;OAEG;IACH,YACI,OAA6B,EACpB,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IACrC,CAAC;IAkED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAEhD,+FAA+F;QAC/F,kGAAkG;QAClG,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,YAAY,IAAI,IAAI,CAAC;QAChC,CAAC;QAED,8FAA8F;QAC9F,OAAO,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,CAAM,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,KAAK,CAAC,SAAS,CAAC,GAAW;QACvB,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAEzB,OAAO;YACH,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI;SAC1C,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACpC,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QAExD,0HAA0H;QAC1H,wHAAwH;QACxH,qEAAqE;QACrE,kIAAkI;QAClI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACpC,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,OAAO;QACX,CAAC;QAED,cAAc,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,QAAQ,GAAoB,EAAE,CAAC;YAErC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACpC,QAAQ,CAAC,IAAI,CACT,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,cAAc,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,wCAAwC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAC/F,CACJ,CAAC;YACN,CAAC;YAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,CAAC,kBAAkB,CAC7B,OAAqC,EACrC,SAA6C;QAE7C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,MAAM,OAAO,GAAQ,EAAE,CAAC;YACxB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACtB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;gBACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrD,IAAI,MAAM,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC;oBACpE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;gBAC9C,CAAC;YACL,CAAC;YACD,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,aAAa,CACxB,OAAqC,EACrC,KAAK,GAAG,sBAAsB;QAE9B,IAAI,iBAAqC,CAAC;QAE1C,OAAO,IAAI,EAAE,CAAC;YACV,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACnC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC9E,GAAG,OAAO;gBACV,iBAAiB;gBACjB,KAAK;aACR,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;YACZ,IAAI,CAAC,WAAW;gBAAE,MAAM;YACxB,iBAAiB,GAAG,qBAAqB,CAAC;QAC9C,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,EAAE,CACE,GAAG,EACH,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,uBAAuB,GAAG,mGAAmG;SACzI,CAAC,CAAC,CACN,CAAC;QACF,IAAI,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YAC5F,MAAM,IAAI,aAAa,CACnB,6HAA6H,EAC7H,IAAI,CAAC,QAAQ,CAChB,CAAC;QACN,CAAC;QACD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CACL,CAAC;QAEF,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,WAAW,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YACvC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACjB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACnC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAK,KAAoB,CAAC,CAAC;qBAC5C,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;QAElE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;YACzB,GAAG;YACH,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,WAAW,EAAE,UAAU,CAAC,WAAW;SACtC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,cAAc,CAAC,yBAAyB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CACL,CAAC;QAEF,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,IAAI,CAAC,UAAwC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;YAC9C,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;SACvD,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAc,UAAwC,EAAE;QAC1D,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAI,OAAO,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAU,CAAC;YACnF,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;SAC/B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CACH,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAc,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,KAAU,CAAC,CAAC;YACnG,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;SAC/B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,EAAK,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,UAA8C,EAC9C,UAA8B,EAAE;QAEhC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACrC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,cAAc,CAAC,iBAAiB,EAAE,CAAC;QAEpF,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE5F,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;QAE7F,OAAO,cAAc,CAAC,yBAAyB,EAAE,CAAC,WAAW,CAAgB,IAAI,EAAE;YAC/E,GAAG,QAAQ;YACX,aAAa,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,0BAA0B,CAAC,QAAQ,CAAC;YACxE,eAAe,EAAE,cAAc,CAAC,yBAAyB,EAAE,EAAE,IAAI,cAAc,CAAC,WAAW,CAAC,IAAI;SACnG,CAAC,CAAC;IACP,CAAC;IA4DD;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAiB,CAAC,CAAC;IACrD,CAAC;IAED;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,GAAW;QAC9B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;CACJ"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical write path for key-value store records.
|
|
3
|
+
*
|
|
4
|
+
* When a content type is provided, the value passes through unchanged — it is the caller's
|
|
5
|
+
* responsibility to supply a String/Buffer/Stream (the frontend validates this).
|
|
6
|
+
*
|
|
7
|
+
* When no content type is provided, it is inferred from the value's shape:
|
|
8
|
+
* - Buffer / typed array / ArrayBuffer / stream → `application/octet-stream` (passthrough)
|
|
9
|
+
* - `string` → `text/plain; charset=utf-8` (passthrough)
|
|
10
|
+
* - anything else → `application/json; charset=utf-8` (serialized via `jsonStringifyExtended`)
|
|
11
|
+
*
|
|
12
|
+
* Does NOT drain streams — that is storage mechanics and stays in the storage backend.
|
|
13
|
+
*
|
|
14
|
+
* Backend-independent.
|
|
15
|
+
*/
|
|
16
|
+
export declare function serializeValue(value: unknown, contentType?: string): {
|
|
17
|
+
value: Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
|
|
18
|
+
contentType: string;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Parses a Buffer or ArrayBuffer using the provided content type header.
|
|
22
|
+
*
|
|
23
|
+
* - application/json is returned as a parsed object.
|
|
24
|
+
* - application/*xml and text/* are returned as strings.
|
|
25
|
+
* - everything else is returned as original body.
|
|
26
|
+
*
|
|
27
|
+
* If the header includes a charset, the body will be stringified only
|
|
28
|
+
* if the charset represents a known encoding to Node.js or Browser.
|
|
29
|
+
*
|
|
30
|
+
* Backend-independent — this is the canonical read path for the {@link KeyValueStore} frontend.
|
|
31
|
+
*/
|
|
32
|
+
export declare function parseValue(body: Buffer | ArrayBuffer | string, contentTypeHeader: string | null): string | Buffer | ArrayBuffer | Record<string, unknown>;
|
|
33
|
+
//# sourceMappingURL=key_value_store_codec.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"key_value_store_codec.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store_codec.ts"],"names":[],"mappings":"AAUA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,cAAc,CAC1B,KAAK,EAAE,OAAO,EACd,WAAW,CAAC,EAAE,MAAM,GACrB;IACC,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG,eAAe,GAAG,MAAM,GAAG,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC;IAChG,WAAW,EAAE,MAAM,CAAC;CACvB,CAqCA;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,UAAU,CACtB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,EACnC,iBAAiB,EAAE,MAAM,GAAG,IAAI,GACjC,MAAM,GAAG,MAAM,GAAG,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAuBzD"}
|