@crawlee/core 4.0.0-beta.17 → 4.0.0-beta.170

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (287) hide show
  1. package/README.md +14 -14
  2. package/autoscaling/autoscaled_pool.d.ts +73 -177
  3. package/autoscaling/autoscaled_pool.js +182 -329
  4. package/autoscaling/concurrency_system.d.ts +268 -0
  5. package/autoscaling/concurrency_system.js +364 -0
  6. package/autoscaling/cpu_load_signal.d.ts +43 -0
  7. package/autoscaling/cpu_load_signal.js +47 -0
  8. package/autoscaling/event_loop_load_signal.d.ts +51 -0
  9. package/autoscaling/event_loop_load_signal.js +60 -0
  10. package/autoscaling/index.d.ts +6 -1
  11. package/autoscaling/index.js +6 -1
  12. package/autoscaling/load_signal.d.ts +98 -0
  13. package/autoscaling/load_signal.js +103 -0
  14. package/autoscaling/memory_load_signal.d.ts +47 -0
  15. package/autoscaling/memory_load_signal.js +106 -0
  16. package/autoscaling/snapshotter.d.ts +58 -163
  17. package/autoscaling/snapshotter.js +45 -263
  18. package/autoscaling/storage_backend_load_signal.d.ts +56 -0
  19. package/autoscaling/storage_backend_load_signal.js +73 -0
  20. package/autoscaling/system_status.d.ts +70 -92
  21. package/autoscaling/system_status.js +92 -122
  22. package/autoscaling/weighted_avg.d.ts +5 -0
  23. package/autoscaling/weighted_avg.js +14 -0
  24. package/byte_utils.d.ts +17 -0
  25. package/byte_utils.js +42 -0
  26. package/configuration.d.ts +96 -223
  27. package/configuration.js +170 -222
  28. package/cookie_utils.d.ts +3 -2
  29. package/cookie_utils.js +18 -7
  30. package/crawlers/context_pipeline.d.ts +10 -1
  31. package/crawlers/context_pipeline.js +31 -8
  32. package/crawlers/crawler_commons.d.ts +128 -124
  33. package/crawlers/crawler_commons.js +1 -108
  34. package/crawlers/error_snapshotter.d.ts +2 -5
  35. package/crawlers/error_snapshotter.js +5 -6
  36. package/crawlers/error_tracker.d.ts +0 -1
  37. package/crawlers/error_tracker.js +0 -1
  38. package/crawlers/index.d.ts +1 -3
  39. package/crawlers/index.js +0 -3
  40. package/crawlers/internals/types.d.ts +0 -1
  41. package/crawlers/internals/types.js +0 -1
  42. package/crawlers/statistics.d.ts +194 -79
  43. package/crawlers/statistics.js +382 -201
  44. package/debug.d.ts +36 -0
  45. package/debug.js +70 -0
  46. package/enqueue_links/enqueue_links.d.ts +61 -154
  47. package/enqueue_links/enqueue_links.js +40 -232
  48. package/enqueue_links/index.d.ts +0 -1
  49. package/enqueue_links/index.js +0 -1
  50. package/enqueue_links/shared.d.ts +55 -28
  51. package/enqueue_links/shared.js +116 -69
  52. package/errors.d.ts +82 -4
  53. package/errors.js +101 -5
  54. package/events/event_manager.d.ts +35 -9
  55. package/events/event_manager.js +12 -13
  56. package/events/index.d.ts +0 -1
  57. package/events/index.js +0 -1
  58. package/events/local_event_manager.d.ts +15 -3
  59. package/events/local_event_manager.js +39 -13
  60. package/http.d.ts +9 -0
  61. package/http.js +28 -0
  62. package/index.d.ts +7 -5
  63. package/index.js +8 -4
  64. package/iterables.d.ts +79 -0
  65. package/iterables.js +134 -0
  66. package/log.d.ts +77 -1
  67. package/log.js +134 -1
  68. package/memory-storage/consts.d.ts +4 -0
  69. package/memory-storage/consts.js +4 -0
  70. package/memory-storage/index.d.ts +1 -0
  71. package/memory-storage/index.js +1 -0
  72. package/memory-storage/memory-storage.d.ts +44 -0
  73. package/memory-storage/memory-storage.js +160 -0
  74. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  75. package/memory-storage/resource-clients/common/base-client.js +6 -0
  76. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  77. package/memory-storage/resource-clients/dataset.js +106 -0
  78. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  79. package/memory-storage/resource-clients/key-value-store.js +199 -0
  80. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  81. package/memory-storage/resource-clients/request-queue.js +414 -0
  82. package/memory-storage/utils.d.ts +16 -0
  83. package/memory-storage/utils.js +41 -0
  84. package/owned_or_injected.d.ts +58 -0
  85. package/owned_or_injected.js +98 -0
  86. package/package.json +17 -16
  87. package/proxy_configuration.d.ts +22 -128
  88. package/proxy_configuration.js +32 -144
  89. package/recoverable_state.d.ts +84 -51
  90. package/recoverable_state.js +175 -72
  91. package/request.d.ts +57 -16
  92. package/request.js +130 -69
  93. package/router.d.ts +193 -21
  94. package/router.js +188 -43
  95. package/serialization.d.ts +0 -1
  96. package/serialization.js +15 -15
  97. package/service_locator.d.ts +165 -0
  98. package/service_locator.js +253 -0
  99. package/session_pool/consts.d.ts +1 -2
  100. package/session_pool/consts.js +1 -2
  101. package/session_pool/errors.d.ts +0 -1
  102. package/session_pool/errors.js +0 -1
  103. package/session_pool/fingerprint.d.ts +9 -0
  104. package/session_pool/fingerprint.js +30 -0
  105. package/session_pool/index.d.ts +0 -2
  106. package/session_pool/index.js +0 -2
  107. package/session_pool/session.d.ts +40 -90
  108. package/session_pool/session.js +103 -160
  109. package/session_pool/session_pool.d.ts +85 -101
  110. package/session_pool/session_pool.js +254 -210
  111. package/storages/batched_adds.d.ts +37 -0
  112. package/storages/batched_adds.js +73 -0
  113. package/storages/dataset.d.ts +114 -56
  114. package/storages/dataset.js +292 -149
  115. package/storages/index.d.ts +9 -9
  116. package/storages/index.js +7 -9
  117. package/storages/key_value_store.d.ts +186 -48
  118. package/storages/key_value_store.js +452 -170
  119. package/storages/key_value_store_codec.d.ts +32 -0
  120. package/storages/key_value_store_codec.js +113 -0
  121. package/storages/request_dedup_cache.d.ts +22 -0
  122. package/storages/request_dedup_cache.js +48 -0
  123. package/storages/request_list.d.ts +54 -129
  124. package/storages/request_list.js +272 -245
  125. package/storages/request_loader.d.ts +133 -0
  126. package/storages/request_loader.js +36 -0
  127. package/storages/request_manager.d.ts +109 -0
  128. package/storages/request_manager.js +1 -0
  129. package/storages/request_manager_tandem.d.ts +55 -23
  130. package/storages/request_manager_tandem.js +139 -57
  131. package/storages/request_queue.d.ts +286 -48
  132. package/storages/request_queue.js +757 -218
  133. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +43 -91
  134. package/storages/sitemap_request_loader.js +439 -0
  135. package/storages/storage_instance_manager.d.ts +87 -0
  136. package/storages/storage_instance_manager.js +256 -0
  137. package/storages/storage_stats.d.ts +48 -0
  138. package/storages/storage_stats.js +29 -0
  139. package/storages/throttling_request_manager.d.ts +223 -0
  140. package/storages/throttling_request_manager.js +835 -0
  141. package/storages/transaction.d.ts +260 -0
  142. package/storages/transaction.js +287 -0
  143. package/storages/utils.d.ts +59 -11
  144. package/storages/utils.js +75 -15
  145. package/system-info/cpu-info.d.ts +67 -0
  146. package/system-info/cpu-info.js +216 -0
  147. package/system-info/memory-info.d.ts +31 -0
  148. package/system-info/memory-info.js +115 -0
  149. package/system-info/ps-tree.d.ts +17 -0
  150. package/system-info/ps-tree.js +144 -0
  151. package/system-info/runtime.d.ts +14 -0
  152. package/system-info/runtime.js +80 -0
  153. package/typedefs.d.ts +0 -6
  154. package/typedefs.js +0 -1
  155. package/url.d.ts +9 -0
  156. package/url.js +11 -0
  157. package/validators.d.ts +23 -18
  158. package/validators.js +14 -18
  159. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  160. package/autoscaling/autoscaled_pool.js.map +0 -1
  161. package/autoscaling/index.d.ts.map +0 -1
  162. package/autoscaling/index.js.map +0 -1
  163. package/autoscaling/snapshotter.d.ts.map +0 -1
  164. package/autoscaling/snapshotter.js.map +0 -1
  165. package/autoscaling/system_status.d.ts.map +0 -1
  166. package/autoscaling/system_status.js.map +0 -1
  167. package/configuration.d.ts.map +0 -1
  168. package/configuration.js.map +0 -1
  169. package/cookie_utils.d.ts.map +0 -1
  170. package/cookie_utils.js.map +0 -1
  171. package/crawlers/context_pipeline.d.ts.map +0 -1
  172. package/crawlers/context_pipeline.js.map +0 -1
  173. package/crawlers/crawler_commons.d.ts.map +0 -1
  174. package/crawlers/crawler_commons.js.map +0 -1
  175. package/crawlers/crawler_utils.d.ts +0 -10
  176. package/crawlers/crawler_utils.d.ts.map +0 -1
  177. package/crawlers/crawler_utils.js +0 -12
  178. package/crawlers/crawler_utils.js.map +0 -1
  179. package/crawlers/error_snapshotter.d.ts.map +0 -1
  180. package/crawlers/error_snapshotter.js.map +0 -1
  181. package/crawlers/error_tracker.d.ts.map +0 -1
  182. package/crawlers/error_tracker.js.map +0 -1
  183. package/crawlers/index.d.ts.map +0 -1
  184. package/crawlers/index.js.map +0 -1
  185. package/crawlers/internals/types.d.ts.map +0 -1
  186. package/crawlers/internals/types.js.map +0 -1
  187. package/crawlers/statistics.d.ts.map +0 -1
  188. package/crawlers/statistics.js.map +0 -1
  189. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  190. package/enqueue_links/enqueue_links.js.map +0 -1
  191. package/enqueue_links/index.d.ts.map +0 -1
  192. package/enqueue_links/index.js.map +0 -1
  193. package/enqueue_links/shared.d.ts.map +0 -1
  194. package/enqueue_links/shared.js.map +0 -1
  195. package/errors.d.ts.map +0 -1
  196. package/errors.js.map +0 -1
  197. package/events/event_manager.d.ts.map +0 -1
  198. package/events/event_manager.js.map +0 -1
  199. package/events/index.d.ts.map +0 -1
  200. package/events/index.js.map +0 -1
  201. package/events/local_event_manager.d.ts.map +0 -1
  202. package/events/local_event_manager.js.map +0 -1
  203. package/http_clients/base-http-client.d.ts +0 -140
  204. package/http_clients/base-http-client.d.ts.map +0 -1
  205. package/http_clients/base-http-client.js +0 -40
  206. package/http_clients/base-http-client.js.map +0 -1
  207. package/http_clients/form-data-like.d.ts +0 -67
  208. package/http_clients/form-data-like.d.ts.map +0 -1
  209. package/http_clients/form-data-like.js +0 -5
  210. package/http_clients/form-data-like.js.map +0 -1
  211. package/http_clients/got-scraping-http-client.d.ts +0 -20
  212. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  213. package/http_clients/got-scraping-http-client.js +0 -85
  214. package/http_clients/got-scraping-http-client.js.map +0 -1
  215. package/http_clients/index.d.ts +0 -3
  216. package/http_clients/index.d.ts.map +0 -1
  217. package/http_clients/index.js +0 -3
  218. package/http_clients/index.js.map +0 -1
  219. package/index.d.ts.map +0 -1
  220. package/index.js.map +0 -1
  221. package/log.d.ts.map +0 -1
  222. package/log.js.map +0 -1
  223. package/proxy_configuration.d.ts.map +0 -1
  224. package/proxy_configuration.js.map +0 -1
  225. package/recoverable_state.d.ts.map +0 -1
  226. package/recoverable_state.js.map +0 -1
  227. package/request.d.ts.map +0 -1
  228. package/request.js.map +0 -1
  229. package/router.d.ts.map +0 -1
  230. package/router.js.map +0 -1
  231. package/serialization.d.ts.map +0 -1
  232. package/serialization.js.map +0 -1
  233. package/session_pool/consts.d.ts.map +0 -1
  234. package/session_pool/consts.js.map +0 -1
  235. package/session_pool/errors.d.ts.map +0 -1
  236. package/session_pool/errors.js.map +0 -1
  237. package/session_pool/events.d.ts +0 -3
  238. package/session_pool/events.d.ts.map +0 -1
  239. package/session_pool/events.js +0 -3
  240. package/session_pool/events.js.map +0 -1
  241. package/session_pool/index.d.ts.map +0 -1
  242. package/session_pool/index.js.map +0 -1
  243. package/session_pool/session.d.ts.map +0 -1
  244. package/session_pool/session.js.map +0 -1
  245. package/session_pool/session_pool.d.ts.map +0 -1
  246. package/session_pool/session_pool.js.map +0 -1
  247. package/storages/access_checking.d.ts +0 -13
  248. package/storages/access_checking.d.ts.map +0 -1
  249. package/storages/access_checking.js +0 -14
  250. package/storages/access_checking.js.map +0 -1
  251. package/storages/dataset.d.ts.map +0 -1
  252. package/storages/dataset.js.map +0 -1
  253. package/storages/index.d.ts.map +0 -1
  254. package/storages/index.js.map +0 -1
  255. package/storages/key_value_store.d.ts.map +0 -1
  256. package/storages/key_value_store.js.map +0 -1
  257. package/storages/request_list.d.ts.map +0 -1
  258. package/storages/request_list.js.map +0 -1
  259. package/storages/request_list_adapter.d.ts +0 -58
  260. package/storages/request_list_adapter.d.ts.map +0 -1
  261. package/storages/request_list_adapter.js +0 -81
  262. package/storages/request_list_adapter.js.map +0 -1
  263. package/storages/request_manager_tandem.d.ts.map +0 -1
  264. package/storages/request_manager_tandem.js.map +0 -1
  265. package/storages/request_provider.d.ts +0 -371
  266. package/storages/request_provider.d.ts.map +0 -1
  267. package/storages/request_provider.js +0 -585
  268. package/storages/request_provider.js.map +0 -1
  269. package/storages/request_queue.d.ts.map +0 -1
  270. package/storages/request_queue.js.map +0 -1
  271. package/storages/request_queue_v2.d.ts +0 -87
  272. package/storages/request_queue_v2.d.ts.map +0 -1
  273. package/storages/request_queue_v2.js +0 -438
  274. package/storages/request_queue_v2.js.map +0 -1
  275. package/storages/sitemap_request_list.d.ts.map +0 -1
  276. package/storages/sitemap_request_list.js +0 -430
  277. package/storages/sitemap_request_list.js.map +0 -1
  278. package/storages/storage_manager.d.ts +0 -58
  279. package/storages/storage_manager.d.ts.map +0 -1
  280. package/storages/storage_manager.js +0 -105
  281. package/storages/storage_manager.js.map +0 -1
  282. package/storages/utils.d.ts.map +0 -1
  283. package/storages/utils.js.map +0 -1
  284. package/typedefs.d.ts.map +0 -1
  285. package/typedefs.js.map +0 -1
  286. package/validators.d.ts.map +0 -1
  287. package/validators.js.map +0 -1
@@ -0,0 +1,44 @@
1
+ import type * as storage from '@crawlee/types';
2
+ import type { CrawleeLogger } from '@crawlee/types';
3
+ import { RequestQueueBackend } from './resource-clients/request-queue.js';
4
+ export interface MemoryStorageOptions {
5
+ /**
6
+ * Optional logger for MemoryStorageBackend warnings.
7
+ */
8
+ logger?: CrawleeLogger;
9
+ }
10
+ export declare class MemoryStorageBackend implements storage.StorageBackend {
11
+ #private;
12
+ readonly logger?: CrawleeLogger;
13
+ constructor(options?: MemoryStorageOptions);
14
+ /**
15
+ * Return a per-instance unique cache key so that distinct `MemoryStorageBackend` instances get separate
16
+ * cache partitions in the storage backend cache.
17
+ */
18
+ getStorageBackendCacheKey(): string;
19
+ /**
20
+ * Evict a cached backend so that a dropped storage is no longer resolved by `createXBackend`,
21
+ * reported by `storageExists` or visited by `purge`. Returns whether the backend was cached, which
22
+ * tells the caller whether it still owns in-memory state worth clearing.
23
+ *
24
+ * The resource clients own their own entry's lifetime but must not reach into the caches directly.
25
+ * Because a client is only ever constructed by `createXBackend`, which caches it immediately, the
26
+ * entry matching `id` is always the caller itself.
27
+ * @internal
28
+ */
29
+ evictBackend(type: 'Dataset' | 'KeyValueStore' | 'RequestQueue', id: string): boolean;
30
+ createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
31
+ createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
32
+ createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<RequestQueueBackend>;
33
+ storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
34
+ /**
35
+ * Cleans up the run-scoped storages before the run starts. For the in-memory storage this simply
36
+ * resets the in-memory state of the cached backends.
37
+ */
38
+ purge(): Promise<void>;
39
+ /**
40
+ * This method should be called at the end of the process. The in-memory storage holds no resources
41
+ * that outlive the process (no file handles, no cross-process locks), so there is nothing to do.
42
+ */
43
+ teardown(): Promise<void>;
44
+ }
@@ -0,0 +1,160 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { DatasetBackend } from './resource-clients/dataset.js';
3
+ import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
4
+ import { RequestQueueBackend } from './resource-clients/request-queue.js';
5
+ /** The alias the default (unnamed) storage is opened under. */
6
+ const DEFAULT_STORAGE_ALIAS = '__default__';
7
+ export class MemoryStorageBackend {
8
+ logger;
9
+ /**
10
+ * Unique per-instance cache partition key. Mirrors the way `FileSystemStorageBackend` partitions its
11
+ * cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.
12
+ */
13
+ #instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
14
+ #keyValueStoreBackendCache = [];
15
+ #datasetBackendCache = [];
16
+ #requestQueueBackendCache = [];
17
+ constructor(options = {}) {
18
+ this.logger = options.logger;
19
+ }
20
+ /**
21
+ * Return a per-instance unique cache key so that distinct `MemoryStorageBackend` instances get separate
22
+ * cache partitions in the storage backend cache.
23
+ */
24
+ getStorageBackendCacheKey() {
25
+ return this.#instanceCacheKey;
26
+ }
27
+ /**
28
+ * Evict a cached backend so that a dropped storage is no longer resolved by `createXBackend`,
29
+ * reported by `storageExists` or visited by `purge`. Returns whether the backend was cached, which
30
+ * tells the caller whether it still owns in-memory state worth clearing.
31
+ *
32
+ * The resource clients own their own entry's lifetime but must not reach into the caches directly.
33
+ * Because a client is only ever constructed by `createXBackend`, which caches it immediately, the
34
+ * entry matching `id` is always the caller itself.
35
+ * @internal
36
+ */
37
+ evictBackend(type, id) {
38
+ let cache;
39
+ switch (type) {
40
+ case 'Dataset':
41
+ cache = this.#datasetBackendCache;
42
+ break;
43
+ case 'KeyValueStore':
44
+ cache = this.#keyValueStoreBackendCache;
45
+ break;
46
+ case 'RequestQueue':
47
+ cache = this.#requestQueueBackendCache;
48
+ break;
49
+ }
50
+ const index = cache.findIndex((entry) => entry.id === id);
51
+ if (index === -1) {
52
+ return false;
53
+ }
54
+ cache.splice(index, 1);
55
+ return true;
56
+ }
57
+ static #resolveStorageKey(options) {
58
+ // No identifier at all means the default storage, which is opened under the reserved alias —
59
+ // same rule as `resolveStorageIdentifier` in the storage frontends, so that a backend used
60
+ // directly lands on the very storage the frontends would have opened.
61
+ const alias = options.alias || (!options.id && !options.name ? DEFAULT_STORAGE_ALIAS : undefined);
62
+ // `alias` covers the identifier-less case, so one of the three is always set.
63
+ const rawKey = alias ?? options.name ?? options.id;
64
+ // Normalize the internal __default__ alias to the user-facing 'default' name.
65
+ const cacheKey = rawKey === DEFAULT_STORAGE_ALIAS ? 'default' : rawKey;
66
+ return { isAlias: alias !== undefined, cacheKey };
67
+ }
68
+ async createDatasetBackend(options = {}) {
69
+ const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
70
+ const found = this.#datasetBackendCache.find((store) => store.id === cacheKey ||
71
+ store.name?.toLowerCase() === cacheKey.toLowerCase() ||
72
+ store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
73
+ if (found) {
74
+ return found;
75
+ }
76
+ const newStore = new DatasetBackend({
77
+ name: isAlias ? undefined : cacheKey,
78
+ cacheKey,
79
+ storageBackend: this,
80
+ });
81
+ this.#datasetBackendCache.push(newStore);
82
+ return newStore;
83
+ }
84
+ async createKeyValueStoreBackend(options = {}) {
85
+ const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
86
+ const found = this.#keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
87
+ store.name?.toLowerCase() === cacheKey.toLowerCase() ||
88
+ store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
89
+ if (found) {
90
+ return found;
91
+ }
92
+ const newStore = new KeyValueStoreBackend({
93
+ name: isAlias ? undefined : cacheKey,
94
+ cacheKey,
95
+ storageBackend: this,
96
+ });
97
+ this.#keyValueStoreBackendCache.push(newStore);
98
+ return newStore;
99
+ }
100
+ async createRequestQueueBackend(options = {}) {
101
+ const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
102
+ const found = this.#requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
103
+ queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
104
+ queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
105
+ if (found) {
106
+ return found;
107
+ }
108
+ const newStore = new RequestQueueBackend({
109
+ name: isAlias ? undefined : cacheKey,
110
+ cacheKey,
111
+ storageBackend: this,
112
+ });
113
+ this.#requestQueueBackendCache.push(newStore);
114
+ return newStore;
115
+ }
116
+ async storageExists(id, type) {
117
+ let backends;
118
+ switch (type) {
119
+ case 'Dataset':
120
+ backends = this.#datasetBackendCache;
121
+ break;
122
+ case 'KeyValueStore':
123
+ backends = this.#keyValueStoreBackendCache;
124
+ break;
125
+ case 'RequestQueue':
126
+ backends = this.#requestQueueBackendCache;
127
+ break;
128
+ default:
129
+ return false;
130
+ }
131
+ // In-memory storage only knows about backends in its cache.
132
+ return backends.some((store) => store.id === id);
133
+ }
134
+ /**
135
+ * Cleans up the run-scoped storages before the run starts. For the in-memory storage this simply
136
+ * resets the in-memory state of the cached backends.
137
+ */
138
+ async purge() {
139
+ // `#resolveStorageKey` leaves `name` unset for the default and alias-keyed storages, which is what
140
+ // marks them as run-scoped. `'default'` is the exception — it collapses onto the default storage.
141
+ const isRunScoped = (store) => store.name === undefined || store.name === 'default';
142
+ const isDefault = (store) => store.name === 'default' || store.cacheKey === 'default';
143
+ const purgeRunScoped = async (cache, purgeStore) => {
144
+ await Promise.all(cache.filter(isRunScoped).map(async (store) => purgeStore(store)));
145
+ };
146
+ await Promise.all([
147
+ // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
148
+ purgeRunScoped(this.#keyValueStoreBackendCache, async (store) => isDefault(store) ? store.purgeExceptInput() : store.purge()),
149
+ purgeRunScoped(this.#datasetBackendCache, async (store) => store.purge()),
150
+ purgeRunScoped(this.#requestQueueBackendCache, async (store) => store.purge()),
151
+ ]);
152
+ }
153
+ /**
154
+ * This method should be called at the end of the process. The in-memory storage holds no resources
155
+ * that outlive the process (no file handles, no cross-process locks), so there is nothing to do.
156
+ */
157
+ async teardown() {
158
+ // Nothing to tear down for in-memory storage.
159
+ }
160
+ }
@@ -0,0 +1,4 @@
1
+ export declare class BaseClient {
2
+ id: string;
3
+ constructor(id: string);
4
+ }
@@ -0,0 +1,6 @@
1
+ export class BaseClient {
2
+ id;
3
+ constructor(id) {
4
+ this.id = id;
5
+ }
6
+ }
@@ -0,0 +1,40 @@
1
+ import type * as storage from '@crawlee/types';
2
+ import type { Dictionary } from '@crawlee/types';
3
+ import type { MemoryStorageBackend } from '../memory-storage.js';
4
+ import { BaseClient } from './common/base-client.js';
5
+ export interface DatasetBackendOptions {
6
+ id?: string;
7
+ name?: string;
8
+ /**
9
+ * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
10
+ * This allows alias-opened storages to have a cache key that differs from their
11
+ * metadata `name` (which is `undefined` for unnamed storages).
12
+ */
13
+ cacheKey?: string;
14
+ storageBackend: MemoryStorageBackend;
15
+ }
16
+ export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends BaseClient implements storage.DatasetBackend<Data> {
17
+ #private;
18
+ name?: string;
19
+ /**
20
+ * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
21
+ * storages, this is the alias string. Falls back to id.
22
+ */
23
+ cacheKey: string;
24
+ createdAt: Date;
25
+ accessedAt: Date;
26
+ modifiedAt: Date;
27
+ itemCount: number;
28
+ private readonly storageBackend;
29
+ constructor(options: DatasetBackendOptions);
30
+ getMetadata(): Promise<storage.DatasetInfo>;
31
+ drop(): Promise<void>;
32
+ purge(): Promise<void>;
33
+ getData(options?: storage.DatasetBackendListOptions): Promise<storage.PaginatedList<Data>>;
34
+ private getDataPage;
35
+ pushData(items: Data[]): Promise<void>;
36
+ toDatasetInfo(): storage.DatasetInfo;
37
+ private generateLocalEntryName;
38
+ private getStartAndEndIndexes;
39
+ private updateTimestamps;
40
+ }
@@ -0,0 +1,106 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
3
+ import { BaseClient } from './common/base-client.js';
4
+ /**
5
+ * This is what API returns in the x-apify-pagination-limit
6
+ * header when no limit query parameter is used.
7
+ */
8
+ const LIST_ITEMS_LIMIT = 999_999_999_999;
9
+ /**
10
+ * Number of characters of the dataset item entry names.
11
+ * E.g.: 000000019 - 9 digits
12
+ */
13
+ const LOCAL_ENTRY_NAME_DIGITS = 9;
14
+ export class DatasetBackend extends BaseClient {
15
+ name;
16
+ /**
17
+ * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
18
+ * storages, this is the alias string. Falls back to id.
19
+ */
20
+ cacheKey;
21
+ createdAt = new Date();
22
+ accessedAt = new Date();
23
+ modifiedAt = new Date();
24
+ itemCount = 0;
25
+ #datasetEntries = new Map();
26
+ // kept as TS-private: storage-backend tests read this field at runtime
27
+ storageBackend;
28
+ constructor(options) {
29
+ super(options.id ?? randomUUID());
30
+ this.name = options.name;
31
+ this.cacheKey = options.cacheKey ?? this.name ?? this.id;
32
+ this.storageBackend = options.storageBackend;
33
+ }
34
+ async getMetadata() {
35
+ this.updateTimestamps(false);
36
+ return this.toDatasetInfo();
37
+ }
38
+ async drop() {
39
+ if (this.storageBackend.evictBackend('Dataset', this.id)) {
40
+ this.itemCount = 0;
41
+ this.#datasetEntries.clear();
42
+ }
43
+ }
44
+ async purge() {
45
+ this.itemCount = 0;
46
+ this.#datasetEntries.clear();
47
+ this.updateTimestamps(true);
48
+ }
49
+ getData(options = {}) {
50
+ const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
51
+ return this.getDataPage({
52
+ desc,
53
+ offset: offset ?? 0,
54
+ limit: Math.min(limit ?? LIST_ITEMS_LIMIT, LIST_ITEMS_LIMIT),
55
+ });
56
+ }
57
+ async getDataPage(options = {}) {
58
+ const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;
59
+ const [start, end] = this.getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
60
+ const items = [];
61
+ for (let idx = start; idx < end; idx++) {
62
+ const entryNumber = this.generateLocalEntryName(idx);
63
+ items.push(this.#datasetEntries.get(entryNumber));
64
+ }
65
+ this.updateTimestamps(false);
66
+ return {
67
+ count: items.length,
68
+ desc: desc ?? false,
69
+ items: desc ? items.reverse() : items,
70
+ limit,
71
+ offset,
72
+ total: this.itemCount,
73
+ };
74
+ }
75
+ async pushData(items) {
76
+ for (const entry of items) {
77
+ const idx = this.generateLocalEntryName(++this.itemCount);
78
+ this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
79
+ }
80
+ this.updateTimestamps(true);
81
+ }
82
+ toDatasetInfo() {
83
+ return {
84
+ id: this.id,
85
+ accessedAt: this.accessedAt,
86
+ createdAt: this.createdAt,
87
+ itemCount: this.itemCount,
88
+ modifiedAt: this.modifiedAt,
89
+ name: this.name,
90
+ };
91
+ }
92
+ generateLocalEntryName(idx) {
93
+ return idx.toString().padStart(LOCAL_ENTRY_NAME_DIGITS, '0');
94
+ }
95
+ getStartAndEndIndexes(offset, limit = this.itemCount) {
96
+ const start = offset + 1;
97
+ const end = Math.min(offset + limit, this.itemCount) + 1;
98
+ return [start, end];
99
+ }
100
+ updateTimestamps(hasBeenModified) {
101
+ this.accessedAt = new Date();
102
+ if (hasBeenModified) {
103
+ this.modifiedAt = new Date();
104
+ }
105
+ }
106
+ }
@@ -0,0 +1,63 @@
1
+ import type * as storage from '@crawlee/types';
2
+ import type { MemoryStorageBackend } from '../memory-storage.js';
3
+ import { BaseClient } from './common/base-client.js';
4
+ export interface KeyValueStoreBackendOptions {
5
+ name?: string;
6
+ id?: string;
7
+ /**
8
+ * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
9
+ * This allows alias-opened storages to have a cache key that differs from their
10
+ * metadata `name` (which is `undefined` for unnamed storages).
11
+ */
12
+ cacheKey?: string;
13
+ storageBackend: MemoryStorageBackend;
14
+ }
15
+ export interface InternalKeyRecord {
16
+ key: string;
17
+ value: Buffer;
18
+ contentType?: string;
19
+ extension: string;
20
+ }
21
+ export declare class KeyValueStoreBackend extends BaseClient implements storage.KeyValueStoreBackend {
22
+ #private;
23
+ name?: string;
24
+ /**
25
+ * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
26
+ * storages, this is the alias string. Falls back to id.
27
+ */
28
+ cacheKey: string;
29
+ createdAt: Date;
30
+ accessedAt: Date;
31
+ modifiedAt: Date;
32
+ private readonly storageBackend;
33
+ constructor(options: KeyValueStoreBackendOptions);
34
+ getMetadata(): Promise<storage.KeyValueStoreInfo>;
35
+ drop(): Promise<void>;
36
+ purge(): Promise<void>;
37
+ /**
38
+ * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
39
+ * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
40
+ * extension variants) when purging the default store. The in-memory key has no extension, so we
41
+ * preserve the bare `INPUT` key only.
42
+ */
43
+ purgeExceptInput(): Promise<void>;
44
+ listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
45
+ /**
46
+ * In-memory records are not file-backed, so there is no public file URL to return.
47
+ * Always resolves to `undefined`.
48
+ * @param key The key of the record to generate the public URL for.
49
+ */
50
+ getPublicUrl(key: string): Promise<string | undefined>;
51
+ /**
52
+ * Tests whether a record with the given key exists in the key-value store without retrieving its value.
53
+ *
54
+ * @param key The queried record key.
55
+ * @returns `true` if the record exists, `false` if it does not.
56
+ */
57
+ recordExists(key: string): Promise<boolean>;
58
+ getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
59
+ setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
60
+ deleteValue(key: string): Promise<void>;
61
+ toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
62
+ private updateTimestamps;
63
+ }
@@ -0,0 +1,199 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
3
+ import { z } from 'zod';
4
+ import { isStream, toBuffer } from '../utils.js';
5
+ import { BaseClient } from './common/base-client.js';
6
+ import mime from 'mime-types';
7
+ const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
8
+ const keySchema = z.string();
9
+ const inputRecordSchema = z.object({
10
+ key: z.string().min(1),
11
+ value: z.union([
12
+ z.null(),
13
+ z.string(),
14
+ z.number(),
15
+ z.instanceof(Buffer),
16
+ z.instanceof(ArrayBuffer),
17
+ schemas.typedArray,
18
+ // only checks the value is an actual object, not null, nor array
19
+ schemas.plainObject,
20
+ ]),
21
+ contentType: z.string().min(1).optional(),
22
+ });
23
+ /**
24
+ * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
25
+ * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
26
+ */
27
+ const KEY_VALUE_STORE_INPUT_KEY = 'INPUT';
28
+ export class KeyValueStoreBackend extends BaseClient {
29
+ name;
30
+ /**
31
+ * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
32
+ * storages, this is the alias string. Falls back to id.
33
+ */
34
+ cacheKey;
35
+ createdAt = new Date();
36
+ accessedAt = new Date();
37
+ modifiedAt = new Date();
38
+ #keyValueEntries = new Map();
39
+ // kept as TS-private: storage-backend tests read this field at runtime
40
+ storageBackend;
41
+ constructor(options) {
42
+ super(options.id ?? randomUUID());
43
+ this.name = options.name;
44
+ this.cacheKey = options.cacheKey ?? this.name ?? this.id;
45
+ this.storageBackend = options.storageBackend;
46
+ }
47
+ async getMetadata() {
48
+ this.updateTimestamps(false);
49
+ return this.toKeyValueStoreInfo();
50
+ }
51
+ async drop() {
52
+ if (this.storageBackend.evictBackend('KeyValueStore', this.id)) {
53
+ this.#keyValueEntries.clear();
54
+ }
55
+ }
56
+ async purge() {
57
+ this.#keyValueEntries.clear();
58
+ this.updateTimestamps(true);
59
+ }
60
+ /**
61
+ * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
62
+ * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
63
+ * extension variants) when purging the default store. The in-memory key has no extension, so we
64
+ * preserve the bare `INPUT` key only.
65
+ */
66
+ async purgeExceptInput() {
67
+ for (const key of this.#keyValueEntries.keys()) {
68
+ if (key !== KEY_VALUE_STORE_INPUT_KEY) {
69
+ this.#keyValueEntries.delete(key);
70
+ }
71
+ }
72
+ this.updateTimestamps(true);
73
+ }
74
+ async listKeys(options = {}) {
75
+ const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
76
+ const items = [];
77
+ for (const record of this.#keyValueEntries.values()) {
78
+ const size = Buffer.byteLength(record.value);
79
+ items.push({
80
+ key: record.key,
81
+ size,
82
+ contentType: record.contentType ?? 'application/octet-stream',
83
+ });
84
+ }
85
+ // Lexically sort to emulate API.
86
+ items.sort((a, b) => a.key.localeCompare(b.key));
87
+ let filteredItems = items.filter((item) => !prefix || item.key.startsWith(prefix));
88
+ if (exclusiveStartKey) {
89
+ const keyPos = filteredItems.findIndex((item) => item.key === exclusiveStartKey);
90
+ if (keyPos === -1) {
91
+ throw new Error(`exclusiveStartKey "${exclusiveStartKey}" was not found in the key-value store. ` +
92
+ `This is likely a bug — the key may have been deleted between paginated listKeys calls.`);
93
+ }
94
+ filteredItems = filteredItems.slice(keyPos + 1);
95
+ }
96
+ const isTruncated = limit !== undefined && filteredItems.length > limit;
97
+ const pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;
98
+ const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;
99
+ this.updateTimestamps(false);
100
+ return {
101
+ items: pageItems,
102
+ count: pageItems.length,
103
+ limit: limit ?? pageItems.length,
104
+ exclusiveStartKey,
105
+ isTruncated,
106
+ nextExclusiveStartKey,
107
+ };
108
+ }
109
+ /**
110
+ * In-memory records are not file-backed, so there is no public file URL to return.
111
+ * Always resolves to `undefined`.
112
+ * @param key The key of the record to generate the public URL for.
113
+ */
114
+ async getPublicUrl(key) {
115
+ parseArgument(key, keySchema);
116
+ return undefined;
117
+ }
118
+ /**
119
+ * Tests whether a record with the given key exists in the key-value store without retrieving its value.
120
+ *
121
+ * @param key The queried record key.
122
+ * @returns `true` if the record exists, `false` if it does not.
123
+ */
124
+ async recordExists(key) {
125
+ parseArgument(key, keySchema);
126
+ return this.#keyValueEntries.has(key);
127
+ }
128
+ async getValue(key) {
129
+ parseArgument(key, keySchema);
130
+ const entry = this.#keyValueEntries.get(key);
131
+ if (!entry) {
132
+ return undefined;
133
+ }
134
+ // Return raw bytes + verbatim content type. Parsing is the frontend's job (see the
135
+ // KeyValueStore codec). The mime fallback reconstructs the content type for on-disk records.
136
+ const record = {
137
+ key: entry.key,
138
+ value: entry.value,
139
+ // mime.contentType returns `false` for unknown extensions; fall back to undefined so the
140
+ // frontend treats it as "no content type" rather than a bogus value.
141
+ contentType: entry.contentType ?? (mime.contentType(entry.extension) || undefined),
142
+ };
143
+ this.updateTimestamps(false);
144
+ return record;
145
+ }
146
+ async setValue(record) {
147
+ parseArgument(record, inputRecordSchema);
148
+ const { key } = record;
149
+ let { value } = record;
150
+ // The frontend (KeyValueStore codec) serializes the value and resolves its content type
151
+ // before it reaches the backend. We only need it here for on-disk extension bookkeeping.
152
+ const contentType = record.contentType ?? 'application/octet-stream';
153
+ const extension = mime.extension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
154
+ // Draining a stream into a Buffer for storage is the backend's responsibility.
155
+ if (isStream(value)) {
156
+ const chunks = [];
157
+ for await (const chunk of value) {
158
+ chunks.push(chunk);
159
+ }
160
+ value = Buffer.concat(chunks);
161
+ }
162
+ // This backend is a byte transport: it stores and returns raw bytes regardless of the input
163
+ // shape. Streams were drained above; encode strings to UTF-8 bytes and normalize
164
+ // ArrayBuffer / typed-array views to a Buffer over the same memory.
165
+ const normalizedValue = typeof value === 'string'
166
+ ? Buffer.from(value, 'utf-8')
167
+ : toBuffer(value);
168
+ const normalizedRecord = {
169
+ extension,
170
+ key,
171
+ value: normalizedValue,
172
+ contentType,
173
+ };
174
+ this.#keyValueEntries.set(key, normalizedRecord);
175
+ this.updateTimestamps(true);
176
+ }
177
+ async deleteValue(key) {
178
+ parseArgument(key, keySchema);
179
+ if (this.#keyValueEntries.has(key)) {
180
+ this.#keyValueEntries.delete(key);
181
+ this.updateTimestamps(true);
182
+ }
183
+ }
184
+ toKeyValueStoreInfo() {
185
+ return {
186
+ id: this.id,
187
+ name: this.name,
188
+ accessedAt: this.accessedAt,
189
+ createdAt: this.createdAt,
190
+ modifiedAt: this.modifiedAt,
191
+ };
192
+ }
193
+ updateTimestamps(hasBeenModified) {
194
+ this.accessedAt = new Date();
195
+ if (hasBeenModified) {
196
+ this.modifiedAt = new Date();
197
+ }
198
+ }
199
+ }