@crawlee/core 4.0.0-beta.21 → 4.0.0-beta.211

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/README.md +14 -14
  2. package/byte_utils.d.ts +17 -0
  3. package/byte_utils.js +42 -0
  4. package/configuration.d.ts +88 -227
  5. package/configuration.js +162 -226
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +45 -18
  9. package/errors.js +58 -27
  10. package/events/event_manager.d.ts +35 -9
  11. package/events/event_manager.js +12 -13
  12. package/events/index.d.ts +1 -1
  13. package/events/index.js +0 -1
  14. package/events/local_event_manager.d.ts +15 -3
  15. package/events/local_event_manager.js +39 -13
  16. package/events/system_info.d.ts +38 -0
  17. package/events/system_info.js +1 -0
  18. package/index.d.ts +7 -11
  19. package/index.js +8 -10
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  35. package/memory-storage/resource-clients/dataset.js +106 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +56 -0
  37. package/memory-storage/resource-clients/key-value-store.js +180 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  39. package/memory-storage/resource-clients/request-queue.js +414 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned_or_injected.d.ts +58 -0
  43. package/owned_or_injected.js +98 -0
  44. package/package.json +19 -20
  45. package/proxy_configuration.d.ts +28 -77
  46. package/proxy_configuration.js +37 -146
  47. package/recoverable_state.d.ts +106 -52
  48. package/recoverable_state.js +200 -72
  49. package/request.d.ts +48 -93
  50. package/request.js +112 -230
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/service_locator.d.ts +168 -0
  54. package/service_locator.js +255 -0
  55. package/storages/batched_adds.d.ts +37 -0
  56. package/storages/batched_adds.js +73 -0
  57. package/storages/dataset.d.ts +115 -58
  58. package/storages/dataset.js +291 -149
  59. package/storages/index.d.ts +8 -9
  60. package/storages/index.js +4 -9
  61. package/storages/key_value_store.d.ts +186 -71
  62. package/storages/key_value_store.js +449 -194
  63. package/storages/key_value_store_codec.d.ts +32 -0
  64. package/storages/key_value_store_codec.js +113 -0
  65. package/storages/request_dedup_cache.d.ts +22 -0
  66. package/storages/request_dedup_cache.js +48 -0
  67. package/storages/request_list.d.ts +50 -133
  68. package/storages/request_list.js +269 -248
  69. package/storages/request_loader.d.ts +127 -0
  70. package/storages/request_loader.js +36 -0
  71. package/storages/request_manager.d.ts +119 -0
  72. package/storages/request_manager.js +1 -0
  73. package/storages/request_manager_tandem.d.ts +50 -24
  74. package/storages/request_manager_tandem.js +134 -58
  75. package/storages/request_queue.d.ts +293 -48
  76. package/storages/request_queue.js +766 -216
  77. package/storages/storage_instance_manager.d.ts +86 -0
  78. package/storages/storage_instance_manager.js +256 -0
  79. package/storages/storage_stats.d.ts +48 -0
  80. package/storages/storage_stats.js +29 -0
  81. package/storages/transaction.d.ts +271 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +75 -15
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -300
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -533
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -189
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -285
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -181
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -169
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -194
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -213
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -349
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -249
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -269
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -73
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -212
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts.map +0 -1
  168. package/events/event_manager.js.map +0 -1
  169. package/events/index.d.ts.map +0 -1
  170. package/events/index.js.map +0 -1
  171. package/events/local_event_manager.d.ts.map +0 -1
  172. package/events/local_event_manager.js.map +0 -1
  173. package/http_clients/base-http-client.d.ts +0 -12
  174. package/http_clients/base-http-client.d.ts.map +0 -1
  175. package/http_clients/base-http-client.js +0 -44
  176. package/http_clients/base-http-client.js.map +0 -1
  177. package/http_clients/form-data-like.d.ts +0 -67
  178. package/http_clients/form-data-like.d.ts.map +0 -1
  179. package/http_clients/form-data-like.js +0 -5
  180. package/http_clients/form-data-like.js.map +0 -1
  181. package/http_clients/got-scraping-http-client.d.ts +0 -22
  182. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  183. package/http_clients/got-scraping-http-client.js +0 -116
  184. package/http_clients/got-scraping-http-client.js.map +0 -1
  185. package/http_clients/index.d.ts +0 -3
  186. package/http_clients/index.d.ts.map +0 -1
  187. package/http_clients/index.js +0 -3
  188. package/http_clients/index.js.map +0 -1
  189. package/index.d.ts.map +0 -1
  190. package/index.js.map +0 -1
  191. package/log.d.ts.map +0 -1
  192. package/log.js.map +0 -1
  193. package/proxy_configuration.d.ts.map +0 -1
  194. package/proxy_configuration.js.map +0 -1
  195. package/recoverable_state.d.ts.map +0 -1
  196. package/recoverable_state.js.map +0 -1
  197. package/request.d.ts.map +0 -1
  198. package/request.js.map +0 -1
  199. package/router.d.ts +0 -134
  200. package/router.d.ts.map +0 -1
  201. package/router.js +0 -164
  202. package/router.js.map +0 -1
  203. package/serialization.d.ts.map +0 -1
  204. package/serialization.js.map +0 -1
  205. package/session_pool/consts.d.ts +0 -4
  206. package/session_pool/consts.d.ts.map +0 -1
  207. package/session_pool/consts.js +0 -4
  208. package/session_pool/consts.js.map +0 -1
  209. package/session_pool/errors.d.ts +0 -8
  210. package/session_pool/errors.d.ts.map +0 -1
  211. package/session_pool/errors.js +0 -12
  212. package/session_pool/errors.js.map +0 -1
  213. package/session_pool/events.d.ts +0 -3
  214. package/session_pool/events.d.ts.map +0 -1
  215. package/session_pool/events.js +0 -3
  216. package/session_pool/events.js.map +0 -1
  217. package/session_pool/index.d.ts +0 -6
  218. package/session_pool/index.d.ts.map +0 -1
  219. package/session_pool/index.js +0 -6
  220. package/session_pool/index.js.map +0 -1
  221. package/session_pool/session.d.ts +0 -186
  222. package/session_pool/session.d.ts.map +0 -1
  223. package/session_pool/session.js +0 -278
  224. package/session_pool/session.js.map +0 -1
  225. package/session_pool/session_pool.d.ts +0 -258
  226. package/session_pool/session_pool.d.ts.map +0 -1
  227. package/session_pool/session_pool.js +0 -387
  228. package/session_pool/session_pool.js.map +0 -1
  229. package/storages/access_checking.d.ts +0 -13
  230. package/storages/access_checking.d.ts.map +0 -1
  231. package/storages/access_checking.js +0 -14
  232. package/storages/access_checking.js.map +0 -1
  233. package/storages/dataset.d.ts.map +0 -1
  234. package/storages/dataset.js.map +0 -1
  235. package/storages/index.d.ts.map +0 -1
  236. package/storages/index.js.map +0 -1
  237. package/storages/key_value_store.d.ts.map +0 -1
  238. package/storages/key_value_store.js.map +0 -1
  239. package/storages/request_list.d.ts.map +0 -1
  240. package/storages/request_list.js.map +0 -1
  241. package/storages/request_list_adapter.d.ts +0 -58
  242. package/storages/request_list_adapter.d.ts.map +0 -1
  243. package/storages/request_list_adapter.js +0 -81
  244. package/storages/request_list_adapter.js.map +0 -1
  245. package/storages/request_manager_tandem.d.ts.map +0 -1
  246. package/storages/request_manager_tandem.js.map +0 -1
  247. package/storages/request_provider.d.ts +0 -372
  248. package/storages/request_provider.d.ts.map +0 -1
  249. package/storages/request_provider.js +0 -591
  250. package/storages/request_provider.js.map +0 -1
  251. package/storages/request_queue.d.ts.map +0 -1
  252. package/storages/request_queue.js.map +0 -1
  253. package/storages/request_queue_v2.d.ts +0 -87
  254. package/storages/request_queue_v2.d.ts.map +0 -1
  255. package/storages/request_queue_v2.js +0 -438
  256. package/storages/request_queue_v2.js.map +0 -1
  257. package/storages/sitemap_request_list.d.ts +0 -250
  258. package/storages/sitemap_request_list.d.ts.map +0 -1
  259. package/storages/sitemap_request_list.js +0 -431
  260. package/storages/sitemap_request_list.js.map +0 -1
  261. package/storages/storage_manager.d.ts +0 -62
  262. package/storages/storage_manager.d.ts.map +0 -1
  263. package/storages/storage_manager.js +0 -105
  264. package/storages/storage_manager.js.map +0 -1
  265. package/storages/utils.d.ts.map +0 -1
  266. package/storages/utils.js.map +0 -1
  267. package/typedefs.d.ts.map +0 -1
  268. package/typedefs.js.map +0 -1
  269. package/validators.d.ts.map +0 -1
  270. package/validators.js.map +0 -1
@@ -1,5 +1,32 @@
1
- import { Configuration } from '@crawlee/core';
2
- import type { Log } from '@apify/log';
1
+ import type { Configuration } from './configuration.js';
2
+ import type { CrawleeLogger } from './log.js';
3
+ import { KeyValueStore } from './storages/key_value_store.js';
4
+ import type { Awaitable } from '@crawlee/types';
5
+ import type { StandardSchemaV1 } from '@standard-schema/spec';
6
+ /**
7
+ * One direction of the conversion between the state model and its persisted form - either a plain function, or a
8
+ * [Standard Schema](https://standardschema.dev) whose validated output is the result.
9
+ *
10
+ * A schema that fails to validate makes {@link RecoverableState} throw a {@link StateValidationError}. Zod
11
+ * codecs work directly, as their validation *is* the decode direction; use `(state) => codec.encode(state)` for the
12
+ * other one.
13
+ */
14
+ export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;
15
+ /**
16
+ * A {@link StateConversion} for a caller that cannot await one - {@link Statistics}, whose `toJSON()` is
17
+ * synchronous, being the reason this exists.
18
+ *
19
+ * Only the function arm can be narrowed here: a Standard Schema is free to validate asynchronously, so a schema
20
+ * that does is rejected when it runs rather than when it is passed.
21
+ */
22
+ export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;
23
+ /**
24
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
25
+ * the value.
26
+ *
27
+ * @internal
28
+ */
29
+ export declare function convertStateSync<TFrom, TTo>(conversion: SyncStateConversion<TFrom, TTo>, value: TFrom, persistStateKey: string): TTo;
3
30
  export interface RecoverableStatePersistenceOptions {
4
31
  /**
5
32
  * The key under which the state is stored in the KeyValueStore
@@ -10,45 +37,66 @@ export interface RecoverableStatePersistenceOptions {
10
37
  */
11
38
  persistenceEnabled?: boolean;
12
39
  /**
13
- * The name of the KeyValueStore to use for persistence.
14
- * If neither a name nor an id are supplied, the default store will be used.
40
+ * The KeyValueStore to persist into, defaulting to the default store. Accepts a pending
41
+ * {@link KeyValueStore.open} so that callers do not have to be async to point at a specific store.
15
42
  */
16
- persistStateKvsName?: string;
43
+ keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
17
44
  /**
18
- * The identifier of the KeyValueStore to use for persistence.
19
- * If neither a name nor an id are supplied, the default store will be used.
45
+ * Time limit for a single load or save of the state, in milliseconds.
46
+ * @default 60_000
20
47
  */
21
- persistStateKvsId?: string;
48
+ persistenceTimeoutMillis?: number;
22
49
  }
23
50
  /**
24
- * Options for configuring the RecoverableState
51
+ * The fields of {@link RecoverableStateOptions}, without the constraint tying `contentType` to the conversions.
25
52
  */
26
- export interface RecoverableStateOptions<TStateModel = Record<string, unknown>> extends RecoverableStatePersistenceOptions {
53
+ export interface RecoverableStateBaseOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
27
54
  /**
28
- * The default state used if no persisted state is found.
29
- * A deep copy is made each time the state is used.
55
+ * The state used when no persisted state is found, and the state {@link RecoverableState.reset} restores.
56
+ *
57
+ * A plain value is deep-copied with `structuredClone` each time it is used, so pass a factory for a state
58
+ * that `structuredClone` cannot rebuild - one holding class instances, say, or one derived from a schema.
30
59
  */
31
- defaultState: TStateModel;
60
+ defaultState: TStateModel | (() => TStateModel);
32
61
  /**
33
62
  * A logger instance for logging operations related to state persistence
34
63
  */
35
- logger?: Log;
64
+ logger?: CrawleeLogger;
65
+ /**
66
+ * Configuration instance to use when opening the KeyValueStore
67
+ */
68
+ configuration?: Configuration;
36
69
  /**
37
- * Configuration instance to use
70
+ * Optional conversion of the state to a plain JSON-serializable value before it is persisted.
71
+ * If not provided, the state is persisted as is.
72
+ *
73
+ * With {@link RecoverableStateBaseOptions.contentType} set, it has to produce what
74
+ * {@link KeyValueStore.setValue} accepts alongside an explicit content type - a `string`, a `Buffer` or a
75
+ * stream.
38
76
  */
39
- config?: Configuration;
77
+ serialize?: StateConversion<TStateModel, TPersistedState>;
40
78
  /**
41
- * Optional function to transform the state to a JSON string before persistence.
42
- * If not provided, JSON.stringify will be used.
79
+ * Optional conversion of a persisted value back to the state model, and the place to validate a record before
80
+ * trusting it. If not provided, the persisted value is used as is.
81
+ *
82
+ * With {@link RecoverableStateBaseOptions.contentType} set, it receives a `Readable` of the record bytes
83
+ * instead of a parsed value.
43
84
  */
44
- serialize?: (state: TStateModel) => string;
85
+ deserialize?: StateConversion<TPersistedState, TStateModel>;
45
86
  /**
46
- * Optional function to transform a JSON-serialized object back to the state model.
47
- * If not provided, JSON.parse is used.
48
- * It is advisable to perform validation in this function and to throw an exception if it fails.
87
+ * Content type of the persisted record. Setting it hands the record encoding over to
88
+ * {@link RecoverableStateBaseOptions.serialize} and {@link RecoverableStateBaseOptions.deserialize}, both of
89
+ * which are then required - the default JSON codec is bypassed in both directions. Meant for a state too large
90
+ * for `JSON.stringify`, which `serialize` can then stream out instead.
49
91
  */
50
- deserialize?: (serializedState: string) => TStateModel;
92
+ contentType?: string;
51
93
  }
94
+ /**
95
+ * Options for configuring the RecoverableState
96
+ */
97
+ export type RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> = RecoverableStateBaseOptions<TStateModel, TPersistedState> & ({
98
+ contentType?: undefined;
99
+ } | Required<Pick<RecoverableStateBaseOptions<TStateModel, TPersistedState>, 'serialize' | 'deserialize' | 'contentType'>>);
52
100
  /**
53
101
  * A class for managing persistent recoverable state using a plain JavaScript object.
54
102
  *
@@ -59,29 +107,24 @@ export interface RecoverableStateOptions<TStateModel = Record<string, unknown>>
59
107
  * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
60
108
  * The class automatically hooks into the event system to persist state when needed.
61
109
  */
62
- export declare class RecoverableState<TStateModel = Record<string, unknown>> {
63
- private readonly defaultState;
64
- private state;
65
- private readonly persistenceEnabled;
66
- private readonly persistStateKey;
67
- private readonly persistStateKvsName?;
68
- private readonly persistStateKvsId?;
69
- private keyValueStore;
70
- private readonly log;
71
- private readonly config;
72
- private readonly serialize;
73
- private readonly deserialize;
110
+ export declare class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
111
+ #private;
74
112
  /**
75
113
  * Initialize a new recoverable state object.
76
114
  *
77
115
  * @param options Configuration options for the recoverable state
78
116
  */
79
- constructor(options: RecoverableStateOptions<TStateModel>);
117
+ constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
80
118
  /**
81
119
  * Initialize the recoverable state.
82
120
  *
83
- * This method must be called before using the recoverable state. It loads the saved state
84
- * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
121
+ * If persistence is enabled, this method loads the saved state and registers the object to listen for
122
+ * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
123
+ * is no record to restore.
124
+ *
125
+ * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
126
+ * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
127
+ * reloading it would only drop whatever the deserialization leaves out.
85
128
  *
86
129
  * @returns The loaded state object
87
130
  */
@@ -90,34 +133,45 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>> {
90
133
  * Clean up resources used by the recoverable state.
91
134
  *
92
135
  * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
93
- * and persists the current state one last time.
136
+ * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
137
+ * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
138
+ * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
139
+ * persistence window.
94
140
  */
95
141
  teardown(): Promise<void>;
96
142
  /**
97
143
  * Get the current state.
144
+ *
145
+ * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
146
+ * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
147
+ * constructor gets a usable state right away.
98
148
  */
99
149
  get currentValue(): TStateModel;
100
150
  /**
101
- * Reset the state to the default values and clear any persisted state.
151
+ * Reset the in-memory state to the default values, leaving any persisted record alone.
102
152
  *
103
- * Resets the current state to the default state and, if persistence is enabled,
104
- * clears the persisted state from the KeyValueStore.
153
+ * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
105
154
  */
106
- reset(): Promise<void>;
155
+ reset(): void;
156
+ /**
157
+ * Clear the persisted state record, leaving the in-memory state alone.
158
+ *
159
+ * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
160
+ * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
161
+ * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
162
+ * or {@link RecoverableState.teardown} before clearing the record.
163
+ *
164
+ * A no-op if persistence is disabled.
165
+ */
166
+ resetStore(): Promise<void>;
107
167
  /**
108
168
  * Persist the current state to the KeyValueStore.
109
169
  *
110
170
  * This method is typically called in response to a PERSIST_STATE event, but can also be called
111
- * directly when needed.
171
+ * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
172
+ * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
112
173
  *
113
174
  * @param eventData Optional data associated with a PERSIST_STATE event
114
175
  */
115
- persistState(eventData?: {
116
- isMigrating: boolean;
117
- }): Promise<void>;
118
- /**
119
- * Load the saved state from the KeyValueStore
120
- */
121
- private loadSavedState;
176
+ persistState(eventData?: Record<string, unknown>): Promise<void>;
122
177
  }
123
- //# sourceMappingURL=recoverable_state.d.ts.map
@@ -1,5 +1,29 @@
1
- import { Configuration, KeyValueStore } from '@crawlee/core';
2
- import log from '@apify/log';
1
+ import { Readable } from 'node:stream';
2
+ import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
3
+ import { StateValidationError } from './errors.js';
4
+ import { EventType } from './events/event_manager.js';
5
+ import { serviceLocator } from './service_locator.js';
6
+ import { KeyValueStore } from './storages/key_value_store.js';
7
+ const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
8
+ /**
9
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
10
+ * the value.
11
+ *
12
+ * @internal
13
+ */
14
+ export function convertStateSync(conversion, value, persistStateKey) {
15
+ if (typeof conversion === 'function') {
16
+ return conversion(value);
17
+ }
18
+ const result = conversion['~standard'].validate(value);
19
+ if ('then' in result) {
20
+ throw new Error(`The state conversion for '${persistStateKey}' validated asynchronously, which this caller cannot await.`);
21
+ }
22
+ if (result.issues) {
23
+ throw new StateValidationError(persistStateKey, result.issues);
24
+ }
25
+ return result.value;
26
+ }
3
27
  /**
4
28
  * A class for managing persistent recoverable state using a plain JavaScript object.
5
29
  *
@@ -11,130 +35,234 @@ import log from '@apify/log';
11
35
  * The class automatically hooks into the event system to persist state when needed.
12
36
  */
13
37
  export class RecoverableState {
14
- defaultState;
15
- state = null;
16
- persistenceEnabled;
17
- persistStateKey;
18
- persistStateKvsName;
19
- persistStateKvsId;
20
- keyValueStore = null;
21
- log;
22
- config;
23
- serialize;
24
- deserialize;
38
+ #defaultState;
39
+ #state = null;
40
+ #initialized = false;
41
+ #recordLoaded = false;
42
+ #listening = false;
43
+ #persistenceEnabled;
44
+ #persistStateKey;
45
+ #persistenceTimeoutMillis;
46
+ #configuration;
47
+ #keyValueStore;
48
+ #log;
49
+ #serialize;
50
+ #deserialize;
51
+ #contentType;
52
+ #persistStateQuietly;
25
53
  /**
26
54
  * Initialize a new recoverable state object.
27
55
  *
28
56
  * @param options Configuration options for the recoverable state
29
57
  */
30
58
  constructor(options) {
31
- this.defaultState = options.defaultState;
32
- this.persistStateKey = options.persistStateKey;
33
- this.persistenceEnabled = options.persistenceEnabled ?? false;
34
- this.persistStateKvsName = options.persistStateKvsName;
35
- this.persistStateKvsId = options.persistStateKvsId;
36
- this.log = options.logger ?? log.child({ prefix: 'RecoverableState' });
37
- this.config = options.config ?? Configuration.getGlobalConfig();
38
- this.serialize = options.serialize ?? JSON.stringify;
39
- this.deserialize = options.deserialize ?? JSON.parse;
40
- this.persistState = this.persistState.bind(this);
59
+ const { defaultState } = options;
60
+ this.#defaultState =
61
+ typeof defaultState === 'function'
62
+ ? defaultState
63
+ : () => structuredClone(defaultState);
64
+ this.#persistStateKey = options.persistStateKey;
65
+ this.#persistenceEnabled = options.persistenceEnabled ?? false;
66
+ this.#persistenceTimeoutMillis = options.persistenceTimeoutMillis ?? DEFAULT_PERSISTENCE_TIMEOUT_MILLIS;
67
+ this.#configuration = options.configuration;
68
+ this.#keyValueStore = options.keyValueStore ?? null;
69
+ this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
70
+ this.#contentType = options.contentType;
71
+ if (this.#contentType !== undefined && (options.serialize === undefined || options.deserialize === undefined)) {
72
+ throw new Error(`A 'contentType' for the state persisted under key '${this.#persistStateKey}' requires both 'serialize' and 'deserialize' - the record is no longer JSON the default codec can handle.`);
73
+ }
74
+ this.#serialize = this.#toConversion(options.serialize);
75
+ this.#deserialize = this.#toConversion(options.deserialize);
76
+ // A pending open() may reject before anything here awaits it. Observe the rejection so it surfaces on
77
+ // first use rather than as an unhandled rejection - most immediately when persistence is disabled and
78
+ // nothing ever awaits the store.
79
+ if (this.#keyValueStore !== null) {
80
+ Promise.resolve(this.#keyValueStore).then(undefined, () => { });
81
+ }
82
+ // The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch
83
+ // listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.
84
+ this.#persistStateQuietly = async (eventData) => this.persistState(eventData).catch((error) => this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }));
85
+ }
86
+ /** Normalizes a conversion option into a function. Absent conversions pass the value through unchanged. */
87
+ #toConversion(conversion) {
88
+ if (conversion === undefined) {
89
+ return async (value) => value;
90
+ }
91
+ if (typeof conversion === 'function') {
92
+ return async (value) => conversion(value);
93
+ }
94
+ return async (value) => {
95
+ const result = await conversion['~standard'].validate(value);
96
+ if (result.issues) {
97
+ throw new StateValidationError(this.#persistStateKey, result.issues);
98
+ }
99
+ return result.value;
100
+ };
41
101
  }
42
102
  /**
43
103
  * Initialize the recoverable state.
44
104
  *
45
- * This method must be called before using the recoverable state. It loads the saved state
46
- * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
105
+ * If persistence is enabled, this method loads the saved state and registers the object to listen for
106
+ * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
107
+ * is no record to restore.
108
+ *
109
+ * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
110
+ * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
111
+ * reloading it would only drop whatever the deserialization leaves out.
47
112
  *
48
113
  * @returns The loaded state object
49
114
  */
50
115
  async initialize() {
51
- if (this.state !== null && this.state !== undefined) {
116
+ if (this.#initialized) {
52
117
  return this.currentValue;
53
118
  }
54
- if (!this.persistenceEnabled) {
55
- this.state = this.deserialize(this.serialize(this.defaultState));
56
- return this.currentValue;
119
+ if (this.#persistenceEnabled) {
120
+ await this.#openKeyValueStore();
121
+ serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);
122
+ this.#listening = true;
123
+ }
124
+ // Flipped before the record is loaded, so that a caller catching a `StateValidationError` is left with a
125
+ // fully wired object running on the default state rather than a half-initialized one.
126
+ this.#initialized = true;
127
+ this.#state ??= this.#defaultState();
128
+ if (!this.#recordLoaded) {
129
+ this.#recordLoaded = true;
130
+ await this.#loadSavedState();
57
131
  }
58
- this.keyValueStore = await KeyValueStore.open(this.persistStateKvsName ?? this.persistStateKvsId, {
59
- config: this.config,
60
- });
61
- await this.loadSavedState();
62
- // Register for persist state events
63
- const eventManager = this.config.getEventManager();
64
- eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
65
132
  return this.currentValue;
66
133
  }
67
134
  /**
68
135
  * Clean up resources used by the recoverable state.
69
136
  *
70
137
  * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
71
- * and persists the current state one last time.
138
+ * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
139
+ * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
140
+ * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
141
+ * persistence window.
72
142
  */
73
143
  async teardown() {
74
- if (!this.persistenceEnabled || !this.persistState) {
144
+ this.#initialized = false;
145
+ if (!this.#persistenceEnabled) {
75
146
  return;
76
147
  }
77
- const eventManager = this.config.getEventManager();
78
- eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
79
- await this.persistState();
148
+ serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.#persistStateQuietly);
149
+ this.#listening = false;
150
+ await this.#persistStateQuietly();
80
151
  }
81
152
  /**
82
153
  * Get the current state.
154
+ *
155
+ * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
156
+ * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
157
+ * constructor gets a usable state right away.
83
158
  */
84
159
  get currentValue() {
85
- if (this.state === null) {
86
- throw new Error('Recoverable state has not yet been loaded');
160
+ if (this.#state === null) {
161
+ throw new Error('Recoverable state has not yet been loaded - call initialize() or reset() first');
87
162
  }
88
- return this.state;
163
+ return this.#state;
89
164
  }
90
165
  /**
91
- * Reset the state to the default values and clear any persisted state.
166
+ * Reset the in-memory state to the default values, leaving any persisted record alone.
92
167
  *
93
- * Resets the current state to the default state and, if persistence is enabled,
94
- * clears the persisted state from the KeyValueStore.
168
+ * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
95
169
  */
96
- async reset() {
97
- this.state = this.deserialize(this.serialize(this.defaultState));
98
- if (this.persistenceEnabled) {
99
- if (this.keyValueStore === null) {
100
- throw new Error('Recoverable state has not yet been initialized');
101
- }
102
- await this.keyValueStore.setValue(this.persistStateKey, null);
170
+ reset() {
171
+ this.#state = this.#defaultState();
172
+ }
173
+ /**
174
+ * Clear the persisted state record, leaving the in-memory state alone.
175
+ *
176
+ * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
177
+ * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
178
+ * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
179
+ * or {@link RecoverableState.teardown} before clearing the record.
180
+ *
181
+ * A no-op if persistence is disabled.
182
+ */
183
+ async resetStore() {
184
+ if (this.#listening) {
185
+ throw new Error(`Cannot clear the state persisted under key '${this.#persistStateKey}' while it is still being persisted periodically - the next PERSIST_STATE event would write it straight back. Use reset() to reset the state itself, or teardown() before clearing the record.`);
186
+ }
187
+ if (!this.#persistenceEnabled) {
188
+ return;
103
189
  }
190
+ const keyValueStore = await this.#openKeyValueStore();
191
+ await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, null), 'Clearing the persisted state');
104
192
  }
105
193
  /**
106
194
  * Persist the current state to the KeyValueStore.
107
195
  *
108
196
  * This method is typically called in response to a PERSIST_STATE event, but can also be called
109
- * directly when needed.
197
+ * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
198
+ * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
110
199
  *
111
200
  * @param eventData Optional data associated with a PERSIST_STATE event
112
201
  */
113
202
  async persistState(eventData) {
114
- this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
115
- if (this.keyValueStore === null || this.state === null) {
116
- throw new Error('Recoverable state has not yet been initialized');
203
+ if (!this.#persistenceEnabled || this.#state === null) {
204
+ return;
117
205
  }
118
- if (this.persistenceEnabled) {
119
- await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
120
- contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
121
- });
206
+ const keyValueStore = await this.#resolveKeyValueStore();
207
+ if (keyValueStore === null) {
208
+ return;
209
+ }
210
+ this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
211
+ const serializedState = await this.#serialize(this.currentValue);
212
+ await this.#withTimeout(async () => this.#contentType === undefined
213
+ ? keyValueStore.setValue(this.#persistStateKey, serializedState)
214
+ : keyValueStore.setValue(this.#persistStateKey, serializedState, {
215
+ contentType: this.#contentType,
216
+ }), 'Persisting the state');
217
+ }
218
+ /** Falls back to the default store when none was supplied - persistence has to write somewhere. */
219
+ async #openKeyValueStore() {
220
+ this.#keyValueStore ??= KeyValueStore.open(null, {
221
+ configuration: this.#configuration ?? serviceLocator.getConfiguration(),
222
+ });
223
+ return (await this.#resolveKeyValueStore());
224
+ }
225
+ /** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */
226
+ async #resolveKeyValueStore() {
227
+ if (this.#keyValueStore === null) {
228
+ return null;
122
229
  }
230
+ this.#keyValueStore = await this.#keyValueStore;
231
+ return this.#keyValueStore;
123
232
  }
124
233
  /**
125
- * Load the saved state from the KeyValueStore
234
+ * Load the saved state from the KeyValueStore. Leaves the current state alone if there is no record to load.
126
235
  */
127
- async loadSavedState() {
128
- if (this.keyValueStore === null) {
129
- throw new Error('Recoverable state has not yet been initialized');
236
+ async #loadSavedState() {
237
+ if (!this.#persistenceEnabled) {
238
+ return;
130
239
  }
131
- const storedState = await this.keyValueStore.getValue(this.persistStateKey);
132
- if (storedState === null || storedState === undefined) {
133
- this.state = this.deserialize(this.serialize(this.defaultState));
240
+ const keyValueStore = await this.#resolveKeyValueStore();
241
+ if (keyValueStore === null) {
242
+ return;
134
243
  }
135
- else {
136
- this.state = this.deserialize(storedState);
244
+ // With a content type, the record is whatever `serialize` produced, so the codec's parse is skipped and
245
+ // `deserialize` gets the bytes as a stream - the shape a streaming parser wants.
246
+ // TODO: read the record as a stream instead of buffering it and wrapping (https://github.com/apify/crawlee/issues/2929).
247
+ const storedState = await this.#withTimeout(async () => {
248
+ if (this.#contentType === undefined) {
249
+ return keyValueStore.getValue(this.#persistStateKey);
250
+ }
251
+ const record = await keyValueStore.getRecord(this.#persistStateKey);
252
+ if (record === null) {
253
+ return null;
254
+ }
255
+ return Readable.from(Buffer.isBuffer(record.value) ? record.value : Buffer.from(record.value));
256
+ }, 'Loading the persisted state');
257
+ if (storedState === null || storedState === undefined) {
258
+ return;
137
259
  }
260
+ this.#state = await this.#deserialize(storedState);
261
+ }
262
+ async #withTimeout(operation, description) {
263
+ // `@apify/timeout` shares one `AbortController` across nested frames and `KeyValueStore` checks it on
264
+ // every operation, so a teardown-time persist running inside an already-expired request handler timeout
265
+ // would be aborted before it started. Hence a fresh timeout context.
266
+ return timeoutStorage.exit(async () => addTimeoutToPromise(operation, this.#persistenceTimeoutMillis, `${description} under key '${this.#persistStateKey}' timed out after ${this.#persistenceTimeoutMillis / 1000} seconds.`));
138
267
  }
139
268
  }
140
- //# sourceMappingURL=recoverable_state.js.map