@crawlee/core 4.0.0-beta.26 → 4.0.0-beta.260

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. package/README.md +15 -15
  2. package/byte-utils.d.ts +17 -0
  3. package/byte-utils.js +42 -0
  4. package/configuration.d.ts +77 -235
  5. package/configuration.js +159 -227
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +45 -18
  9. package/errors.js +58 -27
  10. package/events/event-manager.d.ts +78 -0
  11. package/events/{event_manager.js → event-manager.js} +12 -13
  12. package/events/index.d.ts +3 -3
  13. package/events/index.js +2 -3
  14. package/events/local-event-manager.d.ts +32 -0
  15. package/events/local-event-manager.js +101 -0
  16. package/events/system-info.d.ts +38 -0
  17. package/events/system-info.js +1 -0
  18. package/index.d.ts +9 -12
  19. package/index.js +10 -11
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +35 -0
  35. package/memory-storage/resource-clients/dataset.js +108 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +54 -0
  37. package/memory-storage/resource-clients/key-value-store.js +179 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +55 -0
  39. package/memory-storage/resource-clients/request-queue.js +413 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned-or-injected.d.ts +58 -0
  43. package/owned-or-injected.js +98 -0
  44. package/package.json +23 -23
  45. package/proxy-configuration.d.ts +118 -0
  46. package/proxy-configuration.js +141 -0
  47. package/recoverable-state.d.ts +177 -0
  48. package/recoverable-state.js +268 -0
  49. package/request.d.ts +48 -93
  50. package/request.js +139 -232
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/service-locator.d.ts +168 -0
  54. package/service-locator.js +255 -0
  55. package/storages/batched-adds.d.ts +39 -0
  56. package/storages/batched-adds.js +78 -0
  57. package/storages/dataset.d.ts +105 -58
  58. package/storages/dataset.js +292 -150
  59. package/storages/index.d.ts +11 -12
  60. package/storages/index.js +7 -12
  61. package/storages/key-value-store-codec.d.ts +32 -0
  62. package/storages/key-value-store-codec.js +108 -0
  63. package/storages/{key_value_store.d.ts → key-value-store.d.ts} +169 -72
  64. package/storages/key-value-store.js +767 -0
  65. package/storages/request-dedup-cache.d.ts +20 -0
  66. package/storages/request-dedup-cache.js +48 -0
  67. package/storages/{request_list.d.ts → request-list.d.ts} +33 -173
  68. package/storages/{request_list.js → request-list.js} +276 -255
  69. package/storages/request-loader.d.ts +127 -0
  70. package/storages/request-loader.js +36 -0
  71. package/storages/request-manager-tandem.d.ts +82 -0
  72. package/storages/request-manager-tandem.js +200 -0
  73. package/storages/request-manager.d.ts +119 -0
  74. package/storages/request-manager.js +1 -0
  75. package/storages/{request_provider.d.ts → request-queue.d.ts} +128 -159
  76. package/storages/request-queue.js +871 -0
  77. package/storages/storage-instance-manager.d.ts +86 -0
  78. package/storages/storage-instance-manager.js +256 -0
  79. package/storages/storage-stats.d.ts +48 -0
  80. package/storages/storage-stats.js +29 -0
  81. package/storages/transaction.d.ts +270 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +76 -16
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -300
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -533
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -189
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -285
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -181
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -169
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -191
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -213
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -349
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -249
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -269
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -73
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -212
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts +0 -52
  168. package/events/event_manager.d.ts.map +0 -1
  169. package/events/event_manager.js.map +0 -1
  170. package/events/index.d.ts.map +0 -1
  171. package/events/index.js.map +0 -1
  172. package/events/local_event_manager.d.ts +0 -27
  173. package/events/local_event_manager.d.ts.map +0 -1
  174. package/events/local_event_manager.js +0 -75
  175. package/events/local_event_manager.js.map +0 -1
  176. package/index.d.ts.map +0 -1
  177. package/index.js.map +0 -1
  178. package/log.d.ts.map +0 -1
  179. package/log.js.map +0 -1
  180. package/proxy_configuration.d.ts +0 -169
  181. package/proxy_configuration.d.ts.map +0 -1
  182. package/proxy_configuration.js +0 -247
  183. package/proxy_configuration.js.map +0 -1
  184. package/recoverable_state.d.ts +0 -123
  185. package/recoverable_state.d.ts.map +0 -1
  186. package/recoverable_state.js +0 -140
  187. package/recoverable_state.js.map +0 -1
  188. package/request.d.ts.map +0 -1
  189. package/request.js.map +0 -1
  190. package/router.d.ts +0 -134
  191. package/router.d.ts.map +0 -1
  192. package/router.js +0 -164
  193. package/router.js.map +0 -1
  194. package/serialization.d.ts.map +0 -1
  195. package/serialization.js.map +0 -1
  196. package/session_pool/consts.d.ts +0 -4
  197. package/session_pool/consts.d.ts.map +0 -1
  198. package/session_pool/consts.js +0 -4
  199. package/session_pool/consts.js.map +0 -1
  200. package/session_pool/errors.d.ts +0 -8
  201. package/session_pool/errors.d.ts.map +0 -1
  202. package/session_pool/errors.js +0 -12
  203. package/session_pool/errors.js.map +0 -1
  204. package/session_pool/events.d.ts +0 -3
  205. package/session_pool/events.d.ts.map +0 -1
  206. package/session_pool/events.js +0 -3
  207. package/session_pool/events.js.map +0 -1
  208. package/session_pool/index.d.ts +0 -6
  209. package/session_pool/index.d.ts.map +0 -1
  210. package/session_pool/index.js +0 -6
  211. package/session_pool/index.js.map +0 -1
  212. package/session_pool/session.d.ts +0 -186
  213. package/session_pool/session.d.ts.map +0 -1
  214. package/session_pool/session.js +0 -278
  215. package/session_pool/session.js.map +0 -1
  216. package/session_pool/session_pool.d.ts +0 -258
  217. package/session_pool/session_pool.d.ts.map +0 -1
  218. package/session_pool/session_pool.js +0 -387
  219. package/session_pool/session_pool.js.map +0 -1
  220. package/storages/access_checking.d.ts +0 -13
  221. package/storages/access_checking.d.ts.map +0 -1
  222. package/storages/access_checking.js +0 -14
  223. package/storages/access_checking.js.map +0 -1
  224. package/storages/dataset.d.ts.map +0 -1
  225. package/storages/dataset.js.map +0 -1
  226. package/storages/index.d.ts.map +0 -1
  227. package/storages/index.js.map +0 -1
  228. package/storages/key_value_store.d.ts.map +0 -1
  229. package/storages/key_value_store.js +0 -512
  230. package/storages/key_value_store.js.map +0 -1
  231. package/storages/request_list.d.ts.map +0 -1
  232. package/storages/request_list.js.map +0 -1
  233. package/storages/request_list_adapter.d.ts +0 -58
  234. package/storages/request_list_adapter.d.ts.map +0 -1
  235. package/storages/request_list_adapter.js +0 -81
  236. package/storages/request_list_adapter.js.map +0 -1
  237. package/storages/request_manager_tandem.d.ts +0 -68
  238. package/storages/request_manager_tandem.d.ts.map +0 -1
  239. package/storages/request_manager_tandem.js +0 -124
  240. package/storages/request_manager_tandem.js.map +0 -1
  241. package/storages/request_provider.d.ts.map +0 -1
  242. package/storages/request_provider.js +0 -591
  243. package/storages/request_provider.js.map +0 -1
  244. package/storages/request_queue.d.ts +0 -128
  245. package/storages/request_queue.d.ts.map +0 -1
  246. package/storages/request_queue.js +0 -320
  247. package/storages/request_queue.js.map +0 -1
  248. package/storages/request_queue_v2.d.ts +0 -87
  249. package/storages/request_queue_v2.d.ts.map +0 -1
  250. package/storages/request_queue_v2.js +0 -438
  251. package/storages/request_queue_v2.js.map +0 -1
  252. package/storages/sitemap_request_list.d.ts +0 -250
  253. package/storages/sitemap_request_list.d.ts.map +0 -1
  254. package/storages/sitemap_request_list.js +0 -431
  255. package/storages/sitemap_request_list.js.map +0 -1
  256. package/storages/storage_manager.d.ts +0 -62
  257. package/storages/storage_manager.d.ts.map +0 -1
  258. package/storages/storage_manager.js +0 -105
  259. package/storages/storage_manager.js.map +0 -1
  260. package/storages/utils.d.ts.map +0 -1
  261. package/storages/utils.js.map +0 -1
  262. package/typedefs.d.ts.map +0 -1
  263. package/typedefs.js.map +0 -1
  264. package/validators.d.ts.map +0 -1
  265. package/validators.js.map +0 -1
@@ -1,258 +0,0 @@
1
- import { EventEmitter } from 'node:events';
2
- import type { Log } from '@apify/log';
3
- import { Configuration } from '../configuration.js';
4
- import type { PersistenceOptions } from '../crawlers/statistics.js';
5
- import type { EventManager } from '../events/event_manager.js';
6
- import { KeyValueStore } from '../storages/key_value_store.js';
7
- import type { SessionOptions } from './session.js';
8
- import { Session } from './session.js';
9
- /**
10
- * Factory user-function which creates customized {@link Session} instances.
11
- */
12
- export interface CreateSession {
13
- /**
14
- * @param sessionPool Pool requesting the new session.
15
- * @param options
16
- */
17
- (sessionPool: SessionPool, options?: {
18
- sessionOptions?: SessionOptions;
19
- }): Session | Promise<Session>;
20
- }
21
- export interface SessionPoolOptions {
22
- /**
23
- * Maximum size of the pool. Indicates how many sessions are rotated.
24
- * @default 1000
25
- */
26
- maxPoolSize?: number;
27
- /** The configuration options for {@link Session} instances. */
28
- sessionOptions?: SessionOptions;
29
- /** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
30
- persistStateKeyValueStoreId?: string;
31
- /**
32
- * Session pool persists it's state under this key in Key value store.
33
- * @default SESSION_POOL_STATE
34
- */
35
- persistStateKey?: string;
36
- /**
37
- * Custom function that should return a `Session` instance, or a promise resolving to such instance.
38
- * Any error thrown from this function will terminate the process.
39
- * Function receives `SessionPool` instance as a parameter
40
- */
41
- createSessionFunction?: CreateSession;
42
- /**
43
- * Specifies which response status codes are considered as blocked.
44
- * Session connected to such request will be marked as retired.
45
- * @default [401, 403, 429]
46
- */
47
- blockedStatusCodes?: number[];
48
- /** @internal */
49
- log?: Log;
50
- /**
51
- * Control how and when to persist the state of the session pool.
52
- */
53
- persistenceOptions?: PersistenceOptions;
54
- }
55
- /**
56
- * Handles the rotation, creation and persistence of user-like sessions.
57
- * Creates a pool of {@link Session} instances, that are randomly rotated.
58
- * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
59
- * Learn more in the {@doclink guides/session-management | Session management guide}.
60
- *
61
- * You can create one by calling the {@link SessionPool.open} function.
62
- *
63
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
64
- * performance with just 2 lines of code.
65
- *
66
- * **Example usage:**
67
- *
68
- * ```javascript
69
- * const crawler = new CheerioCrawler({
70
- * useSessionPool: true,
71
- * persistCookiesPerSession: true,
72
- * // ...
73
- * })
74
- * ```
75
- *
76
- * You can configure the pool with many options. See the {@link SessionPoolOptions}.
77
- * Session pool is by default persisted in default {@link KeyValueStore}.
78
- * If you want to have one pool for all runs you have to specify
79
- * {@link SessionPoolOptions.persistStateKeyValueStoreId}.
80
- *
81
- * **Advanced usage:**
82
- *
83
- * ```javascript
84
- * const sessionPool = await SessionPool.open({
85
- * maxPoolSize: 25,
86
- * sessionOptions:{
87
- * maxAgeSecs: 10,
88
- * maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
89
- * },
90
- * persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
91
- * persistStateKey: 'my-session-pool',
92
- * });
93
- *
94
- * // Get random session from the pool
95
- * const session1 = await sessionPool.getSession();
96
- * const session2 = await sessionPool.getSession();
97
- * const session3 = await sessionPool.getSession();
98
- *
99
- * // Now you can mark the session either failed or successful
100
- *
101
- * // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
102
- * session1.markBad()
103
- *
104
- * // Marks as successful.
105
- * session2.markGood()
106
- *
107
- * // Retires session -> session is removed from the pool
108
- * session3.retire()
109
- *
110
- * ```
111
- *
112
- * **Default session allocation flow:*
113
- * 1. Until the `SessionPool` reaches `maxPoolSize`, new sessions are created, provided to the user and added to the pool
114
- * 2. Blocked/retired sessions stay in the pool but are never provided to the user
115
- * 3. Once the pool is full (live plus blocked session count reaches `maxPoolSize`), a random session from the pool is provided.
116
- * 4. If a blocked session would be picked, instead all blocked sessions are evicted from the pool and a new session is created and provided
117
- *
118
- * @category Scaling
119
- */
120
- export declare class SessionPool extends EventEmitter {
121
- readonly config: Configuration;
122
- protected log: Log;
123
- protected maxPoolSize: number;
124
- protected createSessionFunction: CreateSession;
125
- protected keyValueStore: KeyValueStore;
126
- protected sessions: Session[];
127
- protected sessionMap: Map<string, Session>;
128
- protected sessionOptions: SessionOptions;
129
- protected persistStateKeyValueStoreId?: string;
130
- protected persistStateKey: string;
131
- protected _listener: () => Promise<void>;
132
- protected events: EventManager;
133
- protected readonly blockedStatusCodes: number[];
134
- protected persistenceOptions: PersistenceOptions;
135
- protected isInitialized: boolean;
136
- private queue;
137
- /**
138
- * @internal
139
- */
140
- constructor(options?: SessionPoolOptions, config?: Configuration);
141
- /**
142
- * Gets count of usable sessions in the pool.
143
- */
144
- get usableSessionsCount(): number;
145
- /**
146
- * Gets count of retired sessions in the pool.
147
- */
148
- get retiredSessionsCount(): number;
149
- /**
150
- * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
151
- * It is called automatically by the {@link SessionPool.open} function.
152
- */
153
- initialize(): Promise<void>;
154
- /**
155
- * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
156
- * but this allows you to add more sessions once the max pool size is reached.
157
- * This also allows you to add session with overridden session options (e.g. with specific session id).
158
- * @param [options] The configuration options for the session being added to the session pool.
159
- */
160
- addSession(options?: Session | SessionOptions): Promise<void>;
161
- /**
162
- * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
163
- * but this allows you to add more sessions once the max pool size is reached.
164
- * This also allows you to add session with overridden session options (e.g. with specific session id).
165
- * @param [options] The configuration options for the session being added to the session pool.
166
- */
167
- newSession(sessionOptions?: SessionOptions): Promise<Session>;
168
- /**
169
- * Gets session.
170
- * If there is space for new session, it creates and returns new session.
171
- * If the session pool is full, it picks a session from the pool,
172
- * If the picked session is usable it is returned, otherwise it creates and returns a new one.
173
- */
174
- getSession(): Promise<Session>;
175
- /**
176
- * Gets session based on the provided session id or `undefined.
177
- */
178
- getSession(sessionId: string): Promise<Session>;
179
- /**
180
- * @param options - Override the persistence options provided in the constructor
181
- */
182
- resetStore(options?: PersistenceOptions): Promise<void>;
183
- /**
184
- * Returns an object representing the internal state of the `SessionPool` instance.
185
- * Note that the object's fields can change in future releases.
186
- */
187
- getState(): {
188
- usableSessionsCount: number;
189
- retiredSessionsCount: number;
190
- // @ts-ignore optional peer dependency or compatibility with es2022
191
- sessions: import("@crawlee/types").SessionState[];
192
- };
193
- /**
194
- * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
195
- * The state is persisted automatically in regular intervals.
196
- * @param options - Override the persistence options provided in the constructor
197
- */
198
- persistState(options?: PersistenceOptions): Promise<void>;
199
- /**
200
- * Removes listener from `persistState` event.
201
- * This function should be called after you are done with using the `SessionPool` instance.
202
- */
203
- teardown(): Promise<void>;
204
- /**
205
- * SessionPool should not work before initialization.
206
- */
207
- protected _throwIfNotInitialized(): void;
208
- /**
209
- * Removes retired `Session` instances from `SessionPool`.
210
- */
211
- protected _removeRetiredSessions(): void;
212
- /**
213
- * Adds `Session` instance to `SessionPool`.
214
- * @param newSession `Session` instance to be added.
215
- */
216
- protected _addSession(newSession: Session): void;
217
- /**
218
- * Gets random index.
219
- */
220
- protected _getRandomIndex(): number;
221
- /**
222
- * Creates new session without any extra behavior.
223
- * @param sessionPool
224
- * @param [options]
225
- * @param [options.sessionOptions] The configuration options for the session being created.
226
- * @returns New session.
227
- */
228
- protected _defaultCreateSessionFunction(sessionPool: SessionPool, options?: {
229
- sessionOptions?: SessionOptions;
230
- }): Promise<Session>;
231
- /**
232
- * Creates new session and adds it to the pool.
233
- * @returns Newly created `Session` instance.
234
- */
235
- protected _createSession(): Promise<Session>;
236
- /**
237
- * Decides whether there is enough space for creating new session.
238
- */
239
- protected _hasSpaceForSession(): boolean;
240
- /**
241
- * Picks random session from the `SessionPool`.
242
- * @returns Picked `Session`.
243
- */
244
- protected _pickSession(): Session;
245
- /**
246
- * Potentially loads `SessionPool`.
247
- * If the state was persisted it loads the `SessionPool` from the persisted state.
248
- */
249
- protected _maybeLoadSessionPool(): Promise<void>;
250
- /**
251
- * Opens a SessionPool and returns a promise resolving to an instance
252
- * of the {@link SessionPool} class that is already initialized.
253
- *
254
- * For more details and code examples, see the {@link SessionPool} class.
255
- */
256
- static open(options?: SessionPoolOptions, config?: Configuration): Promise<SessionPool>;
257
- }
258
- //# sourceMappingURL=session_pool.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAG/D,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE9B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAuBrC,QAAQ,CAAC,MAAM;IAtBnB,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,EAAG,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,EAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAChD,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,aAAa,UAAS;IAEhC,OAAO,CAAC,KAAK,CAAoB;IAEjC;;OAEG;gBAEC,OAAO,GAAE,kBAAuB,EACvB,MAAM,gBAAkC;IAsDrD;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBvE;;;;;OAKG;IACG,UAAU,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IASnE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAqCrD;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAQ7C;;;OAGG;IACH,QAAQ;;;;;IAQR;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;cACa,6BAA6B,CACzC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAClD,OAAO,CAAC,OAAO,CAAC;IAWnB;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,WAAW,CAAC;CAKhG"}
@@ -1,387 +0,0 @@
1
- import { EventEmitter } from 'node:events';
2
- import { AsyncQueue } from '@sapphire/async-queue';
3
- import ow from 'ow';
4
- import { Configuration } from '../configuration.js';
5
- import { log as defaultLog } from '../log.js';
6
- import { KeyValueStore } from '../storages/key_value_store.js';
7
- import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
8
- import { Session } from './session.js';
9
- /**
10
- * Handles the rotation, creation and persistence of user-like sessions.
11
- * Creates a pool of {@link Session} instances, that are randomly rotated.
12
- * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
13
- * Learn more in the {@doclink guides/session-management | Session management guide}.
14
- *
15
- * You can create one by calling the {@link SessionPool.open} function.
16
- *
17
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
18
- * performance with just 2 lines of code.
19
- *
20
- * **Example usage:**
21
- *
22
- * ```javascript
23
- * const crawler = new CheerioCrawler({
24
- * useSessionPool: true,
25
- * persistCookiesPerSession: true,
26
- * // ...
27
- * })
28
- * ```
29
- *
30
- * You can configure the pool with many options. See the {@link SessionPoolOptions}.
31
- * Session pool is by default persisted in default {@link KeyValueStore}.
32
- * If you want to have one pool for all runs you have to specify
33
- * {@link SessionPoolOptions.persistStateKeyValueStoreId}.
34
- *
35
- * **Advanced usage:**
36
- *
37
- * ```javascript
38
- * const sessionPool = await SessionPool.open({
39
- * maxPoolSize: 25,
40
- * sessionOptions:{
41
- * maxAgeSecs: 10,
42
- * maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
43
- * },
44
- * persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
45
- * persistStateKey: 'my-session-pool',
46
- * });
47
- *
48
- * // Get random session from the pool
49
- * const session1 = await sessionPool.getSession();
50
- * const session2 = await sessionPool.getSession();
51
- * const session3 = await sessionPool.getSession();
52
- *
53
- * // Now you can mark the session either failed or successful
54
- *
55
- * // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
56
- * session1.markBad()
57
- *
58
- * // Marks as successful.
59
- * session2.markGood()
60
- *
61
- * // Retires session -> session is removed from the pool
62
- * session3.retire()
63
- *
64
- * ```
65
- *
66
- * **Default session allocation flow:*
67
- * 1. Until the `SessionPool` reaches `maxPoolSize`, new sessions are created, provided to the user and added to the pool
68
- * 2. Blocked/retired sessions stay in the pool but are never provided to the user
69
- * 3. Once the pool is full (live plus blocked session count reaches `maxPoolSize`), a random session from the pool is provided.
70
- * 4. If a blocked session would be picked, instead all blocked sessions are evicted from the pool and a new session is created and provided
71
- *
72
- * @category Scaling
73
- */
74
- export class SessionPool extends EventEmitter {
75
- config;
76
- log;
77
- maxPoolSize;
78
- createSessionFunction;
79
- keyValueStore;
80
- sessions = [];
81
- sessionMap = new Map();
82
- sessionOptions;
83
- persistStateKeyValueStoreId;
84
- persistStateKey;
85
- _listener;
86
- events;
87
- blockedStatusCodes;
88
- persistenceOptions;
89
- isInitialized = false;
90
- queue = new AsyncQueue();
91
- /**
92
- * @internal
93
- */
94
- constructor(options = {}, config = Configuration.getGlobalConfig()) {
95
- super();
96
- this.config = config;
97
- ow(options, ow.object.exactShape({
98
- maxPoolSize: ow.optional.number,
99
- persistStateKeyValueStoreId: ow.optional.string,
100
- persistStateKey: ow.optional.string,
101
- createSessionFunction: ow.optional.function,
102
- sessionOptions: ow.optional.object,
103
- blockedStatusCodes: ow.optional.array.ofType(ow.number),
104
- log: ow.optional.object,
105
- persistenceOptions: ow.optional.object,
106
- }));
107
- const { maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey = PERSIST_STATE_KEY, createSessionFunction, sessionOptions = {}, blockedStatusCodes = BLOCKED_STATUS_CODES, log = defaultLog, persistenceOptions = {
108
- enable: true,
109
- }, } = options;
110
- this.config = config;
111
- this.blockedStatusCodes = blockedStatusCodes;
112
- this.events = config.getEventManager();
113
- this.log = log.child({ prefix: 'SessionPool' });
114
- this.persistenceOptions = persistenceOptions;
115
- // Pool Configuration
116
- this.maxPoolSize = maxPoolSize;
117
- this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
118
- // Session configuration
119
- this.sessionOptions = {
120
- ...sessionOptions,
121
- // the log needs to propagate to createSessionFunction as in "new Session({ ...sessionPool.sessionOptions })"
122
- // and can't go inside _defaultCreateSessionFunction
123
- log: this.log,
124
- };
125
- // Session keyValueStore
126
- this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
127
- this.persistStateKey = persistStateKey;
128
- }
129
- /**
130
- * Gets count of usable sessions in the pool.
131
- */
132
- get usableSessionsCount() {
133
- return this.sessions.filter((session) => session.isUsable()).length;
134
- }
135
- /**
136
- * Gets count of retired sessions in the pool.
137
- */
138
- get retiredSessionsCount() {
139
- return this.sessions.filter((session) => !session.isUsable()).length;
140
- }
141
- /**
142
- * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
143
- * It is called automatically by the {@link SessionPool.open} function.
144
- */
145
- async initialize() {
146
- if (this.isInitialized) {
147
- return;
148
- }
149
- this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
150
- if (!this.persistenceOptions.enable) {
151
- this.isInitialized = true;
152
- return;
153
- }
154
- if (!this.persistStateKeyValueStoreId) {
155
- this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
156
- }
157
- // in case of migration happened and SessionPool state should be restored from the keyValueStore.
158
- await this._maybeLoadSessionPool();
159
- this._listener = this.persistState.bind(this);
160
- this.events.on("persistState" /* EventType.PERSIST_STATE */, this._listener);
161
- this.isInitialized = true;
162
- }
163
- /**
164
- * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
165
- * but this allows you to add more sessions once the max pool size is reached.
166
- * This also allows you to add session with overridden session options (e.g. with specific session id).
167
- * @param [options] The configuration options for the session being added to the session pool.
168
- */
169
- async addSession(options = {}) {
170
- this._throwIfNotInitialized();
171
- const { id } = options;
172
- if (id) {
173
- const sessionExists = this.sessionMap.has(id);
174
- if (sessionExists) {
175
- throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
176
- }
177
- }
178
- if (!this._hasSpaceForSession()) {
179
- this._removeRetiredSessions();
180
- }
181
- const newSession = options instanceof Session ? options : await this.createSessionFunction(this, { sessionOptions: options });
182
- this.log.debug(`Adding new Session - ${newSession.id}`);
183
- this._addSession(newSession);
184
- }
185
- /**
186
- * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
187
- * but this allows you to add more sessions once the max pool size is reached.
188
- * This also allows you to add session with overridden session options (e.g. with specific session id).
189
- * @param [options] The configuration options for the session being added to the session pool.
190
- */
191
- async newSession(sessionOptions) {
192
- this._throwIfNotInitialized();
193
- const newSession = await this.createSessionFunction(this, { sessionOptions });
194
- this._addSession(newSession);
195
- return newSession;
196
- }
197
- /**
198
- * Gets session.
199
- * If there is space for new session, it creates and returns new session.
200
- * If the session pool is full, it picks a session from the pool,
201
- * If the picked session is usable it is returned, otherwise it creates and returns a new one.
202
- * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
203
- */
204
- async getSession(sessionId) {
205
- await this.queue.wait();
206
- try {
207
- this._throwIfNotInitialized();
208
- if (sessionId) {
209
- const session = this.sessionMap.get(sessionId);
210
- if (session && session.isUsable())
211
- return session;
212
- return undefined;
213
- }
214
- if (this._hasSpaceForSession()) {
215
- return await this._createSession();
216
- }
217
- const pickedSession = this._pickSession();
218
- if (pickedSession.isUsable()) {
219
- return pickedSession;
220
- }
221
- this._removeRetiredSessions();
222
- return await this._createSession();
223
- }
224
- finally {
225
- this.queue.shift();
226
- }
227
- }
228
- /**
229
- * @param options - Override the persistence options provided in the constructor
230
- */
231
- async resetStore(options) {
232
- if (!this.persistenceOptions.enable && !options?.enable) {
233
- return;
234
- }
235
- await this.keyValueStore?.setValue(this.persistStateKey, null);
236
- }
237
- /**
238
- * Returns an object representing the internal state of the `SessionPool` instance.
239
- * Note that the object's fields can change in future releases.
240
- */
241
- getState() {
242
- return {
243
- usableSessionsCount: this.usableSessionsCount,
244
- retiredSessionsCount: this.retiredSessionsCount,
245
- sessions: this.sessions.map((session) => session.getState()),
246
- };
247
- }
248
- /**
249
- * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
250
- * The state is persisted automatically in regular intervals.
251
- * @param options - Override the persistence options provided in the constructor
252
- */
253
- async persistState(options) {
254
- if (!this.persistenceOptions.enable && !options?.enable) {
255
- return;
256
- }
257
- this.log.debug('Persisting state', {
258
- persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
259
- persistStateKey: this.persistStateKey,
260
- });
261
- // use half the interval of `persistState` to avoid race conditions
262
- const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
263
- const timeoutSecs = persistStateIntervalMillis / 2_000;
264
- await this.keyValueStore
265
- .setValue(this.persistStateKey, this.getState(), {
266
- timeoutSecs,
267
- doNotRetryTimeouts: true,
268
- })
269
- .catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
270
- }
271
- /**
272
- * Removes listener from `persistState` event.
273
- * This function should be called after you are done with using the `SessionPool` instance.
274
- */
275
- async teardown() {
276
- this.events.off("persistState" /* EventType.PERSIST_STATE */, this._listener);
277
- await this.persistState();
278
- }
279
- /**
280
- * SessionPool should not work before initialization.
281
- */
282
- _throwIfNotInitialized() {
283
- if (!this.isInitialized)
284
- throw new Error('SessionPool is not initialized.');
285
- }
286
- /**
287
- * Removes retired `Session` instances from `SessionPool`.
288
- */
289
- _removeRetiredSessions() {
290
- this.sessions = this.sessions.filter((storedSession) => {
291
- if (storedSession.isUsable())
292
- return true;
293
- this.sessionMap.delete(storedSession.id);
294
- this.log.debug(`Removed Session - ${storedSession.id}`);
295
- return false;
296
- });
297
- }
298
- /**
299
- * Adds `Session` instance to `SessionPool`.
300
- * @param newSession `Session` instance to be added.
301
- */
302
- _addSession(newSession) {
303
- this.sessions.push(newSession);
304
- this.sessionMap.set(newSession.id, newSession);
305
- }
306
- /**
307
- * Gets random index.
308
- */
309
- _getRandomIndex() {
310
- return Math.floor(Math.random() * this.sessions.length);
311
- }
312
- /**
313
- * Creates new session without any extra behavior.
314
- * @param sessionPool
315
- * @param [options]
316
- * @param [options.sessionOptions] The configuration options for the session being created.
317
- * @returns New session.
318
- */
319
- async _defaultCreateSessionFunction(sessionPool, options = {}) {
320
- ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
321
- const { sessionOptions = {} } = options;
322
- return new Session({
323
- ...this.sessionOptions,
324
- ...sessionOptions,
325
- sessionPool,
326
- });
327
- }
328
- /**
329
- * Creates new session and adds it to the pool.
330
- * @returns Newly created `Session` instance.
331
- */
332
- async _createSession() {
333
- const newSession = await this.createSessionFunction(this);
334
- this._addSession(newSession);
335
- this.log.debug(`Created new Session - ${newSession.id}`);
336
- return newSession;
337
- }
338
- /**
339
- * Decides whether there is enough space for creating new session.
340
- */
341
- _hasSpaceForSession() {
342
- return this.sessions.length < this.maxPoolSize;
343
- }
344
- /**
345
- * Picks random session from the `SessionPool`.
346
- * @returns Picked `Session`.
347
- */
348
- _pickSession() {
349
- return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm
350
- }
351
- /**
352
- * Potentially loads `SessionPool`.
353
- * If the state was persisted it loads the `SessionPool` from the persisted state.
354
- */
355
- async _maybeLoadSessionPool() {
356
- const loadedSessionPool = await this.keyValueStore.getValue(this.persistStateKey);
357
- if (!loadedSessionPool)
358
- return;
359
- // Invalidate old sessions and load active sessions only
360
- this.log.debug('Recreating state from KeyValueStore', {
361
- persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
362
- persistStateKey: this.persistStateKey,
363
- });
364
- for (const sessionObject of loadedSessionPool.sessions) {
365
- sessionObject.sessionPool = this;
366
- sessionObject.createdAt = new Date(sessionObject.createdAt);
367
- sessionObject.expiresAt = new Date(sessionObject.expiresAt);
368
- const recreatedSession = await this.createSessionFunction(this, { sessionOptions: sessionObject });
369
- if (recreatedSession.isUsable()) {
370
- this._addSession(recreatedSession);
371
- }
372
- }
373
- this.log.debug(`${this.usableSessionsCount} active sessions loaded from KeyValueStore`);
374
- }
375
- /**
376
- * Opens a SessionPool and returns a promise resolving to an instance
377
- * of the {@link SessionPool} class that is already initialized.
378
- *
379
- * For more details and code examples, see the {@link SessionPool} class.
380
- */
381
- static async open(options, config) {
382
- const sessionPool = new SessionPool(options, config);
383
- await sessionPool.initialize();
384
- return sessionPool;
385
- }
386
- }
387
- //# sourceMappingURL=session_pool.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErF,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAuDvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAM,OAAO,WAAY,SAAQ,YAAY;IAuB5B;IAtBH,GAAG,CAAM;IACT,WAAW,CAAS;IACpB,qBAAqB,CAAgB;IACrC,aAAa,CAAiB;IAC9B,QAAQ,GAAc,EAAE,CAAC;IACzB,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;IACxC,cAAc,CAAiB;IAC/B,2BAA2B,CAAU;IACrC,eAAe,CAAS;IACxB,SAAS,CAAuB;IAChC,MAAM,CAAe;IACZ,kBAAkB,CAAW;IACtC,kBAAkB,CAAqB;IACvC,aAAa,GAAG,KAAK,CAAC;IAExB,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IAEjC;;OAEG;IACH,YACI,UAA8B,EAAE,EACvB,SAAS,aAAa,CAAC,eAAe,EAAE;QAEjD,KAAK,EAAE,CAAC;QAFC,WAAM,GAAN,MAAM,CAAkC;QAIjD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvD,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,GAAG,aAAa,EAC3B,2BAA2B,EAC3B,eAAe,GAAG,iBAAiB,EACnC,qBAAqB,EACrB,cAAc,GAAG,EAAE,EACnB,kBAAkB,GAAG,oBAAoB,EACzC,GAAG,GAAG,UAAU,EAChB,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,GACJ,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,6GAA6G;YAC7G,oDAAoD;YACpD,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACzG,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,iIAAiI,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAC3J,CAAC;QACN,CAAC;QAED,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACxD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE,CAAC;YACL,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,UAAU,GACZ,OAAO,YAAY,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/G,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,cAA+B;QAC5C,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAE7B,OAAO,UAAU,CAAC;IACtB,CAAC;IAeD;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAExB,IAAI,CAAC;YACD,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAE9B,IAAI,SAAS,EAAE,CAAC;gBACZ,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE;oBAAE,OAAO,OAAO,CAAC;gBAClD,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1C,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC3B,OAAO,aAAa,CAAC;YACzB,CAAC;YAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACvC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,MAAM,WAAW,GAAG,0BAA0B,GAAG,KAAK,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa;aACnB,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE;YAC7C,WAAW;YACX,kBAAkB,EAAE,IAAI;SAC3B,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,+CAA+C,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CACrG,CAAC;IACV,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAChF,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACO,KAAK,CAAC,6BAA6B,CACzC,WAAwB,EACxB,UAA+C,EAAE;QAEjD,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QAExC,OAAO,IAAI,OAAO,CAAC;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,cAAc;YACjB,WAAW;SACd,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,0EAA0E;IAC5H,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE9G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YACrD,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;YACjC,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;YAEnG,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,mBAAmB,4CAA4C,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAA4B,EAAE,MAAsB;QAClE,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ"}
@@ -1,13 +0,0 @@
1
- import type { Awaitable } from '../typedefs.js';
2
- /**
3
- * Invoke a storage access checker function defined using {@link withCheckedStorageAccess} higher up in the call stack.
4
- */
5
- export declare const checkStorageAccess: () => void | undefined;
6
- /**
7
- * Define a storage access checker function that should be used by calls to {@link checkStorageAccess} in the callbacks.
8
- *
9
- * @param checkFunction The check function that should be invoked by {@link checkStorageAccess} calls
10
- * @param callback The code that should be invoked with the `checkFunction` setting
11
- */
12
- export declare const withCheckedStorageAccess: <T>(checkFunction: () => void, callback: () => Awaitable<T>) => Promise<T>;
13
- //# sourceMappingURL=access_checking.d.ts.map