@crawlee/core 4.0.0-beta.5 → 4.0.0-beta.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +18 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +85 -227
- package/configuration.d.ts.map +1 -1
- package/configuration.js +159 -223
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +8 -8
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +71 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +123 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +19 -27
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -20
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_utils.js +1 -1
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +15 -15
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +21 -18
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +32 -18
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +45 -24
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +25 -8
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +69 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +30 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +44 -0
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +8 -5
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +7 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +33 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +3 -2
- package/index.d.ts.map +1 -1
- package/index.js +2 -1
- package/index.js.map +1 -1
- package/log.d.ts +82 -2
- package/log.d.ts.map +1 -1
- package/log.js +102 -0
- package/log.js.map +1 -1
- package/package.json +9 -10
- package/proxy_configuration.d.ts +14 -148
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +19 -167
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +142 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +74 -8
- package/request.d.ts.map +1 -1
- package/request.js +87 -18
- package/request.js.map +1 -1
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/serialization.js +1 -1
- package/serialization.js.map +1 -1
- package/service_locator.d.ts +162 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +247 -0
- package/service_locator.js.map +1 -0
- package/session_pool/session.d.ts +9 -31
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +17 -21
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +51 -57
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +86 -79
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +63 -19
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +86 -22
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +2 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +2 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +79 -10
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +104 -23
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +9 -9
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +13 -8
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +86 -23
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +114 -74
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +1 -3
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +2 -4
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +3 -3
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +4 -5
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +5 -5
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +10 -7
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts +22 -17
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +58 -52
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +6 -5
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
-
import type { Log } from '@apify/log';
|
|
3
|
-
import { Configuration } from '../configuration.js';
|
|
4
2
|
import type { PersistenceOptions } from '../crawlers/statistics.js';
|
|
5
3
|
import type { EventManager } from '../events/event_manager.js';
|
|
4
|
+
import type { CrawleeLogger } from '../log.js';
|
|
6
5
|
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
7
6
|
import type { SessionOptions } from './session.js';
|
|
8
7
|
import { Session } from './session.js';
|
|
8
|
+
declare const SESSION_REUSE_STRATEGIES: readonly ["random", "round-robin", "use-until-failure"];
|
|
9
|
+
export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];
|
|
9
10
|
/**
|
|
10
11
|
* Factory user-function which creates customized {@link Session} instances.
|
|
11
12
|
*/
|
|
@@ -19,6 +20,12 @@ export interface CreateSession {
|
|
|
19
20
|
}): Session | Promise<Session>;
|
|
20
21
|
}
|
|
21
22
|
export interface SessionPoolOptions {
|
|
23
|
+
/**
|
|
24
|
+
* Unique identifier for this session pool instance. Used to generate a unique
|
|
25
|
+
* persistence key when `persistStateKey` is not provided.
|
|
26
|
+
* If not provided, an auto-incrementing ID is used.
|
|
27
|
+
*/
|
|
28
|
+
id?: string | number;
|
|
22
29
|
/**
|
|
23
30
|
* Maximum size of the pool. Indicates how many sessions are rotated.
|
|
24
31
|
* @default 1000
|
|
@@ -29,24 +36,26 @@ export interface SessionPoolOptions {
|
|
|
29
36
|
/** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
|
|
30
37
|
persistStateKeyValueStoreId?: string;
|
|
31
38
|
/**
|
|
32
|
-
* Session pool persists
|
|
33
|
-
* @default
|
|
39
|
+
* Session pool persists its state under this key in Key value store.
|
|
40
|
+
* @default SDK_SESSION_POOL_STATE_{id}
|
|
34
41
|
*/
|
|
35
42
|
persistStateKey?: string;
|
|
36
43
|
/**
|
|
37
|
-
* Custom function that should return `Session` instance.
|
|
44
|
+
* Custom function that should return a `Session` instance, or a promise resolving to such instance.
|
|
38
45
|
* Any error thrown from this function will terminate the process.
|
|
39
46
|
* Function receives `SessionPool` instance as a parameter
|
|
40
47
|
*/
|
|
41
48
|
createSessionFunction?: CreateSession;
|
|
42
49
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
50
|
+
* Strategy for picking sessions from the pool.
|
|
51
|
+
* - `'random'` (default): fills the pool up to `maxPoolSize`, then picks a random usable session
|
|
52
|
+
* - `'round-robin'`: fills the pool up to `maxPoolSize`, then reuses sessions cycling through them in order
|
|
53
|
+
* - `'use-until-failure'`: always reuses the same session until it is retired, then moves to the next one
|
|
54
|
+
* @default 'random'
|
|
46
55
|
*/
|
|
47
|
-
|
|
56
|
+
sessionReuseStrategy?: SessionReuseStrategy;
|
|
48
57
|
/** @internal */
|
|
49
|
-
log?:
|
|
58
|
+
log?: CrawleeLogger;
|
|
50
59
|
/**
|
|
51
60
|
* Control how and when to persist the state of the session pool.
|
|
52
61
|
*/
|
|
@@ -58,20 +67,8 @@ export interface SessionPoolOptions {
|
|
|
58
67
|
* When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
|
|
59
68
|
* Learn more in the {@doclink guides/session-management | Session management guide}.
|
|
60
69
|
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
* Session pool is already integrated into crawlers, and it can significantly improve your scraper
|
|
64
|
-
* performance with just 2 lines of code.
|
|
65
|
-
*
|
|
66
|
-
* **Example usage:**
|
|
67
|
-
*
|
|
68
|
-
* ```javascript
|
|
69
|
-
* const crawler = new CheerioCrawler({
|
|
70
|
-
* useSessionPool: true,
|
|
71
|
-
* persistCookiesPerSession: true,
|
|
72
|
-
* // ...
|
|
73
|
-
* })
|
|
74
|
-
* ```
|
|
70
|
+
* Session pool is already integrated into crawlers and is always active.
|
|
71
|
+
* All public methods are lazy-initialized — the pool initializes itself on first use.
|
|
75
72
|
*
|
|
76
73
|
* You can configure the pool with many options. See the {@link SessionPoolOptions}.
|
|
77
74
|
* Session pool is by default persisted in default {@link KeyValueStore}.
|
|
@@ -81,7 +78,7 @@ export interface SessionPoolOptions {
|
|
|
81
78
|
* **Advanced usage:**
|
|
82
79
|
*
|
|
83
80
|
* ```javascript
|
|
84
|
-
* const sessionPool =
|
|
81
|
+
* const sessionPool = new SessionPool({
|
|
85
82
|
* maxPoolSize: 25,
|
|
86
83
|
* sessionOptions:{
|
|
87
84
|
* maxAgeSecs: 10,
|
|
@@ -118,39 +115,39 @@ export interface SessionPoolOptions {
|
|
|
118
115
|
* @category Scaling
|
|
119
116
|
*/
|
|
120
117
|
export declare class SessionPool extends EventEmitter {
|
|
121
|
-
|
|
122
|
-
|
|
118
|
+
private static nextId;
|
|
119
|
+
readonly id: string;
|
|
120
|
+
protected log: CrawleeLogger;
|
|
123
121
|
protected maxPoolSize: number;
|
|
124
122
|
protected createSessionFunction: CreateSession;
|
|
125
|
-
protected keyValueStore
|
|
123
|
+
protected keyValueStore?: KeyValueStore;
|
|
126
124
|
protected sessions: Session[];
|
|
127
125
|
protected sessionMap: Map<string, Session>;
|
|
128
126
|
protected sessionOptions: SessionOptions;
|
|
129
127
|
protected persistStateKeyValueStoreId?: string;
|
|
130
128
|
protected persistStateKey: string;
|
|
131
|
-
protected _listener
|
|
129
|
+
protected _listener?: () => Promise<void>;
|
|
132
130
|
protected events: EventManager;
|
|
133
|
-
protected readonly blockedStatusCodes: number[];
|
|
134
131
|
protected persistenceOptions: PersistenceOptions;
|
|
135
|
-
protected
|
|
132
|
+
protected sessionReuseStrategy: SessionReuseStrategy;
|
|
133
|
+
private initPromise?;
|
|
136
134
|
private queue;
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
*/
|
|
140
|
-
constructor(options?: SessionPoolOptions, config?: Configuration);
|
|
135
|
+
private roundRobinIndex;
|
|
136
|
+
constructor(options?: SessionPoolOptions);
|
|
141
137
|
/**
|
|
142
138
|
* Gets count of usable sessions in the pool.
|
|
143
139
|
*/
|
|
144
|
-
|
|
140
|
+
usableSessionsCount(): Promise<number>;
|
|
145
141
|
/**
|
|
146
142
|
* Gets count of retired sessions in the pool.
|
|
147
143
|
*/
|
|
148
|
-
|
|
144
|
+
retiredSessionsCount(): Promise<number>;
|
|
149
145
|
/**
|
|
150
146
|
* Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
|
|
151
|
-
*
|
|
147
|
+
* Called automatically on first use of any public method.
|
|
152
148
|
*/
|
|
153
|
-
|
|
149
|
+
protected ensureInitialized(): Promise<void>;
|
|
150
|
+
private setupPool;
|
|
154
151
|
/**
|
|
155
152
|
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
156
153
|
* but this allows you to add more sessions once the max pool size is reached.
|
|
@@ -158,6 +155,13 @@ export declare class SessionPool extends EventEmitter {
|
|
|
158
155
|
* @param [options] The configuration options for the session being added to the session pool.
|
|
159
156
|
*/
|
|
160
157
|
addSession(options?: Session | SessionOptions): Promise<void>;
|
|
158
|
+
/**
|
|
159
|
+
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
160
|
+
* but this allows you to add more sessions once the max pool size is reached.
|
|
161
|
+
* This also allows you to add session with overridden session options (e.g. with specific session id).
|
|
162
|
+
* @param [options] The configuration options for the session being added to the session pool.
|
|
163
|
+
*/
|
|
164
|
+
newSession(sessionOptions?: SessionOptions): Promise<Session>;
|
|
161
165
|
/**
|
|
162
166
|
* Gets session.
|
|
163
167
|
* If there is space for new session, it creates and returns new session.
|
|
@@ -177,12 +181,12 @@ export declare class SessionPool extends EventEmitter {
|
|
|
177
181
|
* Returns an object representing the internal state of the `SessionPool` instance.
|
|
178
182
|
* Note that the object's fields can change in future releases.
|
|
179
183
|
*/
|
|
180
|
-
getState(): {
|
|
184
|
+
getState(): Promise<{
|
|
181
185
|
usableSessionsCount: number;
|
|
182
186
|
retiredSessionsCount: number;
|
|
183
187
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
184
|
-
sessions: import("
|
|
185
|
-
}
|
|
188
|
+
sessions: import("@crawlee/types").SessionState[];
|
|
189
|
+
}>;
|
|
186
190
|
/**
|
|
187
191
|
* Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
|
|
188
192
|
* The state is persisted automatically in regular intervals.
|
|
@@ -194,10 +198,6 @@ export declare class SessionPool extends EventEmitter {
|
|
|
194
198
|
* This function should be called after you are done with using the `SessionPool` instance.
|
|
195
199
|
*/
|
|
196
200
|
teardown(): Promise<void>;
|
|
197
|
-
/**
|
|
198
|
-
* SessionPool should not work before initialization.
|
|
199
|
-
*/
|
|
200
|
-
protected _throwIfNotInitialized(): void;
|
|
201
201
|
/**
|
|
202
202
|
* Removes retired `Session` instances from `SessionPool`.
|
|
203
203
|
*/
|
|
@@ -220,7 +220,7 @@ export declare class SessionPool extends EventEmitter {
|
|
|
220
220
|
*/
|
|
221
221
|
protected _defaultCreateSessionFunction(sessionPool: SessionPool, options?: {
|
|
222
222
|
sessionOptions?: SessionOptions;
|
|
223
|
-
}): Session
|
|
223
|
+
}): Promise<Session>;
|
|
224
224
|
/**
|
|
225
225
|
* Creates new session and adds it to the pool.
|
|
226
226
|
* @returns Newly created `Session` instance.
|
|
@@ -231,21 +231,15 @@ export declare class SessionPool extends EventEmitter {
|
|
|
231
231
|
*/
|
|
232
232
|
protected _hasSpaceForSession(): boolean;
|
|
233
233
|
/**
|
|
234
|
-
* Picks
|
|
235
|
-
*
|
|
234
|
+
* Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
|
|
235
|
+
* Returns `undefined` when no session should be reused and a new one should be created instead.
|
|
236
236
|
*/
|
|
237
|
-
protected _pickSession(): Session;
|
|
237
|
+
protected _pickSession(): Session | undefined;
|
|
238
238
|
/**
|
|
239
239
|
* Potentially loads `SessionPool`.
|
|
240
240
|
* If the state was persisted it loads the `SessionPool` from the persisted state.
|
|
241
241
|
*/
|
|
242
242
|
protected _maybeLoadSessionPool(): Promise<void>;
|
|
243
|
-
/**
|
|
244
|
-
* Opens a SessionPool and returns a promise resolving to an instance
|
|
245
|
-
* of the {@link SessionPool} class that is already initialized.
|
|
246
|
-
*
|
|
247
|
-
* For more details and code examples, see the {@link SessionPool} class.
|
|
248
|
-
*/
|
|
249
|
-
static open(options?: SessionPoolOptions, config?: Configuration): Promise<SessionPool>;
|
|
250
243
|
}
|
|
244
|
+
export {};
|
|
251
245
|
//# sourceMappingURL=session_pool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,QAAA,MAAM,wBAAwB,yDAA0D,CAAC;AACzF,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;;OAIG;IACH,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAErB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAE5C,gBAAgB;IAChB,GAAG,CAAC,EAAE,aAAa,CAAC;IAEpB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AACH,qBAAa,WAAY,SAAQ,YAAY;IACzC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAK;IAE1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,oBAAoB,EAAE,oBAAoB,CAAC;IAErD,OAAO,CAAC,WAAW,CAAC,CAAgB;IACpC,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,eAAe,CAAK;gBAEhB,OAAO,GAAE,kBAAuB;IAuD5C;;OAEG;IACG,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5C;;OAEG;IACG,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC;IAK7C;;;OAGG;cACa,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;YAOpC,SAAS;IAyBvB;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBvE;;;;;OAKG;IACG,UAAU,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IASnE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAkCrD;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAS7C;;;OAGG;IACG,QAAQ;;;;;IASd;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAyB/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;cACa,6BAA6B,CACzC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAClD,OAAO,CAAC,OAAO,CAAC;IAWnB;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO,GAAG,SAAS;IAmB7C;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;CAwBzD"}
|
|
@@ -1,31 +1,19 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
2
|
import { AsyncQueue } from '@sapphire/async-queue';
|
|
3
3
|
import ow from 'ow';
|
|
4
|
-
import {
|
|
5
|
-
import { log as defaultLog } from '../log.js';
|
|
4
|
+
import { serviceLocator } from '../service_locator.js';
|
|
6
5
|
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
7
|
-
import {
|
|
6
|
+
import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
|
|
8
7
|
import { Session } from './session.js';
|
|
8
|
+
const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
|
|
9
9
|
/**
|
|
10
10
|
* Handles the rotation, creation and persistence of user-like sessions.
|
|
11
11
|
* Creates a pool of {@link Session} instances, that are randomly rotated.
|
|
12
12
|
* When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
|
|
13
13
|
* Learn more in the {@doclink guides/session-management | Session management guide}.
|
|
14
14
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* Session pool is already integrated into crawlers, and it can significantly improve your scraper
|
|
18
|
-
* performance with just 2 lines of code.
|
|
19
|
-
*
|
|
20
|
-
* **Example usage:**
|
|
21
|
-
*
|
|
22
|
-
* ```javascript
|
|
23
|
-
* const crawler = new CheerioCrawler({
|
|
24
|
-
* useSessionPool: true,
|
|
25
|
-
* persistCookiesPerSession: true,
|
|
26
|
-
* // ...
|
|
27
|
-
* })
|
|
28
|
-
* ```
|
|
15
|
+
* Session pool is already integrated into crawlers and is always active.
|
|
16
|
+
* All public methods are lazy-initialized — the pool initializes itself on first use.
|
|
29
17
|
*
|
|
30
18
|
* You can configure the pool with many options. See the {@link SessionPoolOptions}.
|
|
31
19
|
* Session pool is by default persisted in default {@link KeyValueStore}.
|
|
@@ -35,7 +23,7 @@ import { Session } from './session.js';
|
|
|
35
23
|
* **Advanced usage:**
|
|
36
24
|
*
|
|
37
25
|
* ```javascript
|
|
38
|
-
* const sessionPool =
|
|
26
|
+
* const sessionPool = new SessionPool({
|
|
39
27
|
* maxPoolSize: 25,
|
|
40
28
|
* sessionOptions:{
|
|
41
29
|
* maxAgeSecs: 10,
|
|
@@ -72,7 +60,8 @@ import { Session } from './session.js';
|
|
|
72
60
|
* @category Scaling
|
|
73
61
|
*/
|
|
74
62
|
export class SessionPool extends EventEmitter {
|
|
75
|
-
|
|
63
|
+
static nextId = 0;
|
|
64
|
+
id;
|
|
76
65
|
log;
|
|
77
66
|
maxPoolSize;
|
|
78
67
|
createSessionFunction;
|
|
@@ -84,32 +73,30 @@ export class SessionPool extends EventEmitter {
|
|
|
84
73
|
persistStateKey;
|
|
85
74
|
_listener;
|
|
86
75
|
events;
|
|
87
|
-
blockedStatusCodes;
|
|
88
76
|
persistenceOptions;
|
|
89
|
-
|
|
77
|
+
sessionReuseStrategy;
|
|
78
|
+
initPromise;
|
|
90
79
|
queue = new AsyncQueue();
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
*/
|
|
94
|
-
constructor(options = {}, config = Configuration.getGlobalConfig()) {
|
|
80
|
+
roundRobinIndex = 0;
|
|
81
|
+
constructor(options = {}) {
|
|
95
82
|
super();
|
|
96
|
-
this.config = config;
|
|
97
83
|
ow(options, ow.object.exactShape({
|
|
84
|
+
id: ow.optional.any(ow.number, ow.string),
|
|
98
85
|
maxPoolSize: ow.optional.number,
|
|
99
86
|
persistStateKeyValueStoreId: ow.optional.string,
|
|
100
87
|
persistStateKey: ow.optional.string,
|
|
101
88
|
createSessionFunction: ow.optional.function,
|
|
102
89
|
sessionOptions: ow.optional.object,
|
|
103
|
-
blockedStatusCodes: ow.optional.array.ofType(ow.number),
|
|
104
90
|
log: ow.optional.object,
|
|
105
91
|
persistenceOptions: ow.optional.object,
|
|
92
|
+
sessionReuseStrategy: ow.optional.string.oneOf([...SESSION_REUSE_STRATEGIES]),
|
|
106
93
|
}));
|
|
107
|
-
const { maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey
|
|
94
|
+
const { id, maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions = {}, log = serviceLocator.getLogger(), persistenceOptions = {
|
|
108
95
|
enable: true,
|
|
109
|
-
}, } = options;
|
|
110
|
-
this.
|
|
111
|
-
this.
|
|
112
|
-
this.events =
|
|
96
|
+
}, sessionReuseStrategy = 'random', } = options;
|
|
97
|
+
this.id = id != null ? String(id) : String(SessionPool.nextId++);
|
|
98
|
+
this.sessionReuseStrategy = sessionReuseStrategy;
|
|
99
|
+
this.events = serviceLocator.getEventManager();
|
|
113
100
|
this.log = log.child({ prefix: 'SessionPool' });
|
|
114
101
|
this.persistenceOptions = persistenceOptions;
|
|
115
102
|
// Pool Configuration
|
|
@@ -124,33 +111,39 @@ export class SessionPool extends EventEmitter {
|
|
|
124
111
|
};
|
|
125
112
|
// Session keyValueStore
|
|
126
113
|
this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
|
|
127
|
-
this.persistStateKey = persistStateKey
|
|
114
|
+
this.persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
|
|
128
115
|
}
|
|
129
116
|
/**
|
|
130
117
|
* Gets count of usable sessions in the pool.
|
|
131
118
|
*/
|
|
132
|
-
|
|
119
|
+
async usableSessionsCount() {
|
|
120
|
+
await this.ensureInitialized();
|
|
133
121
|
return this.sessions.filter((session) => session.isUsable()).length;
|
|
134
122
|
}
|
|
135
123
|
/**
|
|
136
124
|
* Gets count of retired sessions in the pool.
|
|
137
125
|
*/
|
|
138
|
-
|
|
126
|
+
async retiredSessionsCount() {
|
|
127
|
+
await this.ensureInitialized();
|
|
139
128
|
return this.sessions.filter((session) => !session.isUsable()).length;
|
|
140
129
|
}
|
|
141
130
|
/**
|
|
142
131
|
* Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
|
|
143
|
-
*
|
|
132
|
+
* Called automatically on first use of any public method.
|
|
144
133
|
*/
|
|
145
|
-
async
|
|
146
|
-
if (this.
|
|
147
|
-
|
|
134
|
+
async ensureInitialized() {
|
|
135
|
+
if (!this.initPromise) {
|
|
136
|
+
this.initPromise = this.setupPool();
|
|
148
137
|
}
|
|
149
|
-
|
|
138
|
+
return this.initPromise;
|
|
139
|
+
}
|
|
140
|
+
async setupPool() {
|
|
150
141
|
if (!this.persistenceOptions.enable) {
|
|
151
|
-
this.isInitialized = true;
|
|
152
142
|
return;
|
|
153
143
|
}
|
|
144
|
+
this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
|
|
145
|
+
config: serviceLocator.getConfiguration(),
|
|
146
|
+
});
|
|
154
147
|
if (!this.persistStateKeyValueStoreId) {
|
|
155
148
|
this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
|
|
156
149
|
}
|
|
@@ -158,7 +151,6 @@ export class SessionPool extends EventEmitter {
|
|
|
158
151
|
await this._maybeLoadSessionPool();
|
|
159
152
|
this._listener = this.persistState.bind(this);
|
|
160
153
|
this.events.on("persistState" /* EventType.PERSIST_STATE */, this._listener);
|
|
161
|
-
this.isInitialized = true;
|
|
162
154
|
}
|
|
163
155
|
/**
|
|
164
156
|
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
@@ -167,7 +159,7 @@ export class SessionPool extends EventEmitter {
|
|
|
167
159
|
* @param [options] The configuration options for the session being added to the session pool.
|
|
168
160
|
*/
|
|
169
161
|
async addSession(options = {}) {
|
|
170
|
-
this.
|
|
162
|
+
await this.ensureInitialized();
|
|
171
163
|
const { id } = options;
|
|
172
164
|
if (id) {
|
|
173
165
|
const sessionExists = this.sessionMap.has(id);
|
|
@@ -182,6 +174,18 @@ export class SessionPool extends EventEmitter {
|
|
|
182
174
|
this.log.debug(`Adding new Session - ${newSession.id}`);
|
|
183
175
|
this._addSession(newSession);
|
|
184
176
|
}
|
|
177
|
+
/**
|
|
178
|
+
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
179
|
+
* but this allows you to add more sessions once the max pool size is reached.
|
|
180
|
+
* This also allows you to add session with overridden session options (e.g. with specific session id).
|
|
181
|
+
* @param [options] The configuration options for the session being added to the session pool.
|
|
182
|
+
*/
|
|
183
|
+
async newSession(sessionOptions) {
|
|
184
|
+
await this.ensureInitialized();
|
|
185
|
+
const newSession = await this.createSessionFunction(this, { sessionOptions });
|
|
186
|
+
this._addSession(newSession);
|
|
187
|
+
return newSession;
|
|
188
|
+
}
|
|
185
189
|
/**
|
|
186
190
|
* Gets session.
|
|
187
191
|
* If there is space for new session, it creates and returns new session.
|
|
@@ -190,21 +194,20 @@ export class SessionPool extends EventEmitter {
|
|
|
190
194
|
* @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
|
|
191
195
|
*/
|
|
192
196
|
async getSession(sessionId) {
|
|
197
|
+
await this.ensureInitialized();
|
|
193
198
|
await this.queue.wait();
|
|
194
199
|
try {
|
|
195
|
-
this._throwIfNotInitialized();
|
|
196
200
|
if (sessionId) {
|
|
197
201
|
const session = this.sessionMap.get(sessionId);
|
|
198
|
-
if (session
|
|
202
|
+
if (session?.isUsable())
|
|
199
203
|
return session;
|
|
200
204
|
return undefined;
|
|
201
205
|
}
|
|
202
|
-
if (this._hasSpaceForSession()) {
|
|
203
|
-
return await this._createSession();
|
|
204
|
-
}
|
|
205
206
|
const pickedSession = this._pickSession();
|
|
206
|
-
if (pickedSession
|
|
207
|
+
if (pickedSession)
|
|
207
208
|
return pickedSession;
|
|
209
|
+
if (this._hasSpaceForSession()) {
|
|
210
|
+
return await this._createSession();
|
|
208
211
|
}
|
|
209
212
|
this._removeRetiredSessions();
|
|
210
213
|
return await this._createSession();
|
|
@@ -220,16 +223,18 @@ export class SessionPool extends EventEmitter {
|
|
|
220
223
|
if (!this.persistenceOptions.enable && !options?.enable) {
|
|
221
224
|
return;
|
|
222
225
|
}
|
|
226
|
+
await this.ensureInitialized();
|
|
223
227
|
await this.keyValueStore?.setValue(this.persistStateKey, null);
|
|
224
228
|
}
|
|
225
229
|
/**
|
|
226
230
|
* Returns an object representing the internal state of the `SessionPool` instance.
|
|
227
231
|
* Note that the object's fields can change in future releases.
|
|
228
232
|
*/
|
|
229
|
-
getState() {
|
|
233
|
+
async getState() {
|
|
234
|
+
await this.ensureInitialized();
|
|
230
235
|
return {
|
|
231
|
-
usableSessionsCount: this.usableSessionsCount,
|
|
232
|
-
retiredSessionsCount: this.retiredSessionsCount,
|
|
236
|
+
usableSessionsCount: await this.usableSessionsCount(),
|
|
237
|
+
retiredSessionsCount: await this.retiredSessionsCount(),
|
|
233
238
|
sessions: this.sessions.map((session) => session.getState()),
|
|
234
239
|
};
|
|
235
240
|
}
|
|
@@ -242,15 +247,16 @@ export class SessionPool extends EventEmitter {
|
|
|
242
247
|
if (!this.persistenceOptions.enable && !options?.enable) {
|
|
243
248
|
return;
|
|
244
249
|
}
|
|
250
|
+
await this.ensureInitialized();
|
|
245
251
|
this.log.debug('Persisting state', {
|
|
246
252
|
persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
|
|
247
253
|
persistStateKey: this.persistStateKey,
|
|
248
254
|
});
|
|
249
255
|
// use half the interval of `persistState` to avoid race conditions
|
|
250
|
-
const persistStateIntervalMillis =
|
|
256
|
+
const persistStateIntervalMillis = serviceLocator.getConfiguration().persistStateIntervalMillis;
|
|
251
257
|
const timeoutSecs = persistStateIntervalMillis / 2_000;
|
|
252
258
|
await this.keyValueStore
|
|
253
|
-
|
|
259
|
+
?.setValue(this.persistStateKey, await this.getState(), {
|
|
254
260
|
timeoutSecs,
|
|
255
261
|
doNotRetryTimeouts: true,
|
|
256
262
|
})
|
|
@@ -261,16 +267,14 @@ export class SessionPool extends EventEmitter {
|
|
|
261
267
|
* This function should be called after you are done with using the `SessionPool` instance.
|
|
262
268
|
*/
|
|
263
269
|
async teardown() {
|
|
264
|
-
|
|
270
|
+
if (!this.initPromise)
|
|
271
|
+
return;
|
|
272
|
+
await this.ensureInitialized();
|
|
273
|
+
if (this._listener) {
|
|
274
|
+
this.events.off("persistState" /* EventType.PERSIST_STATE */, this._listener);
|
|
275
|
+
}
|
|
265
276
|
await this.persistState();
|
|
266
277
|
}
|
|
267
|
-
/**
|
|
268
|
-
* SessionPool should not work before initialization.
|
|
269
|
-
*/
|
|
270
|
-
_throwIfNotInitialized() {
|
|
271
|
-
if (!this.isInitialized)
|
|
272
|
-
throw new Error('SessionPool is not initialized.');
|
|
273
|
-
}
|
|
274
278
|
/**
|
|
275
279
|
* Removes retired `Session` instances from `SessionPool`.
|
|
276
280
|
*/
|
|
@@ -304,7 +308,7 @@ export class SessionPool extends EventEmitter {
|
|
|
304
308
|
* @param [options.sessionOptions] The configuration options for the session being created.
|
|
305
309
|
* @returns New session.
|
|
306
310
|
*/
|
|
307
|
-
_defaultCreateSessionFunction(sessionPool, options = {}) {
|
|
311
|
+
async _defaultCreateSessionFunction(sessionPool, options = {}) {
|
|
308
312
|
ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
|
|
309
313
|
const { sessionOptions = {} } = options;
|
|
310
314
|
return new Session({
|
|
@@ -330,18 +334,32 @@ export class SessionPool extends EventEmitter {
|
|
|
330
334
|
return this.sessions.length < this.maxPoolSize;
|
|
331
335
|
}
|
|
332
336
|
/**
|
|
333
|
-
* Picks
|
|
334
|
-
*
|
|
337
|
+
* Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
|
|
338
|
+
* Returns `undefined` when no session should be reused and a new one should be created instead.
|
|
335
339
|
*/
|
|
336
340
|
_pickSession() {
|
|
337
|
-
|
|
341
|
+
if (this.sessionReuseStrategy !== 'use-until-failure' && this._hasSpaceForSession())
|
|
342
|
+
return undefined;
|
|
343
|
+
if (this.sessionReuseStrategy === 'use-until-failure') {
|
|
344
|
+
return this.sessions.find((session) => session.isUsable());
|
|
345
|
+
}
|
|
346
|
+
let picked;
|
|
347
|
+
if (this.sessionReuseStrategy === 'round-robin') {
|
|
348
|
+
const index = this.roundRobinIndex % this.sessions.length;
|
|
349
|
+
this.roundRobinIndex = index + 1;
|
|
350
|
+
picked = this.sessions[index];
|
|
351
|
+
}
|
|
352
|
+
else {
|
|
353
|
+
picked = this.sessions[this._getRandomIndex()];
|
|
354
|
+
}
|
|
355
|
+
return picked.isUsable() ? picked : undefined;
|
|
338
356
|
}
|
|
339
357
|
/**
|
|
340
358
|
* Potentially loads `SessionPool`.
|
|
341
359
|
* If the state was persisted it loads the `SessionPool` from the persisted state.
|
|
342
360
|
*/
|
|
343
361
|
async _maybeLoadSessionPool() {
|
|
344
|
-
const loadedSessionPool = await this.keyValueStore
|
|
362
|
+
const loadedSessionPool = await this.keyValueStore?.getValue(this.persistStateKey);
|
|
345
363
|
if (!loadedSessionPool)
|
|
346
364
|
return;
|
|
347
365
|
// Invalidate old sessions and load active sessions only
|
|
@@ -358,18 +376,7 @@ export class SessionPool extends EventEmitter {
|
|
|
358
376
|
this._addSession(recreatedSession);
|
|
359
377
|
}
|
|
360
378
|
}
|
|
361
|
-
this.log.debug(`${this.
|
|
362
|
-
}
|
|
363
|
-
/**
|
|
364
|
-
* Opens a SessionPool and returns a promise resolving to an instance
|
|
365
|
-
* of the {@link SessionPool} class that is already initialized.
|
|
366
|
-
*
|
|
367
|
-
* For more details and code examples, see the {@link SessionPool} class.
|
|
368
|
-
*/
|
|
369
|
-
static async open(options, config) {
|
|
370
|
-
const sessionPool = new SessionPool(options, config);
|
|
371
|
-
await sessionPool.initialize();
|
|
372
|
-
return sessionPool;
|
|
379
|
+
this.log.debug(`${this.sessions.length} active sessions loaded from KeyValueStore`);
|
|
373
380
|
}
|
|
374
381
|
}
|
|
375
382
|
//# sourceMappingURL=session_pool.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,MAAM,IAAI,CAAC;AAMpB,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAE/D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,wBAAwB,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,mBAAmB,CAAU,CAAC;AAiEzF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AACH,MAAM,OAAO,WAAY,SAAQ,YAAY;IACjC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAEjB,EAAE,CAAS;IACV,GAAG,CAAgB;IACnB,WAAW,CAAS;IACpB,qBAAqB,CAAgB;IACrC,aAAa,CAAiB;IAC9B,QAAQ,GAAc,EAAE,CAAC;IACzB,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;IACxC,cAAc,CAAiB;IAC/B,2BAA2B,CAAU;IACrC,eAAe,CAAS;IACxB,SAAS,CAAuB;IAChC,MAAM,CAAe;IACrB,kBAAkB,CAAqB;IACvC,oBAAoB,CAAuB;IAE7C,WAAW,CAAiB;IAC5B,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IACzB,eAAe,GAAG,CAAC,CAAC;IAE5B,YAAY,UAA8B,EAAE;QACxC,KAAK,EAAE,CAAC;QAER,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC;YACzC,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,wBAAwB,CAAC,CAAC;SAChF,CAAC,CACL,CAAC;QAEF,MAAM,EACF,EAAE,EACF,WAAW,GAAG,aAAa,EAC3B,2BAA2B,EAC3B,eAAe,EACf,qBAAqB,EACrB,cAAc,GAAG,EAAE,EACnB,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,EAChC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,oBAAoB,GAAG,QAAQ,GAClC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAC/C,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,6GAA6G;YAC7G,oDAAoD;YACpD,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,IAAI,GAAG,iBAAiB,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;IAChF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB;QACrB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB;QACtB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QACxC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,SAAS;QACnB,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YAClC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CACzC,IAAI,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC,IAAI,EAClF;YACI,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE;SAC5C,CACJ,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,iIAAiI,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAC3J,CAAC;QACN,CAAC;QAED,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE,CAAC;YACL,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,UAAU,GACZ,OAAO,YAAY,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/G,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,cAA+B;QAC5C,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAE7B,OAAO,UAAU,CAAC;IACtB,CAAC;IAeD;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC;YACD,IAAI,SAAS,EAAE,CAAC;gBACZ,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC/C,IAAI,OAAO,EAAE,QAAQ,EAAE;oBAAE,OAAO,OAAO,CAAC;gBACxC,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1C,IAAI,aAAa;gBAAE,OAAO,aAAa,CAAC;YAExC,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,CAAC;YAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACvC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO;YACH,mBAAmB,EAAE,MAAM,IAAI,CAAC,mBAAmB,EAAE;YACrD,oBAAoB,EAAE,MAAM,IAAI,CAAC,oBAAoB,EAAE;YACvD,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,0BAA0B,GAAG,cAAc,CAAC,gBAAgB,EAAE,CAAC,0BAA0B,CAAC;QAChG,MAAM,WAAW,GAAG,0BAA0B,GAAG,KAAK,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa;YACpB,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,EAAE;YACpD,WAAW;YACX,kBAAkB,EAAE,IAAI;SAC3B,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,+CAA+C,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CACrG,CAAC;IACV,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO;QAC9B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACO,KAAK,CAAC,6BAA6B,CACzC,WAAwB,EACxB,UAA+C,EAAE;QAEjD,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QAExC,OAAO,IAAI,OAAO,CAAC;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,cAAc;YACjB,WAAW;SACd,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,IAAI,IAAI,CAAC,oBAAoB,KAAK,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAAE,OAAO,SAAS,CAAC;QAEtG,IAAI,IAAI,CAAC,oBAAoB,KAAK,mBAAmB,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,MAAe,CAAC;QACpB,IAAI,IAAI,CAAC,oBAAoB,KAAK,aAAa,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC1D,IAAI,CAAC,eAAe,GAAG,KAAK,GAAG,CAAC,CAAC;YACjC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACJ,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAClD,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE/G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YACrD,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;YACjC,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;YAEnG,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,4CAA4C,CAAC,CAAC;IACxF,CAAC"}
|