@crawlee/basic 4.0.0-beta.211 → 4.0.0-beta.213
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/autoscaling/autoscaled_pool.d.ts +0 -20
- package/internals/autoscaling/autoscaled_pool.js +11 -12
- package/internals/autoscaling/concurrency_system.d.ts +0 -38
- package/internals/autoscaling/concurrency_system.js +51 -53
- package/internals/autoscaling/memory_load_signal.d.ts +0 -2
- package/internals/autoscaling/memory_load_signal.js +4 -4
- package/internals/autoscaling/system_status.d.ts +0 -10
- package/internals/autoscaling/system_status.js +5 -5
- package/internals/basic-crawler.d.ts +0 -88
- package/internals/basic-crawler.js +73 -68
- package/internals/crawlers/context_pipeline.js +3 -3
- package/internals/crawlers/error_tracker.d.ts +0 -1
- package/internals/crawlers/error_tracker.js +3 -3
- package/internals/crawlers/statistics.d.ts +0 -2
- package/internals/crawlers/statistics.js +11 -11
- package/internals/crawling_request.d.ts +1 -1
- package/internals/crawling_request.js +12 -12
- package/internals/router.d.ts +0 -9
- package/internals/router.js +5 -5
- package/internals/session_pool/session.d.ts +0 -5
- package/internals/session_pool/session.js +3 -3
- package/internals/session_pool/session_pool.d.ts +0 -51
- package/internals/session_pool/session_pool.js +36 -36
- package/internals/sitemap_request_loader.d.ts +0 -39
- package/internals/sitemap_request_loader.js +18 -18
- package/internals/throttling_request_manager.d.ts +0 -1
- package/internals/throttling_request_manager.js +5 -5
- package/package.json +7 -7
|
@@ -40,14 +40,14 @@ class ContextPipelineImpl extends ContextPipeline {
|
|
|
40
40
|
return new ContextPipelineImpl(middleware, this);
|
|
41
41
|
}
|
|
42
42
|
chain(other) {
|
|
43
|
-
const otherMiddlewares = Array.from(other
|
|
43
|
+
const otherMiddlewares = Array.from(other.#middlewareChain()).reverse();
|
|
44
44
|
let result = this;
|
|
45
45
|
for (const middleware of otherMiddlewares) {
|
|
46
46
|
result = result.compose(middleware);
|
|
47
47
|
}
|
|
48
48
|
return result;
|
|
49
49
|
}
|
|
50
|
-
|
|
50
|
+
*#middlewareChain() {
|
|
51
51
|
let step = this;
|
|
52
52
|
while (step !== undefined) {
|
|
53
53
|
yield step.#middleware;
|
|
@@ -58,7 +58,7 @@ class ContextPipelineImpl extends ContextPipeline {
|
|
|
58
58
|
* @inheritdoc
|
|
59
59
|
*/
|
|
60
60
|
async call(crawlingContext, finalContextConsumer, onError) {
|
|
61
|
-
const middlewares = Array.from(this
|
|
61
|
+
const middlewares = Array.from(this.#middlewareChain()).reverse();
|
|
62
62
|
const cleanupStack = [];
|
|
63
63
|
const onCleanup = (cleanup) => {
|
|
64
64
|
cleanupStack.push(cleanup);
|
|
@@ -39,7 +39,6 @@ export declare class ErrorTracker {
|
|
|
39
39
|
/** The total number of errors passed to the tracker, including the same error seen repeatedly. */
|
|
40
40
|
get total(): number;
|
|
41
41
|
constructor(options?: Partial<ErrorTrackerOptions>);
|
|
42
|
-
private updateGroup;
|
|
43
42
|
add(error: ErrnoException): void;
|
|
44
43
|
/**
|
|
45
44
|
* This method is async, because it captures a snapshot of the error context.
|
|
@@ -229,7 +229,7 @@ export class ErrorTracker {
|
|
|
229
229
|
this.#result = Object.create(null);
|
|
230
230
|
this.#total = 0;
|
|
231
231
|
}
|
|
232
|
-
updateGroup(error) {
|
|
232
|
+
#updateGroup(error) {
|
|
233
233
|
let group = this.#result;
|
|
234
234
|
if (this.#options.showStackTrace) {
|
|
235
235
|
group = getStackTraceGroup(error, group, this.#options.showFullStack);
|
|
@@ -248,7 +248,7 @@ export class ErrorTracker {
|
|
|
248
248
|
}
|
|
249
249
|
add(error) {
|
|
250
250
|
this.#total++;
|
|
251
|
-
this
|
|
251
|
+
this.#updateGroup(error);
|
|
252
252
|
if (typeof error.cause === 'object' && error.cause !== null) {
|
|
253
253
|
this.add(error.cause);
|
|
254
254
|
}
|
|
@@ -259,7 +259,7 @@ export class ErrorTracker {
|
|
|
259
259
|
*/
|
|
260
260
|
async addAsync(error, context) {
|
|
261
261
|
this.#total++;
|
|
262
|
-
const group = this
|
|
262
|
+
const group = this.#updateGroup(error);
|
|
263
263
|
// Capture a snapshot (screenshot and HTML) on the first occurrence of an error
|
|
264
264
|
if (group.count === 1 && context) {
|
|
265
265
|
await this.#captureSnapshot(group, error, context).catch(() => { });
|
|
@@ -100,7 +100,6 @@ export declare class Statistics<StateExtension extends object = {}, PersistedSta
|
|
|
100
100
|
* Statistic instance id.
|
|
101
101
|
*/
|
|
102
102
|
readonly id: string;
|
|
103
|
-
private readonly log;
|
|
104
103
|
/**
|
|
105
104
|
* Current statistic state used for doing calculations on {@link Statistics.calculate} calls
|
|
106
105
|
*/
|
|
@@ -156,7 +155,6 @@ export declare class Statistics<StateExtension extends object = {}, PersistedSta
|
|
|
156
155
|
* Stops logging and remove event listeners, then persist
|
|
157
156
|
*/
|
|
158
157
|
stopCapturing(): Promise<void>;
|
|
159
|
-
private saveRetryCountForRequest;
|
|
160
158
|
/**
|
|
161
159
|
* Persist internal state to the key value store.
|
|
162
160
|
*
|
|
@@ -178,7 +178,7 @@ export class Statistics {
|
|
|
178
178
|
#logIntervalMillis;
|
|
179
179
|
#logMessage;
|
|
180
180
|
#requestsInProgress = new Map();
|
|
181
|
-
log;
|
|
181
|
+
#log;
|
|
182
182
|
#logInterval;
|
|
183
183
|
/**
|
|
184
184
|
* Current statistic state used for doing calculations on {@link Statistics.calculate} calls
|
|
@@ -200,7 +200,7 @@ export class Statistics {
|
|
|
200
200
|
const { logIntervalSecs, logMessage, log, keyValueStore, persistenceOptions, saveErrorSnapshots, id, stateExtension, } = parseArgument(options, statisticsOptionsSchema);
|
|
201
201
|
this.id = id ?? String(Statistics.#id++);
|
|
202
202
|
this.#persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
|
|
203
|
-
this
|
|
203
|
+
this.#log = (log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
|
|
204
204
|
this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
|
|
205
205
|
this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
|
|
206
206
|
this.#logIntervalMillis = logIntervalSecs * 1000;
|
|
@@ -224,7 +224,7 @@ export class Statistics {
|
|
|
224
224
|
persistStateKey: this.#persistStateKey,
|
|
225
225
|
persistenceEnabled: persistenceOptions.enable,
|
|
226
226
|
keyValueStore,
|
|
227
|
-
logger: this
|
|
227
|
+
logger: this.#log,
|
|
228
228
|
defaultState: () => this.#defaultState(),
|
|
229
229
|
serialize: (state) => this.#serializeState(state),
|
|
230
230
|
deserialize: (persistedState) => this.#deserializeState(persistedState),
|
|
@@ -337,7 +337,7 @@ export class Statistics {
|
|
|
337
337
|
const durationMillis = record.finish();
|
|
338
338
|
this.state.requestsSucceeded++;
|
|
339
339
|
this.state.requestTotalSucceededDurationMillis += durationMillis;
|
|
340
|
-
this
|
|
340
|
+
this.#saveRetryCountForRequest(retryCount);
|
|
341
341
|
if (durationMillis < this.state.requestMinDurationMillis)
|
|
342
342
|
this.state.requestMinDurationMillis = durationMillis;
|
|
343
343
|
if (durationMillis > this.state.requestMaxDurationMillis)
|
|
@@ -351,7 +351,7 @@ export class Statistics {
|
|
|
351
351
|
return;
|
|
352
352
|
this.state.requestTotalFailedDurationMillis += record.finish();
|
|
353
353
|
this.state.requestsFailed++;
|
|
354
|
-
this
|
|
354
|
+
this.#saveRetryCountForRequest(retryCount);
|
|
355
355
|
this.#requestsInProgress.delete(id);
|
|
356
356
|
}
|
|
357
357
|
/**
|
|
@@ -396,7 +396,7 @@ export class Statistics {
|
|
|
396
396
|
this.state.crawlerStartedAt = new Date();
|
|
397
397
|
}
|
|
398
398
|
this.#logInterval = setInterval(() => {
|
|
399
|
-
this
|
|
399
|
+
this.#log.info(this.#logMessage, {
|
|
400
400
|
...this.calculate(),
|
|
401
401
|
retryHistogram: this.requestRetryHistogram,
|
|
402
402
|
});
|
|
@@ -410,7 +410,7 @@ export class Statistics {
|
|
|
410
410
|
this.state.crawlerFinishedAt = new Date();
|
|
411
411
|
await this.#recoverableState.teardown();
|
|
412
412
|
}
|
|
413
|
-
saveRetryCountForRequest(retryCount) {
|
|
413
|
+
#saveRetryCountForRequest(retryCount) {
|
|
414
414
|
if (retryCount > 0)
|
|
415
415
|
this.state.requestsRetries++;
|
|
416
416
|
this.requestRetryHistogram[retryCount] ??= 0;
|
|
@@ -425,7 +425,7 @@ export class Statistics {
|
|
|
425
425
|
async persistState() {
|
|
426
426
|
await this.#recoverableState
|
|
427
427
|
.persistState()
|
|
428
|
-
.catch((error) => this
|
|
428
|
+
.catch((error) => this.#log.warning(`Failed to persist the statistics to ${this.#persistStateKey}`, { error }));
|
|
429
429
|
}
|
|
430
430
|
/** Rebuilds the state from a persisted record. */
|
|
431
431
|
#deserializeState(persistedState) {
|
|
@@ -436,13 +436,13 @@ export class Statistics {
|
|
|
436
436
|
if (!restored.success) {
|
|
437
437
|
// Statistics are bookkeeping - a record that cannot be made sense of is worth a warning and a fresh
|
|
438
438
|
// start, not a failed crawl.
|
|
439
|
-
this
|
|
439
|
+
this.#log.warning('Received invalid state from Key-value store, starting the statistics from scratch.', {
|
|
440
440
|
persistStateKey: this.#persistStateKey,
|
|
441
441
|
issues: restored.error.issues,
|
|
442
442
|
});
|
|
443
443
|
return this.#defaultState();
|
|
444
444
|
}
|
|
445
|
-
this
|
|
445
|
+
this.#log.debug('Recreating state from KeyValueStore', { persistStateKey: this.#persistStateKey });
|
|
446
446
|
return { ...this.#defaultState(), ...restored.data, ...this.#restoreStateExtension(persistedState) };
|
|
447
447
|
}
|
|
448
448
|
/**
|
|
@@ -474,7 +474,7 @@ export class Statistics {
|
|
|
474
474
|
catch (error) {
|
|
475
475
|
// Same policy as a built-in field that cannot be made sense of, but scoped to the custom ones - a
|
|
476
476
|
// corrupt counter of your own is no reason to throw away the crawler's.
|
|
477
|
-
this
|
|
477
|
+
this.#log.warning('Received invalid custom statistics fields from Key-value store, starting those from scratch.', { persistStateKey: this.#persistStateKey, error });
|
|
478
478
|
return {};
|
|
479
479
|
}
|
|
480
480
|
}
|
|
@@ -41,8 +41,8 @@ export interface CrawlingRequestData {
|
|
|
41
41
|
* @category Sources
|
|
42
42
|
*/
|
|
43
43
|
export declare class CrawlingRequest<UserData extends Dictionary = Dictionary> extends Request<UserData> {
|
|
44
|
+
#private;
|
|
44
45
|
static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawlingRequest<UserData>;
|
|
45
|
-
private get crawleeData();
|
|
46
46
|
/**
|
|
47
47
|
* Tells the crawler processing this request to skip the navigation and process the request directly.
|
|
48
48
|
*
|
|
@@ -23,7 +23,7 @@ export class CrawlingRequest extends Request {
|
|
|
23
23
|
static fromSchema(schema) {
|
|
24
24
|
return super.fromSchema(schema);
|
|
25
25
|
}
|
|
26
|
-
get crawleeData() {
|
|
26
|
+
get #crawleeData() {
|
|
27
27
|
return (this.userData.__crawlee ??= {});
|
|
28
28
|
}
|
|
29
29
|
/**
|
|
@@ -34,45 +34,45 @@ export class CrawlingRequest extends Request {
|
|
|
34
34
|
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
35
35
|
*/
|
|
36
36
|
get skipNavigation() {
|
|
37
|
-
return this
|
|
37
|
+
return this.#crawleeData.skipNavigation ?? false;
|
|
38
38
|
}
|
|
39
39
|
set skipNavigation(value) {
|
|
40
|
-
this
|
|
40
|
+
this.#crawleeData.skipNavigation = value;
|
|
41
41
|
}
|
|
42
42
|
/**
|
|
43
43
|
* Depth of the request in the current crawl tree.
|
|
44
44
|
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
45
45
|
*/
|
|
46
46
|
get crawlDepth() {
|
|
47
|
-
return this
|
|
47
|
+
return this.#crawleeData.crawlDepth ?? 0;
|
|
48
48
|
}
|
|
49
49
|
set crawlDepth(value) {
|
|
50
|
-
this
|
|
50
|
+
this.#crawleeData.crawlDepth = value;
|
|
51
51
|
}
|
|
52
52
|
/** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
|
|
53
53
|
get sessionId() {
|
|
54
|
-
return this
|
|
54
|
+
return this.#crawleeData.sessionId;
|
|
55
55
|
}
|
|
56
56
|
set sessionId(value) {
|
|
57
|
-
this
|
|
57
|
+
this.#crawleeData.sessionId = value;
|
|
58
58
|
}
|
|
59
59
|
/** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
|
|
60
60
|
get maxRetries() {
|
|
61
|
-
return this
|
|
61
|
+
return this.#crawleeData.maxRetries;
|
|
62
62
|
}
|
|
63
63
|
set maxRetries(value) {
|
|
64
|
-
this
|
|
64
|
+
this.#crawleeData.maxRetries = value;
|
|
65
65
|
}
|
|
66
66
|
/** Describes the request's current lifecycle state. */
|
|
67
67
|
get state() {
|
|
68
|
-
return this
|
|
68
|
+
return this.#crawleeData.state ?? RequestState.UNPROCESSED;
|
|
69
69
|
}
|
|
70
70
|
set state(value) {
|
|
71
|
-
this
|
|
71
|
+
this.#crawleeData.state = value;
|
|
72
72
|
}
|
|
73
73
|
/** The strategy the request was enqueued under, if it came from `enqueueLinks`. @internal */
|
|
74
74
|
get enqueueStrategy() {
|
|
75
|
-
return this
|
|
75
|
+
return this.#crawleeData.enqueueStrategy;
|
|
76
76
|
}
|
|
77
77
|
/**
|
|
78
78
|
* Stores information about an error that occurred during processing of this request.
|
package/internals/router.d.ts
CHANGED
|
@@ -272,15 +272,6 @@ export declare class Router<Context extends RestrictedCrawlingContext, Routes ex
|
|
|
272
272
|
* Returns route handler for given label. If no label is provided, the default request handler will be returned.
|
|
273
273
|
*/
|
|
274
274
|
getHandler(label?: string | symbol): (ctx: Context) => Awaitable<void>;
|
|
275
|
-
/**
|
|
276
|
-
* Validates `request.userData` against the schema registered for its label (if any), replacing it with
|
|
277
|
-
* the parsed value. Throws a {@link RequestValidationError} when validation fails.
|
|
278
|
-
*/
|
|
279
|
-
private validateRequest;
|
|
280
|
-
/**
|
|
281
|
-
* Throws when the label already exists in our registry.
|
|
282
|
-
*/
|
|
283
|
-
private validate;
|
|
284
275
|
/**
|
|
285
276
|
* Creates new router instance. This instance can then serve as a `requestHandler` of your crawler.
|
|
286
277
|
*
|
package/internals/router.js
CHANGED
|
@@ -172,7 +172,7 @@ export class Router {
|
|
|
172
172
|
*/
|
|
173
173
|
constructor() { }
|
|
174
174
|
addHandler(label, handler, options = {}) {
|
|
175
|
-
this
|
|
175
|
+
this.#validate(label);
|
|
176
176
|
this.#routes.set(label, handler);
|
|
177
177
|
if (options.requestHandlerTimeoutSecs !== undefined) {
|
|
178
178
|
this.#timeouts.set(label, options.requestHandlerTimeoutSecs);
|
|
@@ -187,7 +187,7 @@ export class Router {
|
|
|
187
187
|
* through to it.
|
|
188
188
|
*/
|
|
189
189
|
addDefaultHandler(handler, options = {}) {
|
|
190
|
-
this
|
|
190
|
+
this.#validate(defaultRoute);
|
|
191
191
|
this.#routes.set(defaultRoute, handler);
|
|
192
192
|
if (options.requestHandlerTimeoutSecs !== undefined) {
|
|
193
193
|
this.#timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
|
|
@@ -259,7 +259,7 @@ export class Router {
|
|
|
259
259
|
* Validates `request.userData` against the schema registered for its label (if any), replacing it with
|
|
260
260
|
* the parsed value. Throws a {@link RequestValidationError} when validation fails.
|
|
261
261
|
*/
|
|
262
|
-
async validateRequest(context) {
|
|
262
|
+
async #validateRequest(context) {
|
|
263
263
|
const label = context.request.label;
|
|
264
264
|
const schema = this.getSchema(label);
|
|
265
265
|
if (schema) {
|
|
@@ -269,7 +269,7 @@ export class Router {
|
|
|
269
269
|
/**
|
|
270
270
|
* Throws when the label already exists in our registry.
|
|
271
271
|
*/
|
|
272
|
-
validate(label) {
|
|
272
|
+
#validate(label) {
|
|
273
273
|
if (this.#routes.has(label)) {
|
|
274
274
|
const message = label === defaultRoute
|
|
275
275
|
? `Default route is already defined!`
|
|
@@ -300,7 +300,7 @@ export class Router {
|
|
|
300
300
|
const func = async function (context) {
|
|
301
301
|
const { url, loadedUrl, label } = context.request;
|
|
302
302
|
context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
|
|
303
|
-
await router
|
|
303
|
+
await router.#validateRequest(context);
|
|
304
304
|
for (const middleware of router.#middlewares) {
|
|
305
305
|
await middleware(context);
|
|
306
306
|
}
|
|
@@ -148,9 +148,4 @@ export declare class Session implements ISession {
|
|
|
148
148
|
* Sets a cookie within this session for the specific URL.
|
|
149
149
|
*/
|
|
150
150
|
setCookie(rawCookie: string, url: string): Promise<void>;
|
|
151
|
-
/**
|
|
152
|
-
* Makes an unusable session's state terminal. Unlike `retire()`, this leaves the error score alone so that
|
|
153
|
-
* `isBlocked()` keeps telling a burned identity from one that merely ran out of uses.
|
|
154
|
-
*/
|
|
155
|
-
private maybeSelfRetire;
|
|
156
151
|
}
|
|
@@ -146,7 +146,7 @@ export class Session {
|
|
|
146
146
|
if (this.#errorScore > 0) {
|
|
147
147
|
this.#errorScore -= this.#errorScoreDecrement;
|
|
148
148
|
}
|
|
149
|
-
this
|
|
149
|
+
this.#maybeSelfRetire();
|
|
150
150
|
}
|
|
151
151
|
/**
|
|
152
152
|
* Gets session state for persistence in KeyValueStore.
|
|
@@ -191,7 +191,7 @@ export class Session {
|
|
|
191
191
|
markBad() {
|
|
192
192
|
this.#errorScore += 1;
|
|
193
193
|
this.#usageCount += 1;
|
|
194
|
-
this
|
|
194
|
+
this.#maybeSelfRetire();
|
|
195
195
|
}
|
|
196
196
|
/**
|
|
197
197
|
* Returns cookies saved with the session in the typical
|
|
@@ -217,7 +217,7 @@ export class Session {
|
|
|
217
217
|
* Makes an unusable session's state terminal. Unlike `retire()`, this leaves the error score alone so that
|
|
218
218
|
* `isBlocked()` keeps telling a burned identity from one that merely ran out of uses.
|
|
219
219
|
*/
|
|
220
|
-
maybeSelfRetire() {
|
|
220
|
+
#maybeSelfRetire() {
|
|
221
221
|
if (!this.isUsable()) {
|
|
222
222
|
this.#retired = true;
|
|
223
223
|
}
|
|
@@ -133,12 +133,6 @@ export declare class SessionPool implements ISessionPool {
|
|
|
133
133
|
* Gets count of retired sessions in the pool.
|
|
134
134
|
*/
|
|
135
135
|
retiredSessionsCount(): Promise<number>;
|
|
136
|
-
/**
|
|
137
|
-
* Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
|
|
138
|
-
* Called automatically on first use of any public method.
|
|
139
|
-
*/
|
|
140
|
-
private ensureInitialized;
|
|
141
|
-
private setupPool;
|
|
142
136
|
/**
|
|
143
137
|
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
144
138
|
* but this allows you to add more sessions once the max pool size is reached.
|
|
@@ -195,50 +189,5 @@ export declare class SessionPool implements ISessionPool {
|
|
|
195
189
|
* Using the pool again afterwards initializes it anew.
|
|
196
190
|
*/
|
|
197
191
|
teardown(): Promise<void>;
|
|
198
|
-
/**
|
|
199
|
-
* Removes retired `Session` instances from `SessionPool`.
|
|
200
|
-
*/
|
|
201
|
-
private removeRetiredSessions;
|
|
202
|
-
/**
|
|
203
|
-
* Adds `Session` instance to `SessionPool`.
|
|
204
|
-
* @param newSession `Session` instance to be added.
|
|
205
|
-
*/
|
|
206
|
-
private registerSession;
|
|
207
|
-
/**
|
|
208
|
-
* Gets random index.
|
|
209
|
-
*/
|
|
210
|
-
private getRandomIndex;
|
|
211
|
-
/**
|
|
212
|
-
* Creates new session without any extra behavior.
|
|
213
|
-
* @param [options]
|
|
214
|
-
* @param [options.sessionOptions] The configuration options for the session being created.
|
|
215
|
-
* @returns New session.
|
|
216
|
-
*/
|
|
217
|
-
private defaultCreateSessionFunction;
|
|
218
|
-
/**
|
|
219
|
-
* Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
|
|
220
|
-
* the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
|
|
221
|
-
*
|
|
222
|
-
* A default {@link SessionFingerprint} is generated up front (host OS as
|
|
223
|
-
* `platform`, a random valid `browser`/`device` for that platform). Pool-wide
|
|
224
|
-
* and per-call options override it, and a persisted fingerprint coming
|
|
225
|
-
* through `maybeLoadSessionPool` naturally wins because it arrives in
|
|
226
|
-
* `perCallOptions`.
|
|
227
|
-
*/
|
|
228
|
-
private invokeCreateSessionFunction;
|
|
229
|
-
/**
|
|
230
|
-
* Creates new session and adds it to the pool.
|
|
231
|
-
* @returns Newly created `Session` instance.
|
|
232
|
-
*/
|
|
233
|
-
private createSession;
|
|
234
|
-
/**
|
|
235
|
-
* Decides whether there is enough space for creating new session.
|
|
236
|
-
*/
|
|
237
|
-
private hasSpaceForSession;
|
|
238
|
-
/**
|
|
239
|
-
* Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
|
|
240
|
-
* Returns `undefined` when no session should be reused and a new one should be created instead.
|
|
241
|
-
*/
|
|
242
|
-
private pickSession;
|
|
243
192
|
}
|
|
244
193
|
export {};
|
|
@@ -151,7 +151,7 @@ export class SessionPool {
|
|
|
151
151
|
this.#persistenceOptions = persistenceOptions;
|
|
152
152
|
// Pool Configuration
|
|
153
153
|
this.#maxPoolSize = maxPoolSize;
|
|
154
|
-
this.#createSessionFunction = createSessionFunction || this
|
|
154
|
+
this.#createSessionFunction = createSessionFunction || this.#defaultCreateSessionFunction;
|
|
155
155
|
// Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
|
|
156
156
|
// `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
|
|
157
157
|
// having to know about it.
|
|
@@ -163,7 +163,7 @@ export class SessionPool {
|
|
|
163
163
|
this.#persistStateKeyValueStoreId = persistStateKeyValueStoreId;
|
|
164
164
|
this.#persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
|
|
165
165
|
this.#stateCodec = buildSessionPoolStateCodec({
|
|
166
|
-
recreateSession: (sessionState) => this
|
|
166
|
+
recreateSession: (sessionState) => this.#invokeCreateSessionFunction(sessionState),
|
|
167
167
|
toRecord: (sessions) => this.#buildPersistedState(sessions),
|
|
168
168
|
log: this.#log,
|
|
169
169
|
});
|
|
@@ -185,27 +185,27 @@ export class SessionPool {
|
|
|
185
185
|
* Gets count of usable sessions in the pool.
|
|
186
186
|
*/
|
|
187
187
|
async usableSessionsCount() {
|
|
188
|
-
await this
|
|
188
|
+
await this.#ensureInitialized();
|
|
189
189
|
return this.#state.currentValue.sessions.filter((session) => session.isUsable()).length;
|
|
190
190
|
}
|
|
191
191
|
/**
|
|
192
192
|
* Gets count of retired sessions in the pool.
|
|
193
193
|
*/
|
|
194
194
|
async retiredSessionsCount() {
|
|
195
|
-
await this
|
|
195
|
+
await this.#ensureInitialized();
|
|
196
196
|
return this.#state.currentValue.sessions.filter((session) => !session.isUsable()).length;
|
|
197
197
|
}
|
|
198
198
|
/**
|
|
199
199
|
* Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
|
|
200
200
|
* Called automatically on first use of any public method.
|
|
201
201
|
*/
|
|
202
|
-
async ensureInitialized() {
|
|
202
|
+
async #ensureInitialized() {
|
|
203
203
|
if (!this.#initPromise) {
|
|
204
|
-
this.#initPromise = this
|
|
204
|
+
this.#initPromise = this.#setupPool();
|
|
205
205
|
}
|
|
206
206
|
return this.#initPromise;
|
|
207
207
|
}
|
|
208
|
-
async setupPool() {
|
|
208
|
+
async #setupPool() {
|
|
209
209
|
await this.#state.initialize();
|
|
210
210
|
for (const session of this.#state.currentValue.sessions) {
|
|
211
211
|
this.#sessionMap.set(session.id, session);
|
|
@@ -221,7 +221,7 @@ export class SessionPool {
|
|
|
221
221
|
* @param [options] The configuration options for the session being added to the session pool.
|
|
222
222
|
*/
|
|
223
223
|
async addSession(options = {}) {
|
|
224
|
-
await this
|
|
224
|
+
await this.#ensureInitialized();
|
|
225
225
|
const { id } = options;
|
|
226
226
|
if (id) {
|
|
227
227
|
const existingSession = this.#sessionMap.get(id);
|
|
@@ -232,12 +232,12 @@ export class SessionPool {
|
|
|
232
232
|
this.#removeSession(existingSession);
|
|
233
233
|
}
|
|
234
234
|
}
|
|
235
|
-
if (!this
|
|
236
|
-
this
|
|
235
|
+
if (!this.#hasSpaceForSession()) {
|
|
236
|
+
this.#removeRetiredSessions();
|
|
237
237
|
}
|
|
238
|
-
const newSession = options instanceof Session ? options : await this
|
|
238
|
+
const newSession = options instanceof Session ? options : await this.#invokeCreateSessionFunction(options);
|
|
239
239
|
this.#log.debug(`Adding new Session - ${newSession.id}`);
|
|
240
|
-
this
|
|
240
|
+
this.#registerSession(newSession);
|
|
241
241
|
}
|
|
242
242
|
/**
|
|
243
243
|
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
@@ -246,9 +246,9 @@ export class SessionPool {
|
|
|
246
246
|
* @param [options] The configuration options for the session being added to the session pool.
|
|
247
247
|
*/
|
|
248
248
|
async newSession(sessionOptions) {
|
|
249
|
-
await this
|
|
250
|
-
const newSession = await this
|
|
251
|
-
this
|
|
249
|
+
await this.#ensureInitialized();
|
|
250
|
+
const newSession = await this.#invokeCreateSessionFunction(sessionOptions);
|
|
251
|
+
this.#registerSession(newSession);
|
|
252
252
|
return newSession;
|
|
253
253
|
}
|
|
254
254
|
/**
|
|
@@ -259,7 +259,7 @@ export class SessionPool {
|
|
|
259
259
|
* @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
|
|
260
260
|
*/
|
|
261
261
|
async getSession(sessionId) {
|
|
262
|
-
await this
|
|
262
|
+
await this.#ensureInitialized();
|
|
263
263
|
await this.#queue.wait();
|
|
264
264
|
try {
|
|
265
265
|
if (sessionId) {
|
|
@@ -268,14 +268,14 @@ export class SessionPool {
|
|
|
268
268
|
return session;
|
|
269
269
|
return undefined;
|
|
270
270
|
}
|
|
271
|
-
const pickedSession = this
|
|
271
|
+
const pickedSession = this.#pickSession();
|
|
272
272
|
if (pickedSession)
|
|
273
273
|
return pickedSession;
|
|
274
|
-
if (this
|
|
275
|
-
return await this
|
|
274
|
+
if (this.#hasSpaceForSession()) {
|
|
275
|
+
return await this.#createSession();
|
|
276
276
|
}
|
|
277
|
-
this
|
|
278
|
-
return await this
|
|
277
|
+
this.#removeRetiredSessions();
|
|
278
|
+
return await this.#createSession();
|
|
279
279
|
}
|
|
280
280
|
finally {
|
|
281
281
|
this.#queue.shift();
|
|
@@ -306,7 +306,7 @@ export class SessionPool {
|
|
|
306
306
|
* @internal
|
|
307
307
|
*/
|
|
308
308
|
async getState() {
|
|
309
|
-
await this
|
|
309
|
+
await this.#ensureInitialized();
|
|
310
310
|
return this.#buildPersistedState(this.#state.currentValue.sessions);
|
|
311
311
|
}
|
|
312
312
|
/**
|
|
@@ -317,7 +317,7 @@ export class SessionPool {
|
|
|
317
317
|
if (!this.#persistenceOptions.enable) {
|
|
318
318
|
return;
|
|
319
319
|
}
|
|
320
|
-
await this
|
|
320
|
+
await this.#ensureInitialized();
|
|
321
321
|
this.#log.debug('Persisting state', {
|
|
322
322
|
persistStateKeyValueStoreId: this.#persistStateKeyValueStoreId,
|
|
323
323
|
persistStateKey: this.#persistStateKey,
|
|
@@ -337,7 +337,7 @@ export class SessionPool {
|
|
|
337
337
|
async teardown() {
|
|
338
338
|
if (!this.#initPromise)
|
|
339
339
|
return;
|
|
340
|
-
await this
|
|
340
|
+
await this.#ensureInitialized();
|
|
341
341
|
await this.#state.teardown();
|
|
342
342
|
// The next use initializes the pool again, restoring the record and resuming the periodic persistence -
|
|
343
343
|
// a crawler that is run twice gets a persisting pool both times.
|
|
@@ -346,7 +346,7 @@ export class SessionPool {
|
|
|
346
346
|
/**
|
|
347
347
|
* Removes retired `Session` instances from `SessionPool`.
|
|
348
348
|
*/
|
|
349
|
-
removeRetiredSessions() {
|
|
349
|
+
#removeRetiredSessions() {
|
|
350
350
|
this.#state.currentValue.sessions = this.#state.currentValue.sessions.filter((storedSession) => {
|
|
351
351
|
if (storedSession.isUsable())
|
|
352
352
|
return true;
|
|
@@ -367,14 +367,14 @@ export class SessionPool {
|
|
|
367
367
|
* Adds `Session` instance to `SessionPool`.
|
|
368
368
|
* @param newSession `Session` instance to be added.
|
|
369
369
|
*/
|
|
370
|
-
registerSession(newSession) {
|
|
370
|
+
#registerSession(newSession) {
|
|
371
371
|
this.#state.currentValue.sessions.push(newSession);
|
|
372
372
|
this.#sessionMap.set(newSession.id, newSession);
|
|
373
373
|
}
|
|
374
374
|
/**
|
|
375
375
|
* Gets random index.
|
|
376
376
|
*/
|
|
377
|
-
getRandomIndex() {
|
|
377
|
+
#getRandomIndex() {
|
|
378
378
|
return Math.floor(Math.random() * this.#state.currentValue.sessions.length);
|
|
379
379
|
}
|
|
380
380
|
/**
|
|
@@ -383,7 +383,7 @@ export class SessionPool {
|
|
|
383
383
|
* @param [options.sessionOptions] The configuration options for the session being created.
|
|
384
384
|
* @returns New session.
|
|
385
385
|
*/
|
|
386
|
-
async defaultCreateSessionFunction(options = {}) {
|
|
386
|
+
async #defaultCreateSessionFunction(options = {}) {
|
|
387
387
|
const { sessionOptions } = parseArgument(options, createSessionOptionsSchema);
|
|
388
388
|
return new Session(sessionOptions);
|
|
389
389
|
}
|
|
@@ -397,7 +397,7 @@ export class SessionPool {
|
|
|
397
397
|
* through `maybeLoadSessionPool` naturally wins because it arrives in
|
|
398
398
|
* `perCallOptions`.
|
|
399
399
|
*/
|
|
400
|
-
async invokeCreateSessionFunction(perCallOptions) {
|
|
400
|
+
async #invokeCreateSessionFunction(perCallOptions) {
|
|
401
401
|
const sessionOptions = {
|
|
402
402
|
fingerprint: createDefaultSessionFingerprint(),
|
|
403
403
|
...this.#sessionOptions,
|
|
@@ -409,24 +409,24 @@ export class SessionPool {
|
|
|
409
409
|
* Creates new session and adds it to the pool.
|
|
410
410
|
* @returns Newly created `Session` instance.
|
|
411
411
|
*/
|
|
412
|
-
async createSession() {
|
|
413
|
-
const newSession = await this
|
|
414
|
-
this
|
|
412
|
+
async #createSession() {
|
|
413
|
+
const newSession = await this.#invokeCreateSessionFunction();
|
|
414
|
+
this.#registerSession(newSession);
|
|
415
415
|
this.#log.debug(`Created new Session - ${newSession.id}`);
|
|
416
416
|
return newSession;
|
|
417
417
|
}
|
|
418
418
|
/**
|
|
419
419
|
* Decides whether there is enough space for creating new session.
|
|
420
420
|
*/
|
|
421
|
-
hasSpaceForSession() {
|
|
421
|
+
#hasSpaceForSession() {
|
|
422
422
|
return this.#state.currentValue.sessions.length < this.#maxPoolSize;
|
|
423
423
|
}
|
|
424
424
|
/**
|
|
425
425
|
* Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
|
|
426
426
|
* Returns `undefined` when no session should be reused and a new one should be created instead.
|
|
427
427
|
*/
|
|
428
|
-
pickSession() {
|
|
429
|
-
if (this.#sessionReuseStrategy !== 'use-until-failure' && this
|
|
428
|
+
#pickSession() {
|
|
429
|
+
if (this.#sessionReuseStrategy !== 'use-until-failure' && this.#hasSpaceForSession())
|
|
430
430
|
return undefined;
|
|
431
431
|
if (this.#sessionReuseStrategy === 'use-until-failure') {
|
|
432
432
|
return this.#state.currentValue.sessions.find((session) => session.isUsable());
|
|
@@ -438,7 +438,7 @@ export class SessionPool {
|
|
|
438
438
|
picked = this.#state.currentValue.sessions[index];
|
|
439
439
|
}
|
|
440
440
|
else {
|
|
441
|
-
picked = this.#state.currentValue.sessions[this
|
|
441
|
+
picked = this.#state.currentValue.sessions[this.#getRandomIndex()];
|
|
442
442
|
}
|
|
443
443
|
return picked.isUsable() ? picked : undefined;
|
|
444
444
|
}
|