@crawlee/core 3.0.0-alpha.4 → 3.0.0-alpha.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +10 -0
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +50 -1
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +1 -1
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +5 -5
- package/configuration.d.ts.map +1 -1
- package/configuration.js +12 -16
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +3 -1
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_utils.d.ts +0 -5
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +3 -13
- package/crawlers/crawler_utils.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +14 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +31 -2
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +1 -1
- package/enqueue_links/shared.d.ts.map +1 -1
- package/events/event_manager.d.ts +9 -3
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +5 -4
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +0 -1
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +0 -6
- package/events/local_event_manager.js.map +1 -1
- package/index.mjs +1 -4
- package/package.json +5 -14
- package/proxy_configuration.d.ts +8 -8
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +6 -6
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts +17 -2
- package/request.d.ts.map +1 -1
- package/request.js +71 -3
- package/request.js.map +1 -1
- package/storages/dataset.d.ts +1 -42
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +1 -49
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +0 -1
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +0 -2
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +1 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +3 -2
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +8 -0
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +11 -2
- package/storages/request_list.js.map +1 -1
- package/storages/request_queue.d.ts +5 -2
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +40 -18
- package/storages/request_queue.js.map +1 -1
- package/storages/storage_manager.d.ts +4 -4
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/cache_container.d.ts +0 -18
- package/cache_container.d.ts.map +0 -1
- package/cache_container.js +0 -42
- package/cache_container.js.map +0 -1
- package/errors.d.ts +0 -8
- package/errors.d.ts.map +0 -1
- package/errors.js +0 -12
- package/errors.js.map +0 -1
- package/storages/storage.d.ts +0 -260
- package/storages/storage.d.ts.map +0 -1
- package/storages/storage.js +0 -3
- package/storages/storage.js.map +0 -1
|
@@ -94,6 +94,11 @@ export interface AutoscaledPoolOptions {
|
|
|
94
94
|
* by the pool.
|
|
95
95
|
*/
|
|
96
96
|
systemStatusOptions?: SystemStatusOptions;
|
|
97
|
+
/**
|
|
98
|
+
* The maximum number of tasks per minute the pool can run.
|
|
99
|
+
* By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
|
|
100
|
+
*/
|
|
101
|
+
maxTasksPerMinute?: number;
|
|
97
102
|
log?: Log;
|
|
98
103
|
}
|
|
99
104
|
/**
|
|
@@ -157,6 +162,7 @@ export declare class AutoscaledPool {
|
|
|
157
162
|
private readonly runTaskFunction;
|
|
158
163
|
private readonly isFinishedFunction;
|
|
159
164
|
private readonly isTaskReadyFunction;
|
|
165
|
+
private readonly maxTasksPerMinute;
|
|
160
166
|
private _minConcurrency;
|
|
161
167
|
private _maxConcurrency;
|
|
162
168
|
private _desiredConcurrency;
|
|
@@ -172,6 +178,8 @@ export declare class AutoscaledPool {
|
|
|
172
178
|
private maybeRunInterval;
|
|
173
179
|
private queryingIsTaskReady;
|
|
174
180
|
private queryingIsFinished;
|
|
181
|
+
private tasksDonePerSecondInterval?;
|
|
182
|
+
private _tasksPerMinute;
|
|
175
183
|
constructor(options: AutoscaledPoolOptions, config?: Configuration);
|
|
176
184
|
/**
|
|
177
185
|
* Gets the minimum number of tasks running in parallel.
|
|
@@ -282,5 +290,7 @@ export declare class AutoscaledPool {
|
|
|
282
290
|
* Cleans up resources.
|
|
283
291
|
*/
|
|
284
292
|
protected _destroy(): Promise<void>;
|
|
293
|
+
protected _incrementTasksDonePerSecond(intervalCallback: () => void): void;
|
|
294
|
+
protected get _isOverMaxRequestLimit(): boolean;
|
|
285
295
|
}
|
|
286
296
|
//# sourceMappingURL=autoscaled_pool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autoscaled_pool.d.ts","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAe,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,UAAU,EAAgB,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEhF,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5C;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAE1C,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,cAAc;
|
|
1
|
+
{"version":3,"file":"autoscaled_pool.d.ts","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAe,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,UAAU,EAAgB,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEhF,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5C;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAE1C;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,cAAc;IAqCnB,OAAO,CAAC,QAAQ,CAAC,MAAM;IApC3B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAG1B,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAyB;IACzD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAyB;IAC5D,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAyB;IAC7D,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAG3C,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,OAAO,CAA0C;IACzD,OAAO,CAAC,MAAM,CAA6C;IAC3D,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,WAAW,CAAoB;IACvC,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,gBAAgB,CAAoB;IAC5C,OAAO,CAAC,mBAAmB,CAAW;IACtC,OAAO,CAAC,kBAAkB,CAAW;IACrC,OAAO,CAAC,0BAA0B,CAAC,CAAmB;IACtD,OAAO,CAAC,eAAe,CAAiD;gBAGpE,OAAO,EAAE,qBAAqB,EACb,MAAM,gBAAkC;IAmF7D;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED;;;;;OAKG;IACH,IAAI,cAAc,CAAC,KAAK,EAAE,MAAM,EAG/B;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED;;OAEG;IACH,IAAI,cAAc,CAAC,KAAK,EAAE,MAAM,EAG/B;IAED;;;OAGG;IACH,IAAI,kBAAkB,IAAI,MAAM,CAE/B;IAED;;;OAGG;IACH,IAAI,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAGnC;IAED;;OAEG;IACH,IAAI,kBAAkB,IAAI,MAAM,CAE/B;IAED;;;OAGG;IACG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IA6B1B;;;;;;;;;;OAUG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B;;;;;;;;;;OAUG;IACG,KAAK,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBhD;;;;;OAKG;IACH,MAAM,IAAI,IAAI;IAId;;;;;;;OAOG;cACa,aAAa,CAAC,gBAAgB,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqG3E;;;;OAIG;IACH,SAAS,CAAC,UAAU,CAAC,gBAAgB,EAAE,MAAM,IAAI;IA4CjD;;;;;OAKG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,GAAG,IAAI;IAUlD;;;;;OAKG;IACH,SAAS,CAAC,UAAU,CAAC,YAAY,EAAE,UAAU,GAAG,IAAI;IAUpD;;;;;OAKG;cACa,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB7C;;OAEG;cACa,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAUzC,SAAS,CAAC,4BAA4B,CAAC,gBAAgB,EAAE,MAAM,IAAI;IAQnE,SAAS,KAAK,sBAAsB,YAMnC;CACJ"}
|
|
@@ -132,6 +132,12 @@ class AutoscaledPool {
|
|
|
132
132
|
writable: true,
|
|
133
133
|
value: void 0
|
|
134
134
|
});
|
|
135
|
+
Object.defineProperty(this, "maxTasksPerMinute", {
|
|
136
|
+
enumerable: true,
|
|
137
|
+
configurable: true,
|
|
138
|
+
writable: true,
|
|
139
|
+
value: void 0
|
|
140
|
+
});
|
|
135
141
|
// Internal properties.
|
|
136
142
|
Object.defineProperty(this, "_minConcurrency", {
|
|
137
143
|
enumerable: true,
|
|
@@ -223,6 +229,18 @@ class AutoscaledPool {
|
|
|
223
229
|
writable: true,
|
|
224
230
|
value: void 0
|
|
225
231
|
});
|
|
232
|
+
Object.defineProperty(this, "tasksDonePerSecondInterval", {
|
|
233
|
+
enumerable: true,
|
|
234
|
+
configurable: true,
|
|
235
|
+
writable: true,
|
|
236
|
+
value: void 0
|
|
237
|
+
});
|
|
238
|
+
Object.defineProperty(this, "_tasksPerMinute", {
|
|
239
|
+
enumerable: true,
|
|
240
|
+
configurable: true,
|
|
241
|
+
writable: true,
|
|
242
|
+
value: Array.from({ length: 60 }, () => 0)
|
|
243
|
+
});
|
|
226
244
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
227
245
|
runTaskFunction: ow_1.default.function,
|
|
228
246
|
isFinishedFunction: ow_1.default.function,
|
|
@@ -240,8 +258,9 @@ class AutoscaledPool {
|
|
|
240
258
|
systemStatusOptions: ow_1.default.optional.object,
|
|
241
259
|
snapshotterOptions: ow_1.default.optional.object,
|
|
242
260
|
log: ow_1.default.optional.object,
|
|
261
|
+
maxTasksPerMinute: ow_1.default.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
|
|
243
262
|
}));
|
|
244
|
-
const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 1000, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.90, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = log_1.log, } = options;
|
|
263
|
+
const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 1000, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.90, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = log_1.log, maxTasksPerMinute = Infinity, } = options;
|
|
245
264
|
this.log = log.child({ prefix: 'AutoscaledPool' });
|
|
246
265
|
// Configurable properties.
|
|
247
266
|
this.desiredConcurrencyRatio = desiredConcurrencyRatio;
|
|
@@ -254,6 +273,7 @@ class AutoscaledPool {
|
|
|
254
273
|
this.runTaskFunction = runTaskFunction;
|
|
255
274
|
this.isFinishedFunction = isFinishedFunction;
|
|
256
275
|
this.isTaskReadyFunction = isTaskReadyFunction;
|
|
276
|
+
this.maxTasksPerMinute = maxTasksPerMinute;
|
|
257
277
|
// Internal properties.
|
|
258
278
|
this._minConcurrency = minConcurrency;
|
|
259
279
|
this._maxConcurrency = maxConcurrency;
|
|
@@ -265,6 +285,7 @@ class AutoscaledPool {
|
|
|
265
285
|
this.reject = null;
|
|
266
286
|
this._autoscale = this._autoscale.bind(this);
|
|
267
287
|
this._maybeRunTask = this._maybeRunTask.bind(this);
|
|
288
|
+
this._incrementTasksDonePerSecond = this._incrementTasksDonePerSecond.bind(this);
|
|
268
289
|
// Create instances with correct options.
|
|
269
290
|
const ssoCopy = { ...systemStatusOptions };
|
|
270
291
|
ssoCopy.snapshotter ?? (ssoCopy.snapshotter = new snapshotter_1.Snapshotter({
|
|
@@ -343,6 +364,10 @@ class AutoscaledPool {
|
|
|
343
364
|
// this._maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results
|
|
344
365
|
// in the actor getting stuck and even after scaling up it never triggers another promise.
|
|
345
366
|
this.maybeRunInterval = (0, utilities_1.betterSetInterval)(this._maybeRunTask, this.maybeRunIntervalMillis);
|
|
367
|
+
if (this.maxTasksPerMinute !== Infinity) {
|
|
368
|
+
// Start the interval that resets the counter of tasks per minute.
|
|
369
|
+
this.tasksDonePerSecondInterval = (0, utilities_1.betterSetInterval)(this._incrementTasksDonePerSecond, 1000);
|
|
370
|
+
}
|
|
346
371
|
try {
|
|
347
372
|
await this.poolPromise;
|
|
348
373
|
}
|
|
@@ -475,9 +500,17 @@ class AutoscaledPool {
|
|
|
475
500
|
// No tasks could mean that we're finished with all tasks.
|
|
476
501
|
return this._maybeFinish();
|
|
477
502
|
}
|
|
503
|
+
// - we have already reached the maximum tasks per minute
|
|
504
|
+
// we need to check this *after* checking if a task is ready to prevent hanging the pool
|
|
505
|
+
// for an extra minute if there are no more tasks
|
|
506
|
+
if (this._isOverMaxRequestLimit) {
|
|
507
|
+
this.log.perf('Task will not run. Maximum tasks per minute reached.');
|
|
508
|
+
return done();
|
|
509
|
+
}
|
|
478
510
|
try {
|
|
479
511
|
// Everything's fine. Run task.
|
|
480
512
|
this._currentConcurrency++;
|
|
513
|
+
this._tasksPerMinute[0]++;
|
|
481
514
|
// Try to run next task to build up concurrency,
|
|
482
515
|
// but defer it so it doesn't create a cycle.
|
|
483
516
|
setImmediate(this._maybeRunTask);
|
|
@@ -516,6 +549,9 @@ class AutoscaledPool {
|
|
|
516
549
|
// Don't scale if paused.
|
|
517
550
|
if (this.isStopped)
|
|
518
551
|
return intervalCallback();
|
|
552
|
+
// Don't scale if we've hit the maximum requests per minute
|
|
553
|
+
if (this._isOverMaxRequestLimit)
|
|
554
|
+
return intervalCallback();
|
|
519
555
|
// Only scale up if:
|
|
520
556
|
// - system has not been overloaded lately.
|
|
521
557
|
const systemStatus = this.systemStatus.getHistoricalStatus();
|
|
@@ -616,9 +652,22 @@ class AutoscaledPool {
|
|
|
616
652
|
this.reject = null;
|
|
617
653
|
(0, utilities_1.betterClearInterval)(this.autoscaleInterval);
|
|
618
654
|
(0, utilities_1.betterClearInterval)(this.maybeRunInterval);
|
|
655
|
+
if (this.tasksDonePerSecondInterval)
|
|
656
|
+
(0, utilities_1.betterClearInterval)(this.tasksDonePerSecondInterval);
|
|
619
657
|
if (this.snapshotter)
|
|
620
658
|
await this.snapshotter.stop();
|
|
621
659
|
}
|
|
660
|
+
_incrementTasksDonePerSecond(intervalCallback) {
|
|
661
|
+
this._tasksPerMinute.unshift(0);
|
|
662
|
+
this._tasksPerMinute.pop();
|
|
663
|
+
return intervalCallback();
|
|
664
|
+
}
|
|
665
|
+
get _isOverMaxRequestLimit() {
|
|
666
|
+
if (this.maxTasksPerMinute === Infinity) {
|
|
667
|
+
return false;
|
|
668
|
+
}
|
|
669
|
+
return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
|
|
670
|
+
}
|
|
622
671
|
}
|
|
623
672
|
exports.AutoscaledPool = AutoscaledPool;
|
|
624
673
|
//# sourceMappingURL=autoscaled_pool.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autoscaled_pool.js","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":";;;;AACA,4CAAqD;AACrD,gDAA4F;AAC5F,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAC3C,+CAAgE;AAChE,mDAAgF;AAgHhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,cAAc;IAgCvB,YACI,OAA8B,EACb,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAjCrB;;;;;WAA0B;QAE1B,2BAA2B;QAC3B;;;;;WAAiD;QACjD;;;;;WAA0C;QAC1C;;;;;WAA4C;QAC5C;;;;;WAAgD;QAChD;;;;;WAA+C;QAC/C;;;;;WAAiD;QACjD;;;;;WAA2C;QAC3C;;;;;WAAyD;QACzD;;;;;WAA4D;QAC5D;;;;;WAA6D;QAE7D,uBAAuB;QACvB;;;;;WAAgC;QAChC;;;;;WAAgC;QAChC;;;;;WAAoC;QACpC;;;;mBAA8B,CAAC;WAAC;QAChC;;;;mBAAoB,KAAK;WAAC;QAC1B;;;;mBAA0B,CAAC;WAAC;QAC5B;;;;mBAAoD,IAAI;WAAC;QACzD;;;;mBAAsD,IAAI;WAAC;QAC3D;;;;;WAAiC;QACjC;;;;;WAAmC;QACnC;;;;;WAAuC;QACvC;;;;;WAA6C;QAC7C;;;;;WAA4C;QAC5C;;;;;WAAsC;QACtC;;;;;WAAqC;QAMjC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ;YAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ;YAC/B,mBAAmB,EAAE,YAAE,CAAC,QAAQ;YAChC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC3C,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACxC,mBAAmB,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,eAAe,CAAC;YAC1D,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GAAG,IAAI,EACrB,cAAc,GAAG,CAAC,EAClB,kBAAkB,EAClB,uBAAuB,GAAG,IAAI,EAC9B,gBAAgB,GAAG,IAAI,EACvB,kBAAkB,GAAG,IAAI,EACzB,oBAAoB,GAAG,GAAG,EAC1B,mBAAmB,GAAG,EAAE,EACxB,eAAe,GAAG,CAAC,EACnB,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,EACnB,kBAAkB,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAEnD,2BAA2B;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,GAAG,IAAI,CAAC;QAC1D,IAAI,CAAC,qBAAqB,GAAG,mBAAoB,GAAG,IAAI,CAAC;QACzD,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,IAAI,cAAc,CAAC;QAChE,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEnD,yCAAyC;QACzC,MAAM,OAAO,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;QAC3C,OAAO,CAAC,WAAW,KAAnB,OAAO,CAAC,WAAW,GAAK,IAAI,yBAAW,CAAC;YACpC,GAAG,kBAAkB;YACrB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE;SACzC,CAAC,EAAC;QACH,OAAO,CAAC,MAAM,KAAd,OAAO,CAAC,MAAM,GAAK,IAAI,CAAC,MAAM,EAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;;OAKG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB,CAAC,KAAa;QAChC,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE/B,0FAA0F;QAC1F,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAE1F,4FAA4F;QAC5F,gGAAgG;QAChG,0FAA0F;QAC1F,IAAI,CAAC,gBAAgB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAE3F,IAAI;YACA,MAAM,IAAI,CAAC,WAAW,CAAC;SAC1B;gBAAS;YACN,qDAAqD;YACrD,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SAC3C;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SACzB;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,WAAoB;QAC5B,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,OAAuB,CAAC;YAC5B,IAAI,WAAW,EAAE;gBACb,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0CAA0C;0BAC1D,MAAM,WAAW,sCAAsC,CAAC,CAAC;oBAC/D,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;aACnB;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,IAAI,CAAC,mBAAmB,IAAI,CAAC,EAAE;oBAC/B,4DAA4D;oBAC5D,IAAI,OAAO;wBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;oBACnC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;iBACb;YACL,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,MAAM;QACF,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,aAAa,CAAC,gBAA6B;QACvD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3C,8FAA8F;QAC9F,MAAM,IAAI,GAAG,gBAAgB,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE5C,kCAAkC;QAClC,kCAAkC;QAClC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAC/D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,wCAAwC;QACxC,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC9D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,yCAAyC;QACzC,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE;YACtD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,mEAAmE;QACnE,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAC3D,MAAM,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC;QACvC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,eAAe,EAAE;YACnE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,6CAA6C,EAAE,aAAa,CAAC,CAAC;YAC5E,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,qBAAqB;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,WAAW,CAAC;QAChB,IAAI;YACA,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC3C,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClD,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;SACpC;QACD,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxD,IAAI,EAAE,CAAC;YACP,0DAA0D;YAC1D,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9B;QAED,IAAI;YACA,+BAA+B;YAC/B,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,gDAAgD;YAChD,6CAA6C;YAC7C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEjC,sFAAsF;YACtF,IAAI,EAAE,CAAC;YAEP,4BAA4B;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAEjC,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE;gBAC5B,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,EAC5B,IAAI,CAAC,iBAAiB,EACtB,mCAAmC,IAAI,CAAC,iBAAiB,GAAG,IAAI,WAAW,CAC9E,CAAC;aACL;iBAAM;gBACH,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aAChC;YAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,4CAA4C;YAC5C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACxC,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;IACL,CAAC;IAED;;;;OAIG;IACO,UAAU,CAAC,gBAA4B;QAC7C,yBAAyB;QACzB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE9C,oBAAoB;QACpB,2CAA2C;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,CAAC;QAC7D,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC;QACtC,0CAA0C;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QACtE,iFAAiF;QACjF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAClG,MAAM,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,IAAI,qBAAqB,CAAC;QAE1F,IAAI,YAAY,IAAI,aAAa,IAAI,+BAA+B;YAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElG,wBAAwB;QACxB,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC;QACzC,gCAAgC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QAEtE,IAAI,kBAAkB,IAAI,aAAa;YAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAEvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,qBAAqB,EAAE;gBACzD,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE;oBACnB,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,YAAY;iBACf,CAAC,CAAC;aACN;SACJ;QAED,8BAA8B;QAC9B,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACO,QAAQ,CAAC,YAAwB;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;YACzB,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,UAAU,CAAC,YAAwB;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC3E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,IAAI,CAAC,mBAAmB,GAAG,CAAC;YAAE,OAAO;QAEzC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI;YACA,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;SACnC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;CACJ;AA7dD,wCA6dC"}
|
|
1
|
+
{"version":3,"file":"autoscaled_pool.js","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":";;;;AACA,4CAAqD;AACrD,gDAA4F;AAC5F,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAC3C,+CAAgE;AAChE,mDAAgF;AAsHhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,cAAc;IAmCvB,YACI,OAA8B,EACb,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QApCrB;;;;;WAA0B;QAE1B,2BAA2B;QAC3B;;;;;WAAiD;QACjD;;;;;WAA0C;QAC1C;;;;;WAA4C;QAC5C;;;;;WAAgD;QAChD;;;;;WAA+C;QAC/C;;;;;WAAiD;QACjD;;;;;WAA2C;QAC3C;;;;;WAAyD;QACzD;;;;;WAA4D;QAC5D;;;;;WAA6D;QAC7D;;;;;WAA2C;QAE3C,uBAAuB;QACvB;;;;;WAAgC;QAChC;;;;;WAAgC;QAChC;;;;;WAAoC;QACpC;;;;mBAA8B,CAAC;WAAC;QAChC;;;;mBAAoB,KAAK;WAAC;QAC1B;;;;mBAA0B,CAAC;WAAC;QAC5B;;;;mBAAoD,IAAI;WAAC;QACzD;;;;mBAAsD,IAAI;WAAC;QAC3D;;;;;WAAiC;QACjC;;;;;WAAmC;QACnC;;;;;WAAuC;QACvC;;;;;WAA6C;QAC7C;;;;;WAA4C;QAC5C;;;;;WAAsC;QACtC;;;;;WAAqC;QACrC;;;;;WAAsD;QACtD;;;;mBAAoC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;WAAC;QAMpE,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ;YAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ;YAC/B,mBAAmB,EAAE,YAAE,CAAC,QAAQ;YAChC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC3C,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACxC,mBAAmB,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,eAAe,CAAC;YAC1D,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;SACzF,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GAAG,IAAI,EACrB,cAAc,GAAG,CAAC,EAClB,kBAAkB,EAClB,uBAAuB,GAAG,IAAI,EAC9B,gBAAgB,GAAG,IAAI,EACvB,kBAAkB,GAAG,IAAI,EACzB,oBAAoB,GAAG,GAAG,EAC1B,mBAAmB,GAAG,EAAE,EACxB,eAAe,GAAG,CAAC,EACnB,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,EACnB,kBAAkB,EAClB,GAAG,GAAG,SAAU,EAChB,iBAAiB,GAAG,QAAQ,GAC/B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAEnD,2BAA2B;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,GAAG,IAAI,CAAC;QAC1D,IAAI,CAAC,qBAAqB,GAAG,mBAAoB,GAAG,IAAI,CAAC;QACzD,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,IAAI,cAAc,CAAC;QAChE,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,4BAA4B,GAAG,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjF,yCAAyC;QACzC,MAAM,OAAO,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;QAC3C,OAAO,CAAC,WAAW,KAAnB,OAAO,CAAC,WAAW,GAAK,IAAI,yBAAW,CAAC;YACpC,GAAG,kBAAkB;YACrB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE;SACzC,CAAC,EAAC;QACH,OAAO,CAAC,MAAM,KAAd,OAAO,CAAC,MAAM,GAAK,IAAI,CAAC,MAAM,EAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;;OAKG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB,CAAC,KAAa;QAChC,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE/B,0FAA0F;QAC1F,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAE1F,4FAA4F;QAC5F,gGAAgG;QAChG,0FAA0F;QAC1F,IAAI,CAAC,gBAAgB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAE3F,IAAI,IAAI,CAAC,iBAAiB,KAAK,QAAQ,EAAE;YACrC,kEAAkE;YAClE,IAAI,CAAC,0BAA0B,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,4BAA4B,EAAE,IAAI,CAAC,CAAC;SAChG;QAED,IAAI;YACA,MAAM,IAAI,CAAC,WAAW,CAAC;SAC1B;gBAAS;YACN,qDAAqD;YACrD,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SAC3C;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SACzB;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,WAAoB;QAC5B,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,OAAuB,CAAC;YAC5B,IAAI,WAAW,EAAE;gBACb,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0CAA0C;0BAC1D,MAAM,WAAW,sCAAsC,CAAC,CAAC;oBAC/D,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;aACnB;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,IAAI,CAAC,mBAAmB,IAAI,CAAC,EAAE;oBAC/B,4DAA4D;oBAC5D,IAAI,OAAO;wBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;oBACnC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;iBACb;YACL,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,MAAM;QACF,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,aAAa,CAAC,gBAA6B;QACvD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3C,8FAA8F;QAC9F,MAAM,IAAI,GAAG,gBAAgB,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE5C,kCAAkC;QAClC,kCAAkC;QAClC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAC/D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,wCAAwC;QACxC,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC9D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,yCAAyC;QACzC,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE;YACtD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,mEAAmE;QACnE,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAC3D,MAAM,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC;QACvC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,eAAe,EAAE;YACnE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,6CAA6C,EAAE,aAAa,CAAC,CAAC;YAC5E,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,qBAAqB;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,WAAW,CAAC;QAChB,IAAI;YACA,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC3C,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClD,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;SACpC;QACD,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxD,IAAI,EAAE,CAAC;YACP,0DAA0D;YAC1D,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9B;QAED,yDAAyD;QACzD,wFAAwF;QACxF,iDAAiD;QACjD,IAAI,IAAI,CAAC,sBAAsB,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;YACtE,OAAO,IAAI,EAAE,CAAC;SACjB;QAED,IAAI;YACA,+BAA+B;YAC/B,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,gDAAgD;YAChD,6CAA6C;YAC7C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEjC,sFAAsF;YACtF,IAAI,EAAE,CAAC;YAEP,4BAA4B;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAEjC,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE;gBAC5B,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,EAC5B,IAAI,CAAC,iBAAiB,EACtB,mCAAmC,IAAI,CAAC,iBAAiB,GAAG,IAAI,WAAW,CAC9E,CAAC;aACL;iBAAM;gBACH,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aAChC;YAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,4CAA4C;YAC5C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACxC,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;IACL,CAAC;IAED;;;;OAIG;IACO,UAAU,CAAC,gBAA4B;QAC7C,yBAAyB;QACzB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE9C,2DAA2D;QAC3D,IAAI,IAAI,CAAC,sBAAsB;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE3D,oBAAoB;QACpB,2CAA2C;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,CAAC;QAC7D,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC;QACtC,0CAA0C;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QACtE,iFAAiF;QACjF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAClG,MAAM,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,IAAI,qBAAqB,CAAC;QAE1F,IAAI,YAAY,IAAI,aAAa,IAAI,+BAA+B;YAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElG,wBAAwB;QACxB,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC;QACzC,gCAAgC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QAEtE,IAAI,kBAAkB,IAAI,aAAa;YAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAEvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,qBAAqB,EAAE;gBACzD,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE;oBACnB,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,YAAY;iBACf,CAAC,CAAC;aACN;SACJ;QAED,8BAA8B;QAC9B,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACO,QAAQ,CAAC,YAAwB;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;YACzB,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,UAAU,CAAC,YAAwB;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC3E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,IAAI,CAAC,mBAAmB,GAAG,CAAC;YAAE,OAAO;QAEzC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI;YACA,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;SACnC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,0BAA0B;YAAE,IAAA,+BAAmB,EAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC1F,IAAI,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;IAES,4BAA4B,CAAC,gBAA4B;QAC/D,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEhC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;QAE3B,OAAO,gBAAgB,EAAE,CAAC;IAC9B,CAAC;IAED,IAAc,sBAAsB;QAChC,IAAI,IAAI,CAAC,iBAAiB,KAAK,QAAQ,EAAE;YACrC,OAAO,KAAK,CAAC;SAChB;QAED,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC;IAC/F,CAAC;CACJ;AAtgBD,wCAsgBC"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Log } from '@apify/log';
|
|
2
2
|
import { BetterIntervalID } from '@apify/utilities';
|
|
3
|
+
import { StorageClient } from '@crawlee/types';
|
|
3
4
|
import { Configuration } from '../configuration';
|
|
4
5
|
import { SystemInfo } from './system_status';
|
|
5
|
-
import { StorageClient } from '../storages/storage';
|
|
6
6
|
import { EventManager } from '../events/event_manager';
|
|
7
7
|
export interface SnapshotterOptions {
|
|
8
8
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAuB,gBAAgB,EAAqB,MAAM,kBAAkB,CAAC;AAG5F,OAAO,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAuB,gBAAgB,EAAqB,MAAM,kBAAkB,CAAC;AAG5F,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAa,MAAM,yBAAyB,CAAC;AAMlE,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,aAAK,cAAc,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AACrF,aAAK,WAAW,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAC1H,aAAK,iBAAiB,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAAC;AAC5F,aAAK,cAAc,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,mBAAmB,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9F;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IAExB,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA0C5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAe5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAU3B;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;;OAGG;IACH,kBAAkB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAIhD;;;OAGG;IACH,YAAY,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI1C;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAA;KAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE;IAoBvG;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU;IAchD;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU;IAmBvD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAuB5D;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU;IAY7C;;;;;;;OAOG;IACH,SAAS,CAAC,eAAe,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAwBzD;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,GAAG,iBAAiB,EAAE,GAAG,cAAc,EAAE,EAAE,GAAG,EAAE,IAAI;IAUzH;;OAEG;YACW,cAAc;CAG/B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,gDAA4F;AAC5F,0CAA+C;AAC/C,oDAAoB;
|
|
1
|
+
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,gDAA4F;AAC5F,0CAA+C;AAC/C,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA2C;AAI3C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AA2DlD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAa,WAAW;IAuBpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QAzB5C;;;;;WAAS;QACT;;;;;WAAsB;QACtB;;;;;WAAsB;QACtB;;;;;WAAqB;QACrB;;;;;WAAwC;QACxC;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;;WAAyB;QACzB;;;;;WAA2B;QAC3B;;;;;WAAwB;QACxB;;;;;WAAwB;QAExB;;;;mBAA8B,EAAE;WAAC;QACjC;;;;mBAA0C,EAAE;WAAC;QAC7C;;;;mBAAoC,EAAE;WAAC;QACvC;;;;mBAAoC,EAAE;WAAC;QAEvC;;;;mBAAsC,IAAK;WAAC;QAC5C;;;;mBAAmC,IAAK;WAAC;QAEzC;;;;mBAAkD,IAAI;WAAC;QAMnD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,SAAU,EAChB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,GACrC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAE5C,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,mFAAmF;QACnF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACnD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAE,CAAC,CAAC;QACvF,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,0EAA0E,CAAC,CAAC,CAAC,8BAA8B;QAE5M,iCAAiC;QACjC,4BAA4B;QAE5B,sBAAsB;QACtB,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACrH,IAAI,CAAC,cAAc,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5G,IAAI,CAAC,MAAM,CAAC,EAAE,iCAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,EAAE,iCAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,GAAG,iCAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,GAAG,iCAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7D,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QACtD,6BAA6B;IACjC,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACO,UAAU,CAAgC,SAAc,EAAE,oBAA6B;QAC7F,IAAI,CAAC,oBAAoB;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,OAAO,GAAG,EAAE,EAAE;YACV,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,oBAAoB,EAAE;gBAC3D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;aAC5B;iBAAM;gBACH,MAAM;aACT;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,UAAsB;QAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAmB;YAC7B,SAAS;YACT,YAAY,EAAE,eAAgB,GAAG,IAAI,CAAC,cAAe,GAAG,IAAI,CAAC,kBAAkB;YAC/E,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,IAAI,CAAC,kCAAkC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,kCAAkC,GAAG,mCAAmC;YAAE,OAAO;QAEnJ,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAe,CAAC;QAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,cAAe,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QAClG,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAgB,GAAG,qBAAqB,CAAC;QAEpE,IAAI,kBAAkB,EAAE;YACpB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAgB,GAAG,IAAI,CAAC,cAAe,CAAC,GAAG,GAAG,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;YAChE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,mCAAmC;kBAC9C,SAAS,IAAI,CAAC,eAAgB,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAe,CAAC,QAAQ,cAAc,2CAA2C,CAAC,CAAC;YAC5I,IAAI,CAAC,kCAAkC,GAAG,SAAS,CAAC;SACvD;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,CAAC;SACpB,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrF,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC;YAEhF,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YAChE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;SACxE;QAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACnB,SAAS;YACT,YAAY,EAAE,eAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,eAAgB,GAAG,GAAG,CAAC;SAC/C,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CAAC,gBAA+B;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,wCAAwC;QACzG,MAAM,eAAe,GAAG,cAAc,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,mBAAmB,EAAE,eAAe;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;YACjD,MAAM,KAAK,GAAG,eAAe,GAAG,mBAAmB,CAAC;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;SAClE;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,SAAoF,EAAE,GAAS;QACrH,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,qBAAqB;gBAAE,QAAQ,EAAE,CAAC;;gBACtF,MAAM;SACd;QACD,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AA/RD,kCA+RC"}
|
package/configuration.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import {
|
|
3
|
-
import { AsyncLocalStorage } from 'async_hooks';
|
|
2
|
+
import { MemoryStorage, MemoryStorageOptions } from '@crawlee/memory-storage';
|
|
3
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
4
|
+
import { StorageClient } from '@crawlee/types';
|
|
4
5
|
import { Dictionary } from './typedefs';
|
|
5
|
-
import { StorageClient } from './storages/storage';
|
|
6
6
|
import { EventManager } from './events';
|
|
7
7
|
export interface ConfigurationOptions {
|
|
8
8
|
storageClient?: StorageClient;
|
|
@@ -121,10 +121,10 @@ export declare class Configuration {
|
|
|
121
121
|
getStorageClient(): StorageClient;
|
|
122
122
|
getEventManager(): EventManager;
|
|
123
123
|
/**
|
|
124
|
-
* Creates an instance of
|
|
124
|
+
* Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
|
|
125
125
|
* @internal
|
|
126
126
|
*/
|
|
127
|
-
|
|
127
|
+
createMemoryStorage(options?: MemoryStorageOptions): MemoryStorage;
|
|
128
128
|
useStorageClient(client: StorageClient): void;
|
|
129
129
|
useEventManager(events: EventManager): void;
|
|
130
130
|
/**
|
package/configuration.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAW,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,YAAY,EAAqB,MAAM,UAAU,CAAC;AAG3D,MAAM,WAAW,oBAAoB;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,OAAO,CAyBpB;IAEF;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAGE;IAEjC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAgB;IAE3C,OAAO,CAAC,MAAM,CAAC,YAAY,CAAkD;IAE7E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAerB;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,OAAO,CAAC,OAAO,CAAoF;IACnG,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAgB;IAE5C;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAO9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IAkBzG,OAAO,CAAC,aAAa;IAarB;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAatE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;;;OAIG;IACH,MAAM,CAAC,eAAe,IAAK,aAAa;IASxC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;CAGlC"}
|
package/configuration.js
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Configuration = void 0;
|
|
4
4
|
const consts_1 = require("@apify/consts");
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
const
|
|
5
|
+
const memory_storage_1 = require("@crawlee/memory-storage");
|
|
6
|
+
const node_async_hooks_1 = require("node:async_hooks");
|
|
7
|
+
const node_events_1 = require("node:events");
|
|
8
8
|
const typedefs_1 = require("./typedefs");
|
|
9
|
-
const
|
|
9
|
+
const events_1 = require("./events");
|
|
10
10
|
/**
|
|
11
11
|
* `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
|
|
12
12
|
*
|
|
@@ -73,7 +73,7 @@ class Configuration {
|
|
|
73
73
|
});
|
|
74
74
|
this.options = new Map((0, typedefs_1.entries)(options));
|
|
75
75
|
// Increase the global limit for event emitter memory leak warnings.
|
|
76
|
-
|
|
76
|
+
node_events_1.EventEmitter.defaultMaxListeners = 50;
|
|
77
77
|
}
|
|
78
78
|
/**
|
|
79
79
|
* Returns configured value. First checks the environment variables, then provided configuration,
|
|
@@ -125,7 +125,7 @@ class Configuration {
|
|
|
125
125
|
return this.options.get('storageClient');
|
|
126
126
|
}
|
|
127
127
|
const options = this.options.get('storageClientOptions');
|
|
128
|
-
return this.
|
|
128
|
+
return this.createMemoryStorage(options);
|
|
129
129
|
}
|
|
130
130
|
getEventManager() {
|
|
131
131
|
if (this.options.has('eventManager')) {
|
|
@@ -134,25 +134,21 @@ class Configuration {
|
|
|
134
134
|
if (this.services.has('eventManager')) {
|
|
135
135
|
return this.services.get('eventManager');
|
|
136
136
|
}
|
|
137
|
-
const eventManager = new
|
|
137
|
+
const eventManager = new events_1.LocalEventManager(this);
|
|
138
138
|
this.services.set('eventManager', eventManager);
|
|
139
139
|
return eventManager;
|
|
140
140
|
}
|
|
141
141
|
/**
|
|
142
|
-
* Creates an instance of
|
|
142
|
+
* Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
|
|
143
143
|
* @internal
|
|
144
144
|
*/
|
|
145
|
-
|
|
146
|
-
const cacheKey = `
|
|
145
|
+
createMemoryStorage(options = {}) {
|
|
146
|
+
const cacheKey = `MemoryStorage-${JSON.stringify(options)}`;
|
|
147
147
|
if (this.services.has(cacheKey)) {
|
|
148
148
|
return this.services.get(cacheKey);
|
|
149
149
|
}
|
|
150
|
-
const storage = new
|
|
150
|
+
const storage = new memory_storage_1.MemoryStorage(options);
|
|
151
151
|
this.services.set(cacheKey, storage);
|
|
152
|
-
process.on('exit', () => {
|
|
153
|
-
// TODO this is not public API, need to update storage local with some teardown
|
|
154
|
-
storage.dbConnections.closeAllConnections();
|
|
155
|
-
});
|
|
156
152
|
return storage;
|
|
157
153
|
}
|
|
158
154
|
useStorageClient(client) {
|
|
@@ -279,6 +275,6 @@ Object.defineProperty(Configuration, "storage", {
|
|
|
279
275
|
enumerable: true,
|
|
280
276
|
configurable: true,
|
|
281
277
|
writable: true,
|
|
282
|
-
value: new
|
|
278
|
+
value: new node_async_hooks_1.AsyncLocalStorage()
|
|
283
279
|
});
|
|
284
280
|
//# sourceMappingURL=configuration.js.map
|
package/configuration.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;AAAA,0CAAyD;AACzD,
|
|
1
|
+
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;AAAA,0CAAyD;AACzD,4DAA8E;AAC9E,uDAAqD;AACrD,6CAA2C;AAE3C,yCAAiD;AACjD,qCAA2D;AA2B3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAa,aAAa;IAsEtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAP9C;;;;;WAAmG;QACnG;;;;mBAAmB,IAAI,GAAG,EAAmB;WAAC;QAO1C,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;QAEzC,oEAAoE;QACpE,0BAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,kBAAkB;QAClB,MAAM,MAAM,GAAG,aAAa,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;QAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErC,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAEO,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACnF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;SACvD;QAED,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;;AAxNL,sCAyNC;AAxNG;;GAEG;AACH;;;;WAAyB;QACrB,0CAA0C;QAC1C,sBAAsB,EAAE,sBAAsB;QAE9C,wBAAwB,EAAE,kBAAkB;QAC5C,gCAAgC,EAAE,wBAAwB;QAC1D,8BAA8B,EAAE,uBAAuB;QACvD,kCAAkC,EAAE,2BAA2B;QAC/D,mCAAmC,EAAE,4BAA4B;QACjE,kCAAkC,EAAE,4BAA4B;QAChE,eAAe,EAAE,UAAU;QAC3B,cAAc,EAAE,SAAS;QACzB,kBAAkB,EAAE,YAAY;QAChC,mBAAmB,EAAE,aAAa;QAClC,oBAAoB,EAAE,eAAe;QACrC,mBAAmB,EAAE,cAAc;QACnC,oBAAoB,EAAE,eAAe;QACrC,oBAAoB,EAAE,eAAe;QACrC,gBAAgB,EAAE,WAAW;QAE7B,wCAAwC;QACxC,uCAAuC;QACvC,8BAA8B;QAC9B,sBAAsB;QACtB,wDAAwD;KAC3D;GAAC;AAEF;;GAEG;AACH;;;;WAAkC,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC1F,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;QACjB,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAA4B,CAAC;GAAC;AAEjC;;;;WAAwC,EAAE;GAAC;AAE3C;;;;WAA8B,CAAC,WAAW,EAAE,cAAc,EAAE,eAAe,CAAC;GAAC;AAE7E;;;;WAA0B;QACtB,sBAAsB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,0BAA0B,CAAC;QAC3E,gBAAgB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,kBAAkB,CAAC;QAC7D,qBAAqB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,wBAAwB,CAAC;QACxE,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,QAAQ,EAAE,OAAO;QACjB,aAAa,EAAE,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACtD,SAAS,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,UAAU,CAAC;QAC/C,aAAa,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACvD,YAAY,EAAE,uBAAc,CAAC,iBAAQ,CAAC,aAAa,CAAC;QACpD,yBAAyB,EAAE,MAAO;QAClC,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;KACnC;GAAC;AAEF;;;GAGG;AACH;;;;WAAiB,IAAI,oCAAiB,EAAiB;GAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Log } from '../log';
|
|
1
2
|
import { ProxyInfo } from '../proxy_configuration';
|
|
2
3
|
import { Request } from '../request';
|
|
3
4
|
import { Session } from '../session_pool/session';
|
|
@@ -13,10 +14,11 @@ export interface CrawlingContext extends Record<PropertyKey, unknown> {
|
|
|
13
14
|
* and configured by the {@link ProxyConfiguration} class.
|
|
14
15
|
*/
|
|
15
16
|
proxyInfo?: ProxyInfo;
|
|
17
|
+
log: Log;
|
|
16
18
|
}
|
|
17
19
|
export interface CrawlerHandleFailedRequestInput extends CrawlingContext {
|
|
18
20
|
/**
|
|
19
|
-
* The Error thrown by `
|
|
21
|
+
* The Error thrown by `requestHandler`.
|
|
20
22
|
*/
|
|
21
23
|
error: Error;
|
|
22
24
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,MAAM,WAAW,eAAgB,SAAQ,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;IACjE,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,MAAM,WAAW,eAAgB,SAAQ,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;IACjE,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,+BAAgC,SAAQ,eAAe;IACpE;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC;CAChB"}
|
|
@@ -7,11 +7,6 @@ export declare function handleRequestTimeout({ session, errorMessage }: {
|
|
|
7
7
|
session?: Session;
|
|
8
8
|
errorMessage: string;
|
|
9
9
|
}): void;
|
|
10
|
-
/**
|
|
11
|
-
* Handles blocked request
|
|
12
|
-
* @internal
|
|
13
|
-
*/
|
|
14
|
-
export declare function throwOnBlockedRequest(session: Session, statusCode: number): void;
|
|
15
10
|
/**
|
|
16
11
|
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
17
12
|
* if we see two cookies with same keys but different casing.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,QAK1G;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CAsBzE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,SAAK,EAAE,aAAa,SAAK,GAAG,MAAM,CAmBvF"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.diffCookies = exports.mergeCookies = exports.
|
|
3
|
+
exports.diffCookies = exports.mergeCookies = exports.handleRequestTimeout = void 0;
|
|
4
|
+
const timeout_1 = require("@apify/timeout");
|
|
4
5
|
const tough_cookie_1 = require("tough-cookie");
|
|
5
6
|
const log_1 = require("../log");
|
|
6
7
|
/**
|
|
@@ -11,20 +12,9 @@ function handleRequestTimeout({ session, errorMessage }) {
|
|
|
11
12
|
session?.markBad();
|
|
12
13
|
const timeoutMillis = errorMessage.match(/(\d+)\s?ms/)?.[1]; // first capturing group
|
|
13
14
|
const timeoutSecs = Number(timeoutMillis) / 1000;
|
|
14
|
-
throw new
|
|
15
|
+
throw new timeout_1.TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
|
|
15
16
|
}
|
|
16
17
|
exports.handleRequestTimeout = handleRequestTimeout;
|
|
17
|
-
/**
|
|
18
|
-
* Handles blocked request
|
|
19
|
-
* @internal
|
|
20
|
-
*/
|
|
21
|
-
function throwOnBlockedRequest(session, statusCode) {
|
|
22
|
-
const isBlocked = session.retireOnBlockedStatusCodes(statusCode);
|
|
23
|
-
if (isBlocked) {
|
|
24
|
-
throw new Error(`Request blocked - received ${statusCode} status code.`);
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
exports.throwOnBlockedRequest = throwOnBlockedRequest;
|
|
28
18
|
/**
|
|
29
19
|
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
30
20
|
* if we see two cookies with same keys but different casing.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":";;;AAAA,+CAAiD;AACjD,gCAA6B;AAG7B;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAA+C;IACvG,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,IAAI,
|
|
1
|
+
{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":";;;AAAA,4CAA8C;AAC9C,+CAAiD;AACjD,gCAA6B;AAG7B;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAA+C;IACvG,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,IAAI,sBAAY,CAAC,8BAA8B,WAAW,WAAW,CAAC,CAAC;AACjF,CAAC;AALD,oDAKC;AAED;;;;GAIG;AACH,SAAgB,YAAY,CAAC,GAAW,EAAE,aAAuB;IAC7D,MAAM,GAAG,GAAG,IAAI,wBAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE;QAC7D,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAsB;QAE1F,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE;YAChC,MAAM,MAAM,GAAG,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC;YAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE;gBAClB,SAAG,CAAC,UAAU,CAAC,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CAAC,CAAC;aAC1H;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAClC;KACJ;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AAtBD,oCAsBC;AAED;;GAEG;AACH,SAAgB,WAAW,CAAC,GAAW,EAAE,aAAa,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE;IAC3E,IAAI,aAAa,KAAK,aAAa,IAAI,CAAC,aAAa,EAAE;QACnD,OAAO,EAAE,CAAC;KACb;IAED,IAAI,CAAC,aAAa,EAAE;QAChB,OAAO,aAAa,CAAC;KACxB;IAED,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC,CAAC;IACrH,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC,CAAC;IAErH,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACxC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxF,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,wBAAS,EAAE,CAAC;IAC5B,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IAE1D,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AAnBD,kCAmBC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import { BatchAddRequestsResult } from '@crawlee/types';
|
|
1
2
|
import { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform } from './shared';
|
|
2
3
|
import { RequestQueue } from '../storages/request_queue';
|
|
3
|
-
import { BatchAddRequestsResult } from '../storages/storage';
|
|
4
4
|
export interface EnqueueLinksOptions {
|
|
5
5
|
/** Limit the count of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
|
|
6
6
|
limit?: number;
|
|
@@ -135,4 +135,17 @@ export declare enum EnqueueStrategy {
|
|
|
135
135
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
136
136
|
*/
|
|
137
137
|
export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
|
|
138
|
+
/**
|
|
139
|
+
* @internal
|
|
140
|
+
*/
|
|
141
|
+
export declare function resolveBaseUrl({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
|
|
142
|
+
/**
|
|
143
|
+
* @internal
|
|
144
|
+
*/
|
|
145
|
+
export interface ResolveBaseUrl {
|
|
146
|
+
userProvidedBaseUrl?: string;
|
|
147
|
+
enqueueStrategy?: EnqueueLinksOptions['strategy'];
|
|
148
|
+
originalRequestUrl: string;
|
|
149
|
+
finalRequestUrl?: string;
|
|
150
|
+
}
|
|
138
151
|
//# sourceMappingURL=enqueue_links.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAGzD,MAAM,WAAW,mBAAmB;IAChC,qHAAqH;IACrH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;CAC3E;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IACX;;OAEG;IACH,aAAa,mBAAmB;IAChC;;;;OAIG;IACH,YAAY,kBAAkB;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAqGhG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC3B,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA4BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.enqueueLinks = exports.EnqueueStrategy = void 0;
|
|
3
|
+
exports.resolveBaseUrl = exports.enqueueLinks = exports.EnqueueStrategy = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const tldts_1 = require("tldts");
|
|
6
6
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
@@ -80,7 +80,9 @@ async function enqueueLinks(options) {
|
|
|
80
80
|
if (regexps?.length) {
|
|
81
81
|
urlPatternObjects.push(...(0, shared_1.constructRegExpObjectsFromRegExps)(regexps));
|
|
82
82
|
}
|
|
83
|
-
|
|
83
|
+
if (!urlPatternObjects.length) {
|
|
84
|
+
options.strategy ?? (options.strategy = EnqueueStrategy.SameSubdomain);
|
|
85
|
+
}
|
|
84
86
|
if (options.baseUrl) {
|
|
85
87
|
switch (options.strategy) {
|
|
86
88
|
case EnqueueStrategy.SameSubdomain:
|
|
@@ -120,4 +122,31 @@ async function enqueueLinks(options) {
|
|
|
120
122
|
return requestQueue.addRequests(requests);
|
|
121
123
|
}
|
|
122
124
|
exports.enqueueLinks = enqueueLinks;
|
|
125
|
+
/**
|
|
126
|
+
* @internal
|
|
127
|
+
*/
|
|
128
|
+
function resolveBaseUrl({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
|
|
129
|
+
// User provided base url takes priority
|
|
130
|
+
if (userProvidedBaseUrl) {
|
|
131
|
+
return userProvidedBaseUrl;
|
|
132
|
+
}
|
|
133
|
+
const originalUrlOrigin = new URL(originalRequestUrl).origin;
|
|
134
|
+
const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
|
|
135
|
+
// We can assume users want to go off the domain in this case
|
|
136
|
+
if (enqueueStrategy === EnqueueStrategy.All) {
|
|
137
|
+
return finalUrlOrigin ?? originalUrlOrigin;
|
|
138
|
+
}
|
|
139
|
+
// Ensure links use the same hostname
|
|
140
|
+
if (enqueueStrategy === EnqueueStrategy.SameHostname) {
|
|
141
|
+
const originalHostname = (0, tldts_1.getDomain)(originalUrlOrigin, { mixedInputs: false });
|
|
142
|
+
const finalHostname = (0, tldts_1.getDomain)(finalUrlOrigin, { mixedInputs: false });
|
|
143
|
+
if (originalHostname === finalHostname) {
|
|
144
|
+
return finalUrlOrigin;
|
|
145
|
+
}
|
|
146
|
+
return undefined;
|
|
147
|
+
}
|
|
148
|
+
// Always enqueue urls that are from the same origin
|
|
149
|
+
return originalUrlOrigin;
|
|
150
|
+
}
|
|
151
|
+
exports.resolveBaseUrl = resolveBaseUrl;
|
|
123
152
|
//# sourceMappingURL=enqueue_links.js.map
|