@crawlee/core 3.0.0-alpha.4 → 3.0.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +10 -0
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +50 -1
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +2 -0
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +13 -0
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +31 -2
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/index.mjs +1 -0
- package/package.json +4 -4
- package/proxy_configuration.d.ts +8 -8
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +6 -6
- package/proxy_configuration.js.map +1 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +3 -2
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +8 -0
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +11 -2
- package/storages/request_list.js.map +1 -1
- package/storages/request_queue.d.ts +3 -0
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +34 -0
- package/storages/request_queue.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -94,6 +94,11 @@ export interface AutoscaledPoolOptions {
|
|
|
94
94
|
* by the pool.
|
|
95
95
|
*/
|
|
96
96
|
systemStatusOptions?: SystemStatusOptions;
|
|
97
|
+
/**
|
|
98
|
+
* The maximum number of tasks per minute the pool can run.
|
|
99
|
+
* By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
|
|
100
|
+
*/
|
|
101
|
+
maxTasksPerMinute?: number;
|
|
97
102
|
log?: Log;
|
|
98
103
|
}
|
|
99
104
|
/**
|
|
@@ -157,6 +162,7 @@ export declare class AutoscaledPool {
|
|
|
157
162
|
private readonly runTaskFunction;
|
|
158
163
|
private readonly isFinishedFunction;
|
|
159
164
|
private readonly isTaskReadyFunction;
|
|
165
|
+
private readonly maxTasksPerMinute;
|
|
160
166
|
private _minConcurrency;
|
|
161
167
|
private _maxConcurrency;
|
|
162
168
|
private _desiredConcurrency;
|
|
@@ -172,6 +178,8 @@ export declare class AutoscaledPool {
|
|
|
172
178
|
private maybeRunInterval;
|
|
173
179
|
private queryingIsTaskReady;
|
|
174
180
|
private queryingIsFinished;
|
|
181
|
+
private tasksDonePerSecondInterval?;
|
|
182
|
+
private _tasksPerMinute;
|
|
175
183
|
constructor(options: AutoscaledPoolOptions, config?: Configuration);
|
|
176
184
|
/**
|
|
177
185
|
* Gets the minimum number of tasks running in parallel.
|
|
@@ -282,5 +290,7 @@ export declare class AutoscaledPool {
|
|
|
282
290
|
* Cleans up resources.
|
|
283
291
|
*/
|
|
284
292
|
protected _destroy(): Promise<void>;
|
|
293
|
+
protected _incrementTasksDonePerSecond(intervalCallback: () => void): void;
|
|
294
|
+
protected get _isOverMaxRequestLimit(): boolean;
|
|
285
295
|
}
|
|
286
296
|
//# sourceMappingURL=autoscaled_pool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autoscaled_pool.d.ts","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAe,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,UAAU,EAAgB,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEhF,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5C;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAE1C,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,cAAc;
|
|
1
|
+
{"version":3,"file":"autoscaled_pool.d.ts","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAe,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,UAAU,EAAgB,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEhF,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5C;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAE1C;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,cAAc;IAqCnB,OAAO,CAAC,QAAQ,CAAC,MAAM;IApC3B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAG1B,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAyB;IACzD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAyB;IAC5D,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAyB;IAC7D,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAG3C,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,OAAO,CAA0C;IACzD,OAAO,CAAC,MAAM,CAA6C;IAC3D,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,WAAW,CAAoB;IACvC,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,gBAAgB,CAAoB;IAC5C,OAAO,CAAC,mBAAmB,CAAW;IACtC,OAAO,CAAC,kBAAkB,CAAW;IACrC,OAAO,CAAC,0BAA0B,CAAC,CAAmB;IACtD,OAAO,CAAC,eAAe,CAAiD;gBAGpE,OAAO,EAAE,qBAAqB,EACb,MAAM,gBAAkC;IAmF7D;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED;;;;;OAKG;IACH,IAAI,cAAc,CAAC,KAAK,EAAE,MAAM,EAG/B;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED;;OAEG;IACH,IAAI,cAAc,CAAC,KAAK,EAAE,MAAM,EAG/B;IAED;;;OAGG;IACH,IAAI,kBAAkB,IAAI,MAAM,CAE/B;IAED;;;OAGG;IACH,IAAI,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAGnC;IAED;;OAEG;IACH,IAAI,kBAAkB,IAAI,MAAM,CAE/B;IAED;;;OAGG;IACG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IA6B1B;;;;;;;;;;OAUG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B;;;;;;;;;;OAUG;IACG,KAAK,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBhD;;;;;OAKG;IACH,MAAM,IAAI,IAAI;IAId;;;;;;;OAOG;cACa,aAAa,CAAC,gBAAgB,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqG3E;;;;OAIG;IACH,SAAS,CAAC,UAAU,CAAC,gBAAgB,EAAE,MAAM,IAAI;IA4CjD;;;;;OAKG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,GAAG,IAAI;IAUlD;;;;;OAKG;IACH,SAAS,CAAC,UAAU,CAAC,YAAY,EAAE,UAAU,GAAG,IAAI;IAUpD;;;;;OAKG;cACa,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB7C;;OAEG;cACa,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAUzC,SAAS,CAAC,4BAA4B,CAAC,gBAAgB,EAAE,MAAM,IAAI;IAQnE,SAAS,KAAK,sBAAsB,YAMnC;CACJ"}
|
|
@@ -132,6 +132,12 @@ class AutoscaledPool {
|
|
|
132
132
|
writable: true,
|
|
133
133
|
value: void 0
|
|
134
134
|
});
|
|
135
|
+
Object.defineProperty(this, "maxTasksPerMinute", {
|
|
136
|
+
enumerable: true,
|
|
137
|
+
configurable: true,
|
|
138
|
+
writable: true,
|
|
139
|
+
value: void 0
|
|
140
|
+
});
|
|
135
141
|
// Internal properties.
|
|
136
142
|
Object.defineProperty(this, "_minConcurrency", {
|
|
137
143
|
enumerable: true,
|
|
@@ -223,6 +229,18 @@ class AutoscaledPool {
|
|
|
223
229
|
writable: true,
|
|
224
230
|
value: void 0
|
|
225
231
|
});
|
|
232
|
+
Object.defineProperty(this, "tasksDonePerSecondInterval", {
|
|
233
|
+
enumerable: true,
|
|
234
|
+
configurable: true,
|
|
235
|
+
writable: true,
|
|
236
|
+
value: void 0
|
|
237
|
+
});
|
|
238
|
+
Object.defineProperty(this, "_tasksPerMinute", {
|
|
239
|
+
enumerable: true,
|
|
240
|
+
configurable: true,
|
|
241
|
+
writable: true,
|
|
242
|
+
value: Array.from({ length: 60 }, () => 0)
|
|
243
|
+
});
|
|
226
244
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
227
245
|
runTaskFunction: ow_1.default.function,
|
|
228
246
|
isFinishedFunction: ow_1.default.function,
|
|
@@ -240,8 +258,9 @@ class AutoscaledPool {
|
|
|
240
258
|
systemStatusOptions: ow_1.default.optional.object,
|
|
241
259
|
snapshotterOptions: ow_1.default.optional.object,
|
|
242
260
|
log: ow_1.default.optional.object,
|
|
261
|
+
maxTasksPerMinute: ow_1.default.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
|
|
243
262
|
}));
|
|
244
|
-
const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 1000, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.90, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = log_1.log, } = options;
|
|
263
|
+
const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 1000, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.90, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = log_1.log, maxTasksPerMinute = Infinity, } = options;
|
|
245
264
|
this.log = log.child({ prefix: 'AutoscaledPool' });
|
|
246
265
|
// Configurable properties.
|
|
247
266
|
this.desiredConcurrencyRatio = desiredConcurrencyRatio;
|
|
@@ -254,6 +273,7 @@ class AutoscaledPool {
|
|
|
254
273
|
this.runTaskFunction = runTaskFunction;
|
|
255
274
|
this.isFinishedFunction = isFinishedFunction;
|
|
256
275
|
this.isTaskReadyFunction = isTaskReadyFunction;
|
|
276
|
+
this.maxTasksPerMinute = maxTasksPerMinute;
|
|
257
277
|
// Internal properties.
|
|
258
278
|
this._minConcurrency = minConcurrency;
|
|
259
279
|
this._maxConcurrency = maxConcurrency;
|
|
@@ -265,6 +285,7 @@ class AutoscaledPool {
|
|
|
265
285
|
this.reject = null;
|
|
266
286
|
this._autoscale = this._autoscale.bind(this);
|
|
267
287
|
this._maybeRunTask = this._maybeRunTask.bind(this);
|
|
288
|
+
this._incrementTasksDonePerSecond = this._incrementTasksDonePerSecond.bind(this);
|
|
268
289
|
// Create instances with correct options.
|
|
269
290
|
const ssoCopy = { ...systemStatusOptions };
|
|
270
291
|
ssoCopy.snapshotter ?? (ssoCopy.snapshotter = new snapshotter_1.Snapshotter({
|
|
@@ -343,6 +364,10 @@ class AutoscaledPool {
|
|
|
343
364
|
// this._maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results
|
|
344
365
|
// in the actor getting stuck and even after scaling up it never triggers another promise.
|
|
345
366
|
this.maybeRunInterval = (0, utilities_1.betterSetInterval)(this._maybeRunTask, this.maybeRunIntervalMillis);
|
|
367
|
+
if (this.maxTasksPerMinute !== Infinity) {
|
|
368
|
+
// Start the interval that resets the counter of tasks per minute.
|
|
369
|
+
this.tasksDonePerSecondInterval = (0, utilities_1.betterSetInterval)(this._incrementTasksDonePerSecond, 1000);
|
|
370
|
+
}
|
|
346
371
|
try {
|
|
347
372
|
await this.poolPromise;
|
|
348
373
|
}
|
|
@@ -475,9 +500,17 @@ class AutoscaledPool {
|
|
|
475
500
|
// No tasks could mean that we're finished with all tasks.
|
|
476
501
|
return this._maybeFinish();
|
|
477
502
|
}
|
|
503
|
+
// - we have already reached the maximum tasks per minute
|
|
504
|
+
// we need to check this *after* checking if a task is ready to prevent hanging the pool
|
|
505
|
+
// for an extra minute if there are no more tasks
|
|
506
|
+
if (this._isOverMaxRequestLimit) {
|
|
507
|
+
this.log.perf('Task will not run. Maximum tasks per minute reached.');
|
|
508
|
+
return done();
|
|
509
|
+
}
|
|
478
510
|
try {
|
|
479
511
|
// Everything's fine. Run task.
|
|
480
512
|
this._currentConcurrency++;
|
|
513
|
+
this._tasksPerMinute[0]++;
|
|
481
514
|
// Try to run next task to build up concurrency,
|
|
482
515
|
// but defer it so it doesn't create a cycle.
|
|
483
516
|
setImmediate(this._maybeRunTask);
|
|
@@ -516,6 +549,9 @@ class AutoscaledPool {
|
|
|
516
549
|
// Don't scale if paused.
|
|
517
550
|
if (this.isStopped)
|
|
518
551
|
return intervalCallback();
|
|
552
|
+
// Don't scale if we've hit the maximum requests per minute
|
|
553
|
+
if (this._isOverMaxRequestLimit)
|
|
554
|
+
return intervalCallback();
|
|
519
555
|
// Only scale up if:
|
|
520
556
|
// - system has not been overloaded lately.
|
|
521
557
|
const systemStatus = this.systemStatus.getHistoricalStatus();
|
|
@@ -616,9 +652,22 @@ class AutoscaledPool {
|
|
|
616
652
|
this.reject = null;
|
|
617
653
|
(0, utilities_1.betterClearInterval)(this.autoscaleInterval);
|
|
618
654
|
(0, utilities_1.betterClearInterval)(this.maybeRunInterval);
|
|
655
|
+
if (this.tasksDonePerSecondInterval)
|
|
656
|
+
(0, utilities_1.betterClearInterval)(this.tasksDonePerSecondInterval);
|
|
619
657
|
if (this.snapshotter)
|
|
620
658
|
await this.snapshotter.stop();
|
|
621
659
|
}
|
|
660
|
+
_incrementTasksDonePerSecond(intervalCallback) {
|
|
661
|
+
this._tasksPerMinute.unshift(0);
|
|
662
|
+
this._tasksPerMinute.pop();
|
|
663
|
+
return intervalCallback();
|
|
664
|
+
}
|
|
665
|
+
get _isOverMaxRequestLimit() {
|
|
666
|
+
if (this.maxTasksPerMinute === Infinity) {
|
|
667
|
+
return false;
|
|
668
|
+
}
|
|
669
|
+
return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
|
|
670
|
+
}
|
|
622
671
|
}
|
|
623
672
|
exports.AutoscaledPool = AutoscaledPool;
|
|
624
673
|
//# sourceMappingURL=autoscaled_pool.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autoscaled_pool.js","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":";;;;AACA,4CAAqD;AACrD,gDAA4F;AAC5F,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAC3C,+CAAgE;AAChE,mDAAgF;AAgHhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,cAAc;IAgCvB,YACI,OAA8B,EACb,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAjCrB;;;;;WAA0B;QAE1B,2BAA2B;QAC3B;;;;;WAAiD;QACjD;;;;;WAA0C;QAC1C;;;;;WAA4C;QAC5C;;;;;WAAgD;QAChD;;;;;WAA+C;QAC/C;;;;;WAAiD;QACjD;;;;;WAA2C;QAC3C;;;;;WAAyD;QACzD;;;;;WAA4D;QAC5D;;;;;WAA6D;QAE7D,uBAAuB;QACvB;;;;;WAAgC;QAChC;;;;;WAAgC;QAChC;;;;;WAAoC;QACpC;;;;mBAA8B,CAAC;WAAC;QAChC;;;;mBAAoB,KAAK;WAAC;QAC1B;;;;mBAA0B,CAAC;WAAC;QAC5B;;;;mBAAoD,IAAI;WAAC;QACzD;;;;mBAAsD,IAAI;WAAC;QAC3D;;;;;WAAiC;QACjC;;;;;WAAmC;QACnC;;;;;WAAuC;QACvC;;;;;WAA6C;QAC7C;;;;;WAA4C;QAC5C;;;;;WAAsC;QACtC;;;;;WAAqC;QAMjC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ;YAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ;YAC/B,mBAAmB,EAAE,YAAE,CAAC,QAAQ;YAChC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC3C,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACxC,mBAAmB,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,eAAe,CAAC;YAC1D,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GAAG,IAAI,EACrB,cAAc,GAAG,CAAC,EAClB,kBAAkB,EAClB,uBAAuB,GAAG,IAAI,EAC9B,gBAAgB,GAAG,IAAI,EACvB,kBAAkB,GAAG,IAAI,EACzB,oBAAoB,GAAG,GAAG,EAC1B,mBAAmB,GAAG,EAAE,EACxB,eAAe,GAAG,CAAC,EACnB,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,EACnB,kBAAkB,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAEnD,2BAA2B;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,GAAG,IAAI,CAAC;QAC1D,IAAI,CAAC,qBAAqB,GAAG,mBAAoB,GAAG,IAAI,CAAC;QACzD,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,IAAI,cAAc,CAAC;QAChE,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEnD,yCAAyC;QACzC,MAAM,OAAO,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;QAC3C,OAAO,CAAC,WAAW,KAAnB,OAAO,CAAC,WAAW,GAAK,IAAI,yBAAW,CAAC;YACpC,GAAG,kBAAkB;YACrB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE;SACzC,CAAC,EAAC;QACH,OAAO,CAAC,MAAM,KAAd,OAAO,CAAC,MAAM,GAAK,IAAI,CAAC,MAAM,EAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;;OAKG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB,CAAC,KAAa;QAChC,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE/B,0FAA0F;QAC1F,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAE1F,4FAA4F;QAC5F,gGAAgG;QAChG,0FAA0F;QAC1F,IAAI,CAAC,gBAAgB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAE3F,IAAI;YACA,MAAM,IAAI,CAAC,WAAW,CAAC;SAC1B;gBAAS;YACN,qDAAqD;YACrD,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SAC3C;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SACzB;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,WAAoB;QAC5B,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,OAAuB,CAAC;YAC5B,IAAI,WAAW,EAAE;gBACb,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0CAA0C;0BAC1D,MAAM,WAAW,sCAAsC,CAAC,CAAC;oBAC/D,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;aACnB;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,IAAI,CAAC,mBAAmB,IAAI,CAAC,EAAE;oBAC/B,4DAA4D;oBAC5D,IAAI,OAAO;wBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;oBACnC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;iBACb;YACL,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,MAAM;QACF,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,aAAa,CAAC,gBAA6B;QACvD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3C,8FAA8F;QAC9F,MAAM,IAAI,GAAG,gBAAgB,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE5C,kCAAkC;QAClC,kCAAkC;QAClC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAC/D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,wCAAwC;QACxC,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC9D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,yCAAyC;QACzC,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE;YACtD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,mEAAmE;QACnE,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAC3D,MAAM,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC;QACvC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,eAAe,EAAE;YACnE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,6CAA6C,EAAE,aAAa,CAAC,CAAC;YAC5E,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,qBAAqB;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,WAAW,CAAC;QAChB,IAAI;YACA,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC3C,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClD,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;SACpC;QACD,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxD,IAAI,EAAE,CAAC;YACP,0DAA0D;YAC1D,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9B;QAED,IAAI;YACA,+BAA+B;YAC/B,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,gDAAgD;YAChD,6CAA6C;YAC7C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEjC,sFAAsF;YACtF,IAAI,EAAE,CAAC;YAEP,4BAA4B;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAEjC,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE;gBAC5B,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,EAC5B,IAAI,CAAC,iBAAiB,EACtB,mCAAmC,IAAI,CAAC,iBAAiB,GAAG,IAAI,WAAW,CAC9E,CAAC;aACL;iBAAM;gBACH,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aAChC;YAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,4CAA4C;YAC5C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACxC,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;IACL,CAAC;IAED;;;;OAIG;IACO,UAAU,CAAC,gBAA4B;QAC7C,yBAAyB;QACzB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE9C,oBAAoB;QACpB,2CAA2C;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,CAAC;QAC7D,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC;QACtC,0CAA0C;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QACtE,iFAAiF;QACjF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAClG,MAAM,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,IAAI,qBAAqB,CAAC;QAE1F,IAAI,YAAY,IAAI,aAAa,IAAI,+BAA+B;YAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElG,wBAAwB;QACxB,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC;QACzC,gCAAgC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QAEtE,IAAI,kBAAkB,IAAI,aAAa;YAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAEvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,qBAAqB,EAAE;gBACzD,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE;oBACnB,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,YAAY;iBACf,CAAC,CAAC;aACN;SACJ;QAED,8BAA8B;QAC9B,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACO,QAAQ,CAAC,YAAwB;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;YACzB,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,UAAU,CAAC,YAAwB;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC3E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,IAAI,CAAC,mBAAmB,GAAG,CAAC;YAAE,OAAO;QAEzC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI;YACA,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;SACnC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;CACJ;AA7dD,wCA6dC"}
|
|
1
|
+
{"version":3,"file":"autoscaled_pool.js","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":";;;;AACA,4CAAqD;AACrD,gDAA4F;AAC5F,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAC3C,+CAAgE;AAChE,mDAAgF;AAsHhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,cAAc;IAmCvB,YACI,OAA8B,EACb,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QApCrB;;;;;WAA0B;QAE1B,2BAA2B;QAC3B;;;;;WAAiD;QACjD;;;;;WAA0C;QAC1C;;;;;WAA4C;QAC5C;;;;;WAAgD;QAChD;;;;;WAA+C;QAC/C;;;;;WAAiD;QACjD;;;;;WAA2C;QAC3C;;;;;WAAyD;QACzD;;;;;WAA4D;QAC5D;;;;;WAA6D;QAC7D;;;;;WAA2C;QAE3C,uBAAuB;QACvB;;;;;WAAgC;QAChC;;;;;WAAgC;QAChC;;;;;WAAoC;QACpC;;;;mBAA8B,CAAC;WAAC;QAChC;;;;mBAAoB,KAAK;WAAC;QAC1B;;;;mBAA0B,CAAC;WAAC;QAC5B;;;;mBAAoD,IAAI;WAAC;QACzD;;;;mBAAsD,IAAI;WAAC;QAC3D;;;;;WAAiC;QACjC;;;;;WAAmC;QACnC;;;;;WAAuC;QACvC;;;;;WAA6C;QAC7C;;;;;WAA4C;QAC5C;;;;;WAAsC;QACtC;;;;;WAAqC;QACrC;;;;;WAAsD;QACtD;;;;mBAAoC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;WAAC;QAMpE,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ;YAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ;YAC/B,mBAAmB,EAAE,YAAE,CAAC,QAAQ;YAChC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC3C,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACxC,mBAAmB,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,eAAe,CAAC;YAC1D,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;SACzF,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GAAG,IAAI,EACrB,cAAc,GAAG,CAAC,EAClB,kBAAkB,EAClB,uBAAuB,GAAG,IAAI,EAC9B,gBAAgB,GAAG,IAAI,EACvB,kBAAkB,GAAG,IAAI,EACzB,oBAAoB,GAAG,GAAG,EAC1B,mBAAmB,GAAG,EAAE,EACxB,eAAe,GAAG,CAAC,EACnB,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,EACnB,kBAAkB,EAClB,GAAG,GAAG,SAAU,EAChB,iBAAiB,GAAG,QAAQ,GAC/B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAEnD,2BAA2B;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,GAAG,IAAI,CAAC;QAC1D,IAAI,CAAC,qBAAqB,GAAG,mBAAoB,GAAG,IAAI,CAAC;QACzD,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,IAAI,cAAc,CAAC;QAChE,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,4BAA4B,GAAG,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjF,yCAAyC;QACzC,MAAM,OAAO,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;QAC3C,OAAO,CAAC,WAAW,KAAnB,OAAO,CAAC,WAAW,GAAK,IAAI,yBAAW,CAAC;YACpC,GAAG,kBAAkB;YACrB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE;SACzC,CAAC,EAAC;QACH,OAAO,CAAC,MAAM,KAAd,OAAO,CAAC,MAAM,GAAK,IAAI,CAAC,MAAM,EAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;;OAKG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB,CAAC,KAAa;QAChC,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE/B,0FAA0F;QAC1F,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAE1F,4FAA4F;QAC5F,gGAAgG;QAChG,0FAA0F;QAC1F,IAAI,CAAC,gBAAgB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAE3F,IAAI,IAAI,CAAC,iBAAiB,KAAK,QAAQ,EAAE;YACrC,kEAAkE;YAClE,IAAI,CAAC,0BAA0B,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,4BAA4B,EAAE,IAAI,CAAC,CAAC;SAChG;QAED,IAAI;YACA,MAAM,IAAI,CAAC,WAAW,CAAC;SAC1B;gBAAS;YACN,qDAAqD;YACrD,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SAC3C;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SACzB;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,WAAoB;QAC5B,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,OAAuB,CAAC;YAC5B,IAAI,WAAW,EAAE;gBACb,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0CAA0C;0BAC1D,MAAM,WAAW,sCAAsC,CAAC,CAAC;oBAC/D,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;aACnB;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,IAAI,CAAC,mBAAmB,IAAI,CAAC,EAAE;oBAC/B,4DAA4D;oBAC5D,IAAI,OAAO;wBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;oBACnC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;iBACb;YACL,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,MAAM;QACF,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,aAAa,CAAC,gBAA6B;QACvD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3C,8FAA8F;QAC9F,MAAM,IAAI,GAAG,gBAAgB,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE5C,kCAAkC;QAClC,kCAAkC;QAClC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAC/D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,wCAAwC;QACxC,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC9D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,yCAAyC;QACzC,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE;YACtD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,mEAAmE;QACnE,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAC3D,MAAM,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC;QACvC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,eAAe,EAAE;YACnE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,6CAA6C,EAAE,aAAa,CAAC,CAAC;YAC5E,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,qBAAqB;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,WAAW,CAAC;QAChB,IAAI;YACA,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC3C,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClD,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;SACpC;QACD,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxD,IAAI,EAAE,CAAC;YACP,0DAA0D;YAC1D,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9B;QAED,yDAAyD;QACzD,wFAAwF;QACxF,iDAAiD;QACjD,IAAI,IAAI,CAAC,sBAAsB,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;YACtE,OAAO,IAAI,EAAE,CAAC;SACjB;QAED,IAAI;YACA,+BAA+B;YAC/B,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,gDAAgD;YAChD,6CAA6C;YAC7C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEjC,sFAAsF;YACtF,IAAI,EAAE,CAAC;YAEP,4BAA4B;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAEjC,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE;gBAC5B,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,EAC5B,IAAI,CAAC,iBAAiB,EACtB,mCAAmC,IAAI,CAAC,iBAAiB,GAAG,IAAI,WAAW,CAC9E,CAAC;aACL;iBAAM;gBACH,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aAChC;YAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,4CAA4C;YAC5C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACxC,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;IACL,CAAC;IAED;;;;OAIG;IACO,UAAU,CAAC,gBAA4B;QAC7C,yBAAyB;QACzB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE9C,2DAA2D;QAC3D,IAAI,IAAI,CAAC,sBAAsB;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE3D,oBAAoB;QACpB,2CAA2C;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,CAAC;QAC7D,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC;QACtC,0CAA0C;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QACtE,iFAAiF;QACjF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAClG,MAAM,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,IAAI,qBAAqB,CAAC;QAE1F,IAAI,YAAY,IAAI,aAAa,IAAI,+BAA+B;YAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElG,wBAAwB;QACxB,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC;QACzC,gCAAgC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QAEtE,IAAI,kBAAkB,IAAI,aAAa;YAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAEvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,qBAAqB,EAAE;gBACzD,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE;oBACnB,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,YAAY;iBACf,CAAC,CAAC;aACN;SACJ;QAED,8BAA8B;QAC9B,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACO,QAAQ,CAAC,YAAwB;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;YACzB,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,UAAU,CAAC,YAAwB;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC3E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,IAAI,CAAC,mBAAmB,GAAG,CAAC;YAAE,OAAO;QAEzC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI;YACA,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;SACnC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,0BAA0B;YAAE,IAAA,+BAAmB,EAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC1F,IAAI,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;IAES,4BAA4B,CAAC,gBAA4B;QAC/D,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEhC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;QAE3B,OAAO,gBAAgB,EAAE,CAAC;IAC9B,CAAC;IAED,IAAc,sBAAsB;QAChC,IAAI,IAAI,CAAC,iBAAiB,KAAK,QAAQ,EAAE;YACrC,OAAO,KAAK,CAAC;SAChB;QAED,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC;IAC/F,CAAC;CACJ;AAtgBD,wCAsgBC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Log } from '../log';
|
|
1
2
|
import { ProxyInfo } from '../proxy_configuration';
|
|
2
3
|
import { Request } from '../request';
|
|
3
4
|
import { Session } from '../session_pool/session';
|
|
@@ -13,6 +14,7 @@ export interface CrawlingContext extends Record<PropertyKey, unknown> {
|
|
|
13
14
|
* and configured by the {@link ProxyConfiguration} class.
|
|
14
15
|
*/
|
|
15
16
|
proxyInfo?: ProxyInfo;
|
|
17
|
+
log: Log;
|
|
16
18
|
}
|
|
17
19
|
export interface CrawlerHandleFailedRequestInput extends CrawlingContext {
|
|
18
20
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,MAAM,WAAW,eAAgB,SAAQ,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;IACjE,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,MAAM,WAAW,eAAgB,SAAQ,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;IACjE,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,+BAAgC,SAAQ,eAAe;IACpE;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC;CAChB"}
|
|
@@ -135,4 +135,17 @@ export declare enum EnqueueStrategy {
|
|
|
135
135
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
136
136
|
*/
|
|
137
137
|
export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
|
|
138
|
+
/**
|
|
139
|
+
* @internal
|
|
140
|
+
*/
|
|
141
|
+
export declare function resolveBaseUrl({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
|
|
142
|
+
/**
|
|
143
|
+
* @internal
|
|
144
|
+
*/
|
|
145
|
+
export interface ResolveBaseUrl {
|
|
146
|
+
userProvidedBaseUrl?: string;
|
|
147
|
+
enqueueStrategy?: EnqueueLinksOptions['strategy'];
|
|
148
|
+
originalRequestUrl: string;
|
|
149
|
+
finalRequestUrl?: string;
|
|
150
|
+
}
|
|
138
151
|
//# sourceMappingURL=enqueue_links.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,MAAM,WAAW,mBAAmB;IAChC,qHAAqH;IACrH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;CAC3E;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IACX;;OAEG;IACH,aAAa,mBAAmB;IAChC;;;;OAIG;IACH,YAAY,kBAAkB;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,MAAM,WAAW,mBAAmB;IAChC,qHAAqH;IACrH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;CAC3E;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IACX;;OAEG;IACH,aAAa,mBAAmB;IAChC;;;;OAIG;IACH,YAAY,kBAAkB;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAqGhG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC3B,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA4BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.enqueueLinks = exports.EnqueueStrategy = void 0;
|
|
3
|
+
exports.resolveBaseUrl = exports.enqueueLinks = exports.EnqueueStrategy = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const tldts_1 = require("tldts");
|
|
6
6
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
@@ -80,7 +80,9 @@ async function enqueueLinks(options) {
|
|
|
80
80
|
if (regexps?.length) {
|
|
81
81
|
urlPatternObjects.push(...(0, shared_1.constructRegExpObjectsFromRegExps)(regexps));
|
|
82
82
|
}
|
|
83
|
-
|
|
83
|
+
if (!urlPatternObjects.length) {
|
|
84
|
+
options.strategy ?? (options.strategy = EnqueueStrategy.SameSubdomain);
|
|
85
|
+
}
|
|
84
86
|
if (options.baseUrl) {
|
|
85
87
|
switch (options.strategy) {
|
|
86
88
|
case EnqueueStrategy.SameSubdomain:
|
|
@@ -120,4 +122,31 @@ async function enqueueLinks(options) {
|
|
|
120
122
|
return requestQueue.addRequests(requests);
|
|
121
123
|
}
|
|
122
124
|
exports.enqueueLinks = enqueueLinks;
|
|
125
|
+
/**
|
|
126
|
+
* @internal
|
|
127
|
+
*/
|
|
128
|
+
function resolveBaseUrl({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
|
|
129
|
+
// User provided base url takes priority
|
|
130
|
+
if (userProvidedBaseUrl) {
|
|
131
|
+
return userProvidedBaseUrl;
|
|
132
|
+
}
|
|
133
|
+
const originalUrlOrigin = new URL(originalRequestUrl).origin;
|
|
134
|
+
const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
|
|
135
|
+
// We can assume users want to go off the domain in this case
|
|
136
|
+
if (enqueueStrategy === EnqueueStrategy.All) {
|
|
137
|
+
return finalUrlOrigin ?? originalUrlOrigin;
|
|
138
|
+
}
|
|
139
|
+
// Ensure links use the same hostname
|
|
140
|
+
if (enqueueStrategy === EnqueueStrategy.SameHostname) {
|
|
141
|
+
const originalHostname = (0, tldts_1.getDomain)(originalUrlOrigin, { mixedInputs: false });
|
|
142
|
+
const finalHostname = (0, tldts_1.getDomain)(finalUrlOrigin, { mixedInputs: false });
|
|
143
|
+
if (originalHostname === finalHostname) {
|
|
144
|
+
return finalUrlOrigin;
|
|
145
|
+
}
|
|
146
|
+
return undefined;
|
|
147
|
+
}
|
|
148
|
+
// Always enqueue urls that are from the same origin
|
|
149
|
+
return originalUrlOrigin;
|
|
150
|
+
}
|
|
151
|
+
exports.resolveBaseUrl = resolveBaseUrl;
|
|
123
152
|
//# sourceMappingURL=enqueue_links.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAC7B,qCAWkB;AAuGlB,IAAY,eAeX;AAfD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IACX;;OAEG;IACH,mDAAgC,CAAA;IAChC;;;;OAIG;IACH,iDAA8B,CAAA;AAClC,CAAC,EAfW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAe1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,aAAa;YAC7B,eAAe,CAAC,YAAY;SAC/B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAC7B,qCAWkB;AAuGlB,IAAY,eAeX;AAfD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IACX;;OAEG;IACH,mDAAgC,CAAA;IAChC;;;;OAIG;IACH,iDAA8B,CAAA;AAClC,CAAC,EAfW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAe1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,aAAa;YAC7B,eAAe,CAAC,YAAY;SAC/B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,aAAa,EAAC;KACtD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,aAAa;gBAC9B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC1E,MAAM;YACV,KAAK,eAAe,CAAC,YAAY,CAAC,CAAC;gBAC/B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAErC,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,CAAC,CAAC;IAEhD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAC1G;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AArGD,oCAqGC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,EAC3B,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,IAAI,iBAAiB,CAAC;KAC9C;IAED,qCAAqC;IACrC,IAAI,eAAe,KAAK,eAAe,CAAC,YAAY,EAAE;QAClD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE;YACpC,OAAO,cAAc,CAAC;SACzB;QAED,OAAO,SAAS,CAAC;KACpB;IAED,oDAAoD;IACpD,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAjCD,wCAiCC"}
|
package/index.mjs
CHANGED
|
@@ -55,6 +55,7 @@ export const maybeStringify = mod.maybeStringify;
|
|
|
55
55
|
export const mergeCookies = mod.mergeCookies;
|
|
56
56
|
export const openDataset = mod.openDataset;
|
|
57
57
|
export const pushData = mod.pushData;
|
|
58
|
+
export const resolveBaseUrl = mod.resolveBaseUrl;
|
|
58
59
|
export const serializeArray = mod.serializeArray;
|
|
59
60
|
export const storage = mod.storage;
|
|
60
61
|
export const throwOnBlockedRequest = mod.throwOnBlockedRequest;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.0-alpha.
|
|
3
|
+
"version": "3.0.0-alpha.5",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -57,9 +57,9 @@
|
|
|
57
57
|
"@apify/timeout": "^0.2.1",
|
|
58
58
|
"@apify/utilities": "^1.2.5",
|
|
59
59
|
"@apify/storage-local": "^2.0.3-beta.3",
|
|
60
|
-
"@crawlee/browser-pool": "^3.0.0-alpha.
|
|
61
|
-
"@crawlee/storage": "^3.0.0-alpha.
|
|
62
|
-
"@crawlee/utils": "^3.0.0-alpha.
|
|
60
|
+
"@crawlee/browser-pool": "^3.0.0-alpha.5",
|
|
61
|
+
"@crawlee/storage": "^3.0.0-alpha.5",
|
|
62
|
+
"@crawlee/utils": "^3.0.0-alpha.5",
|
|
63
63
|
"@types/mime-types": "^2.1.1",
|
|
64
64
|
"@types/stream-json": "^1.7.1",
|
|
65
65
|
"content-type": "^1.0.4",
|
package/proxy_configuration.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Configuration } from './configuration';
|
|
2
2
|
export interface ProxyConfigurationFunction {
|
|
3
|
-
(sessionId: string | number): string
|
|
3
|
+
(sessionId: string | number): string | Promise<string>;
|
|
4
4
|
}
|
|
5
5
|
export interface ProxyConfigurationOptions {
|
|
6
6
|
/**
|
|
@@ -11,7 +11,7 @@ export interface ProxyConfigurationOptions {
|
|
|
11
11
|
proxyUrls?: string[];
|
|
12
12
|
/**
|
|
13
13
|
* Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
|
|
14
|
-
* and should always return stringified proxy URL.
|
|
14
|
+
* and should always return stringified proxy URL. Can be asynchronous.
|
|
15
15
|
* This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
|
|
16
16
|
*/
|
|
17
17
|
newUrlFunction?: ProxyConfigurationFunction;
|
|
@@ -151,7 +151,7 @@ export declare class ProxyConfiguration {
|
|
|
151
151
|
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
152
152
|
* @return Represents information about used proxy and its configuration.
|
|
153
153
|
*/
|
|
154
|
-
newProxyInfo(sessionId?: string | number): ProxyInfo
|
|
154
|
+
newProxyInfo(sessionId?: string | number): Promise<ProxyInfo>;
|
|
155
155
|
/**
|
|
156
156
|
* Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
|
|
157
157
|
* @param [sessionId]
|
|
@@ -165,7 +165,7 @@ export declare class ProxyConfiguration {
|
|
|
165
165
|
* @return A string with a proxy URL, including authentication credentials and port number.
|
|
166
166
|
* For example, `http://bob:password123@proxy.example.com:8000`
|
|
167
167
|
*/
|
|
168
|
-
newUrl(sessionId?: string | number): string
|
|
168
|
+
newUrl(sessionId?: string | number): Promise<string>;
|
|
169
169
|
/**
|
|
170
170
|
* Handles custom url rotation with session
|
|
171
171
|
*/
|
|
@@ -173,9 +173,9 @@ export declare class ProxyConfiguration {
|
|
|
173
173
|
/**
|
|
174
174
|
* Calls the custom newUrlFunction and checks format of its return value
|
|
175
175
|
*/
|
|
176
|
-
protected _callNewUrlFunction(sessionId?: string): string
|
|
177
|
-
protected _throwNewUrlFunctionInvalid(err: Error):
|
|
178
|
-
protected _throwCannotCombineCustomMethods():
|
|
179
|
-
protected _throwNoOptionsProvided():
|
|
176
|
+
protected _callNewUrlFunction(sessionId?: string): Promise<string>;
|
|
177
|
+
protected _throwNewUrlFunctionInvalid(err: Error): never;
|
|
178
|
+
protected _throwCannotCombineCustomMethods(): never;
|
|
179
|
+
protected _throwNoOptionsProvided(): never;
|
|
180
180
|
}
|
|
181
181
|
//# sourceMappingURL=proxy_configuration.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,0BAA0B;IACvC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,0BAA0B;IACvC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,yBAAyB;IACtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;;;OAIG;IACH,cAAc,CAAC,EAAE,0BAA0B,CAAC;CAC/C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,kBAAkB;IA+B0B,QAAQ,CAAC,MAAM;IA9BpE,gBAAgB,UAAS;IACzB,SAAS,CAAC,kBAAkB,SAAK;IACjC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,aAAa,sBAA6B;IACpD,SAAS,CAAC,cAAc,CAAC,EAAE,0BAA0B,CAAC;IACtD,SAAS,CAAC,GAAG,2BAA+C;IAE5D;;;;;;;;;;;;;;;;;;;;;;OAsBG;gBACS,OAAO,GAAE,yBAA8B,EAAW,MAAM,gBAAkC,EAAE,gBAAgB,UAAO;IAe/H;;;;;;;;;;;;;;;;OAgBG;IACG,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAgBnE;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAU1D;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM;IAiBtD;;OAEG;cACa,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM;IAYtD,SAAS,CAAC,2BAA2B,CAAC,GAAG,EAAE,KAAK,GAAI,KAAK;IAIzD,SAAS,CAAC,gCAAgC,IAAK,KAAK;IAIpD,SAAS,CAAC,uBAAuB,IAAK,KAAK;CAG9C"}
|
package/proxy_configuration.js
CHANGED
|
@@ -129,10 +129,10 @@ class ProxyConfiguration {
|
|
|
129
129
|
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
130
130
|
* @return Represents information about used proxy and its configuration.
|
|
131
131
|
*/
|
|
132
|
-
newProxyInfo(sessionId) {
|
|
132
|
+
async newProxyInfo(sessionId) {
|
|
133
133
|
if (typeof sessionId === 'number')
|
|
134
134
|
sessionId = `${sessionId}`;
|
|
135
|
-
const url = this.newUrl(sessionId);
|
|
135
|
+
const url = await this.newUrl(sessionId);
|
|
136
136
|
const { username, password, port, hostname } = new URL(url);
|
|
137
137
|
return {
|
|
138
138
|
sessionId,
|
|
@@ -156,7 +156,7 @@ class ProxyConfiguration {
|
|
|
156
156
|
* @return A string with a proxy URL, including authentication credentials and port number.
|
|
157
157
|
* For example, `http://bob:password123@proxy.example.com:8000`
|
|
158
158
|
*/
|
|
159
|
-
newUrl(sessionId) {
|
|
159
|
+
async newUrl(sessionId) {
|
|
160
160
|
if (typeof sessionId === 'number')
|
|
161
161
|
sessionId = `${sessionId}`;
|
|
162
162
|
if (this.newUrlFunction) {
|
|
@@ -184,16 +184,16 @@ class ProxyConfiguration {
|
|
|
184
184
|
/**
|
|
185
185
|
* Calls the custom newUrlFunction and checks format of its return value
|
|
186
186
|
*/
|
|
187
|
-
_callNewUrlFunction(sessionId) {
|
|
187
|
+
async _callNewUrlFunction(sessionId) {
|
|
188
188
|
let proxyUrl;
|
|
189
189
|
try {
|
|
190
|
-
proxyUrl = this.newUrlFunction(sessionId);
|
|
190
|
+
proxyUrl = await this.newUrlFunction(sessionId);
|
|
191
191
|
new URL(proxyUrl); // eslint-disable-line no-new
|
|
192
|
+
return proxyUrl;
|
|
192
193
|
}
|
|
193
194
|
catch (err) {
|
|
194
195
|
this._throwNewUrlFunctionInvalid(err);
|
|
195
196
|
}
|
|
196
|
-
return proxyUrl;
|
|
197
197
|
}
|
|
198
198
|
_throwNewUrlFunctionInvalid(err) {
|
|
199
199
|
throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,6DAA6B;AAC7B,mDAAgD;AAsFhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,kBAAkB;IAQ3B;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,YAAY,UAAqC,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE,EAAE,gBAAgB,GAAG,IAAI;;;;;mBAAjE;;QA9B9D;;;;mBAAmB,KAAK;WAAC;QACzB;;;;mBAA+B,CAAC;WAAC;QACjC;;;;;WAA+B;QAC/B;;;;mBAA0B,IAAI,GAAG,EAAkB;WAAC;QACpD;;;;;WAAsD;QACtD;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;WAAC;QA0BxD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9C,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACzE,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,YAAY,CAAC,SAA2B;
|
|
1
|
+
{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,6DAA6B;AAC7B,mDAAgD;AAsFhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,kBAAkB;IAQ3B;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,YAAY,UAAqC,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE,EAAE,gBAAgB,GAAG,IAAI;;;;;mBAAjE;;QA9B9D;;;;mBAAmB,KAAK;WAAC;QACzB;;;;mBAA+B,CAAC;WAAC;QACjC;;;;;WAA+B;QAC/B;;;;mBAA0B,IAAI,GAAG,EAAkB;WAAC;QACpD;;;;;WAAsD;QACtD;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;WAAC;QA0BxD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9C,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACzE,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,YAAY,CAAC,SAA2B;QAC1C,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEzC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO;YACH,SAAS;YACT,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,IAAI,EAAE,IAAK;SACd,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,MAAM,CAAC,SAA2B;QACpC,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAE9D,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAE,CAAC;SAC/C;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,gBAAgB,CAAC,SAAkB;QACzC,IAAI,cAAsB,CAAC;QAE3B,IAAI,CAAC,SAAS,EAAE;YACZ,OAAO,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;SAC9E;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACnC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;SACvD;aAAM;YACH,cAAc,GAAG,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;YACrF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;SACrD;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,SAAkB;QAClD,IAAI,QAAgB,CAAC;QAErB,IAAI;YACA,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,SAAU,CAAC,CAAC;YAClD,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;YAChD,OAAO,QAAQ,CAAC;SACnB;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,2BAA2B,CAAC,GAAY,CAAC,CAAC;SAClD;IACL,CAAC;IAES,2BAA2B,CAAC,GAAU;QAC5C,MAAM,IAAI,KAAK,CAAC,mEAAmE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACtG,CAAC;IAES,gCAAgC;QACtC,MAAM,IAAI,KAAK,CAAC,6GAA6G,CAAC,CAAC;IACnI,CAAC;IAES,uBAAuB;QAC7B,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IACpG,CAAC;CACJ;AApJD,gDAoJC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAc,MAAM,aAAa,CAAC;AACpD,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,EAAE,aAAa,EAAuB,MAAM,WAAW,CAAC;AAE/D;;;;GAIG;AACH,eAAO,MAAM,cAAc,yBAA0B;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MAwB5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,aAAa;IAQqB,QAAQ,CAAC,MAAM;IAP1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,MAAM,CAAsB;IAEpC;;OAEG;gBACS,OAAO,EAAE,oBAAoB,EAAW,MAAM,gBAAkC;IAM5F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAO3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAc,MAAM,aAAa,CAAC;AACpD,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,EAAE,aAAa,EAAuB,MAAM,WAAW,CAAC;AAE/D;;;;GAIG;AACH,eAAO,MAAM,cAAc,yBAA0B;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MAwB5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,aAAa;IAQqB,QAAQ,CAAC,MAAM;IAP1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,MAAM,CAAsB;IAEpC;;OAEG;gBACS,OAAO,EAAE,oBAAoB,EAAW,MAAM,gBAAkC;IAM5F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAO3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;OAGG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAKjC;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAIpF,WAAW;IAiBzB;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;CAShH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
|
|
@@ -218,8 +218,9 @@ class KeyValueStore {
|
|
|
218
218
|
validator: ow_1.default.isValid(k, ow_1.default.string.matches(consts_1.KEY_VALUE_STORE_KEY_REGEX)),
|
|
219
219
|
message: 'The "key" argument must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.\'()',
|
|
220
220
|
})));
|
|
221
|
-
if (options.contentType
|
|
222
|
-
|
|
221
|
+
if (options.contentType
|
|
222
|
+
&& !(ow_1.default.isValid(value, ow_1.default.any(ow_1.default.string, ow_1.default.buffer)) || (ow_1.default.isValid(value, ow_1.default.object) && typeof value.pipe === 'function'))) {
|
|
223
|
+
throw new ow_1.ArgumentError('The "value" parameter must be a String, Buffer or Stream when "options.contentType" is specified.', this.setValue);
|
|
223
224
|
}
|
|
224
225
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
225
226
|
contentType: ow_1.default.optional.string.nonEmpty,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AACvC,oDAAiD;AAEjD,uDAA0E;AAG1E;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,aAAa;IAKtB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAPpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAAoC;QAMhC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,IAAI,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW,
|
|
1
|
+
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AACvC,oDAAiD;AAEjD,uDAA0E;AAG1E;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,aAAa;IAKtB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAPpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAAoC;QAMhC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,IAAI,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/D,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,GAAW;QACpB,uGAAuG;QACvG,OAAO,6CAA6C,IAAI,CAAC,EAAE,YAAY,GAAG,EAAE,CAAC;IACjF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;CACJ;AA/MD,sCA+MC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import { Configuration } from '../configuration';
|
|
3
3
|
import { Request, RequestOptions } from '../request';
|
|
4
|
+
import { ProxyConfiguration } from '../proxy_configuration';
|
|
4
5
|
/** @internal */
|
|
5
6
|
export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
|
|
6
7
|
/** @internal */
|
|
@@ -81,6 +82,12 @@ export interface RequestListOptions {
|
|
|
81
82
|
* ```
|
|
82
83
|
*/
|
|
83
84
|
sourcesFunction?: RequestListSourcesFunction;
|
|
85
|
+
/**
|
|
86
|
+
* Used to pass the the proxy configuration for the `requestsFromUrls` objects.
|
|
87
|
+
* Takes advantage of the internal address rotation and authentication process.
|
|
88
|
+
* If undefined, the `requestsFromUrls` requests will be made without proxy.
|
|
89
|
+
*/
|
|
90
|
+
proxyConfiguration?: ProxyConfiguration;
|
|
84
91
|
/**
|
|
85
92
|
* Identifies the key in the default key-value store under which `RequestList` periodically stores its
|
|
86
93
|
* state (i.e. which URLs were crawled and which not).
|
|
@@ -250,6 +257,7 @@ export declare class RequestList {
|
|
|
250
257
|
private keepDuplicateUrls;
|
|
251
258
|
private sources;
|
|
252
259
|
private sourcesFunction?;
|
|
260
|
+
private proxyConfiguration?;
|
|
253
261
|
private events;
|
|
254
262
|
/**
|
|
255
263
|
* @param options All `RequestList` configuration options
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,uBAAuB,CAAC;AAE1D,gBAAgB;AAChB,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAIhE,MAAM,WAAW,kBAAkB;IAC/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,eAAe,CAAC,EAAE,0BAA0B,CAAC;IAE7C;;;;MAIE;IACF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,EAAE,gBAAgB,CAAC;IAEzB;;;;;;;;;;;;;;OAcG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmEG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,GAAG,CAAwC;IAEnD;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,CAAM;IAEzB,kHAAkH;IAClH,OAAO,CAAC,SAAS,CAAK;IAEtB,gGAAgG;IAChG,OAAO,CAAC,gBAAgB,CAA8B;IAEtD;;;OAGG;IACH,UAAU,cAAqB;IAE/B;;;OAGG;IACH,SAAS,cAAqB;IAE9B;;;OAGG;IACH,gBAAgB,UAAQ;IAExB;;;OAGG;IACH,oBAAoB,UAAS;IAE7B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAC9B,OAAO,CAAC,iBAAiB,CAAU;IACnC,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAC,CAA6B;IACrD,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAChD,OAAO,CAAC,MAAM,CAAe;IAE7B;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA6C5C;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBjC;;;;OAIG;cACa,qBAAqB,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/E;;;;;OAKG;cACa,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC;IAmCxD;;;;;;OAMG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAenC;;;;OAIG;cACa,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAOjD;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,gBAAgB,GAAG,IAAI;IAuDvD;;;OAGG;cACa,8BAA8B,IAAI,OAAO,CAAC,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAoBrF;;;OAGG;IACH,QAAQ,IAAI,gBAAgB;IAY5B;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAMjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;;;;;;OAOG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAuBjD;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzD;;;OAGG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrD;;OAEG;cACa,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,cAAc,EAAE;cAmB7E,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAO9D;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoBxF;;;;OAIG;IACH,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM;IAgCpC;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAMxD;;OAEG;IACH,SAAS,CAAC,gCAAgC,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IASnE;;OAEG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAMtC;;OAEG;IACH,MAAM,IAAI,MAAM;IAMhB;;OAEG;IACH,YAAY,IAAI,MAAM;IAMtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;WACU,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IAgBrH;;OAEG;YACW,mBAAmB;CAGpC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,gBAAgB;IAE7B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAElB,+CAA+C;IAC/C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,mFAAmF;IACnF,UAAU,EAAE,MAAM,EAAE,CAAC;CAExB;AAED,oBAAY,MAAM,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC;AACjH,aAAK,cAAc,GAAG;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAClE,oBAAY,0BAA0B,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC"}
|