@crawlee/core 3.0.0-alpha.3 → 3.0.0-alpha.6

package/{dist/autoscaling → autoscaling}/autoscaled_pool.d.ts

@@ -94,6 +94,11 @@ export interface AutoscaledPoolOptions {
      * by the pool.
      */
     systemStatusOptions?: SystemStatusOptions;
+    /**
+     * The maximum number of tasks per minute the pool can run.
+     * By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
+     */
+    maxTasksPerMinute?: number;
     log?: Log;
 }
 /**
@@ -157,6 +162,7 @@ export declare class AutoscaledPool {
     private readonly runTaskFunction;
     private readonly isFinishedFunction;
     private readonly isTaskReadyFunction;
+    private readonly maxTasksPerMinute;
     private _minConcurrency;
     private _maxConcurrency;
     private _desiredConcurrency;
@@ -172,6 +178,8 @@ export declare class AutoscaledPool {
     private maybeRunInterval;
     private queryingIsTaskReady;
     private queryingIsFinished;
+    private tasksDonePerSecondInterval?;
+    private _tasksPerMinute;
     constructor(options: AutoscaledPoolOptions, config?: Configuration);
     /**
      * Gets the minimum number of tasks running in parallel.
@@ -282,5 +290,7 @@ export declare class AutoscaledPool {
      * Cleans up resources.
      */
     protected _destroy(): Promise<void>;
+    protected _incrementTasksDonePerSecond(intervalCallback: () => void): void;
+    protected get _isOverMaxRequestLimit(): boolean;
 }
 //# sourceMappingURL=autoscaled_pool.d.ts.map

package/autoscaling/autoscaled_pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"autoscaled_pool.d.ts","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAe,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,UAAU,EAAgB,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEhF,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5C;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAE1C;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,cAAc;IAqCnB,OAAO,CAAC,QAAQ,CAAC,MAAM;IApC3B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAG1B,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAyB;IACzD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAyB;IAC5D,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAyB;IAC7D,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAG3C,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,OAAO,CAA0C;IACzD,OAAO,CAAC,MAAM,CAA6C;IAC3D,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,WAAW,CAAoB;IACvC,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,gBAAgB,CAAoB;IAC5C,OAAO,CAAC,mBAAmB,CAAW;IACtC,OAAO,CAAC,kBAAkB,CAAW;IACrC,OAAO,CAAC,0BAA0B,CAAC,CAAmB;IACtD,OAAO,CAAC,eAAe,CAAiD;gBAGpE,OAAO,EAAE,qBAAqB,EACb,MAAM,gBAAkC;IAmF7D;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED;;;;;OAKG;IACH,IAAI,cAAc,CAAC,KAAK,EAAE,MAAM,EAG/B;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED;;OAEG;IACH,IAAI,cAAc,CAAC,KAAK,EAAE,MAAM,EAG/B;IAED;;;OAGG;IACH,IAAI,kBAAkB,IAAI,MAAM,CAE/B;IAED;;;OAGG;IACH,IAAI,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAGnC;IAED;;OAEG;IACH,IAAI,kBAAkB,IAAI,MAAM,CAE/B;IAED;;;OAGG;IACG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IA6B1B;;;;;;;;;;OAUG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B;;;;;;;;;;OAUG;IACG,KAAK,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBhD;;;;;OAKG;IACH,MAAM,IAAI,IAAI;IAId;;;;;;;OAOG;cACa,aAAa,CAAC,gBAAgB,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqG3E;;;;OAIG;IACH,SAAS,CAAC,UAAU,CAAC,gBAAgB,EAAE,MAAM,IAAI;IA4CjD;;;;;OAKG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,GAAG,IAAI;IAUlD;;;;;OAKG;IACH,SAAS,CAAC,UAAU,CAAC,YAAY,EAAE,UAAU,GAAG,IAAI;IAUpD;;;;;OAKG;cACa,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB7C;;OAEG;cACa,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAUzC,SAAS,CAAC,4BAA4B,CAAC,gBAAgB,EAAE,MAAM,IAAI;IAQnE,SAAS,KAAK,sBAAsB,YAMnC;CACJ"}

package/{dist/autoscaling → autoscaling}/autoscaled_pool.js

@@ -132,6 +132,12 @@ class AutoscaledPool {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "maxTasksPerMinute", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         // Internal properties.
         Object.defineProperty(this, "_minConcurrency", {
             enumerable: true,
@@ -223,6 +229,18 @@ class AutoscaledPool {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "tasksDonePerSecondInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_tasksPerMinute", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: Array.from({ length: 60 }, () => 0)
+        });
         (0, ow_1.default)(options, ow_1.default.object.exactShape({
             runTaskFunction: ow_1.default.function,
             isFinishedFunction: ow_1.default.function,
@@ -240,8 +258,9 @@ class AutoscaledPool {
             systemStatusOptions: ow_1.default.optional.object,
             snapshotterOptions: ow_1.default.optional.object,
             log: ow_1.default.optional.object,
+            maxTasksPerMinute: ow_1.default.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
         }));
-        const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 1000, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.90, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = log_1.log, } = options;
+        const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 1000, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.90, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = log_1.log, maxTasksPerMinute = Infinity, } = options;
         this.log = log.child({ prefix: 'AutoscaledPool' });
         // Configurable properties.
         this.desiredConcurrencyRatio = desiredConcurrencyRatio;
@@ -254,6 +273,7 @@ class AutoscaledPool {
         this.runTaskFunction = runTaskFunction;
         this.isFinishedFunction = isFinishedFunction;
         this.isTaskReadyFunction = isTaskReadyFunction;
+        this.maxTasksPerMinute = maxTasksPerMinute;
         // Internal properties.
         this._minConcurrency = minConcurrency;
         this._maxConcurrency = maxConcurrency;
@@ -265,6 +285,7 @@ class AutoscaledPool {
         this.reject = null;
         this._autoscale = this._autoscale.bind(this);
         this._maybeRunTask = this._maybeRunTask.bind(this);
+        this._incrementTasksDonePerSecond = this._incrementTasksDonePerSecond.bind(this);
         // Create instances with correct options.
         const ssoCopy = { ...systemStatusOptions };
         ssoCopy.snapshotter ?? (ssoCopy.snapshotter = new snapshotter_1.Snapshotter({
@@ -343,6 +364,10 @@ class AutoscaledPool {
         // this._maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results
         // in the actor getting stuck and even after scaling up it never triggers another promise.
         this.maybeRunInterval = (0, utilities_1.betterSetInterval)(this._maybeRunTask, this.maybeRunIntervalMillis);
+        if (this.maxTasksPerMinute !== Infinity) {
+            // Start the interval that resets the counter of tasks per minute.
+            this.tasksDonePerSecondInterval = (0, utilities_1.betterSetInterval)(this._incrementTasksDonePerSecond, 1000);
+        }
         try {
             await this.poolPromise;
         }
@@ -475,9 +500,17 @@ class AutoscaledPool {
             // No tasks could mean that we're finished with all tasks.
             return this._maybeFinish();
         }
+        // - we have already reached the maximum tasks per minute
+        // we need to check this *after* checking if a task is ready to prevent hanging the pool
+        // for an extra minute if there are no more tasks
+        if (this._isOverMaxRequestLimit) {
+            this.log.perf('Task will not run. Maximum tasks per minute reached.');
+            return done();
+        }
         try {
             // Everything's fine. Run task.
             this._currentConcurrency++;
+            this._tasksPerMinute[0]++;
             // Try to run next task to build up concurrency,
             // but defer it so it doesn't create a cycle.
             setImmediate(this._maybeRunTask);
@@ -516,6 +549,9 @@ class AutoscaledPool {
         // Don't scale if paused.
         if (this.isStopped)
             return intervalCallback();
+        // Don't scale if we've hit the maximum requests per minute
+        if (this._isOverMaxRequestLimit)
+            return intervalCallback();
         // Only scale up if:
         // - system has not been overloaded lately.
         const systemStatus = this.systemStatus.getHistoricalStatus();
@@ -616,9 +652,22 @@ class AutoscaledPool {
         this.reject = null;
         (0, utilities_1.betterClearInterval)(this.autoscaleInterval);
         (0, utilities_1.betterClearInterval)(this.maybeRunInterval);
+        if (this.tasksDonePerSecondInterval)
+            (0, utilities_1.betterClearInterval)(this.tasksDonePerSecondInterval);
         if (this.snapshotter)
             await this.snapshotter.stop();
     }
+    _incrementTasksDonePerSecond(intervalCallback) {
+        this._tasksPerMinute.unshift(0);
+        this._tasksPerMinute.pop();
+        return intervalCallback();
+    }
+    get _isOverMaxRequestLimit() {
+        if (this.maxTasksPerMinute === Infinity) {
+            return false;
+        }
+        return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
+    }
 }
 exports.AutoscaledPool = AutoscaledPool;
 //# sourceMappingURL=autoscaled_pool.js.map

package/autoscaling/autoscaled_pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"autoscaled_pool.js","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":";;;;AACA,4CAAqD;AACrD,gDAA4F;AAC5F,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAC3C,+CAAgE;AAChE,mDAAgF;AAsHhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,cAAc;IAmCvB,YACI,OAA8B,EACb,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QApCrB;;;;;WAA0B;QAE1B,2BAA2B;QAC3B;;;;;WAAiD;QACjD;;;;;WAA0C;QAC1C;;;;;WAA4C;QAC5C;;;;;WAAgD;QAChD;;;;;WAA+C;QAC/C;;;;;WAAiD;QACjD;;;;;WAA2C;QAC3C;;;;;WAAyD;QACzD;;;;;WAA4D;QAC5D;;;;;WAA6D;QAC7D;;;;;WAA2C;QAE3C,uBAAuB;QACvB;;;;;WAAgC;QAChC;;;;;WAAgC;QAChC;;;;;WAAoC;QACpC;;;;mBAA8B,CAAC;WAAC;QAChC;;;;mBAAoB,KAAK;WAAC;QAC1B;;;;mBAA0B,CAAC;WAAC;QAC5B;;;;mBAAoD,IAAI;WAAC;QACzD;;;;mBAAsD,IAAI;WAAC;QAC3D;;;;;WAAiC;QACjC;;;;;WAAmC;QACnC;;;;;WAAuC;QACvC;;;;;WAA6C;QAC7C;;;;;WAA4C;QAC5C;;;;;WAAsC;QACtC;;;;;WAAqC;QACrC;;;;;WAAsD;QACtD;;;;mBAAoC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;WAAC;QAMpE,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ;YAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ;YAC/B,mBAAmB,EAAE,YAAE,CAAC,QAAQ;YAChC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC3C,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACxC,mBAAmB,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,eAAe,CAAC;YAC1D,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;SACzF,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GAAG,IAAI,EACrB,cAAc,GAAG,CAAC,EAClB,kBAAkB,EAClB,uBAAuB,GAAG,IAAI,EAC9B,gBAAgB,GAAG,IAAI,EACvB,kBAAkB,GAAG,IAAI,EACzB,oBAAoB,GAAG,GAAG,EAC1B,mBAAmB,GAAG,EAAE,EACxB,eAAe,GAAG,CAAC,EACnB,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,EACnB,kBAAkB,EAClB,GAAG,GAAG,SAAU,EAChB,iBAAiB,GAAG,QAAQ,GAC/B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAEnD,2BAA2B;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,GAAG,IAAI,CAAC;QAC1D,IAAI,CAAC,qBAAqB,GAAG,mBAAoB,GAAG,IAAI,CAAC;QACzD,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,IAAI,cAAc,CAAC;QAChE,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,4BAA4B,GAAG,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjF,yCAAyC;QACzC,MAAM,OAAO,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;QAC3C,OAAO,CAAC,WAAW,KAAnB,OAAO,CAAC,WAAW,GAAK,IAAI,yBAAW,CAAC;YACpC,GAAG,kBAAkB;YACrB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE;SACzC,CAAC,EAAC;QACH,OAAO,CAAC,MAAM,KAAd,OAAO,CAAC,MAAM,GAAK,IAAI,CAAC,MAAM,EAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;;OAKG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB,CAAC,KAAa;QAChC,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE/B,0FAA0F;QAC1F,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAE1F,4FAA4F;QAC5F,gGAAgG;QAChG,0FAA0F;QAC1F,IAAI,CAAC,gBAAgB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAE3F,IAAI,IAAI,CAAC,iBAAiB,KAAK,QAAQ,EAAE;YACrC,kEAAkE;YAClE,IAAI,CAAC,0BAA0B,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,4BAA4B,EAAE,IAAI,CAAC,CAAC;SAChG;QAED,IAAI;YACA,MAAM,IAAI,CAAC,WAAW,CAAC;SAC1B;gBAAS;YACN,qDAAqD;YACrD,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SAC3C;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SACzB;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,WAAoB;QAC5B,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,OAAuB,CAAC;YAC5B,IAAI,WAAW,EAAE;gBACb,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0CAA0C;0BAC1D,MAAM,WAAW,sCAAsC,CAAC,CAAC;oBAC/D,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;aACnB;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,IAAI,CAAC,mBAAmB,IAAI,CAAC,EAAE;oBAC/B,4DAA4D;oBAC5D,IAAI,OAAO;wBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;oBACnC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;iBACb;YACL,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,MAAM;QACF,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,aAAa,CAAC,gBAA6B;QACvD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3C,8FAA8F;QAC9F,MAAM,IAAI,GAAG,gBAAgB,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE5C,kCAAkC;QAClC,kCAAkC;QAClC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAC/D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,wCAAwC;QACxC,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC9D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,yCAAyC;QACzC,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE;YACtD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,mEAAmE;QACnE,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAC3D,MAAM,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC;QACvC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,eAAe,EAAE;YACnE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,6CAA6C,EAAE,aAAa,CAAC,CAAC;YAC5E,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,qBAAqB;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,WAAW,CAAC;QAChB,IAAI;YACA,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC3C,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClD,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;SACpC;QACD,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxD,IAAI,EAAE,CAAC;YACP,0DAA0D;YAC1D,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9B;QAED,yDAAyD;QACzD,wFAAwF;QACxF,iDAAiD;QACjD,IAAI,IAAI,CAAC,sBAAsB,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;YACtE,OAAO,IAAI,EAAE,CAAC;SACjB;QAED,IAAI;YACA,+BAA+B;YAC/B,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,gDAAgD;YAChD,6CAA6C;YAC7C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEjC,sFAAsF;YACtF,IAAI,EAAE,CAAC;YAEP,4BAA4B;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAEjC,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE;gBAC5B,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,EAC5B,IAAI,CAAC,iBAAiB,EACtB,mCAAmC,IAAI,CAAC,iBAAiB,GAAG,IAAI,WAAW,CAC9E,CAAC;aACL;iBAAM;gBACH,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aAChC;YAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,4CAA4C;YAC5C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACxC,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;IACL,CAAC;IAED;;;;OAIG;IACO,UAAU,CAAC,gBAA4B;QAC7C,yBAAyB;QACzB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE9C,2DAA2D;QAC3D,IAAI,IAAI,CAAC,sBAAsB;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE3D,oBAAoB;QACpB,2CAA2C;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,CAAC;QAC7D,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC;QACtC,0CAA0C;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QACtE,iFAAiF;QACjF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAClG,MAAM,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,IAAI,qBAAqB,CAAC;QAE1F,IAAI,YAAY,IAAI,aAAa,IAAI,+BAA+B;YAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElG,wBAAwB;QACxB,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC;QACzC,gCAAgC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QAEtE,IAAI,kBAAkB,IAAI,aAAa;YAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAEvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,qBAAqB,EAAE;gBACzD,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE;oBACnB,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,YAAY;iBACf,CAAC,CAAC;aACN;SACJ;QAED,8BAA8B;QAC9B,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACO,QAAQ,CAAC,YAAwB;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;YACzB,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,UAAU,CAAC,YAAwB;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC3E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,IAAI,CAAC,mBAAmB,GAAG,CAAC;YAAE,OAAO;QAEzC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI;YACA,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;SACnC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,0BAA0B;YAAE,IAAA,+BAAmB,EAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC1F,IAAI,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;IAES,4BAA4B,CAAC,gBAA4B;QAC/D,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEhC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;QAE3B,OAAO,gBAAgB,EAAE,CAAC;IAC9B,CAAC;IAED,IAAc,sBAAsB;QAChC,IAAI,IAAI,CAAC,iBAAiB,KAAK,QAAQ,EAAE;YACrC,OAAO,KAAK,CAAC;SAChB;QAED,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC;IAC/F,CAAC;CACJ;AAtgBD,wCAsgBC"}

package/{dist/crawlers → crawlers}/crawler_commons.d.ts

@@ -1,3 +1,4 @@
+import { Log } from '../log';
 import { ProxyInfo } from '../proxy_configuration';
 import { Request } from '../request';
 import { Session } from '../session_pool/session';
@@ -13,6 +14,7 @@ export interface CrawlingContext extends Record<PropertyKey, unknown> {
      * and configured by the {@link ProxyConfiguration} class.
      */
     proxyInfo?: ProxyInfo;
+    log: Log;
 }
 export interface CrawlerHandleFailedRequestInput extends CrawlingContext {
     /**

package/crawlers/crawler_commons.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,MAAM,WAAW,eAAgB,SAAQ,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;IACjE,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,+BAAgC,SAAQ,eAAe;IACpE;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC;CAChB"}

package/{dist/enqueue_links → enqueue_links}/enqueue_links.d.ts

@@ -135,4 +135,17 @@ export declare enum EnqueueStrategy {
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
  */
 export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
+/**
+ * @internal
+ */
+export declare function resolveBaseUrl({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
+/**
+ * @internal
+ */
+export interface ResolveBaseUrl {
+    userProvidedBaseUrl?: string;
+    enqueueStrategy?: EnqueueLinksOptions['strategy'];
+    originalRequestUrl: string;
+    finalRequestUrl?: string;
+}
 //# sourceMappingURL=enqueue_links.d.ts.map

package/{dist/enqueue_links → enqueue_links}/enqueue_links.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,MAAM,WAAW,mBAAmB;IAChC,qHAAqH;IACrH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;CAC3E;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IACX;;OAEG;IACH,aAAa,mBAAmB;IAChC;;;;OAIG;IACH,YAAY,kBAAkB;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAmGhG"}
+{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,MAAM,WAAW,mBAAmB;IAChC,qHAAqH;IACrH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;CAC3E;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IACX;;OAEG;IACH,aAAa,mBAAmB;IAChC;;;;OAIG;IACH,YAAY,kBAAkB;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAqGhG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC3B,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA4BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}

package/{dist/enqueue_links → enqueue_links}/enqueue_links.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.enqueueLinks = exports.EnqueueStrategy = void 0;
+exports.resolveBaseUrl = exports.enqueueLinks = exports.EnqueueStrategy = void 0;
 const tslib_1 = require("tslib");
 const tldts_1 = require("tldts");
 const ow_1 = tslib_1.__importDefault(require("ow"));
@@ -80,7 +80,9 @@ async function enqueueLinks(options) {
     if (regexps?.length) {
         urlPatternObjects.push(...(0, shared_1.constructRegExpObjectsFromRegExps)(regexps));
     }
-    options.strategy ?? (options.strategy = urlPatternObjects.length > 0 ? undefined : EnqueueStrategy.SameSubdomain);
+    if (!urlPatternObjects.length) {
+        options.strategy ?? (options.strategy = EnqueueStrategy.SameSubdomain);
+    }
     if (options.baseUrl) {
         switch (options.strategy) {
             case EnqueueStrategy.SameSubdomain:
@@ -120,4 +122,31 @@ async function enqueueLinks(options) {
     return requestQueue.addRequests(requests);
 }
 exports.enqueueLinks = enqueueLinks;
+/**
+ * @internal
+ */
+function resolveBaseUrl({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
+    // User provided base url takes priority
+    if (userProvidedBaseUrl) {
+        return userProvidedBaseUrl;
+    }
+    const originalUrlOrigin = new URL(originalRequestUrl).origin;
+    const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
+    // We can assume users want to go off the domain in this case
+    if (enqueueStrategy === EnqueueStrategy.All) {
+        return finalUrlOrigin ?? originalUrlOrigin;
+    }
+    // Ensure links use the same hostname
+    if (enqueueStrategy === EnqueueStrategy.SameHostname) {
+        const originalHostname = (0, tldts_1.getDomain)(originalUrlOrigin, { mixedInputs: false });
+        const finalHostname = (0, tldts_1.getDomain)(finalUrlOrigin, { mixedInputs: false });
+        if (originalHostname === finalHostname) {
+            return finalUrlOrigin;
+        }
+        return undefined;
+    }
+    // Always enqueue urls that are from the same origin
+    return originalUrlOrigin;
+}
+exports.resolveBaseUrl = resolveBaseUrl;
 //# sourceMappingURL=enqueue_links.js.map

package/enqueue_links/enqueue_links.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAC7B,qCAWkB;AAuGlB,IAAY,eAeX;AAfD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IACX;;OAEG;IACH,mDAAgC,CAAA;IAChC;;;;OAIG;IACH,iDAA8B,CAAA;AAClC,CAAC,EAfW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAe1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,aAAa;YAC7B,eAAe,CAAC,YAAY;SAC/B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,aAAa,EAAC;KACtD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,aAAa;gBAC9B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC1E,MAAM;YACV,KAAK,eAAe,CAAC,YAAY,CAAC,CAAC;gBAC/B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAErC,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,CAAC,CAAC;IAEhD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAC1G;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AArGD,oCAqGC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,EAC3B,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,IAAI,iBAAiB,CAAC;KAC9C;IAED,qCAAqC;IACrC,IAAI,eAAe,KAAK,eAAe,CAAC,YAAY,EAAE;QAClD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE;YACpC,OAAO,cAAc,CAAC;SACzB;QAED,OAAO,SAAS,CAAC;KACpB;IAED,oDAAoD;IACpD,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAjCD,wCAiCC"}

package/{dist/index.mjs → index.mjs}

@@ -55,6 +55,7 @@ export const maybeStringify = mod.maybeStringify;
 export const mergeCookies = mod.mergeCookies;
 export const openDataset = mod.openDataset;
 export const pushData = mod.pushData;
+export const resolveBaseUrl = mod.resolveBaseUrl;
 export const serializeArray = mod.serializeArray;
 export const storage = mod.storage;
 export const throwOnBlockedRequest = mod.throwOnBlockedRequest;

package/package.json

@@ -1,15 +1,15 @@
 {
     "name": "@crawlee/core",
-    "version": "3.0.0-alpha.3",
+    "version": "3.0.0-alpha.6",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
     },
-    "types": "dist/index.d.ts",
+    "types": "index.d.ts",
     "exports": {
         ".": {
-            "import": "./dist/index.mjs",
-            "require": "./dist/index.js"
+            "import": "./index.mjs",
+            "require": "./index.js"
         }
     },
     "keywords": [
@@ -39,13 +39,10 @@
         "url": "https://github.com/apify/apify-js/issues"
     },
     "homepage": "https://sdk.apify.com/",
-    "files": [
-        "dist"
-    ],
     "scripts": {
         "build": "npm run clean && npm run compile && npm run copy",
         "clean": "rimraf ./dist",
-        "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs",
+        "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./index.js ./index.mjs",
         "copy": "ts-node -T ../../scripts/copy.ts"
     },
     "publishConfig": {
@@ -57,12 +54,12 @@
         "@apify/log": "^1.1.6",
         "@apify/ps-tree": "^1.1.4",
         "@apify/pseudo_url": "^1.1.0",
-        "@apify/storage-local": "^2.0.3-beta.3",
         "@apify/timeout": "^0.2.1",
         "@apify/utilities": "^1.2.5",
-        "@crawlee/browser-pool": "^3.0.0-alpha.3",
-        "@crawlee/storage": "^3.0.0-alpha.3",
-        "@crawlee/utils": "^3.0.0-alpha.3",
+        "@apify/storage-local": "^2.0.3-beta.3",
+        "@crawlee/browser-pool": "^3.0.0-alpha.6",
+        "@crawlee/storage": "^3.0.0-alpha.6",
+        "@crawlee/utils": "^3.0.0-alpha.6",
         "@types/mime-types": "^2.1.1",
         "@types/stream-json": "^1.7.1",
         "content-type": "^1.0.4",
@@ -78,6 +75,5 @@
         "tldts": "^5.7.69",
         "tough-cookie": "^4.0.0",
         "ws": "^7.5.3"
-    },
-    "gitHead": "548b65585f29cd3c662acddbb30abfb79d0579e6"
+    }
 }

package/{dist/proxy_configuration.d.ts → proxy_configuration.d.ts}

@@ -1,6 +1,6 @@
 import { Configuration } from './configuration';
 export interface ProxyConfigurationFunction {
-    (sessionId: string | number): string;
+    (sessionId: string | number): string | Promise<string>;
 }
 export interface ProxyConfigurationOptions {
     /**
@@ -11,7 +11,7 @@ export interface ProxyConfigurationOptions {
     proxyUrls?: string[];
     /**
      * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
-     * and should always return stringified proxy URL.
+     * and should always return stringified proxy URL. Can be asynchronous.
      * This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
      */
     newUrlFunction?: ProxyConfigurationFunction;
@@ -151,7 +151,7 @@ export declare class ProxyConfiguration {
      *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
      * @return Represents information about used proxy and its configuration.
      */
-    newProxyInfo(sessionId?: string | number): ProxyInfo;
+    newProxyInfo(sessionId?: string | number): Promise<ProxyInfo>;
     /**
      * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
      * @param [sessionId]
@@ -165,7 +165,7 @@ export declare class ProxyConfiguration {
      * @return A string with a proxy URL, including authentication credentials and port number.
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
-    newUrl(sessionId?: string | number): string;
+    newUrl(sessionId?: string | number): Promise<string>;
     /**
      * Handles custom url rotation with session
      */
@@ -173,9 +173,9 @@ export declare class ProxyConfiguration {
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
-    protected _callNewUrlFunction(sessionId?: string): string | undefined;
-    protected _throwNewUrlFunctionInvalid(err: Error): void;
-    protected _throwCannotCombineCustomMethods(): void;
-    protected _throwNoOptionsProvided(): void;
+    protected _callNewUrlFunction(sessionId?: string): Promise<string>;
+    protected _throwNewUrlFunctionInvalid(err: Error): never;
+    protected _throwCannotCombineCustomMethods(): never;
+    protected _throwNoOptionsProvided(): never;
 }
 //# sourceMappingURL=proxy_configuration.d.ts.map

package/proxy_configuration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,0BAA0B;IACvC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,yBAAyB;IACtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;;;OAIG;IACH,cAAc,CAAC,EAAE,0BAA0B,CAAC;CAC/C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,kBAAkB;IA+B0B,QAAQ,CAAC,MAAM;IA9BpE,gBAAgB,UAAS;IACzB,SAAS,CAAC,kBAAkB,SAAK;IACjC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,aAAa,sBAA6B;IACpD,SAAS,CAAC,cAAc,CAAC,EAAE,0BAA0B,CAAC;IACtD,SAAS,CAAC,GAAG,2BAA+C;IAE5D;;;;;;;;;;;;;;;;;;;;;;OAsBG;gBACS,OAAO,GAAE,yBAA8B,EAAW,MAAM,gBAAkC,EAAE,gBAAgB,UAAO;IAe/H;;;;;;;;;;;;;;;;OAgBG;IACG,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAgBnE;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAU1D;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM;IAiBtD;;OAEG;cACa,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM;IAYtD,SAAS,CAAC,2BAA2B,CAAC,GAAG,EAAE,KAAK,GAAI,KAAK;IAIzD,SAAS,CAAC,gCAAgC,IAAK,KAAK;IAIpD,SAAS,CAAC,uBAAuB,IAAK,KAAK;CAG9C"}