@crawlee/basic 4.0.0-beta.229 → 4.0.0-beta.230
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -20,15 +20,13 @@ export interface TaskLoopPredicates {
|
|
|
20
20
|
*/
|
|
21
21
|
isFinishedFunction?: () => Promise<boolean>;
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/** @internal */
|
|
24
|
+
export interface AutoscaledPoolOptions extends TaskLoopPredicates {
|
|
24
25
|
/**
|
|
25
26
|
* How often the pool should check if a new task is ready, in seconds.
|
|
26
27
|
* @default 0.5
|
|
27
28
|
*/
|
|
28
29
|
maybeRunIntervalSecs?: number;
|
|
29
|
-
}
|
|
30
|
-
/** @internal */
|
|
31
|
-
export interface AutoscaledPoolOptions extends TaskLoopOptions {
|
|
32
30
|
/**
|
|
33
31
|
* The governor that decides whether there is free compute for one more task. Typically a
|
|
34
32
|
* {@link ConcurrencySystem}, but any {@link IConcurrencySystem} works. Share a single instance across
|
|
@@ -5,7 +5,7 @@ import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, Stor
|
|
|
5
5
|
import { RobotsTxtFile } from '@crawlee/utils';
|
|
6
6
|
import type { ReadonlyDeep } from 'type-fest';
|
|
7
7
|
import { z } from 'zod';
|
|
8
|
-
import type {
|
|
8
|
+
import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
|
|
9
9
|
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
10
10
|
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
11
11
|
import type { FinalStatistics } from './autoscaling/system-status.js';
|
|
@@ -174,7 +174,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
174
174
|
* Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
|
|
175
175
|
* shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
|
|
176
176
|
*/
|
|
177
|
-
taskLoopOptions?:
|
|
177
|
+
taskLoopOptions?: TaskLoopPredicates;
|
|
178
178
|
/**
|
|
179
179
|
* A pre-configured concurrency governor — the component that decides whether there is free compute for one more
|
|
180
180
|
* task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.230",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.0.1",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.230",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.230",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.230",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.230",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.0.0",
|
|
51
51
|
"csv-stringify": "^6.5.2",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.5.4"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.230"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "9af978adbda8c2a750920e35b50ae19d6e940ddd"
|
|
70
70
|
}
|