@crawlee/basic 4.0.0-beta.277 → 4.0.0-beta.278
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -38,7 +38,7 @@ const errorTrackerConfig = {
|
|
|
38
38
|
* order here *is* the record's field order (guarded by a test).
|
|
39
39
|
*
|
|
40
40
|
* JSON has no infinity, so the three fields that are `Infinity` until the first request settles are written as
|
|
41
|
-
* `null`. Both
|
|
41
|
+
* `null`. Both the private serialize and deserialize steps of `Statistics` run through this,
|
|
42
42
|
* which is what keeps them describing the same record.
|
|
43
43
|
*
|
|
44
44
|
* Nothing is optional on purpose: the record has always carried every field, so one missing a field is not one of
|
|
@@ -153,7 +153,7 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
|
|
|
153
153
|
*/
|
|
154
154
|
export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager {
|
|
155
155
|
#private;
|
|
156
|
-
constructor(options: ThrottlingRequestManagerOptions<T>,
|
|
156
|
+
constructor(options: ThrottlingRequestManagerOptions<T>, configuration?: Configuration);
|
|
157
157
|
/**
|
|
158
158
|
* The wrapped manager, holding every request whose domain is not throttled. `undefined` until an `inner`
|
|
159
159
|
* passed as a factory is resolved - reading this never forces it, because a getter should not open a queue
|
|
@@ -112,7 +112,7 @@ export class ThrottlingRequestManager {
|
|
|
112
112
|
#throttleBy;
|
|
113
113
|
#maxThrottledDomains;
|
|
114
114
|
#persistStateKey;
|
|
115
|
-
#
|
|
115
|
+
#configuration;
|
|
116
116
|
#subManagers = new Map();
|
|
117
117
|
#domainStates = new Map();
|
|
118
118
|
#log;
|
|
@@ -150,13 +150,13 @@ export class ThrottlingRequestManager {
|
|
|
150
150
|
get #throttlingEnabled() {
|
|
151
151
|
return this.#listedDomains.size > 0 || this.#throttlesEveryDomain;
|
|
152
152
|
}
|
|
153
|
-
constructor(options,
|
|
154
|
-
this.#
|
|
153
|
+
constructor(options, configuration = serviceLocator.getConfiguration()) {
|
|
154
|
+
this.#configuration = configuration;
|
|
155
155
|
parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
|
|
156
156
|
if (options.inner === undefined) {
|
|
157
157
|
// Deferred like any other factory, so a manager nobody fetches from opens no storage. The opener is
|
|
158
158
|
// assigned below and read only when this runs.
|
|
159
|
-
this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#
|
|
159
|
+
this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#configuration });
|
|
160
160
|
}
|
|
161
161
|
else if (typeof options.inner === 'function') {
|
|
162
162
|
this.#innerFactory = options.inner;
|
|
@@ -310,7 +310,7 @@ export class ThrottlingRequestManager {
|
|
|
310
310
|
subManager = this.#requestManagerOpener(
|
|
311
311
|
// Backends use the alias as a directory name, and an IPv6 literal is full of characters
|
|
312
312
|
// Windows will not accept. Ordinary hostnames survive this untouched.
|
|
313
|
-
{ alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.#
|
|
313
|
+
{ alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.#configuration });
|
|
314
314
|
this.#subManagers.set(domain, subManager);
|
|
315
315
|
this.#ensureDomainState(domain);
|
|
316
316
|
}
|
|
@@ -342,7 +342,7 @@ export class ThrottlingRequestManager {
|
|
|
342
342
|
async #ensureSubManagers() {
|
|
343
343
|
this.#subManagersReady ??= (async () => {
|
|
344
344
|
if (this.#throttlesEveryDomain) {
|
|
345
|
-
this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#
|
|
345
|
+
this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#configuration });
|
|
346
346
|
for (const domain of (await this.#domainListStore.getValue(this.#persistStateKey)) ?? []) {
|
|
347
347
|
this.#discoveredDomains.add(domain);
|
|
348
348
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.278",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.278",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.278",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.278",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.278",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.278"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "745f27ebf5c8307d949416c9c884ab941eef1559"
|
|
70
70
|
}
|