@crawlee/basic 4.0.0-beta.86 → 4.0.0-beta.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -643,7 +643,7 @@ export class BasicCrawler {
|
|
|
643
643
|
await purgeDefaultStorages({
|
|
644
644
|
onlyPurgeOnce: true,
|
|
645
645
|
storageBackend: serviceLocator.getStorageBackend(),
|
|
646
|
-
|
|
646
|
+
configuration: serviceLocator.getConfiguration(),
|
|
647
647
|
});
|
|
648
648
|
if (requests) {
|
|
649
649
|
await this.addRequests(requests, addRequestsOptions);
|
|
@@ -754,7 +754,7 @@ export class BasicCrawler {
|
|
|
754
754
|
// The first crawler instance uses the default queue (null identifier);
|
|
755
755
|
// subsequent instances get their own queue via a unique alias so they don't collide.
|
|
756
756
|
const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
|
|
757
|
-
const requestQueue = await RequestQueue.open(identifier, {
|
|
757
|
+
const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
|
|
758
758
|
return this.ownedRequestQueue.set(requestQueue);
|
|
759
759
|
}
|
|
760
760
|
/**
|
|
@@ -794,7 +794,7 @@ export class BasicCrawler {
|
|
|
794
794
|
target.userData = await validateUserData(target.label, schema, target.userData ?? {});
|
|
795
795
|
}
|
|
796
796
|
async useState(defaultValue = {}) {
|
|
797
|
-
const kvs = await KeyValueStore.open(null, {
|
|
797
|
+
const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
|
|
798
798
|
if (this.identity.hasExplicitId) {
|
|
799
799
|
const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
|
|
800
800
|
return kvs.getAutoSavedValue(stateKey, defaultValue);
|
|
@@ -913,7 +913,7 @@ export class BasicCrawler {
|
|
|
913
913
|
*/
|
|
914
914
|
async getDataset(identifier) {
|
|
915
915
|
return Dataset.open(identifier, {
|
|
916
|
-
|
|
916
|
+
configuration: serviceLocator.getConfiguration(),
|
|
917
917
|
});
|
|
918
918
|
}
|
|
919
919
|
/**
|
|
@@ -1042,7 +1042,7 @@ export class BasicCrawler {
|
|
|
1042
1042
|
await this.requestManager.persistState().catch((err) => {
|
|
1043
1043
|
if (err.message.includes('Cannot persist state.')) {
|
|
1044
1044
|
this.log.error("The crawler attempted to persist its request list's state and failed due to missing or " +
|
|
1045
|
-
'invalid
|
|
1045
|
+
'invalid configuration. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
|
|
1046
1046
|
'constructor to ensure your crawling state is persisted through host migrations and restarts.');
|
|
1047
1047
|
}
|
|
1048
1048
|
else {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.87",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^2.0.0",
|
|
43
43
|
"@apify/timeout": "^0.3.2",
|
|
44
44
|
"@apify/utilities": "^2.15.5",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.87",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.87",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.87",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.87",
|
|
49
49
|
"csv-stringify": "^6.5.2",
|
|
50
50
|
"fs-extra": "^11.3.0",
|
|
51
51
|
"ow": "^2.0.0",
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"type-fest": "^4.41.0"
|
|
55
55
|
},
|
|
56
56
|
"optionalDependencies": {
|
|
57
|
-
"@crawlee/impit-client": "^4.0.0-beta.
|
|
57
|
+
"@crawlee/impit-client": "^4.0.0-beta.87"
|
|
58
58
|
},
|
|
59
59
|
"lerna": {
|
|
60
60
|
"command": {
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "1b7604cd77b694460aac4448b4555444fa1b2bdb"
|
|
67
67
|
}
|