@crawlee/core 4.0.0-beta.47 → 4.0.0-beta.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +3 -3
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +89 -158
- package/configuration.d.ts.map +1 -1
- package/configuration.js +162 -130
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +4 -4
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -12
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/statistics.js +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/errors.d.ts.map +1 -1
- package/errors.js +2 -7
- package/errors.js.map +1 -1
- package/events/local_event_manager.js +4 -4
- package/events/local_event_manager.js.map +1 -1
- package/package.json +8 -8
- package/recoverable_state.d.ts.map +1 -1
- package/recoverable_state.js +8 -3
- package/recoverable_state.js.map +1 -1
- package/service_locator.js +1 -1
- package/service_locator.js.map +1 -1
- package/session_pool/session_pool.d.ts +16 -3
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +28 -10
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +13 -19
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +10 -17
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +8 -9
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +9 -11
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_provider.d.ts +11 -15
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +25 -23
- package/storages/request_provider.js.map +1 -1
- package/storages/storage_manager.d.ts +15 -12
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +48 -32
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.js +2 -2
- package/storages/utils.js.map +1 -1
|
@@ -20,8 +20,8 @@ export class StorageManager {
|
|
|
20
20
|
this.StorageConstructor = StorageConstructor;
|
|
21
21
|
this.name = this.StorageConstructor.name;
|
|
22
22
|
}
|
|
23
|
-
static async openStorage(storageClass,
|
|
24
|
-
return this.getManager(storageClass).openStorage(
|
|
23
|
+
static async openStorage(storageClass, identifier, client) {
|
|
24
|
+
return this.getManager(storageClass).openStorage(identifier, client);
|
|
25
25
|
}
|
|
26
26
|
static getManager(storageClass) {
|
|
27
27
|
let storageManager = serviceLocator.getStorageManager(storageClass);
|
|
@@ -31,28 +31,51 @@ export class StorageManager {
|
|
|
31
31
|
}
|
|
32
32
|
return serviceLocator.getStorageManager(storageClass);
|
|
33
33
|
}
|
|
34
|
-
async openStorage(
|
|
34
|
+
async openStorage(identifier, client) {
|
|
35
35
|
await this.storageOpenQueue.wait();
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
idOrName = this.config.get(defaultIdConfigKey);
|
|
39
|
-
}
|
|
40
|
-
const cacheKey = idOrName;
|
|
36
|
+
const resolvedIdentifier = await this._resolveIdentifier(identifier, client);
|
|
37
|
+
const cacheKey = resolvedIdentifier.id ?? resolvedIdentifier.name;
|
|
41
38
|
let storage = this.cache.get(cacheKey);
|
|
42
39
|
if (!storage) {
|
|
43
40
|
client ??= serviceLocator.getStorageClient();
|
|
44
|
-
const
|
|
41
|
+
const subClient = await this._createSubClient(resolvedIdentifier, this.name, client);
|
|
42
|
+
const storageInfo = await subClient.getMetadata();
|
|
45
43
|
storage = new this.StorageConstructor({
|
|
46
|
-
id:
|
|
47
|
-
name:
|
|
48
|
-
|
|
49
|
-
client,
|
|
44
|
+
id: storageInfo.id,
|
|
45
|
+
name: storageInfo.name,
|
|
46
|
+
client: subClient,
|
|
50
47
|
}, this.config);
|
|
51
48
|
this._addStorageToCache(storage);
|
|
52
49
|
}
|
|
53
50
|
this.storageOpenQueue.shift();
|
|
54
51
|
return storage;
|
|
55
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Resolves the user-provided identifier to an unambiguous `StorageIdentifier`.
|
|
55
|
+
*
|
|
56
|
+
* - `null`/`undefined` → uses the default storage ID from config
|
|
57
|
+
* - `StorageIdentifier` object → passed through (with default ID fallback if empty)
|
|
58
|
+
* - `string` → tries to find an existing storage with that ID first;
|
|
59
|
+
* if none exists, treats the string as a name
|
|
60
|
+
*/
|
|
61
|
+
async _resolveIdentifier(identifier, client) {
|
|
62
|
+
if (typeof identifier === 'string') {
|
|
63
|
+
client ??= serviceLocator.getStorageClient();
|
|
64
|
+
if (client.storageExists && (await client.storageExists(identifier, this.name))) {
|
|
65
|
+
return { id: identifier };
|
|
66
|
+
}
|
|
67
|
+
return { name: identifier };
|
|
68
|
+
}
|
|
69
|
+
if (identifier?.id) {
|
|
70
|
+
return { id: identifier.id };
|
|
71
|
+
}
|
|
72
|
+
if (identifier?.name) {
|
|
73
|
+
return { name: identifier.name };
|
|
74
|
+
}
|
|
75
|
+
const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
|
|
76
|
+
const defaultId = this.config[defaultIdConfigKey];
|
|
77
|
+
return { id: defaultId };
|
|
78
|
+
}
|
|
56
79
|
closeStorage(storage) {
|
|
57
80
|
const idKey = storage.id;
|
|
58
81
|
this.cache.delete(idKey);
|
|
@@ -62,26 +85,19 @@ export class StorageManager {
|
|
|
62
85
|
}
|
|
63
86
|
}
|
|
64
87
|
/**
|
|
65
|
-
*
|
|
88
|
+
* Creates a sub-client for the given storage type using the new factory-based StorageClient interface.
|
|
66
89
|
*/
|
|
67
|
-
async
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
const clientName = (storageConstructorName[0].toLowerCase() + storageConstructorName.slice(1));
|
|
79
|
-
// dataset => datasets
|
|
80
|
-
const collectionClientName = `${clientName}s`;
|
|
81
|
-
return {
|
|
82
|
-
createStorageClient: client[clientName].bind(client),
|
|
83
|
-
createStorageCollectionClient: client[collectionClientName].bind(client),
|
|
84
|
-
};
|
|
90
|
+
async _createSubClient(identifier, storageConstructorName, apiClient) {
|
|
91
|
+
switch (storageConstructorName) {
|
|
92
|
+
case 'Dataset':
|
|
93
|
+
return apiClient.createDatasetClient(identifier);
|
|
94
|
+
case 'KeyValueStore':
|
|
95
|
+
return apiClient.createKeyValueStoreClient(identifier);
|
|
96
|
+
case 'RequestQueue':
|
|
97
|
+
return apiClient.createRequestQueueClient(identifier);
|
|
98
|
+
default:
|
|
99
|
+
throw new Error(`Unknown storage type: ${storageConstructorName}`);
|
|
100
|
+
}
|
|
85
101
|
}
|
|
86
102
|
_addStorageToCache(storage) {
|
|
87
103
|
const idKey = storage.id;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAInD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAKvD,MAAM,sBAAsB,GAAG;IAC3B,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,uBAAuB;CAC/B,CAAC;AAOX;;;GAGG;AACH,MAAM,OAAO,cAAc;IAQF;IAPJ,IAAI,CAA+C;IACnD,kBAAkB,CAAoC;IACtD,KAAK,GAAG,IAAI,GAAG,EAAa,CAAC;IAC7B,gBAAgB,GAAG,IAAI,UAAU,EAAE,CAAC;IAErD,YACI,kBAAkC,EACjB,MAAqB;QAArB,WAAM,GAAN,MAAM,CAAe;QAEtC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAoD,CAAC;IAC7F,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,WAAW,CACpB,YAA4B,EAC5B,UAAuC,EACvC,MAAsB;QAEtB,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,UAAU,CAAqB,YAA4B;QAC9D,IAAI,cAAc,GAAG,cAAc,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACpE,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;YAC/B,cAAc,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE,cAAc,CAAC,gBAAgB,EAAE,CAAC,CAAC;YACrF,cAAc,CAAC,iBAAiB,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,cAAc,CAAC,iBAAiB,CAAC,YAAY,CAAsB,CAAC;IAC/E,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,UAA8C,EAAE,MAAsB;QACpF,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAEnC,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAE7E,MAAM,QAAQ,GAAG,kBAAkB,CAAC,EAAE,IAAI,kBAAkB,CAAC,IAAK,CAAC;QACnE,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,KAAK,cAAc,CAAC,gBAAgB,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACrF,MAAM,WAAW,GAAG,MAChB,SACH,CAAC,WAAW,EAAE,CAAC;YAChB,OAAO,GAAG,IAAI,IAAI,CAAC,kBAAkB,CACjC;gBACI,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,SAAS;aACpB,EACD,IAAI,CAAC,MAAM,CACd,CAAC;YACF,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAE9B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,kBAAkB,CAC5B,UAA8C,EAC9C,MAAsB;QAEtB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,KAAK,cAAc,CAAC,gBAAgB,EAAE,CAAC;YAE7C,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC9E,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;YAC9B,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QAChC,CAAC;QAED,IAAI,UAAU,EAAE,EAAE,EAAE,CAAC;YACjB,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;QACjC,CAAC;QAED,IAAI,UAAU,EAAE,IAAI,EAAE,CAAC;YACnB,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;QACrC,CAAC;QAED,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAW,CAAC;QAE5D,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC;IAC7B,CAAC;IAED,YAAY,CAAC,OAAsC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,gBAAgB,CAC5B,UAA6B,EAC7B,sBAA8B,EAC9B,SAAwB;QAExB,QAAQ,sBAAsB,EAAE,CAAC;YAC7B,KAAK,SAAS;gBACV,OAAO,SAAS,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YACrD,KAAK,eAAe;gBAChB,OAAO,SAAS,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;YAC3D,KAAK,cAAc;gBACf,OAAO,SAAS,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;YAC1D;gBACI,MAAM,IAAI,KAAK,CAAC,yBAAyB,sBAAsB,EAAE,CAAC,CAAC;QAC3E,CAAC;IACL,CAAC;IAES,kBAAkB,CAAC,OAAU;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;CACJ"}
|
package/storages/utils.js
CHANGED
|
@@ -13,7 +13,7 @@ export async function purgeDefaultStorages(configOrOptions, client) {
|
|
|
13
13
|
({ client = serviceLocator.getStorageClient() } = options);
|
|
14
14
|
const casted = client;
|
|
15
15
|
// if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
|
|
16
|
-
if (!onlyPurgeOnce || (config.
|
|
16
|
+
if (!onlyPurgeOnce || (config.purgeOnStart && !casted.__purged)) {
|
|
17
17
|
casted.__purged = true;
|
|
18
18
|
await casted.purge?.();
|
|
19
19
|
}
|
|
@@ -28,7 +28,7 @@ export async function purgeDefaultStorages(configOrOptions, client) {
|
|
|
28
28
|
* @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
|
|
29
29
|
*/
|
|
30
30
|
export async function useState(name, defaultValue = {}, options) {
|
|
31
|
-
const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
|
|
31
|
+
const kvStore = await KeyValueStore.open(options?.keyValueStoreName ? { name: options.keyValueStoreName } : null, {
|
|
32
32
|
config: options?.config || serviceLocator.getConfiguration(),
|
|
33
33
|
});
|
|
34
34
|
return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
|
package/storages/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACtF,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACtF,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9G,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE;KAC/D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}
|