@crawlee/core 3.10.1-beta.4 → 3.10.1-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -3
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +7 -6
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +4 -1
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts.map +1 -1
- package/configuration.js +3 -1
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +4 -2
- package/cookie_utils.js.map +1 -1
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +13 -5
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +7 -4
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +9 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js +18 -10
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +3 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +4 -3
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +6 -7
- package/enqueue_links/shared.js.map +1 -1
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +2 -2
- package/events/local_event_manager.js.map +1 -1
- package/log.d.ts +1 -1
- package/log.d.ts.map +1 -1
- package/log.js.map +1 -1
- package/package.json +5 -5
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +1 -1
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts +1 -1
- package/request.d.ts.map +1 -1
- package/request.js +12 -16
- package/request.js.map +1 -1
- package/router.d.ts.map +1 -1
- package/router.js +3 -3
- package/router.js.map +1 -1
- package/serialization.d.ts.map +1 -1
- package/serialization.js.map +1 -1
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +3 -1
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +1 -4
- package/session_pool/session_pool.js.map +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +4 -10
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +4 -0
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +12 -4
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +4 -4
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +10 -8
- package/storages/request_list.js.map +1 -1
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +31 -16
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +13 -10
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +3 -1
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +2 -2
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +11 -11
- package/storages/utils.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.d.ts.map +1 -1
package/storages/utils.js
CHANGED
|
@@ -6,11 +6,13 @@ const node_crypto_1 = tslib_1.__importDefault(require("node:crypto"));
|
|
|
6
6
|
const key_value_store_1 = require("./key_value_store");
|
|
7
7
|
const configuration_1 = require("../configuration");
|
|
8
8
|
async function purgeDefaultStorages(configOrOptions, client) {
|
|
9
|
-
const options = configOrOptions instanceof configuration_1.Configuration
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
const options = configOrOptions instanceof configuration_1.Configuration
|
|
10
|
+
? {
|
|
11
|
+
client,
|
|
12
|
+
config: configOrOptions,
|
|
13
|
+
}
|
|
14
|
+
: configOrOptions ?? {};
|
|
15
|
+
const { config = configuration_1.Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
|
|
14
16
|
({ client = config.getStorageClient() } = options);
|
|
15
17
|
const casted = client;
|
|
16
18
|
// if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
|
|
@@ -30,7 +32,9 @@ exports.purgeDefaultStorages = purgeDefaultStorages;
|
|
|
30
32
|
* @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
|
|
31
33
|
*/
|
|
32
34
|
async function useState(name, defaultValue = {}, options) {
|
|
33
|
-
const kvStore = await key_value_store_1.KeyValueStore.open(options?.keyValueStoreName, {
|
|
35
|
+
const kvStore = await key_value_store_1.KeyValueStore.open(options?.keyValueStoreName, {
|
|
36
|
+
config: options?.config || configuration_1.Configuration.getGlobalConfig(),
|
|
37
|
+
});
|
|
34
38
|
return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
|
|
35
39
|
}
|
|
36
40
|
exports.useState = useState;
|
|
@@ -44,11 +48,7 @@ exports.useState = useState;
|
|
|
44
48
|
* @internal
|
|
45
49
|
*/
|
|
46
50
|
function getRequestId(uniqueKey) {
|
|
47
|
-
const str = node_crypto_1.default
|
|
48
|
-
.createHash('sha256')
|
|
49
|
-
.update(uniqueKey)
|
|
50
|
-
.digest('base64')
|
|
51
|
-
.replace(/[+/=]/g, '');
|
|
51
|
+
const str = node_crypto_1.default.createHash('sha256').update(uniqueKey).digest('base64').replace(/[+/=]/g, '');
|
|
52
52
|
return str.slice(0, 15);
|
|
53
53
|
}
|
|
54
54
|
exports.getRequestId = getRequestId;
|
package/storages/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":";;;;AAAA,sEAAiC;AAIjC,uDAAkD;AAClD,oDAAiD;AAwC1C,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":";;;;AAAA,sEAAiC;AAIjC,uDAAkD;AAClD,oDAAiD;AAwC1C,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,6BAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AArBD,oDAqBC;AAWD;;;;;;;;GAQG;AACI,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,6BAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AATD,4BASC;AAED;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,qBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAJD,oCAIC;AAED;;;GAGG;AACU,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACU,QAAA,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AACH,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACU,QAAA,mCAAmC,GAAG,KAAM,CAAC;AAE1D;;;GAGG;AACU,QAAA,2BAA2B,GAAG,CAAC,CAAC"}
|