@crawlee/core 3.13.5 → 4.0.0-beta.0

package/storages/utils.js

@@ -1,21 +1,14 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.MAX_QUERIES_FOR_CONSISTENCY = exports.API_PROCESSED_REQUESTS_DELAY_MILLIS = exports.QUERY_HEAD_BUFFER = exports.STORAGE_CONSISTENCY_DELAY_MILLIS = exports.QUERY_HEAD_MIN_LENGTH = void 0;
-exports.purgeDefaultStorages = purgeDefaultStorages;
-exports.useState = useState;
-exports.getRequestId = getRequestId;
-const tslib_1 = require("tslib");
-const node_crypto_1 = tslib_1.__importDefault(require("node:crypto"));
-const configuration_1 = require("../configuration");
-const key_value_store_1 = require("./key_value_store");
-async function purgeDefaultStorages(configOrOptions, client) {
-    const options = configOrOptions instanceof configuration_1.Configuration
+import crypto from 'node:crypto';
+import { Configuration } from '../configuration.js';
+import { KeyValueStore } from './key_value_store.js';
+export async function purgeDefaultStorages(configOrOptions, client) {
+    const options = configOrOptions instanceof Configuration
         ? {
             client,
             config: configOrOptions,
         }
         : (configOrOptions ?? {});
-    const { config = configuration_1.Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
+    const { config = Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
     ({ client = config.getStorageClient() } = options);
     const casted = client;
     // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
@@ -33,9 +26,9 @@ async function purgeDefaultStorages(configOrOptions, client) {
  * @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
  * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
  */
-async function useState(name, defaultValue = {}, options) {
-    const kvStore = await key_value_store_1.KeyValueStore.open(options?.keyValueStoreName, {
-        config: options?.config || configuration_1.Configuration.getGlobalConfig(),
+export async function useState(name, defaultValue = {}, options) {
+    const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
+        config: options?.config || Configuration.getGlobalConfig(),
     });
     return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
 }
@@ -48,32 +41,32 @@ async function useState(name, defaultValue = {}, options) {
  *
  * @internal
  */
-function getRequestId(uniqueKey) {
-    const str = node_crypto_1.default.createHash('sha256').update(uniqueKey).digest('base64').replace(/[+/=]/g, '');
+export function getRequestId(uniqueKey) {
+    const str = crypto.createHash('sha256').update(uniqueKey).digest('base64').replace(/[+/=]/g, '');
     return str.slice(0, 15);
 }
 /**
  * When requesting queue head we always fetch requestsInProgressCount * QUERY_HEAD_BUFFER number of requests.
  * @internal
  */
-exports.QUERY_HEAD_MIN_LENGTH = 100;
+export const QUERY_HEAD_MIN_LENGTH = 100;
 /**
  * Indicates how long it usually takes for the underlying storage to propagate all writes
  * to be available to subsequent reads.
  * @internal
  */
-exports.STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
+export const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
 /** @internal */
-exports.QUERY_HEAD_BUFFER = 3;
+export const QUERY_HEAD_BUFFER = 3;
 /**
  * If queue was modified (request added/updated/deleted) before more than API_PROCESSED_REQUESTS_DELAY_MILLIS
  * then we assume the get head operation to be consistent.
  * @internal
  */
-exports.API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
+export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10_000;
 /**
  * How many times we try to get queue head with queueModifiedAt older than API_PROCESSED_REQUESTS_DELAY_MILLIS.
  * @internal
  */
-exports.MAX_QUERIES_FOR_CONSISTENCY = 6;
+export const MAX_QUERIES_FOR_CONSISTENCY = 6;
 //# sourceMappingURL=utils.js.map

package/storages/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":";;;AA6CA,oDAqBC;AAoBD,4BASC;AAWD,oCAIC;;AA9GD,sEAAiC;AAIjC,oDAAiD;AACjD,uDAAkD;AAwC3C,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,6BAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACI,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,6BAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,qBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACU,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACU,QAAA,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AACH,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACU,QAAA,mCAAmC,GAAG,KAAM,CAAC;AAE1D;;;GAGG;AACU,QAAA,2BAA2B,GAAG,CAAC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}