@crawlee/memory-storage 4.0.0-beta.6 → 4.0.0-beta.60

package/utils.js

@@ -1,6 +1,5 @@
 import { createHash } from 'node:crypto';
 import { s } from '@sapphire/shapeshift';
-import defaultLog from '@apify/log';
 import { REQUEST_ID_LENGTH } from './consts.js';
 /**
  * Removes all properties with a null value
@@ -39,5 +38,145 @@ export function isStream(value) {
         value &&
         ['on', 'pipe'].every((key) => key in value && typeof value[key] === 'function'));
 }
-export const memoryStorageLog = defaultLog.child({ prefix: 'MemoryStorage' });
+/**
+ * Creates a hybrid Promise + AsyncIterable for offset-based pagination (Dataset.listItems).
+ *
+ * The returned object can be:
+ * - Awaited directly to get the first page (backward compatible)
+ * - Used with `for await...of` to iterate through all items
+ */
+export function createPaginatedList(getPage, options = {}) {
+    const offset = options.offset ?? 0;
+    // Immediately fetch the first page (Infinity is used when no limit, gets clamped by Math.min downstream)
+    const firstPagePromise = getPage(offset, options.limit ?? Infinity);
+    async function* asyncGenerator() {
+        let currentPage = await firstPagePromise;
+        yield* currentPage.items;
+        const limit = Math.min(options.limit ?? currentPage.total, currentPage.total);
+        let currentOffset = offset + currentPage.items.length;
+        let remainingItems = Math.min(currentPage.total - offset, limit) - currentPage.items.length;
+        while (currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page.
+            remainingItems > 0) {
+            currentPage = await getPage(currentOffset, remainingItems);
+            yield* currentPage.items;
+            currentOffset += currentPage.items.length;
+            remainingItems -= currentPage.items.length;
+        }
+    }
+    return Object.defineProperty(firstPagePromise, Symbol.asyncIterator, {
+        value: asyncGenerator,
+    });
+}
+/**
+ * Creates a hybrid Promise + AsyncIterable for cursor-based pagination (KeyValueStore.listKeys).
+ *
+ * The returned object can be:
+ * - Awaited directly to get the first page (backward compatible)
+ * - Used with `for await...of` to iterate through all keys
+ */
+export function createKeyList(getPage, options = {}) {
+    // Immediately fetch the first page
+    const firstPagePromise = getPage(options.exclusiveStartKey);
+    async function* asyncGenerator() {
+        let currentPage = await firstPagePromise;
+        yield* currentPage.items;
+        let remainingItems = options.limit ? options.limit - currentPage.items.length : undefined;
+        while (currentPage.items.length > 0 &&
+            currentPage.nextExclusiveStartKey !== undefined &&
+            (remainingItems === undefined || remainingItems > 0)) {
+            currentPage = await getPage(currentPage.nextExclusiveStartKey);
+            yield* currentPage.items;
+            if (remainingItems !== undefined) {
+                remainingItems -= currentPage.items.length;
+            }
+        }
+    }
+    return Object.defineProperty(firstPagePromise, Symbol.asyncIterator, {
+        value: asyncGenerator,
+    });
+}
+/**
+ * Creates a hybrid Promise + AsyncIterable that yields only key strings (KeyValueStore.keys).
+ *
+ * The returned object can be:
+ * - Awaited directly to get the first page (backward compatible)
+ * - Used with `for await...of` to iterate through all key strings
+ */
+export function createKeyStringList(getPage, options = {}) {
+    // Immediately fetch the first page
+    const firstPagePromise = getPage(options.exclusiveStartKey);
+    async function* asyncGenerator() {
+        let currentPage = await firstPagePromise;
+        for (const item of currentPage.items) {
+            yield item.key;
+        }
+        let remainingItems = options.limit ? options.limit - currentPage.items.length : undefined;
+        while (currentPage.items.length > 0 &&
+            currentPage.nextExclusiveStartKey !== undefined &&
+            (remainingItems === undefined || remainingItems > 0)) {
+            currentPage = await getPage(currentPage.nextExclusiveStartKey);
+            for (const item of currentPage.items) {
+                yield item.key;
+            }
+            if (remainingItems !== undefined) {
+                remainingItems -= currentPage.items.length;
+            }
+        }
+    }
+    return Object.defineProperty(firstPagePromise, Symbol.asyncIterator, {
+        value: asyncGenerator,
+    });
+}
+/**
+ * Creates a hybrid Promise + AsyncIterable for offset-based pagination with index-value entries (Dataset.listEntries).
+ *
+ * The returned object can be:
+ * - Awaited directly to get the first page with [index, item] tuples (backward compatible)
+ * - Used with `for await...of` to iterate through all entries as [index, item] tuples
+ */
+export function createPaginatedEntryList(getPage, options = {}) {
+    const offset = options.offset ?? 0;
+    // Immediately fetch the first page and transform items to entries
+    const firstPagePromise = getPage(offset, options.limit ?? Infinity).then((result) => ({
+        ...result,
+        items: result.items.map((item, i) => [offset + i, item]),
+    }));
+    async function* asyncGenerator() {
+        let currentIndex = offset;
+        for await (const item of createPaginatedList(getPage, options)) {
+            yield [currentIndex++, item];
+        }
+    }
+    return Object.defineProperty(firstPagePromise, Symbol.asyncIterator, {
+        value: asyncGenerator,
+    });
+}
+/**
+ * Creates an object that acts as both a lazy Promise and an AsyncIterable.
+ * - When awaited, it triggers `promiseFactory` (bulk fetch, cached after first call).
+ * - When iterated with `for await...of`, it uses `iteratorFactory` (streaming, no bulk fetch).
+ */
+export function createLazyIterablePromise(promiseFactory, iteratorFactory) {
+    let cached = null;
+    function getOrCreate() {
+        if (!cached) {
+            cached = promiseFactory();
+        }
+        return cached;
+    }
+    const result = {
+        then(onfulfilled, onrejected) {
+            return getOrCreate().then(onfulfilled, onrejected);
+        },
+        catch(onrejected) {
+            return getOrCreate().catch(onrejected);
+        },
+        finally(onfinally) {
+            return getOrCreate().finally(onfinally);
+        },
+        [Symbol.asyncIterator]: iteratorFactory,
+        [Symbol.toStringTag]: 'Promise',
+    };
+    return result;
+}
 //# sourceMappingURL=utils.js.map

package/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAEzC,OAAO,UAAU,MAAM,YAAY,CAAC;AAEpC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAI,MAAS;IAC7C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACjE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,CAAC,cAAc,CAAC,MAAiC,EAAE,GAAG,CAAC,CAAC;QACvF,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB;IAClD,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC3B,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAE/B,OAAO,GAAG,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAc;IACnC,IAAI,CAAC;QACD,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEpF,OAAO,IAAI,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAU;IAC/B,OAAO,CACH,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK;QACL,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,CAClF,CAAC;AACN,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAEzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAI,MAAS;IAC7C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACjE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,CAAC,cAAc,CAAC,MAAiC,EAAE,GAAG,CAAC,CAAC;QACvF,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB;IAClD,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC3B,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAE/B,OAAO,GAAG,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAc;IACnC,IAAI,CAAC;QACD,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEpF,OAAO,IAAI,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAU;IAC/B,OAAO,CACH,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK;QACL,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,CAClF,CAAC;AACN,CAAC;AAqBD;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAC/B,OAAgF,EAChF,UAA+C,EAAE;IAEjD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IAEnC,yGAAyG;IACzG,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;IAEpE,KAAK,SAAS,CAAC,CAAC,cAAc;QAC1B,IAAI,WAAW,GAAG,MAAM,gBAAgB,CAAC;QACzC,KAAK,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;QAEzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;QAC9E,IAAI,aAAa,GAAG,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC;QACtD,IAAI,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC;QAE5F,OACI,WAAW,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,uEAAuE;YACvG,cAAc,GAAG,CAAC,EACpB,CAAC;YACC,WAAW,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;YAC3D,KAAK,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;YACzB,aAAa,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC;YAC1C,cAAc,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC,cAAc,CAAC,gBAAgB,EAAE,MAAM,CAAC,aAAa,EAAE;QACjE,KAAK,EAAE,cAAc;KACxB,CAA+D,CAAC;AACrE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CACzB,OAAqF,EACrF,UAA0D,EAAE;IAE5D,mCAAmC;IACnC,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE5D,KAAK,SAAS,CAAC,CAAC,cAAc;QAC1B,IAAI,WAAW,GAAG,MAAM,gBAAgB,CAAC;QACzC,KAAK,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;QAEzB,IAAI,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAE1F,OACI,WAAW,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YAC5B,WAAW,CAAC,qBAAqB,KAAK,SAAS;YAC/C,CAAC,cAAc,KAAK,SAAS,IAAI,cAAc,GAAG,CAAC,CAAC,EACtD,CAAC;YACC,WAAW,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;YAC/D,KAAK,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;YACzB,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;gBAC/B,cAAc,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC;YAC/C,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC,cAAc,CAAC,gBAAgB,EAAE,MAAM,CAAC,aAAa,EAAE;QACjE,KAAK,EAAE,cAAc;KACxB,CAAgG,CAAC;AACtG,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAC/B,OAAqF,EACrF,UAA0D,EAAE;IAE5D,mCAAmC;IACnC,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE5D,KAAK,SAAS,CAAC,CAAC,cAAc;QAC1B,IAAI,WAAW,GAAG,MAAM,gBAAgB,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,GAAG,CAAC;QACnB,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAE1F,OACI,WAAW,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YAC5B,WAAW,CAAC,qBAAqB,KAAK,SAAS;YAC/C,CAAC,cAAc,KAAK,SAAS,IAAI,cAAc,GAAG,CAAC,CAAC,EACtD,CAAC;YACC,WAAW,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;YAC/D,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;gBACnC,MAAM,IAAI,CAAC,GAAG,CAAC;YACnB,CAAC;YACD,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;gBAC/B,cAAc,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC;YAC/C,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC,cAAc,CAAC,gBAAgB,EAAE,MAAM,CAAC,aAAa,EAAE;QACjE,KAAK,EAAE,cAAc;KACxB,CAAyE,CAAC;AAC/E,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CACpC,OAAgF,EAChF,UAA+C,EAAE;IAEjD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IAEnC,kEAAkE;IAClE,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAClF,GAAG,MAAM;QACT,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAmB,CAAC;KAC7E,CAAC,CAAC,CAAC;IAEJ,KAAK,SAAS,CAAC,CAAC,cAAc;QAC1B,IAAI,YAAY,GAAG,MAAM,CAAC;QAC1B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC;YAC7D,MAAM,CAAC,YAAY,EAAE,EAAE,IAAI,CAAC,CAAC;QACjC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC,cAAc,CAAC,gBAAgB,EAAE,MAAM,CAAC,aAAa,EAAE;QACjE,KAAK,EAAE,cAAc;KACxB,CAAmF,CAAC;AACzF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CACrC,cAAuC,EACvC,eAA+C;IAE/C,IAAI,MAAM,GAA6B,IAAI,CAAC;IAC5C,SAAS,WAAW;QAChB,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,GAAG,cAAc,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG;QACX,IAAI,CACA,WAA4E,EAC5E,UAAuE;YAEvE,OAAO,WAAW,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QACvD,CAAC;QACD,KAAK,CACD,UAAqE;YAErE,OAAO,WAAW,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,CAAC,SAA+B;YACnC,OAAO,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,eAAe;QACvC,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,SAAkB;KAC3C,CAAC;IAEF,OAAO,MAAqD,CAAC;AACjE,CAAC"}

package/resource-clients/dataset-collection.d.ts

@@ -1,14 +0,0 @@
-import type * as storage from '@crawlee/types';
-import type { MemoryStorage } from '../index.js';
-export interface DatasetCollectionClientOptions {
-    baseStorageDirectory: string;
-    client: MemoryStorage;
-}
-export declare class DatasetCollectionClient implements storage.DatasetCollectionClient {
-    private readonly datasetsDirectory;
-    private readonly client;
-    constructor({ baseStorageDirectory, client }: DatasetCollectionClientOptions);
-    list(): ReturnType<storage.DatasetCollectionClient['list']>;
-    getOrCreate(name?: string): Promise<storage.DatasetInfo>;
-}
-//# sourceMappingURL=dataset-collection.d.ts.map

package/resource-clients/dataset-collection.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"dataset-collection.d.ts","sourceRoot":"","sources":["../../src/resource-clients/dataset-collection.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAK/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAGjD,MAAM,WAAW,8BAA8B;IAC3C,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,uBAAwB,YAAW,OAAO,CAAC,uBAAuB;IAC3E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAAE,8BAA8B;IAKtE,IAAI,IAAI,UAAU,CAAC,OAAO,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;IAa3D,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;CA6BjE"}

package/resource-clients/dataset-collection.js

@@ -1,49 +0,0 @@
-import { resolve } from 'node:path';
-import { s } from '@sapphire/shapeshift';
-import { scheduleBackgroundTask } from '../background-handler/index.js';
-import { findOrCacheDatasetByPossibleId } from '../cache-helpers.js';
-import { DatasetClient } from './dataset.js';
-export class DatasetCollectionClient {
-    datasetsDirectory;
-    client;
-    constructor({ baseStorageDirectory, client }) {
-        this.datasetsDirectory = resolve(baseStorageDirectory);
-        this.client = client;
-    }
-    async list() {
-        return {
-            total: this.client.datasetClientsHandled.length,
-            count: this.client.datasetClientsHandled.length,
-            offset: 0,
-            limit: this.client.datasetClientsHandled.length,
-            desc: false,
-            items: this.client.datasetClientsHandled
-                .map((store) => store.toDatasetInfo())
-                .sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()),
-        };
-    }
-    async getOrCreate(name) {
-        s.string().optional().parse(name);
-        if (name) {
-            const found = await findOrCacheDatasetByPossibleId(this.client, name);
-            if (found) {
-                return found.toDatasetInfo();
-            }
-        }
-        const newStore = new DatasetClient({ name, baseStorageDirectory: this.datasetsDirectory, client: this.client });
-        this.client.datasetClientsHandled.push(newStore);
-        // Schedule the worker to write to the disk
-        const datasetInfo = newStore.toDatasetInfo();
-        scheduleBackgroundTask({
-            action: 'update-metadata',
-            entityType: 'datasets',
-            entityDirectory: newStore.datasetDirectory,
-            id: datasetInfo.name ?? datasetInfo.id,
-            data: datasetInfo,
-            writeMetadata: this.client.writeMetadata,
-            persistStorage: this.client.persistStorage,
-        });
-        return datasetInfo;
-    }
-}
-//# sourceMappingURL=dataset-collection.js.map

package/resource-clients/dataset-collection.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"dataset-collection.js","sourceRoot":"","sources":["../../src/resource-clients/dataset-collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAEzC,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAC;AAErE,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAO7C,MAAM,OAAO,uBAAuB;IACf,iBAAiB,CAAS;IAC1B,MAAM,CAAgB;IAEvC,YAAY,EAAE,oBAAoB,EAAE,MAAM,EAAkC;QACxE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI;QACN,OAAO;YACH,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB;iBACnC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;iBACrC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SACrE,CAAC;IACN,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAa;QAC3B,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,KAAK,GAAG,MAAM,8BAA8B,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAEtE,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC,aAAa,EAAE,CAAC;YACjC,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,aAAa,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAChH,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEjD,2CAA2C;QAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QAE7C,sBAAsB,CAAC;YACnB,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,UAAU;YACtB,eAAe,EAAE,QAAQ,CAAC,gBAAgB;YAC1C,EAAE,EAAE,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE;YACtC,IAAI,EAAE,WAAW;YACjB,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;YACxC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;SAC7C,CAAC,CAAC;QAEH,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ"}

package/resource-clients/key-value-store-collection.d.ts

@@ -1,14 +0,0 @@
-import type * as storage from '@crawlee/types';
-import type { MemoryStorage } from '../index.js';
-export interface KeyValueStoreCollectionClientOptions {
-    baseStorageDirectory: string;
-    client: MemoryStorage;
-}
-export declare class KeyValueStoreCollectionClient implements storage.KeyValueStoreCollectionClient {
-    private readonly keyValueStoresDirectory;
-    private readonly client;
-    constructor({ baseStorageDirectory, client }: KeyValueStoreCollectionClientOptions);
-    list(): ReturnType<storage.KeyValueStoreCollectionClient['list']>;
-    getOrCreate(name?: string): Promise<storage.KeyValueStoreInfo>;
-}
-//# sourceMappingURL=key-value-store-collection.d.ts.map

package/resource-clients/key-value-store-collection.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"key-value-store-collection.d.ts","sourceRoot":"","sources":["../../src/resource-clients/key-value-store-collection.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAK/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAGjD,MAAM,WAAW,oCAAoC;IACjD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,6BAA8B,YAAW,OAAO,CAAC,6BAA6B;IACvF,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAAE,oCAAoC;IAK5E,IAAI,IAAI,UAAU,CAAC,OAAO,CAAC,6BAA6B,CAAC,MAAM,CAAC,CAAC;IAajE,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;CAiCvE"}

package/resource-clients/key-value-store-collection.js

@@ -1,53 +0,0 @@
-import { resolve } from 'node:path';
-import { s } from '@sapphire/shapeshift';
-import { scheduleBackgroundTask } from '../background-handler/index.js';
-import { findOrCacheKeyValueStoreByPossibleId } from '../cache-helpers.js';
-import { KeyValueStoreClient } from './key-value-store.js';
-export class KeyValueStoreCollectionClient {
-    keyValueStoresDirectory;
-    client;
-    constructor({ baseStorageDirectory, client }) {
-        this.keyValueStoresDirectory = resolve(baseStorageDirectory);
-        this.client = client;
-    }
-    async list() {
-        return {
-            total: this.client.keyValueStoresHandled.length,
-            count: this.client.keyValueStoresHandled.length,
-            offset: 0,
-            limit: this.client.keyValueStoresHandled.length,
-            desc: false,
-            items: this.client.keyValueStoresHandled
-                .map((store) => store.toKeyValueStoreInfo())
-                .sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()),
-        };
-    }
-    async getOrCreate(name) {
-        s.string().optional().parse(name);
-        if (name) {
-            const found = await findOrCacheKeyValueStoreByPossibleId(this.client, name);
-            if (found) {
-                return found.toKeyValueStoreInfo();
-            }
-        }
-        const newStore = new KeyValueStoreClient({
-            name,
-            baseStorageDirectory: this.keyValueStoresDirectory,
-            client: this.client,
-        });
-        this.client.keyValueStoresHandled.push(newStore);
-        // Schedule the worker to write to the disk
-        const kvStoreInfo = newStore.toKeyValueStoreInfo();
-        scheduleBackgroundTask({
-            action: 'update-metadata',
-            entityType: 'keyValueStores',
-            entityDirectory: newStore.keyValueStoreDirectory,
-            id: kvStoreInfo.name ?? kvStoreInfo.id,
-            data: kvStoreInfo,
-            writeMetadata: this.client.writeMetadata,
-            persistStorage: this.client.persistStorage,
-        });
-        return kvStoreInfo;
-    }
-}
-//# sourceMappingURL=key-value-store-collection.js.map

package/resource-clients/key-value-store-collection.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"key-value-store-collection.js","sourceRoot":"","sources":["../../src/resource-clients/key-value-store-collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAEzC,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,oCAAoC,EAAE,MAAM,qBAAqB,CAAC;AAE3E,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAO3D,MAAM,OAAO,6BAA6B;IACrB,uBAAuB,CAAS;IAChC,MAAM,CAAgB;IAEvC,YAAY,EAAE,oBAAoB,EAAE,MAAM,EAAwC;QAC9E,IAAI,CAAC,uBAAuB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI;QACN,OAAO;YACH,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB;iBACnC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC;iBAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SACrE,CAAC;IACN,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAa;QAC3B,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,KAAK,GAAG,MAAM,oCAAoC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAE5E,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC,mBAAmB,EAAE,CAAC;YACvC,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,mBAAmB,CAAC;YACrC,IAAI;YACJ,oBAAoB,EAAE,IAAI,CAAC,uBAAuB;YAClD,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEjD,2CAA2C;QAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QAEnD,sBAAsB,CAAC;YACnB,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,gBAAgB;YAC5B,eAAe,EAAE,QAAQ,CAAC,sBAAsB;YAChD,EAAE,EAAE,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE;YACtC,IAAI,EAAE,WAAW;YACjB,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;YACxC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;SAC7C,CAAC,CAAC;QAEH,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ"}

package/resource-clients/request-queue-collection.d.ts

@@ -1,14 +0,0 @@
-import type * as storage from '@crawlee/types';
-import type { MemoryStorage } from '../index.js';
-export interface RequestQueueCollectionClientOptions {
-    baseStorageDirectory: string;
-    client: MemoryStorage;
-}
-export declare class RequestQueueCollectionClient implements storage.RequestQueueCollectionClient {
-    private readonly requestQueuesDirectory;
-    private readonly client;
-    constructor({ baseStorageDirectory, client }: RequestQueueCollectionClientOptions);
-    list(): ReturnType<storage.RequestQueueCollectionClient['list']>;
-    getOrCreate(name?: string): Promise<storage.RequestQueueInfo>;
-}
-//# sourceMappingURL=request-queue-collection.d.ts.map

package/resource-clients/request-queue-collection.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"request-queue-collection.d.ts","sourceRoot":"","sources":["../../src/resource-clients/request-queue-collection.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAK/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAGjD,MAAM,WAAW,mCAAmC;IAChD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,4BAA6B,YAAW,OAAO,CAAC,4BAA4B;IACrF,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAAE,mCAAmC;IAK3E,IAAI,IAAI,UAAU,CAAC,OAAO,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC;IAahE,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC;CAiCtE"}

package/resource-clients/request-queue-collection.js

@@ -1,53 +0,0 @@
-import { resolve } from 'node:path';
-import { s } from '@sapphire/shapeshift';
-import { scheduleBackgroundTask } from '../background-handler/index.js';
-import { findRequestQueueByPossibleId } from '../cache-helpers.js';
-import { RequestQueueClient } from './request-queue.js';
-export class RequestQueueCollectionClient {
-    requestQueuesDirectory;
-    client;
-    constructor({ baseStorageDirectory, client }) {
-        this.requestQueuesDirectory = resolve(baseStorageDirectory);
-        this.client = client;
-    }
-    async list() {
-        return {
-            total: this.client.requestQueuesHandled.length,
-            count: this.client.requestQueuesHandled.length,
-            offset: 0,
-            limit: this.client.requestQueuesHandled.length,
-            desc: false,
-            items: this.client.requestQueuesHandled
-                .map((store) => store.toRequestQueueInfo())
-                .sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()),
-        };
-    }
-    async getOrCreate(name) {
-        s.string().optional().parse(name);
-        if (name) {
-            const found = await findRequestQueueByPossibleId(this.client, name);
-            if (found) {
-                return found.toRequestQueueInfo();
-            }
-        }
-        const newStore = new RequestQueueClient({
-            name,
-            baseStorageDirectory: this.requestQueuesDirectory,
-            client: this.client,
-        });
-        this.client.requestQueuesHandled.push(newStore);
-        // Schedule the worker to write to the disk
-        const queueInfo = newStore.toRequestQueueInfo();
-        scheduleBackgroundTask({
-            action: 'update-metadata',
-            entityType: 'requestQueues',
-            entityDirectory: newStore.requestQueueDirectory,
-            id: queueInfo.name ?? queueInfo.id,
-            data: queueInfo,
-            writeMetadata: this.client.writeMetadata,
-            persistStorage: this.client.persistStorage,
-        });
-        return queueInfo;
-    }
-}
-//# sourceMappingURL=request-queue-collection.js.map

package/resource-clients/request-queue-collection.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"request-queue-collection.js","sourceRoot":"","sources":["../../src/resource-clients/request-queue-collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAEzC,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,4BAA4B,EAAE,MAAM,qBAAqB,CAAC;AAEnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,MAAM,OAAO,4BAA4B;IACpB,sBAAsB,CAAS;IAC/B,MAAM,CAAgB;IAEvC,YAAY,EAAE,oBAAoB,EAAE,MAAM,EAAuC;QAC7E,IAAI,CAAC,sBAAsB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI;QACN,OAAO;YACH,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,MAAM;YAC9C,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,MAAM;YAC9C,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,MAAM;YAC9C,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB;iBAClC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC;iBAC1C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SACrE,CAAC;IACN,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAa;QAC3B,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,KAAK,GAAG,MAAM,4BAA4B,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAEpE,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,KAAK,CAAC,kBAAkB,EAAE,CAAC;YACtC,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;YACpC,IAAI;YACJ,oBAAoB,EAAE,IAAI,CAAC,sBAAsB;YACjD,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEhD,2CAA2C;QAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC,kBAAkB,EAAE,CAAC;QAEhD,sBAAsB,CAAC;YACnB,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,eAAe;YAC3B,eAAe,EAAE,QAAQ,CAAC,qBAAqB;YAC/C,EAAE,EAAE,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,EAAE;YAClC,IAAI,EAAE,SAAS;YACf,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;YACxC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;SAC7C,CAAC,CAAC;QAEH,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ"}