@crawlee/memory-storage 3.0.0-alpha.34

package/memory-storage.js

@@ -0,0 +1,102 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryStorage = void 0;
+const shapeshift_1 = require("@sapphire/shapeshift");
+const node_path_1 = require("node:path");
+const dataset_1 = require("./resource-clients/dataset");
+const dataset_collection_1 = require("./resource-clients/dataset-collection");
+const key_value_store_1 = require("./resource-clients/key-value-store");
+const key_value_store_collection_1 = require("./resource-clients/key-value-store-collection");
+const request_queue_1 = require("./resource-clients/request-queue");
+const request_queue_collection_1 = require("./resource-clients/request-queue-collection");
+const instance_1 = require("./workers/instance");
+class MemoryStorage {
+    constructor(options = {}) {
+        Object.defineProperty(this, "localDataDirectory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "datasetsDirectory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "keyValueStoresDirectory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "requestQueuesDirectory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "keyValueStoresHandled", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "datasetClientsHandled", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "requestQueuesHandled", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        shapeshift_1.s.object({
+            localDataDirectory: shapeshift_1.s.string.optional,
+        }).parse(options);
+        this.localDataDirectory = options.localDataDirectory ?? process.env.CRAWLEE_STORAGE_DIR ?? './memory_storage';
+        this.datasetsDirectory = (0, node_path_1.resolve)(this.localDataDirectory, 'datasets');
+        this.keyValueStoresDirectory = (0, node_path_1.resolve)(this.localDataDirectory, 'key_value_stores');
+        this.requestQueuesDirectory = (0, node_path_1.resolve)(this.localDataDirectory, 'request_queues');
+        (0, instance_1.initWorkerIfNeeded)();
+    }
+    datasets() {
+        return new dataset_collection_1.DatasetCollectionClient({
+            baseStorageDirectory: this.datasetsDirectory,
+            client: this,
+        });
+    }
+    dataset(id) {
+        shapeshift_1.s.string.parse(id);
+        return new dataset_1.DatasetClient({ id, baseStorageDirectory: this.datasetsDirectory, client: this });
+    }
+    keyValueStores() {
+        return new key_value_store_collection_1.KeyValueStoreCollectionClient({
+            baseStorageDirectory: this.keyValueStoresDirectory,
+            client: this,
+        });
+    }
+    keyValueStore(id) {
+        shapeshift_1.s.string.parse(id);
+        return new key_value_store_1.KeyValueStoreClient({ id, baseStorageDirectory: this.keyValueStoresDirectory, client: this });
+    }
+    requestQueues() {
+        return new request_queue_collection_1.RequestQueueCollectionClient({
+            baseStorageDirectory: this.requestQueuesDirectory,
+            client: this,
+        });
+    }
+    requestQueue(id, options = {}) {
+        shapeshift_1.s.string.parse(id);
+        shapeshift_1.s.object({
+            clientKey: shapeshift_1.s.string.optional,
+            timeoutSecs: shapeshift_1.s.number.optional,
+        }).parse(options);
+        return new request_queue_1.RequestQueueClient({ id, baseStorageDirectory: this.requestQueuesDirectory, client: this, ...options });
+    }
+}
+exports.MemoryStorage = MemoryStorage;
+//# sourceMappingURL=memory-storage.js.map

package/memory-storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-storage.js","sourceRoot":"","sources":["../src/memory-storage.ts"],"names":[],"mappings":";;;AAEA,qDAAyC;AACzC,yCAAoC;AACpC,wDAA2D;AAC3D,8EAAgF;AAChF,wEAAyE;AACzE,8FAA8F;AAC9F,oEAAsE;AACtE,0FAA2F;AAC3F,iDAAwD;AAUxD,MAAa,aAAa;IAUtB,YAAY,UAAgC,EAAE;QAT9C;;;;;WAAoC;QACpC;;;;;WAAmC;QACnC;;;;;WAAyC;QACzC;;;;;WAAwC;QAExC;;;;mBAAwD,EAAE;WAAC;QAC3D;;;;mBAAkD,EAAE;WAAC;QACrD;;;;mBAAsD,EAAE;WAAC;QAGrD,cAAC,CAAC,MAAM,CAAC;YACL,kBAAkB,EAAE,cAAC,CAAC,MAAM,CAAC,QAAQ;SACxC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,kBAAkB,CAAC;QAC9G,IAAI,CAAC,iBAAiB,GAAG,IAAA,mBAAO,EAAC,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,IAAA,mBAAO,EAAC,IAAI,CAAC,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;QACpF,IAAI,CAAC,sBAAsB,GAAG,IAAA,mBAAO,EAAC,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;QAEjF,IAAA,6BAAkB,GAAE,CAAC;IACzB,CAAC;IAED,QAAQ;QACJ,OAAO,IAAI,4CAAuB,CAAC;YAC/B,oBAAoB,EAAE,IAAI,CAAC,iBAAiB;YAC5C,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;IACP,CAAC;IAED,OAAO,CAAuC,EAAU;QACpD,cAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAEnB,OAAO,IAAI,uBAAa,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,cAAc;QACV,OAAO,IAAI,0DAA6B,CAAC;YACrC,oBAAoB,EAAE,IAAI,CAAC,uBAAuB;YAClD,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;IACP,CAAC;IAED,aAAa,CAAC,EAAU;QACpB,cAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAEnB,OAAO,IAAI,qCAAmB,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,CAAC,uBAAuB,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7G,CAAC;IAED,aAAa;QACT,OAAO,IAAI,uDAA4B,CAAC;YACpC,oBAAoB,EAAE,IAAI,CAAC,sBAAsB;YACjD,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;IACP,CAAC;IAED,YAAY,CAAC,EAAU,EAAE,UAAuC,EAAE;QAC9D,cAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnB,cAAC,CAAC,MAAM,CAAC;YACL,SAAS,EAAE,cAAC,CAAC,MAAM,CAAC,QAAQ;YAC5B,WAAW,EAAE,cAAC,CAAC,MAAM,CAAC,QAAQ;SACjC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,OAAO,IAAI,kCAAkB,CAAC,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACvH,CAAC;CACJ;AAjED,sCAiEC"}

package/package.json

@@ -0,0 +1,60 @@
+{
+    "name": "@crawlee/memory-storage",
+    "version": "3.0.0-alpha.34",
+    "description": "A simple in-memory storage implementation of the Apify API",
+    "engines": {
+        "node": ">= 16"
+    },
+    "main": "index.js",
+    "module": "index.mjs",
+    "types": "index.d.ts",
+    "exports": {
+        ".": {
+            "import": "./index.mjs",
+            "require": "./index.js",
+            "types": "./index.d.ts"
+        },
+        "./package.json": "./package.json"
+    },
+    "keywords": [
+        "apify",
+        "api",
+        "memory"
+    ],
+    "author": {
+        "name": "Apify",
+        "email": "support@apify.com",
+        "url": "https://apify.com"
+    },
+    "contributors": [
+        "Vlad Frangu <kingdgrizzle@gmail.com>"
+    ],
+    "license": "Apache-2.0",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/apify/apify-js"
+    },
+    "bugs": {
+        "url": "https://github.com/apify/apify-js/issues"
+    },
+    "homepage": "https://sdk.apify.com/",
+    "scripts": {
+        "build": "npm run clean && npm run compile && npm run copy",
+        "clean": "rimraf ./dist",
+        "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./index.js ./index.mjs",
+        "copy": "ts-node -T ../../scripts/copy.ts"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "dependencies": {
+        "@apify/log": "^1.2.0",
+        "@crawlee/types": "^3.0.0-alpha.34",
+        "@crawlee/utils": "^3.0.0-alpha.34",
+        "@sapphire/shapeshift": "^3.0.0",
+        "content-type": "^1.0.4",
+        "fs-extra": "^10.1.0",
+        "mime-types": "^2.1.35",
+        "tslib": "^2.4.0"
+    }
+}

package/resource-clients/common/base-client.d.ts

@@ -0,0 +1,8 @@
+import { StorageTypes } from '../../consts';
+export declare class BaseClient {
+    id: string;
+    constructor(id: string);
+    protected throwOnNonExisting(clientType: StorageTypes): never;
+    protected throwOnDuplicateEntry(clientType: StorageTypes, keyName: string, value: string): never;
+}
+//# sourceMappingURL=base-client.d.ts.map

package/resource-clients/common/base-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-client.d.ts","sourceRoot":"","sources":["../../../src/resource-clients/common/base-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,qBAAa,UAAU;IACnB,EAAE,EAAE,MAAM,CAAC;gBAEC,EAAE,EAAE,MAAM;IAItB,SAAS,CAAC,kBAAkB,CAAC,UAAU,EAAE,YAAY,GAAG,KAAK;IAI7D,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK;CAGnG"}

package/resource-clients/common/base-client.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseClient = void 0;
+class BaseClient {
+    constructor(id) {
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.id = id;
+    }
+    throwOnNonExisting(clientType) {
+        throw new Error(`${clientType} with id: ${this.id} does not exist.`);
+    }
+    throwOnDuplicateEntry(clientType, keyName, value) {
+        throw new Error(`${clientType} with ${keyName}: ${value} already exists.`);
+    }
+}
+exports.BaseClient = BaseClient;
+//# sourceMappingURL=base-client.js.map

package/resource-clients/common/base-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-client.js","sourceRoot":"","sources":["../../../src/resource-clients/common/base-client.ts"],"names":[],"mappings":";;;AAEA,MAAa,UAAU;IAGnB,YAAY,EAAU;QAFtB;;;;;WAAW;QAGP,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACjB,CAAC;IAES,kBAAkB,CAAC,UAAwB;QACjD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,aAAa,IAAI,CAAC,EAAE,kBAAkB,CAAC,CAAC;IACzE,CAAC;IAES,qBAAqB,CAAC,UAAwB,EAAE,OAAe,EAAE,KAAa;QACpF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,SAAS,OAAO,KAAK,KAAK,kBAAkB,CAAC,CAAC;IAC/E,CAAC;CACJ;AAdD,gCAcC"}

package/resource-clients/dataset-collection.d.ts

@@ -0,0 +1,14 @@
+import type * as storage from '@crawlee/types';
+import { MemoryStorage } from '../index';
+export interface DatasetCollectionClientOptions {
+    baseStorageDirectory: string;
+    client: MemoryStorage;
+}
+export declare class DatasetCollectionClient implements storage.DatasetCollectionClient {
+    private readonly datasetsDirectory;
+    private readonly client;
+    constructor({ baseStorageDirectory, client }: DatasetCollectionClientOptions);
+    list(): ReturnType<storage.DatasetCollectionClient['list']>;
+    getOrCreate(name?: string): Promise<storage.DatasetInfo>;
+}
+//# sourceMappingURL=dataset-collection.d.ts.map

package/resource-clients/dataset-collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-collection.d.ts","sourceRoot":"","sources":["../../src/resource-clients/dataset-collection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,MAAM,WAAW,8BAA8B;IAC3C,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,uBAAwB,YAAW,OAAO,CAAC,uBAAuB;IAC3E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAAE,8BAA8B;IAKtE,IAAI,IAAI,UAAU,CAAC,OAAO,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;IAa3D,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;CA2BjE"}

package/resource-clients/dataset-collection.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatasetCollectionClient = void 0;
+const shapeshift_1 = require("@sapphire/shapeshift");
+const path_1 = require("path");
+const instance_1 = require("../workers/instance");
+const dataset_1 = require("./dataset");
+class DatasetCollectionClient {
+    constructor({ baseStorageDirectory, client }) {
+        Object.defineProperty(this, "datasetsDirectory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.datasetsDirectory = (0, path_1.resolve)(baseStorageDirectory);
+        this.client = client;
+    }
+    async list() {
+        return {
+            total: this.client.datasetClientsHandled.length,
+            count: this.client.datasetClientsHandled.length,
+            offset: 0,
+            limit: this.client.datasetClientsHandled.length,
+            desc: false,
+            items: this.client.datasetClientsHandled.map((store) => store.toDatasetInfo())
+                .sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()),
+        };
+    }
+    async getOrCreate(name) {
+        shapeshift_1.s.string.optional.parse(name);
+        if (name) {
+            const found = this.client.datasetClientsHandled.find((store) => store.name === name);
+            if (found) {
+                return found.toDatasetInfo();
+            }
+        }
+        const newStore = new dataset_1.DatasetClient({ name, baseStorageDirectory: this.datasetsDirectory, client: this.client });
+        this.client.datasetClientsHandled.push(newStore);
+        // Schedule the worker to write to the disk
+        const datasetInfo = newStore.toDatasetInfo();
+        // eslint-disable-next-line dot-notation
+        (0, instance_1.sendWorkerMessage)({
+            action: 'update-metadata',
+            entityType: 'datasets',
+            entityDirectory: newStore.datasetDirectory,
+            id: datasetInfo.name ?? datasetInfo.id,
+            data: datasetInfo,
+        });
+        return datasetInfo;
+    }
+}
+exports.DatasetCollectionClient = DatasetCollectionClient;
+//# sourceMappingURL=dataset-collection.js.map

package/resource-clients/dataset-collection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-collection.js","sourceRoot":"","sources":["../../src/resource-clients/dataset-collection.ts"],"names":[],"mappings":";;;AACA,qDAAyC;AACzC,+BAA+B;AAE/B,kDAAwD;AACxD,uCAA0C;AAO1C,MAAa,uBAAuB;IAIhC,YAAY,EAAE,oBAAoB,EAAE,MAAM,EAAkC;QAH5E;;;;;WAA2C;QAC3C;;;;;WAAuC;QAGnC,IAAI,CAAC,iBAAiB,GAAG,IAAA,cAAO,EAAC,oBAAoB,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI;QACN,OAAO;YACH,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,GAAG,CACxC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;iBAChC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SACrE,CAAC;IACN,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAa;QAC3B,cAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE9B,IAAI,IAAI,EAAE;YACN,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;YAErF,IAAI,KAAK,EAAE;gBACP,OAAO,KAAK,CAAC,aAAa,EAAE,CAAC;aAChC;SACJ;QAED,MAAM,QAAQ,GAAG,IAAI,uBAAa,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAChH,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEjD,2CAA2C;QAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;QAC7C,wCAAwC;QACxC,IAAA,4BAAiB,EAAC;YACd,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,UAAU;YACtB,eAAe,EAAE,QAAQ,CAAC,gBAAgB;YAC1C,EAAE,EAAE,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE;YACtC,IAAI,EAAE,WAAW;SACpB,CAAC,CAAC;QAEH,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ;AAjDD,0DAiDC"}

package/resource-clients/dataset.d.ts

@@ -0,0 +1,42 @@
+/// <reference types="node" />
+import type * as storage from '@crawlee/types';
+import { Dictionary } from '@crawlee/utils';
+import { MemoryStorage } from '../index';
+import { BaseClient } from './common/base-client';
+export interface DatasetClientOptions {
+    id?: string;
+    name?: string;
+    baseStorageDirectory: string;
+    client: MemoryStorage;
+}
+export declare class DatasetClient<Data extends Dictionary = Dictionary> extends BaseClient implements storage.DatasetClient<Data> {
+    name?: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    itemCount: number;
+    datasetDirectory: string;
+    private readonly datasetEntries;
+    private readonly client;
+    constructor(options: DatasetClientOptions);
+    get(): Promise<storage.DatasetInfo | undefined>;
+    update(newFields?: storage.DatasetClientUpdateOptions): Promise<storage.DatasetInfo>;
+    delete(): Promise<void>;
+    downloadItems(): Promise<Buffer>;
+    listItems(options?: storage.DatasetClientListOptions): Promise<storage.PaginatedList<Data>>;
+    pushItems(items: string | Data | string[] | Data[]): Promise<void>;
+    toDatasetInfo(): storage.DatasetInfo;
+    private generateLocalEntryName;
+    private getStartAndEndIndexes;
+    /**
+     * To emulate API and split arrays of items into individual dataset items,
+     * we need to normalize the input items - which can be strings, objects
+     * or arrays of those - into objects, so that we can save them one by one
+     * later. We could potentially do this directly with strings, but let's
+     * not optimize prematurely.
+     */
+    private normalizeItems;
+    private normalizeItem;
+    private updateTimestamps;
+}
+//# sourceMappingURL=dataset.d.ts.map

package/resource-clients/dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAM5C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAelD,MAAM,WAAW,oBAAoB;IACjC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,aAAa,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,UAAW,YAAW,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC;IACtH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IACxB,SAAS,SAAK;IACd,gBAAgB,EAAE,MAAM,CAAC;IAEzB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA2B;IAC1D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,OAAO,EAAE,oBAAoB;IAOnC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAW/C,MAAM,CAAC,SAAS,GAAE,OAAO,CAAC,0BAA+B,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;IAsCxF,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAYvB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;IAIhC,SAAS,CAAC,OAAO,GAAE,OAAO,CAAC,wBAA6B,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAuC/F,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAqCxE,aAAa,IAAI,OAAO,CAAC,WAAW;IAWpC,OAAO,CAAC,sBAAsB;IAI9B,OAAO,CAAC,qBAAqB;IAM7B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,aAAa;IAgBrB,OAAO,CAAC,gBAAgB;CAgB3B"}

package/resource-clients/dataset.js

@@ -0,0 +1,235 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatasetClient = void 0;
+const shapeshift_1 = require("@sapphire/shapeshift");
+const node_crypto_1 = require("node:crypto");
+const promises_1 = require("node:fs/promises");
+const node_path_1 = require("node:path");
+const fs_extra_1 = require("fs-extra");
+const consts_1 = require("../consts");
+const base_client_1 = require("./common/base-client");
+const instance_1 = require("../workers/instance");
+/**
+ * This is what API returns in the x-apify-pagination-limit
+ * header when no limit query parameter is used.
+ */
+const LIST_ITEMS_LIMIT = 999999999999;
+/**
+ * Number of characters of the dataset item file names.
+ * E.g.: 000000019.json - 9 digits
+ */
+const LOCAL_ENTRY_NAME_DIGITS = 9;
+class DatasetClient extends base_client_1.BaseClient {
+    constructor(options) {
+        super(options.id ?? (0, node_crypto_1.randomUUID)());
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "createdAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Date()
+        });
+        Object.defineProperty(this, "accessedAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Date()
+        });
+        Object.defineProperty(this, "modifiedAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Date()
+        });
+        Object.defineProperty(this, "itemCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "datasetDirectory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "datasetEntries", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.name = options.name;
+        this.datasetDirectory = (0, node_path_1.resolve)(options.baseStorageDirectory, this.name ?? this.id);
+        this.client = options.client;
+    }
+    async get() {
+        const found = this.client.datasetClientsHandled.find((store) => store.id === this.id);
+        if (found) {
+            found.updateTimestamps(false);
+            return found.toDatasetInfo();
+        }
+        return undefined;
+    }
+    async update(newFields = {}) {
+        const parsed = shapeshift_1.s.object({
+            name: shapeshift_1.s.string.lengthGreaterThan(0).optional,
+        }).parse(newFields);
+        // Check by id
+        const existingStoreById = this.client.datasetClientsHandled.find((store) => store.id === this.id);
+        if (!existingStoreById) {
+            this.throwOnNonExisting(consts_1.StorageTypes.Dataset);
+        }
+        // Skip if no changes
+        if (!parsed.name) {
+            return existingStoreById.toDatasetInfo();
+        }
+        // Check that name is not in use already
+        const existingStoreByName = this.client.datasetClientsHandled.find((store) => store.name?.toLowerCase() === parsed.name.toLowerCase());
+        if (existingStoreByName) {
+            this.throwOnDuplicateEntry(consts_1.StorageTypes.Dataset, 'name', parsed.name);
+        }
+        existingStoreById.name = parsed.name;
+        const previousDir = existingStoreById.datasetDirectory;
+        existingStoreById.datasetDirectory = (0, node_path_1.resolve)(this.client.datasetsDirectory, parsed.name ?? existingStoreById.name ?? existingStoreById.id);
+        await (0, fs_extra_1.move)(previousDir, existingStoreById.datasetDirectory, { overwrite: true });
+        // Update timestamps
+        existingStoreById.updateTimestamps(true);
+        return existingStoreById.toDatasetInfo();
+    }
+    async delete() {
+        const storeIndex = this.client.datasetClientsHandled.findIndex((store) => store.id === this.id);
+        if (storeIndex !== -1) {
+            const [oldClient] = this.client.datasetClientsHandled.splice(storeIndex, 1);
+            oldClient.itemCount = 0;
+            oldClient.datasetEntries.clear();
+            await (0, promises_1.rm)(oldClient.datasetDirectory, { recursive: true });
+        }
+    }
+    async downloadItems() {
+        throw new Error('This method is not implemented in @crawlee/memory-storage');
+    }
+    async listItems(options = {}) {
+        const { limit = LIST_ITEMS_LIMIT, offset = 0, desc, } = shapeshift_1.s.object({
+            desc: shapeshift_1.s.boolean.optional,
+            limit: shapeshift_1.s.number.int.optional,
+            offset: shapeshift_1.s.number.int.optional,
+        }).parse(options);
+        // Check by id
+        const existingStoreById = this.client.datasetClientsHandled.find((store) => store.id === this.id);
+        if (!existingStoreById) {
+            this.throwOnNonExisting(consts_1.StorageTypes.Dataset);
+        }
+        const [start, end] = this.getStartAndEndIndexes(offset, limit);
+        const items = [];
+        for (let idx = start; idx < end; idx++) {
+            const entryNumber = this.generateLocalEntryName(idx);
+            items.push(existingStoreById.datasetEntries.get(entryNumber));
+        }
+        existingStoreById.updateTimestamps(false);
+        return {
+            count: items.length,
+            desc: desc ?? false,
+            items: desc ? items.reverse() : items,
+            limit,
+            offset,
+            total: this.itemCount,
+        };
+    }
+    async pushItems(items) {
+        const rawItems = shapeshift_1.s.union(shapeshift_1.s.string, shapeshift_1.s.object({}).passthrough, shapeshift_1.s.array(shapeshift_1.s.union(shapeshift_1.s.string, shapeshift_1.s.object({}).passthrough))).parse(items);
+        // Check by id
+        const existingStoreById = this.client.datasetClientsHandled.find((store) => store.id === this.id);
+        if (!existingStoreById) {
+            this.throwOnNonExisting(consts_1.StorageTypes.Dataset);
+        }
+        const normalized = this.normalizeItems(rawItems);
+        const addedIds = [];
+        for (const entry of normalized) {
+            const idx = this.generateLocalEntryName(++existingStoreById.itemCount);
+            existingStoreById.datasetEntries.set(idx, entry);
+            addedIds.push(idx);
+        }
+        const dataEntries = addedIds.map((id) => [id, existingStoreById.datasetEntries.get(id)]);
+        existingStoreById.updateTimestamps(true);
+        (0, instance_1.sendWorkerMessage)({
+            data: dataEntries,
+            action: 'update-entries',
+            entityType: 'datasets',
+            entityDirectory: existingStoreById.datasetDirectory,
+            id: existingStoreById.name ?? existingStoreById.id,
+        });
+    }
+    toDatasetInfo() {
+        return {
+            id: this.id,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            itemCount: this.itemCount,
+            modifiedAt: this.modifiedAt,
+            name: this.name,
+        };
+    }
+    generateLocalEntryName(idx) {
+        return idx.toString().padStart(LOCAL_ENTRY_NAME_DIGITS, '0');
+    }
+    getStartAndEndIndexes(offset, limit = this.itemCount) {
+        const start = offset + 1;
+        const end = Math.min(offset + limit, this.itemCount) + 1;
+        return [start, end];
+    }
+    /**
+     * To emulate API and split arrays of items into individual dataset items,
+     * we need to normalize the input items - which can be strings, objects
+     * or arrays of those - into objects, so that we can save them one by one
+     * later. We could potentially do this directly with strings, but let's
+     * not optimize prematurely.
+     */
+    normalizeItems(items) {
+        if (typeof items === 'string') {
+            items = JSON.parse(items);
+        }
+        return Array.isArray(items)
+            ? items.map((item) => this.normalizeItem(item))
+            : [this.normalizeItem(items)];
+    }
+    normalizeItem(item) {
+        if (typeof item === 'string') {
+            item = JSON.parse(item);
+        }
+        if (Array.isArray(item)) {
+            throw new Error(`Each dataset item can only be a single JSON object, not an array. Received: [${item.join(',\n')}]`);
+        }
+        if (typeof item !== 'object' || item === null) {
+            throw new Error(`Each dataset item must be a JSON object. Received: ${item}`);
+        }
+        return item;
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+        const data = this.toDatasetInfo();
+        (0, instance_1.sendWorkerMessage)({
+            action: 'update-metadata',
+            data,
+            entityType: 'datasets',
+            entityDirectory: this.datasetDirectory,
+            id: this.name ?? this.id,
+        });
+    }
+}
+exports.DatasetClient = DatasetClient;
+//# sourceMappingURL=dataset.js.map

package/resource-clients/dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/resource-clients/dataset.ts"],"names":[],"mappings":";;;AAEA,qDAAyC;AACzC,6CAAyC;AACzC,+CAAsC;AACtC,yCAAoC;AACpC,uCAAgC;AAEhC,sCAAyC;AACzC,sDAAkD;AAClD,kDAAwD;AAExD;;;GAGG;AACH,MAAM,gBAAgB,GAAG,YAAe,CAAC;AAEzC;;;GAGG;AACH,MAAM,uBAAuB,GAAG,CAAC,CAAC;AASlC,MAAa,aAAoD,SAAQ,wBAAU;IAW/E,YAAY,OAA6B;QACrC,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,IAAA,wBAAU,GAAE,CAAC,CAAC;QAXtC;;;;;WAAc;QACd;;;;mBAAY,IAAI,IAAI,EAAE;WAAC;QACvB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;mBAAY,CAAC;WAAC;QACd;;;;;WAAyB;QAEzB;;;;mBAAkC,IAAI,GAAG,EAAgB;WAAC;QAC1D;;;;;WAAuC;QAInC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,gBAAgB,GAAG,IAAA,mBAAO,EAAC,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,GAAG;QACL,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAEtF,IAAI,KAAK,EAAE;YACP,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC,aAAa,EAAE,CAAC;SAChC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,YAAgD,EAAE;QAC3D,MAAM,MAAM,GAAG,cAAC,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,cAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,QAAQ;SAC/C,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpB,cAAc;QACd,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,OAAO,CAAC,CAAC;SACjD;QAED,qBAAqB;QACrB,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YACd,OAAO,iBAAiB,CAAC,aAAa,EAAE,CAAC;SAC5C;QAED,wCAAwC;QACxC,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,MAAM,CAAC,IAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAExI,IAAI,mBAAmB,EAAE;YACrB,IAAI,CAAC,qBAAqB,CAAC,qBAAY,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;SACzE;QAED,iBAAiB,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QAErC,MAAM,WAAW,GAAG,iBAAiB,CAAC,gBAAgB,CAAC;QAEvD,iBAAiB,CAAC,gBAAgB,GAAG,IAAA,mBAAO,EAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,IAAI,iBAAiB,CAAC,IAAI,IAAI,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAE3I,MAAM,IAAA,eAAI,EAAC,WAAW,EAAE,iBAAiB,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEjF,oBAAoB;QACpB,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAEzC,OAAO,iBAAiB,CAAC,aAAa,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM;QACR,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhG,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE;YACnB,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAC5E,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC;YACxB,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAEjC,MAAM,IAAA,aAAE,EAAC,SAAS,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SAC7D;IACL,CAAC;IAED,KAAK,CAAC,aAAa;QACf,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,UAA4C,EAAE;QAC1D,MAAM,EACF,KAAK,GAAG,gBAAgB,EACxB,MAAM,GAAG,CAAC,EACV,IAAI,GACP,GAAG,cAAC,CAAC,MAAM,CAAC;YACT,IAAI,EAAE,cAAC,CAAC,OAAO,CAAC,QAAQ;YACxB,KAAK,EAAE,cAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ;YAC5B,MAAM,EAAE,cAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ;SAChC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,cAAc;QACd,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,OAAO,CAAC,CAAC;SACjD;QAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAE/D,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,KAAK,IAAI,GAAG,GAAG,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,EAAE;YACpC,MAAM,WAAW,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAE,CAAC,CAAC;SAClE;QAED,iBAAiB,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE1C,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,IAAI,KAAK;YACnB,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK;YACrC,KAAK;YACL,MAAM;YACN,KAAK,EAAE,IAAI,CAAC,SAAS;SACxB,CAAC;IACN,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAwC;QACpD,MAAM,QAAQ,GAAG,cAAC,CAAC,KAAK,CACpB,cAAC,CAAC,MAAM,EACR,cAAC,CAAC,MAAM,CAAO,EAAU,CAAC,CAAC,WAAW,EACtC,cAAC,CAAC,KAAK,CAAC,cAAC,CAAC,KAAK,CAAC,cAAC,CAAC,MAAM,EAAE,cAAC,CAAC,MAAM,CAAO,EAAU,CAAC,CAAC,WAAW,CAAC,CAAC,CACrE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEf,cAAc;QACd,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,OAAO,CAAC,CAAC;SACjD;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAEjD,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,sBAAsB,CAAC,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEvE,iBAAiB,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACjD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACtB;QAED,MAAM,WAAW,GAA2B,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,iBAAiB,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC,CAAC;QAElH,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACzC,IAAA,4BAAiB,EAAC;YACd,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,gBAAgB;YACxB,UAAU,EAAE,UAAU;YACtB,eAAe,EAAE,iBAAiB,CAAC,gBAAgB;YACnD,EAAE,EAAE,iBAAiB,CAAC,IAAI,IAAI,iBAAiB,CAAC,EAAE;SACrD,CAAC,CAAC;IACP,CAAC;IAED,aAAa;QACT,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;SAClB,CAAC;IACN,CAAC;IAEO,sBAAsB,CAAC,GAAW;QACtC,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC;IACjE,CAAC;IAEO,qBAAqB,CAAC,MAAc,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAChE,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAU,CAAC;IACjC,CAAC;IAED;;;;;;OAMG;IACK,cAAc,CAAC,KAAwC;QAC3D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC3B,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;SAC7B;QAED,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACvB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,IAAmB;QACrC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;YAC1B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAS,CAAC;SACnC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,gFAAgF,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACxH;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE;YAC3C,MAAM,IAAI,KAAK,CAAC,sDAAsD,IAAI,EAAE,CAAC,CAAC;SACjF;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,gBAAgB,CAAC,eAAwB;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,eAAe,EAAE;YACjB,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;SAChC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAA,4BAAiB,EAAC;YACd,MAAM,EAAE,iBAAiB;YACzB,IAAI;YACJ,UAAU,EAAE,UAAU;YACtB,eAAe,EAAE,IAAI,CAAC,gBAAgB;YACtC,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE;SAC3B,CAAC,CAAC;IACP,CAAC;CACJ;AArOD,sCAqOC"}

package/resource-clients/key-value-store-collection.d.ts

@@ -0,0 +1,14 @@
+import type * as storage from '@crawlee/types';
+import { MemoryStorage } from '../index';
+export interface KeyValueStoreCollectionClientOptions {
+    baseStorageDirectory: string;
+    client: MemoryStorage;
+}
+export declare class KeyValueStoreCollectionClient implements storage.KeyValueStoreCollectionClient {
+    private readonly keyValueStoresDirectory;
+    private readonly client;
+    constructor({ baseStorageDirectory, client }: KeyValueStoreCollectionClientOptions);
+    list(): ReturnType<storage.KeyValueStoreCollectionClient['list']>;
+    getOrCreate(name?: string): Promise<storage.KeyValueStoreInfo>;
+}
+//# sourceMappingURL=key-value-store-collection.d.ts.map

package/resource-clients/key-value-store-collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"key-value-store-collection.d.ts","sourceRoot":"","sources":["../../src/resource-clients/key-value-store-collection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,MAAM,WAAW,oCAAoC;IACjD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,6BAA8B,YAAW,OAAO,CAAC,6BAA6B;IACvF,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAAE,oCAAoC;IAK5E,IAAI,IAAI,UAAU,CAAC,OAAO,CAAC,6BAA6B,CAAC,MAAM,CAAC,CAAC;IAajE,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;CA2BvE"}