@crawlee/memory-storage 3.0.0-alpha.34

package/resource-clients/key-value-store-collection.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KeyValueStoreCollectionClient = void 0;
+const shapeshift_1 = require("@sapphire/shapeshift");
+const node_path_1 = require("node:path");
+const instance_1 = require("../workers/instance");
+const key_value_store_1 = require("./key-value-store");
+class KeyValueStoreCollectionClient {
+    constructor({ baseStorageDirectory, client }) {
+        Object.defineProperty(this, "keyValueStoresDirectory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.keyValueStoresDirectory = (0, node_path_1.resolve)(baseStorageDirectory);
+        this.client = client;
+    }
+    async list() {
+        return {
+            total: this.client.keyValueStoresHandled.length,
+            count: this.client.keyValueStoresHandled.length,
+            offset: 0,
+            limit: this.client.keyValueStoresHandled.length,
+            desc: false,
+            items: this.client.keyValueStoresHandled.map((store) => store.toKeyValueStoreInfo())
+                .sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()),
+        };
+    }
+    async getOrCreate(name) {
+        shapeshift_1.s.string.optional.parse(name);
+        if (name) {
+            const found = this.client.keyValueStoresHandled.find((store) => store.name === name);
+            if (found) {
+                return found.toKeyValueStoreInfo();
+            }
+        }
+        const newStore = new key_value_store_1.KeyValueStoreClient({ name, baseStorageDirectory: this.keyValueStoresDirectory, client: this.client });
+        this.client.keyValueStoresHandled.push(newStore);
+        // Schedule the worker to write to the disk
+        const kvStoreInfo = newStore.toKeyValueStoreInfo();
+        // eslint-disable-next-line dot-notation
+        (0, instance_1.sendWorkerMessage)({
+            action: 'update-metadata',
+            entityType: 'keyValueStores',
+            entityDirectory: newStore.keyValueStoreDirectory,
+            id: kvStoreInfo.name ?? kvStoreInfo.id,
+            data: kvStoreInfo,
+        });
+        return kvStoreInfo;
+    }
+}
+exports.KeyValueStoreCollectionClient = KeyValueStoreCollectionClient;
+//# sourceMappingURL=key-value-store-collection.js.map

package/resource-clients/key-value-store-collection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"key-value-store-collection.js","sourceRoot":"","sources":["../../src/resource-clients/key-value-store-collection.ts"],"names":[],"mappings":";;;AACA,qDAAyC;AACzC,yCAAoC;AAEpC,kDAAwD;AACxD,uDAAwD;AAOxD,MAAa,6BAA6B;IAItC,YAAY,EAAE,oBAAoB,EAAE,MAAM,EAAwC;QAHlF;;;;;WAAiD;QACjD;;;;;WAAuC;QAGnC,IAAI,CAAC,uBAAuB,GAAG,IAAA,mBAAO,EAAC,oBAAoB,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI;QACN,OAAO;YACH,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM;YAC/C,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,GAAG,CACxC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC;iBACtC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SACrE,CAAC;IACN,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAa;QAC3B,cAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE9B,IAAI,IAAI,EAAE;YACN,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;YAErF,IAAI,KAAK,EAAE;gBACP,OAAO,KAAK,CAAC,mBAAmB,EAAE,CAAC;aACtC;SACJ;QAED,MAAM,QAAQ,GAAG,IAAI,qCAAmB,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,CAAC,uBAAuB,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5H,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEjD,2CAA2C;QAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QACnD,wCAAwC;QACxC,IAAA,4BAAiB,EAAC;YACd,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,gBAAgB;YAC5B,eAAe,EAAE,QAAQ,CAAC,sBAAsB;YAChD,EAAE,EAAE,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE;YACtC,IAAI,EAAE,WAAW;SACpB,CAAC,CAAC;QAEH,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ;AAjDD,sEAiDC"}

package/resource-clients/key-value-store.d.ts

@@ -0,0 +1,36 @@
+/// <reference types="node" />
+import type * as storage from '@crawlee/types';
+import { MemoryStorage } from '../index';
+import { BaseClient } from './common/base-client';
+export interface KeyValueStoreClientOptions {
+    name?: string;
+    id?: string;
+    baseStorageDirectory: string;
+    client: MemoryStorage;
+}
+export interface InternalKeyRecord {
+    key: string;
+    value: Buffer | string;
+    contentType?: string;
+    extension: string;
+}
+export declare class KeyValueStoreClient extends BaseClient {
+    name?: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    keyValueStoreDirectory: string;
+    private readonly keyValueEntries;
+    private readonly client;
+    constructor(options: KeyValueStoreClientOptions);
+    get(): Promise<storage.KeyValueStoreInfo | undefined>;
+    update(newFields?: storage.KeyValueStoreClientUpdateOptions): Promise<storage.KeyValueStoreInfo>;
+    delete(): Promise<void>;
+    listKeys(options?: storage.KeyValueStoreClientListOptions): Promise<storage.KeyValueStoreClientListData>;
+    getRecord(key: string, options?: storage.KeyValueStoreClientGetRecordOptions): Promise<storage.KeyValueStoreRecord | undefined>;
+    setRecord(record: storage.KeyValueStoreRecord): Promise<void>;
+    deleteRecord(key: string): Promise<void>;
+    toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
+    private updateTimestamps;
+}
+//# sourceMappingURL=key-value-store.d.ts.map

package/resource-clients/key-value-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"key-value-store.d.ts","sourceRoot":"","sources":["../../src/resource-clients/key-value-store.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAQ/C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAKlD,MAAM,WAAW,0BAA0B;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,mBAAoB,SAAQ,UAAU;IAC/C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IACxB,sBAAsB,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAwC;IACxE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,OAAO,EAAE,0BAA0B;IAOzC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;IAWrD,MAAM,CAAC,SAAS,GAAE,OAAO,CAAC,gCAAqC,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAsCpG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAWvB,QAAQ,CAAC,OAAO,GAAE,OAAO,CAAC,8BAAmC,GAAG,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC;IA2D5G,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,mCAAwC,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IA0CnI,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsE7D,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA4B9C,mBAAmB,IAAI,OAAO,CAAC,iBAAiB;IAWhD,OAAO,CAAC,gBAAgB;CAgB3B"}

package/resource-clients/key-value-store.js

@@ -0,0 +1,300 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KeyValueStoreClient = void 0;
+const tslib_1 = require("tslib");
+const shapeshift_1 = require("@sapphire/shapeshift");
+const mime_types_1 = tslib_1.__importDefault(require("mime-types"));
+const node_crypto_1 = require("node:crypto");
+const node_stream_1 = require("node:stream");
+const node_path_1 = require("node:path");
+const promises_1 = require("node:fs/promises");
+const fs_extra_1 = require("fs-extra");
+const body_parser_1 = require("../body-parser");
+const consts_1 = require("../consts");
+const utils_1 = require("../utils");
+const base_client_1 = require("./common/base-client");
+const instance_1 = require("../workers/instance");
+const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
+class KeyValueStoreClient extends base_client_1.BaseClient {
+    constructor(options) {
+        super(options.id ?? (0, node_crypto_1.randomUUID)());
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "createdAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Date()
+        });
+        Object.defineProperty(this, "accessedAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Date()
+        });
+        Object.defineProperty(this, "modifiedAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Date()
+        });
+        Object.defineProperty(this, "keyValueStoreDirectory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "keyValueEntries", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.name = options.name;
+        this.keyValueStoreDirectory = (0, node_path_1.resolve)(options.baseStorageDirectory, this.name ?? this.id);
+        this.client = options.client;
+    }
+    async get() {
+        const found = this.client.keyValueStoresHandled.find((store) => store.id === this.id);
+        if (found) {
+            found.updateTimestamps(false);
+            return found.toKeyValueStoreInfo();
+        }
+        return undefined;
+    }
+    async update(newFields = {}) {
+        const parsed = shapeshift_1.s.object({
+            name: shapeshift_1.s.string.lengthGreaterThan(0).optional,
+        }).parse(newFields);
+        // Check by id
+        const existingStoreById = this.client.keyValueStoresHandled.find((store) => store.id === this.id);
+        if (!existingStoreById) {
+            this.throwOnNonExisting(consts_1.StorageTypes.KeyValueStore);
+        }
+        // Skip if no changes
+        if (!parsed.name) {
+            return existingStoreById.toKeyValueStoreInfo();
+        }
+        // Check that name is not in use already
+        const existingStoreByName = this.client.keyValueStoresHandled.find((store) => store.name?.toLowerCase() === parsed.name.toLowerCase());
+        if (existingStoreByName) {
+            this.throwOnDuplicateEntry(consts_1.StorageTypes.KeyValueStore, 'name', parsed.name);
+        }
+        existingStoreById.name = parsed.name;
+        const previousDir = existingStoreById.keyValueStoreDirectory;
+        existingStoreById.keyValueStoreDirectory = (0, node_path_1.resolve)(this.client.keyValueStoresDirectory, parsed.name ?? existingStoreById.name ?? existingStoreById.id);
+        await (0, fs_extra_1.move)(previousDir, existingStoreById.keyValueStoreDirectory, { overwrite: true });
+        // Update timestamps
+        existingStoreById.updateTimestamps(true);
+        return existingStoreById.toKeyValueStoreInfo();
+    }
+    async delete() {
+        const storeIndex = this.client.keyValueStoresHandled.findIndex((store) => store.id === this.id);
+        if (storeIndex !== -1) {
+            const [oldClient] = this.client.keyValueStoresHandled.splice(storeIndex, 1);
+            oldClient.keyValueEntries.clear();
+            await (0, promises_1.rm)(oldClient.keyValueStoreDirectory, { recursive: true });
+        }
+    }
+    async listKeys(options = {}) {
+        const { limit = consts_1.DEFAULT_API_PARAM_LIMIT, exclusiveStartKey, } = shapeshift_1.s.object({
+            limit: shapeshift_1.s.number.greaterThan(0).optional,
+            exclusiveStartKey: shapeshift_1.s.string.optional,
+        }).parse(options);
+        // Check by id
+        const existingStoreById = this.client.keyValueStoresHandled.find((store) => store.id === this.id);
+        if (!existingStoreById) {
+            this.throwOnNonExisting(consts_1.StorageTypes.KeyValueStore);
+        }
+        const items = [];
+        for (const record of existingStoreById.keyValueEntries.values()) {
+            const size = Buffer.byteLength(record.value);
+            items.push({
+                key: record.key,
+                size,
+            });
+        }
+        // Lexically sort to emulate API.
+        // TODO(vladfrangu): ensure the sorting works the same way as before (if it matters)
+        items.sort((a, b) => {
+            return a.key.localeCompare(b.key);
+        });
+        let truncatedItems = items;
+        if (exclusiveStartKey) {
+            const keyPos = items.findIndex((item) => item.key === exclusiveStartKey);
+            if (keyPos !== -1)
+                truncatedItems = items.slice(keyPos + 1);
+        }
+        const limitedItems = truncatedItems.slice(0, limit);
+        const lastItemInStore = items[items.length - 1];
+        const lastSelectedItem = limitedItems[limitedItems.length - 1];
+        const isLastSelectedItemAbsolutelyLast = lastItemInStore === lastSelectedItem;
+        const nextExclusiveStartKey = isLastSelectedItemAbsolutelyLast
+            ? undefined
+            : lastSelectedItem.key;
+        existingStoreById.updateTimestamps(false);
+        return {
+            count: items.length,
+            limit,
+            exclusiveStartKey,
+            isTruncated: !isLastSelectedItemAbsolutelyLast,
+            nextExclusiveStartKey,
+            items: limitedItems,
+        };
+    }
+    async getRecord(key, options = {}) {
+        shapeshift_1.s.string.parse(key);
+        shapeshift_1.s.object({
+            buffer: shapeshift_1.s.boolean.optional,
+            // These options are ignored, but kept here
+            // for validation consistency with API client.
+            stream: shapeshift_1.s.boolean.optional,
+            disableRedirect: shapeshift_1.s.boolean.optional,
+        }).parse(options);
+        // Check by id
+        const existingStoreById = this.client.keyValueStoresHandled.find((store) => store.id === this.id);
+        if (!existingStoreById) {
+            this.throwOnNonExisting(consts_1.StorageTypes.KeyValueStore);
+        }
+        const entry = existingStoreById.keyValueEntries.get(key);
+        if (!entry) {
+            return undefined;
+        }
+        const record = {
+            key: entry.key,
+            value: entry.value,
+            contentType: entry.contentType ?? mime_types_1.default.contentType(entry.extension),
+        };
+        if (options.stream) {
+            record.value = node_stream_1.Readable.from(record.value);
+        }
+        else if (options.buffer) {
+            record.value = Buffer.from(record.value);
+        }
+        else {
+            record.value = (0, body_parser_1.maybeParseBody)(record.value, record.contentType);
+        }
+        existingStoreById.updateTimestamps(false);
+        return record;
+    }
+    async setRecord(record) {
+        shapeshift_1.s.object({
+            key: shapeshift_1.s.string.lengthGreaterThan(0),
+            value: shapeshift_1.s.union(shapeshift_1.s.null, shapeshift_1.s.string, shapeshift_1.s.number, shapeshift_1.s.object({}).passthrough),
+            contentType: shapeshift_1.s.string.lengthGreaterThan(0).optional,
+        }).parse(record);
+        // Check by id
+        const existingStoreById = this.client.keyValueStoresHandled.find((store) => store.id === this.id);
+        if (!existingStoreById) {
+            this.throwOnNonExisting(consts_1.StorageTypes.KeyValueStore);
+        }
+        const { key } = record;
+        let { value, contentType } = record;
+        const valueIsStream = (0, utils_1.isStream)(value);
+        const isValueStreamOrBuffer = valueIsStream || (0, utils_1.isBuffer)(value);
+        // To allow saving Objects to JSON without providing content type
+        if (!contentType) {
+            if (isValueStreamOrBuffer)
+                contentType = 'application/octet-stream';
+            else if (typeof value === 'string')
+                contentType = 'text/plain; charset=utf-8';
+            else
+                contentType = 'application/json; charset=utf-8';
+        }
+        const extension = mime_types_1.default.extension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
+        const isContentTypeJson = extension === 'json';
+        if (isContentTypeJson && !isValueStreamOrBuffer && typeof value !== 'string') {
+            try {
+                value = JSON.stringify(value, null, 2);
+            }
+            catch (err) {
+                const msg = `The record value cannot be stringified to JSON. Please provide other content type.\nCause: ${err.message}`;
+                throw new Error(msg);
+            }
+        }
+        if (valueIsStream) {
+            const chunks = [];
+            for await (const chunk of value) {
+                chunks.push(chunk);
+            }
+            value = Buffer.concat(chunks);
+        }
+        const entry = {
+            extension,
+            key,
+            value,
+            contentType,
+        };
+        existingStoreById.keyValueEntries.set(key, entry);
+        existingStoreById.updateTimestamps(true);
+        (0, instance_1.sendWorkerMessage)({
+            action: 'update-entries',
+            data: {
+                action: 'set',
+                record: entry,
+            },
+            entityType: 'keyValueStores',
+            entityDirectory: existingStoreById.keyValueStoreDirectory,
+            id: existingStoreById.name ?? existingStoreById.id,
+        });
+    }
+    async deleteRecord(key) {
+        shapeshift_1.s.string.parse(key);
+        // Check by id
+        const existingStoreById = this.client.keyValueStoresHandled.find((store) => store.id === this.id);
+        if (!existingStoreById) {
+            this.throwOnNonExisting(consts_1.StorageTypes.KeyValueStore);
+        }
+        const entry = existingStoreById.keyValueEntries.get(key);
+        if (entry) {
+            existingStoreById.keyValueEntries.delete(key);
+            existingStoreById.updateTimestamps(true);
+            (0, instance_1.sendWorkerMessage)({
+                action: 'update-entries',
+                data: {
+                    action: 'delete',
+                    record: entry,
+                },
+                entityType: 'keyValueStores',
+                entityDirectory: existingStoreById.keyValueStoreDirectory,
+                id: existingStoreById.name ?? existingStoreById.id,
+            });
+        }
+    }
+    toKeyValueStoreInfo() {
+        return {
+            id: this.id,
+            name: this.name,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            modifiedAt: this.modifiedAt,
+            userId: '1',
+        };
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+        const data = this.toKeyValueStoreInfo();
+        (0, instance_1.sendWorkerMessage)({
+            action: 'update-metadata',
+            data,
+            entityType: 'keyValueStores',
+            entityDirectory: this.keyValueStoreDirectory,
+            id: this.name ?? this.id,
+        });
+    }
+}
+exports.KeyValueStoreClient = KeyValueStoreClient;
+//# sourceMappingURL=key-value-store.js.map

package/resource-clients/key-value-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"key-value-store.js","sourceRoot":"","sources":["../../src/resource-clients/key-value-store.ts"],"names":[],"mappings":";;;;AACA,qDAAyC;AACzC,oEAA8B;AAC9B,6CAAyC;AACzC,6CAAuC;AACvC,yCAAoC;AACpC,+CAAsC;AACtC,uCAAgC;AAEhC,gDAAgD;AAChD,sCAAkE;AAClE,oCAA8C;AAC9C,sDAAkD;AAClD,kDAAwD;AAExD,MAAM,4BAA4B,GAAG,KAAK,CAAC;AAgB3C,MAAa,mBAAoB,SAAQ,wBAAU;IAU/C,YAAY,OAAmC;QAC3C,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,IAAA,wBAAU,GAAE,CAAC,CAAC;QAVtC;;;;;WAAc;QACd;;;;mBAAY,IAAI,IAAI,EAAE;WAAC;QACvB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;mBAAa,IAAI,IAAI,EAAE;WAAC;QACxB;;;;;WAA+B;QAE/B;;;;mBAAmC,IAAI,GAAG,EAA6B;WAAC;QACxE;;;;;WAAuC;QAInC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,sBAAsB,GAAG,IAAA,mBAAO,EAAC,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1F,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,GAAG;QACL,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAEtF,IAAI,KAAK,EAAE;YACP,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC,mBAAmB,EAAE,CAAC;SACtC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,YAAsD,EAAE;QACjE,MAAM,MAAM,GAAG,cAAC,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,cAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,QAAQ;SAC/C,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpB,cAAc;QACd,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,qBAAqB;QACrB,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YACd,OAAO,iBAAiB,CAAC,mBAAmB,EAAE,CAAC;SAClD;QAED,wCAAwC;QACxC,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,MAAM,CAAC,IAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAExI,IAAI,mBAAmB,EAAE;YACrB,IAAI,CAAC,qBAAqB,CAAC,qBAAY,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;SAC/E;QAED,iBAAiB,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QAErC,MAAM,WAAW,GAAG,iBAAiB,CAAC,sBAAsB,CAAC;QAE7D,iBAAiB,CAAC,sBAAsB,GAAG,IAAA,mBAAO,EAAC,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE,MAAM,CAAC,IAAI,IAAI,iBAAiB,CAAC,IAAI,IAAI,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAEvJ,MAAM,IAAA,eAAI,EAAC,WAAW,EAAE,iBAAiB,CAAC,sBAAsB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvF,oBAAoB;QACpB,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAEzC,OAAO,iBAAiB,CAAC,mBAAmB,EAAE,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,MAAM;QACR,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhG,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE;YACnB,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAC5E,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAElC,MAAM,IAAA,aAAE,EAAC,SAAS,CAAC,sBAAsB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACnE;IACL,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,UAAkD,EAAE;QAC/D,MAAM,EACF,KAAK,GAAG,gCAAuB,EAC/B,iBAAiB,GACpB,GAAG,cAAC,CAAC,MAAM,CAAC;YACT,KAAK,EAAE,cAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ;YACvC,iBAAiB,EAAE,cAAC,CAAC,MAAM,CAAC,QAAQ;SACvC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,cAAc;QACd,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,MAAM,KAAK,GAAG,EAAE,CAAC;QAEjB,KAAK,MAAM,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE;YAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC;gBACP,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,IAAI;aACP,CAAC,CAAC;SACN;QAED,iCAAiC;QACjC,oFAAoF;QACpF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAChB,OAAO,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,iBAAiB,EAAE;YACnB,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,iBAAiB,CAAC,CAAC;YACzE,IAAI,MAAM,KAAK,CAAC,CAAC;gBAAE,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;SAC/D;QAED,MAAM,YAAY,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAEpD,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChD,MAAM,gBAAgB,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/D,MAAM,gCAAgC,GAAG,eAAe,KAAK,gBAAgB,CAAC;QAC9E,MAAM,qBAAqB,GAAG,gCAAgC;YAC1D,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC;QAE3B,iBAAiB,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE1C,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,KAAK;YACL,iBAAiB;YACjB,WAAW,EAAE,CAAC,gCAAgC;YAC9C,qBAAqB;YACrB,KAAK,EAAE,YAAY;SACtB,CAAC;IACN,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAAuD,EAAE;QAClF,cAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpB,cAAC,CAAC,MAAM,CAAC;YACL,MAAM,EAAE,cAAC,CAAC,OAAO,CAAC,QAAQ;YAC1B,2CAA2C;YAC3C,8CAA8C;YAC9C,MAAM,EAAE,cAAC,CAAC,OAAO,CAAC,QAAQ;YAC1B,eAAe,EAAE,cAAC,CAAC,OAAO,CAAC,QAAQ;SACtC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElB,cAAc;QACd,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEzD,IAAI,CAAC,KAAK,EAAE;YACR,OAAO,SAAS,CAAC;SACpB;QAED,MAAM,MAAM,GAAgC;YACxC,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,oBAAI,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAW;SAChF,CAAC;QAEF,IAAI,OAAO,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,GAAG,sBAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SAC9C;aAAM,IAAI,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SAC5C;aAAM;YACH,MAAM,CAAC,KAAK,GAAG,IAAA,4BAAc,EAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,WAAY,CAAC,CAAC;SACpE;QAED,iBAAiB,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE1C,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAmC;QAC/C,cAAC,CAAC,MAAM,CAAC;YACL,GAAG,EAAE,cAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAClC,KAAK,EAAE,cAAC,CAAC,KAAK,CAAC,cAAC,CAAC,IAAI,EAAE,cAAC,CAAC,MAAM,EAAE,cAAC,CAAC,MAAM,EAAE,cAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC;YACpE,WAAW,EAAE,cAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,QAAQ;SACtD,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAEjB,cAAc;QACd,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;QACvB,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;QAEpC,MAAM,aAAa,GAAG,IAAA,gBAAQ,EAAC,KAAK,CAAC,CAAC;QAEtC,MAAM,qBAAqB,GAAG,aAAa,IAAI,IAAA,gBAAQ,EAAC,KAAK,CAAC,CAAC;QAC/D,iEAAiE;QACjE,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,qBAAqB;gBAAE,WAAW,GAAG,0BAA0B,CAAC;iBAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,WAAW,GAAG,2BAA2B,CAAC;;gBACzE,WAAW,GAAG,iCAAiC,CAAC;SACxD;QAED,MAAM,SAAS,GAAG,oBAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,4BAA4B,CAAC;QAE9E,MAAM,iBAAiB,GAAG,SAAS,KAAK,MAAM,CAAC;QAE/C,IAAI,iBAAiB,IAAI,CAAC,qBAAqB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC1E,IAAI;gBACA,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;aAC1C;YAAC,OAAO,GAAQ,EAAE;gBACf,MAAM,GAAG,GAAG,8FAA8F,GAAG,CAAC,OAAO,EAAE,CAAC;gBACxH,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;aACxB;SACJ;QAED,IAAI,aAAa,EAAE;YACf,MAAM,MAAM,GAAG,EAAE,CAAC;YAClB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE;gBAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACtB;YACD,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;SACjC;QAED,MAAM,KAAK,GAAG;YACV,SAAS;YACT,GAAG;YACH,KAAK;YACL,WAAW;SACd,CAAC;QAEF,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAElD,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACzC,IAAA,4BAAiB,EAAC;YACd,MAAM,EAAE,gBAAgB;YACxB,IAAI,EAAE;gBACF,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,KAAK;aAChB;YACD,UAAU,EAAE,gBAAgB;YAC5B,eAAe,EAAE,iBAAiB,CAAC,sBAAsB;YACzD,EAAE,EAAE,iBAAiB,CAAC,IAAI,IAAI,iBAAiB,CAAC,EAAE;SACrD,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,cAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEpB,cAAc;QACd,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QAElG,IAAI,CAAC,iBAAiB,EAAE;YACpB,IAAI,CAAC,kBAAkB,CAAC,qBAAY,CAAC,aAAa,CAAC,CAAC;SACvD;QAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEzD,IAAI,KAAK,EAAE;YACP,iBAAiB,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9C,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACzC,IAAA,4BAAiB,EAAC;gBACd,MAAM,EAAE,gBAAgB;gBACxB,IAAI,EAAE;oBACF,MAAM,EAAE,QAAQ;oBAChB,MAAM,EAAE,KAAK;iBAChB;gBACD,UAAU,EAAE,gBAAgB;gBAC5B,eAAe,EAAE,iBAAiB,CAAC,sBAAsB;gBACzD,EAAE,EAAE,iBAAiB,CAAC,IAAI,IAAI,iBAAiB,CAAC,EAAE;aACrD,CAAC,CAAC;SACN;IACL,CAAC;IAED,mBAAmB;QACf,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,MAAM,EAAE,GAAG;SACd,CAAC;IACN,CAAC;IAEO,gBAAgB,CAAC,eAAwB;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,eAAe,EAAE;YACjB,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;SAChC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACxC,IAAA,4BAAiB,EAAC;YACd,MAAM,EAAE,iBAAiB;YACzB,IAAI;YACJ,UAAU,EAAE,gBAAgB;YAC5B,eAAe,EAAE,IAAI,CAAC,sBAAsB;YAC5C,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE;SAC3B,CAAC,CAAC;IACP,CAAC;CACJ;AA/SD,kDA+SC"}

package/resource-clients/request-queue-collection.d.ts

@@ -0,0 +1,14 @@
+import type * as storage from '@crawlee/types';
+import { MemoryStorage } from '../index';
+export interface RequestQueueCollectionClientOptions {
+    baseStorageDirectory: string;
+    client: MemoryStorage;
+}
+export declare class RequestQueueCollectionClient implements storage.RequestQueueCollectionClient {
+    private readonly requestQueuesDirectory;
+    private readonly client;
+    constructor({ baseStorageDirectory, client }: RequestQueueCollectionClientOptions);
+    list(): ReturnType<storage.RequestQueueCollectionClient['list']>;
+    getOrCreate(name?: string): Promise<storage.RequestQueueInfo>;
+}
+//# sourceMappingURL=request-queue-collection.d.ts.map

package/resource-clients/request-queue-collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request-queue-collection.d.ts","sourceRoot":"","sources":["../../src/resource-clients/request-queue-collection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,MAAM,WAAW,mCAAmC;IAChD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,4BAA6B,YAAW,OAAO,CAAC,4BAA4B;IACrF,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAAE,mCAAmC;IAK3E,IAAI,IAAI,UAAU,CAAC,OAAO,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC;IAahE,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC;CA2BtE"}

package/resource-clients/request-queue-collection.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequestQueueCollectionClient = void 0;
+const shapeshift_1 = require("@sapphire/shapeshift");
+const node_path_1 = require("node:path");
+const instance_1 = require("../workers/instance");
+const request_queue_1 = require("./request-queue");
+class RequestQueueCollectionClient {
+    constructor({ baseStorageDirectory, client }) {
+        Object.defineProperty(this, "requestQueuesDirectory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.requestQueuesDirectory = (0, node_path_1.resolve)(baseStorageDirectory);
+        this.client = client;
+    }
+    async list() {
+        return {
+            total: this.client.requestQueuesHandled.length,
+            count: this.client.requestQueuesHandled.length,
+            offset: 0,
+            limit: this.client.requestQueuesHandled.length,
+            desc: false,
+            items: this.client.requestQueuesHandled.map((store) => store.toRequestQueueInfo())
+                .sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()),
+        };
+    }
+    async getOrCreate(name) {
+        shapeshift_1.s.string.optional.parse(name);
+        if (name) {
+            const found = this.client.requestQueuesHandled.find((store) => store.name === name);
+            if (found) {
+                return found.toRequestQueueInfo();
+            }
+        }
+        const newStore = new request_queue_1.RequestQueueClient({ name, baseStorageDirectory: this.requestQueuesDirectory, client: this.client });
+        this.client.requestQueuesHandled.push(newStore);
+        // Schedule the worker to write to the disk
+        const queueInfo = newStore.toRequestQueueInfo();
+        // eslint-disable-next-line dot-notation
+        (0, instance_1.sendWorkerMessage)({
+            action: 'update-metadata',
+            entityType: 'requestQueues',
+            entityDirectory: newStore.requestQueueDirectory,
+            id: queueInfo.name ?? queueInfo.id,
+            data: queueInfo,
+        });
+        return queueInfo;
+    }
+}
+exports.RequestQueueCollectionClient = RequestQueueCollectionClient;
+//# sourceMappingURL=request-queue-collection.js.map

package/resource-clients/request-queue-collection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request-queue-collection.js","sourceRoot":"","sources":["../../src/resource-clients/request-queue-collection.ts"],"names":[],"mappings":";;;AACA,qDAAyC;AACzC,yCAAoC;AAEpC,kDAAwD;AACxD,mDAAqD;AAOrD,MAAa,4BAA4B;IAIrC,YAAY,EAAE,oBAAoB,EAAE,MAAM,EAAuC;QAHjF;;;;;WAAgD;QAChD;;;;;WAAuC;QAGnC,IAAI,CAAC,sBAAsB,GAAG,IAAA,mBAAO,EAAC,oBAAoB,CAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI;QACN,OAAO;YACH,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,MAAM;YAC9C,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,MAAM;YAC9C,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,MAAM;YAC9C,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,GAAG,CACvC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC;iBACrC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SACrE,CAAC;IACN,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAa;QAC3B,cAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE9B,IAAI,IAAI,EAAE;YACN,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;YAEpF,IAAI,KAAK,EAAE;gBACP,OAAO,KAAK,CAAC,kBAAkB,EAAE,CAAC;aACrC;SACJ;QAED,MAAM,QAAQ,GAAG,IAAI,kCAAkB,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1H,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEhD,2CAA2C;QAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC,kBAAkB,EAAE,CAAC;QAChD,wCAAwC;QACxC,IAAA,4BAAiB,EAAC;YACd,MAAM,EAAE,iBAAiB;YACzB,UAAU,EAAE,eAAe;YAC3B,eAAe,EAAE,QAAQ,CAAC,qBAAqB;YAC/C,EAAE,EAAE,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,EAAE;YAClC,IAAI,EAAE,SAAS;SAClB,CAAC,CAAC;QAEH,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ;AAjDD,oEAiDC"}

package/resource-clients/request-queue.d.ts

@@ -0,0 +1,48 @@
+import type * as storage from '@crawlee/types';
+import { MemoryStorage } from '../index';
+import { BaseClient } from './common/base-client';
+export interface RequestQueueClientOptions {
+    name?: string;
+    id?: string;
+    baseStorageDirectory: string;
+    client: MemoryStorage;
+}
+export interface InternalRequest {
+    id: string;
+    orderNo: number | null;
+    url: string;
+    uniqueKey: string;
+    method: Exclude<storage.RequestOptions['method'], undefined>;
+    retryCount: number;
+    json: string;
+}
+export declare class RequestQueueClient extends BaseClient implements storage.RequestQueueClient {
+    name?: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    handledRequestCount: number;
+    pendingRequestCount: number;
+    requestQueueDirectory: string;
+    private readonly requests;
+    private readonly client;
+    constructor(options: RequestQueueClientOptions);
+    get(): Promise<storage.RequestQueueInfo | undefined>;
+    update(newFields: {
+        name?: string | undefined;
+    }): Promise<storage.RequestQueueInfo | undefined>;
+    delete(): Promise<void>;
+    listHead(options?: storage.ListOptions): Promise<storage.QueueHead>;
+    addRequest(request: storage.RequestSchema, options?: storage.RequestOptions): Promise<storage.QueueOperationInfo>;
+    batchAddRequests(requests: storage.RequestSchema[], options?: storage.RequestOptions): Promise<storage.BatchAddRequestsResult>;
+    getRequest(id: string): Promise<storage.RequestOptions | undefined>;
+    updateRequest(request: storage.UpdateRequestSchema, options?: storage.RequestOptions): Promise<storage.QueueOperationInfo>;
+    deleteRequest(id: string): Promise<void>;
+    toRequestQueueInfo(): storage.RequestQueueInfo;
+    private updateTimestamps;
+    private updateItems;
+    private _jsonToRequest;
+    private _createInternalRequest;
+    private _calculateOrderNo;
+}
+//# sourceMappingURL=request-queue.d.ts.map

package/resource-clients/request-queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request-queue.d.ts","sourceRoot":"","sources":["../../src/resource-clients/request-queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAM/C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAsBlD,MAAM,WAAW,yBAAyB;IACtC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,oBAAoB,EAAE,MAAM,CAAC;IAC7B,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;IAC7D,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,kBAAmB,SAAQ,UAAW,YAAW,OAAO,CAAC,kBAAkB;IACpF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,OAAc;IACvB,UAAU,OAAc;IACxB,UAAU,OAAc;IACxB,mBAAmB,SAAK;IACxB,mBAAmB,SAAK;IACxB,qBAAqB,EAAE,MAAM,CAAC;IAE9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAC/D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,OAAO,EAAE,yBAAyB;IAOxC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAWpD,MAAM,CAAC,SAAS,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAuC/F,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAYvB,QAAQ,CAAC,OAAO,GAAE,OAAO,CAAC,WAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;IAiCvE,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,EAAE,OAAO,GAAE,OAAO,CAAC,cAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAuCrH,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,EAAE,EAAE,OAAO,GAAE,OAAO,CAAC,cAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC;IAiDlI,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAcnE,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,mBAAmB,EAAE,OAAO,GAAE,OAAO,CAAC,cAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC;IA6C9H,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB9C,kBAAkB,IAAI,OAAO,CAAC,gBAAgB;IAgB9C,OAAO,CAAC,gBAAgB;IAiBxB,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,iBAAiB;CAO5B"}