@crawlee/core 4.0.0-beta.17 → 4.0.0-beta.171

package/memory-storage/resource-clients/dataset.js

@@ -0,0 +1,106 @@
+import { randomUUID } from 'node:crypto';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { BaseClient } from './common/base-client.js';
+/**
+ * This is what API returns in the x-apify-pagination-limit
+ * header when no limit query parameter is used.
+ */
+const LIST_ITEMS_LIMIT = 999_999_999_999;
+/**
+ * Number of characters of the dataset item entry names.
+ * E.g.: 000000019 - 9 digits
+ */
+const LOCAL_ENTRY_NAME_DIGITS = 9;
+export class DatasetBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    itemCount = 0;
+    #datasetEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toDatasetInfo();
+    }
+    async drop() {
+        if (this.storageBackend.evictBackend('Dataset', this.id)) {
+            this.itemCount = 0;
+            this.#datasetEntries.clear();
+        }
+    }
+    async purge() {
+        this.itemCount = 0;
+        this.#datasetEntries.clear();
+        this.updateTimestamps(true);
+    }
+    getData(options = {}) {
+        const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
+        return this.getDataPage({
+            desc,
+            offset: offset ?? 0,
+            limit: Math.min(limit ?? LIST_ITEMS_LIMIT, LIST_ITEMS_LIMIT),
+        });
+    }
+    async getDataPage(options = {}) {
+        const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;
+        const [start, end] = this.getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
+        const items = [];
+        for (let idx = start; idx < end; idx++) {
+            const entryNumber = this.generateLocalEntryName(idx);
+            items.push(this.#datasetEntries.get(entryNumber));
+        }
+        this.updateTimestamps(false);
+        return {
+            count: items.length,
+            desc: desc ?? false,
+            items: desc ? items.reverse() : items,
+            limit,
+            offset,
+            total: this.itemCount,
+        };
+    }
+    async pushData(items) {
+        for (const entry of items) {
+            const idx = this.generateLocalEntryName(++this.itemCount);
+            this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
+        }
+        this.updateTimestamps(true);
+    }
+    toDatasetInfo() {
+        return {
+            id: this.id,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            itemCount: this.itemCount,
+            modifiedAt: this.modifiedAt,
+            name: this.name,
+        };
+    }
+    generateLocalEntryName(idx) {
+        return idx.toString().padStart(LOCAL_ENTRY_NAME_DIGITS, '0');
+    }
+    getStartAndEndIndexes(offset, limit = this.itemCount) {
+        const start = offset + 1;
+        const end = Math.min(offset + limit, this.itemCount) + 1;
+        return [start, end];
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+}

package/memory-storage/resource-clients/key-value-store.d.ts

@@ -0,0 +1,63 @@
+import type * as storage from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface KeyValueStoreBackendOptions {
+    name?: string;
+    id?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export interface InternalKeyRecord {
+    key: string;
+    value: Buffer;
+    contentType?: string;
+    extension: string;
+}
+export declare class KeyValueStoreBackend extends BaseClient implements storage.KeyValueStoreBackend {
+    #private;
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    private readonly storageBackend;
+    constructor(options: KeyValueStoreBackendOptions);
+    getMetadata(): Promise<storage.KeyValueStoreInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    /**
+     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
+     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
+     * extension variants) when purging the default store. The in-memory key has no extension, so we
+     * preserve the bare `INPUT` key only.
+     */
+    purgeExceptInput(): Promise<void>;
+    listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
+    /**
+     * In-memory records are not file-backed, so there is no public file URL to return.
+     * Always resolves to `undefined`.
+     * @param key The key of the record to generate the public URL for.
+     */
+    getPublicUrl(key: string): Promise<string | undefined>;
+    /**
+     * Tests whether a record with the given key exists in the key-value store without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` if it does not.
+     */
+    recordExists(key: string): Promise<boolean>;
+    getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
+    setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
+    deleteValue(key: string): Promise<void>;
+    toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
+    private updateTimestamps;
+}

package/memory-storage/resource-clients/key-value-store.js

@@ -0,0 +1,199 @@
+import { randomUUID } from 'node:crypto';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
+import { isStream, toBuffer } from '../utils.js';
+import { BaseClient } from './common/base-client.js';
+import mime from 'mime-types';
+const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
+const keySchema = z.string();
+const inputRecordSchema = z.object({
+    key: z.string().min(1),
+    value: z.union([
+        z.null(),
+        z.string(),
+        z.number(),
+        z.instanceof(Buffer),
+        z.instanceof(ArrayBuffer),
+        schemas.typedArray,
+        // only checks the value is an actual object, not null, nor array
+        schemas.plainObject,
+    ]),
+    contentType: z.string().min(1).optional(),
+});
+/**
+ * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
+ * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
+ */
+const KEY_VALUE_STORE_INPUT_KEY = 'INPUT';
+export class KeyValueStoreBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    #keyValueEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toKeyValueStoreInfo();
+    }
+    async drop() {
+        if (this.storageBackend.evictBackend('KeyValueStore', this.id)) {
+            this.#keyValueEntries.clear();
+        }
+    }
+    async purge() {
+        this.#keyValueEntries.clear();
+        this.updateTimestamps(true);
+    }
+    /**
+     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
+     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
+     * extension variants) when purging the default store. The in-memory key has no extension, so we
+     * preserve the bare `INPUT` key only.
+     */
+    async purgeExceptInput() {
+        for (const key of this.#keyValueEntries.keys()) {
+            if (key !== KEY_VALUE_STORE_INPUT_KEY) {
+                this.#keyValueEntries.delete(key);
+            }
+        }
+        this.updateTimestamps(true);
+    }
+    async listKeys(options = {}) {
+        const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
+        const items = [];
+        for (const record of this.#keyValueEntries.values()) {
+            const size = Buffer.byteLength(record.value);
+            items.push({
+                key: record.key,
+                size,
+                contentType: record.contentType ?? 'application/octet-stream',
+            });
+        }
+        // Lexically sort to emulate API.
+        items.sort((a, b) => a.key.localeCompare(b.key));
+        let filteredItems = items.filter((item) => !prefix || item.key.startsWith(prefix));
+        if (exclusiveStartKey) {
+            const keyPos = filteredItems.findIndex((item) => item.key === exclusiveStartKey);
+            if (keyPos === -1) {
+                throw new Error(`exclusiveStartKey "${exclusiveStartKey}" was not found in the key-value store. ` +
+                    `This is likely a bug — the key may have been deleted between paginated listKeys calls.`);
+            }
+            filteredItems = filteredItems.slice(keyPos + 1);
+        }
+        const isTruncated = limit !== undefined && filteredItems.length > limit;
+        const pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;
+        const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;
+        this.updateTimestamps(false);
+        return {
+            items: pageItems,
+            count: pageItems.length,
+            limit: limit ?? pageItems.length,
+            exclusiveStartKey,
+            isTruncated,
+            nextExclusiveStartKey,
+        };
+    }
+    /**
+     * In-memory records are not file-backed, so there is no public file URL to return.
+     * Always resolves to `undefined`.
+     * @param key The key of the record to generate the public URL for.
+     */
+    async getPublicUrl(key) {
+        parseArgument(key, keySchema);
+        return undefined;
+    }
+    /**
+     * Tests whether a record with the given key exists in the key-value store without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` if it does not.
+     */
+    async recordExists(key) {
+        parseArgument(key, keySchema);
+        return this.#keyValueEntries.has(key);
+    }
+    async getValue(key) {
+        parseArgument(key, keySchema);
+        const entry = this.#keyValueEntries.get(key);
+        if (!entry) {
+            return undefined;
+        }
+        // Return raw bytes + verbatim content type. Parsing is the frontend's job (see the
+        // KeyValueStore codec). The mime fallback reconstructs the content type for on-disk records.
+        const record = {
+            key: entry.key,
+            value: entry.value,
+            // mime.contentType returns `false` for unknown extensions; fall back to undefined so the
+            // frontend treats it as "no content type" rather than a bogus value.
+            contentType: entry.contentType ?? (mime.contentType(entry.extension) || undefined),
+        };
+        this.updateTimestamps(false);
+        return record;
+    }
+    async setValue(record) {
+        parseArgument(record, inputRecordSchema);
+        const { key } = record;
+        let { value } = record;
+        // The frontend (KeyValueStore codec) serializes the value and resolves its content type
+        // before it reaches the backend. We only need it here for on-disk extension bookkeeping.
+        const contentType = record.contentType ?? 'application/octet-stream';
+        const extension = mime.extension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
+        // Draining a stream into a Buffer for storage is the backend's responsibility.
+        if (isStream(value)) {
+            const chunks = [];
+            for await (const chunk of value) {
+                chunks.push(chunk);
+            }
+            value = Buffer.concat(chunks);
+        }
+        // This backend is a byte transport: it stores and returns raw bytes regardless of the input
+        // shape. Streams were drained above; encode strings to UTF-8 bytes and normalize
+        // ArrayBuffer / typed-array views to a Buffer over the same memory.
+        const normalizedValue = typeof value === 'string'
+            ? Buffer.from(value, 'utf-8')
+            : toBuffer(value);
+        const normalizedRecord = {
+            extension,
+            key,
+            value: normalizedValue,
+            contentType,
+        };
+        this.#keyValueEntries.set(key, normalizedRecord);
+        this.updateTimestamps(true);
+    }
+    async deleteValue(key) {
+        parseArgument(key, keySchema);
+        if (this.#keyValueEntries.has(key)) {
+            this.#keyValueEntries.delete(key);
+            this.updateTimestamps(true);
+        }
+    }
+    toKeyValueStoreInfo() {
+        return {
+            id: this.id,
+            name: this.name,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            modifiedAt: this.modifiedAt,
+        };
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+}

package/memory-storage/resource-clients/request-queue.d.ts

@@ -0,0 +1,77 @@
+import type * as storage from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface RequestQueueBackendOptions {
+    name?: string;
+    id?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export interface InternalRequest {
+    id: string;
+    orderNo: number | null;
+    url: string;
+    uniqueKey: string;
+    method: storage.RequestSchema['method'];
+    retryCount: number;
+    json: string;
+}
+export declare class RequestQueueBackend extends BaseClient implements storage.RequestQueueBackend {
+    #private;
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    handledRequestCount: number;
+    pendingRequestCount: number;
+    private readonly storageBackend;
+    constructor(options: RequestQueueBackendOptions);
+    getMetadata(): Promise<storage.RequestQueueInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    private requestKeyIterator;
+    /**
+     * Scans the queue and returns the pending head — requests that are neither handled nor currently
+     * in progress — ordered by `orderNo`, deduplicated.
+     *
+     * When `detectInProgressRequests` is set, the result also carries an `hasInProgressRequests` flag
+     * telling whether any unhandled-but-in-progress request was skipped along the way. It lets
+     * {@link isFinished} distinguish "no work left at all" from "work remains, but it is currently being
+     * processed". Without it, a consumer with concurrency could consider the queue finished and shut the
+     * crawler down while it is still handling the last requests.
+     *
+     * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
+     * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
+     * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
+     * soon as the page is filled, keeping those calls O(head) instead of O(pending).
+     */
+    private listPendingHead;
+    fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;
+    addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;
+    getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;
+    markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined>;
+    reclaimRequest(request: storage.UpdateRequestSchema, options?: storage.RequestQueueOperationOptions): Promise<storage.QueueOperationInfo | undefined>;
+    isEmpty(): Promise<boolean>;
+    isFinished(): Promise<boolean>;
+    /**
+     * Returns all pending (not yet handled, not currently in progress) requests in the queue, ordered
+     * the same way {@link fetchNextRequest} would hand them out. This does not mutate the queue,
+     * nothing is marked in progress.
+     */
+    listItems(): Promise<storage.UpdateRequestSchema[]>;
+    toRequestQueueInfo(): storage.RequestQueueInfo;
+    private updateTimestamps;
+    private jsonToRequest;
+    private createInternalRequest;
+    private calculateOrderNo;
+}