@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/memory-storage/resource-clients/dataset.js

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { BaseClient } from './common/base-client.js';
 /**
  * This is what API returns in the x-apify-pagination-limit
@@ -22,54 +22,46 @@ export class DatasetBackend extends BaseClient {
     accessedAt = new Date();
     modifiedAt = new Date();
     itemCount = 0;
-    datasetEntries = new Map();
-    storageBackend;
+    #datasetEntries = new Map();
+    #storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
         this.name = options.name;
         this.cacheKey = options.cacheKey ?? this.name ?? this.id;
-        this.storageBackend = options.storageBackend;
+        this.#storageBackend = options.storageBackend;
     }
     async getMetadata() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return this.toDatasetInfo();
     }
     async drop() {
-        const storeIndex = this.storageBackend.datasetBackendCache.findIndex((store) => store.id === this.id);
-        if (storeIndex !== -1) {
-            const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);
-            oldBackend.itemCount = 0;
-            oldBackend.datasetEntries.clear();
+        if (this.#storageBackend.evictBackend('Dataset', this.id)) {
+            this.itemCount = 0;
+            this.#datasetEntries.clear();
         }
     }
     async purge() {
         this.itemCount = 0;
-        this.datasetEntries.clear();
-        this.updateTimestamps(true);
+        this.#datasetEntries.clear();
+        this.#updateTimestamps(true);
     }
     getData(options = {}) {
-        const { desc, limit, offset } = s
-            .object({
-            desc: s.boolean().optional(),
-            limit: s.number().int().optional(),
-            offset: s.number().int().optional(),
-        })
-            .parse(options);
-        return this.getDataPage({
+        const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
+        return this.#getDataPage({
             desc,
             offset: offset ?? 0,
             limit: Math.min(limit ?? LIST_ITEMS_LIMIT, LIST_ITEMS_LIMIT),
         });
     }
-    async getDataPage(options = {}) {
+    async #getDataPage(options = {}) {
         const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;
-        const [start, end] = this.getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
+        const [start, end] = this.#getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
         const items = [];
         for (let idx = start; idx < end; idx++) {
-            const entryNumber = this.generateLocalEntryName(idx);
-            items.push(this.datasetEntries.get(entryNumber));
+            const entryNumber = this.#generateLocalEntryName(idx);
+            items.push(this.#datasetEntries.get(entryNumber));
         }
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return {
             count: items.length,
             desc: desc ?? false,
@@ -81,10 +73,10 @@ export class DatasetBackend extends BaseClient {
     }
     async pushData(items) {
         for (const entry of items) {
-            const idx = this.generateLocalEntryName(++this.itemCount);
-            this.datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
+            const idx = this.#generateLocalEntryName(++this.itemCount);
+            this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
         }
-        this.updateTimestamps(true);
+        this.#updateTimestamps(true);
     }
     toDatasetInfo() {
         return {
@@ -96,15 +88,15 @@ export class DatasetBackend extends BaseClient {
             name: this.name,
         };
     }
-    generateLocalEntryName(idx) {
+    #generateLocalEntryName(idx) {
         return idx.toString().padStart(LOCAL_ENTRY_NAME_DIGITS, '0');
     }
-    getStartAndEndIndexes(offset, limit = this.itemCount) {
+    #getStartAndEndIndexes(offset, limit = this.itemCount) {
         const start = offset + 1;
         const end = Math.min(offset + limit, this.itemCount) + 1;
         return [start, end];
     }
-    updateTimestamps(hasBeenModified) {
+    #updateTimestamps(hasBeenModified) {
         this.accessedAt = new Date();
         if (hasBeenModified) {
             this.modifiedAt = new Date();

package/memory-storage/resource-clients/key-value-store.d.ts

@@ -19,6 +19,7 @@ export interface InternalKeyRecord {
     extension: string;
 }
 export declare class KeyValueStoreBackend extends BaseClient implements storage.KeyValueStoreBackend {
+    #private;
     name?: string;
     /**
      * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
@@ -28,19 +29,10 @@ export declare class KeyValueStoreBackend extends BaseClient implements storage.
     createdAt: Date;
     accessedAt: Date;
     modifiedAt: Date;
-    private readonly keyValueEntries;
-    private readonly storageBackend;
     constructor(options: KeyValueStoreBackendOptions);
     getMetadata(): Promise<storage.KeyValueStoreInfo>;
     drop(): Promise<void>;
     purge(): Promise<void>;
-    /**
-     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
-     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
-     * extension variants) when purging the default store. The in-memory key has no extension, so we
-     * preserve the bare `INPUT` key only.
-     */
-    purgeExceptInput(): Promise<void>;
     listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
     /**
      * In-memory records are not file-backed, so there is no public file URL to return.
@@ -59,5 +51,4 @@ export declare class KeyValueStoreBackend extends BaseClient implements storage.
     setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
     deleteValue(key: string): Promise<void>;
     toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
-    private updateTimestamps;
 }

package/memory-storage/resource-clients/key-value-store.js

@@ -1,14 +1,25 @@
 import { randomUUID } from 'node:crypto';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { isStream, toBuffer } from '../utils.js';
 import { BaseClient } from './common/base-client.js';
 import mime from 'mime-types';
 const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
-/**
- * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
- * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
- */
-const KEY_VALUE_STORE_INPUT_KEY = 'INPUT';
+const keySchema = z.string();
+const inputRecordSchema = z.object({
+    key: z.string().min(1),
+    value: z.union([
+        z.null(),
+        z.string(),
+        z.number(),
+        z.instanceof(Buffer),
+        z.instanceof(ArrayBuffer),
+        schemas.typedArray,
+        // only checks the value is an actual object, not null, nor array
+        schemas.plainObject,
+    ]),
+    contentType: z.string().min(1).optional(),
+});
 export class KeyValueStoreBackend extends BaseClient {
     name;
     /**
@@ -19,53 +30,31 @@ export class KeyValueStoreBackend extends BaseClient {
     createdAt = new Date();
     accessedAt = new Date();
     modifiedAt = new Date();
-    keyValueEntries = new Map();
-    storageBackend;
+    #keyValueEntries = new Map();
+    #storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
         this.name = options.name;
         this.cacheKey = options.cacheKey ?? this.name ?? this.id;
-        this.storageBackend = options.storageBackend;
+        this.#storageBackend = options.storageBackend;
     }
     async getMetadata() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return this.toKeyValueStoreInfo();
     }
     async drop() {
-        const storeIndex = this.storageBackend.keyValueStoreBackendCache.findIndex((store) => store.id === this.id);
-        if (storeIndex !== -1) {
-            const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);
-            oldBackend.keyValueEntries.clear();
+        if (this.#storageBackend.evictBackend('KeyValueStore', this.id)) {
+            this.#keyValueEntries.clear();
         }
     }
     async purge() {
-        this.keyValueEntries.clear();
-        this.updateTimestamps(true);
-    }
-    /**
-     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
-     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
-     * extension variants) when purging the default store. The in-memory key has no extension, so we
-     * preserve the bare `INPUT` key only.
-     */
-    async purgeExceptInput() {
-        for (const key of this.keyValueEntries.keys()) {
-            if (key !== KEY_VALUE_STORE_INPUT_KEY) {
-                this.keyValueEntries.delete(key);
-            }
-        }
-        this.updateTimestamps(true);
+        this.#keyValueEntries.clear();
+        this.#updateTimestamps(true);
     }
     async listKeys(options = {}) {
-        const { prefix, exclusiveStartKey, limit } = s
-            .object({
-            prefix: s.string().optional(),
-            exclusiveStartKey: s.string().optional(),
-            limit: s.number().int().greaterThan(0).optional(),
-        })
-            .parse(options);
+        const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
         const items = [];
-        for (const record of this.keyValueEntries.values()) {
+        for (const record of this.#keyValueEntries.values()) {
             const size = Buffer.byteLength(record.value);
             items.push({
                 key: record.key,
@@ -87,7 +76,7 @@ export class KeyValueStoreBackend extends BaseClient {
         const isTruncated = limit !== undefined && filteredItems.length > limit;
         const pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;
         const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return {
             items: pageItems,
             count: pageItems.length,
@@ -103,7 +92,7 @@ export class KeyValueStoreBackend extends BaseClient {
      * @param key The key of the record to generate the public URL for.
      */
     async getPublicUrl(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         return undefined;
     }
     /**
@@ -113,12 +102,12 @@ export class KeyValueStoreBackend extends BaseClient {
      * @returns `true` if the record exists, `false` if it does not.
      */
     async recordExists(key) {
-        s.string().parse(key);
-        return this.keyValueEntries.has(key);
+        parseArgument(key, keySchema);
+        return this.#keyValueEntries.has(key);
     }
     async getValue(key) {
-        s.string().parse(key);
-        const entry = this.keyValueEntries.get(key);
+        parseArgument(key, keySchema);
+        const entry = this.#keyValueEntries.get(key);
         if (!entry) {
             return undefined;
         }
@@ -131,24 +120,11 @@ export class KeyValueStoreBackend extends BaseClient {
             // frontend treats it as "no content type" rather than a bogus value.
             contentType: entry.contentType ?? (mime.contentType(entry.extension) || undefined),
         };
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return record;
     }
     async setValue(record) {
-        s.object({
-            key: s.string().lengthGreaterThan(0),
-            value: s.union([
-                s.null(),
-                s.string(),
-                s.number(),
-                s.instance(Buffer),
-                s.instance(ArrayBuffer),
-                s.typedArray(),
-                // disabling validation will make shapeshift only check the object given is an actual object, not null, nor array
-                s.object({}).setValidationEnabled(false),
-            ]),
-            contentType: s.string().lengthGreaterThan(0).optional(),
-        }).parse(record);
+        parseArgument(record, inputRecordSchema);
         const { key } = record;
         let { value } = record;
         // The frontend (KeyValueStore codec) serializes the value and resolves its content type
@@ -169,20 +145,20 @@ export class KeyValueStoreBackend extends BaseClient {
         const normalizedValue = typeof value === 'string'
             ? Buffer.from(value, 'utf-8')
             : toBuffer(value);
-        const _record = {
+        const normalizedRecord = {
             extension,
             key,
             value: normalizedValue,
             contentType,
         };
-        this.keyValueEntries.set(key, _record);
-        this.updateTimestamps(true);
+        this.#keyValueEntries.set(key, normalizedRecord);
+        this.#updateTimestamps(true);
     }
     async deleteValue(key) {
-        s.string().parse(key);
-        if (this.keyValueEntries.has(key)) {
-            this.keyValueEntries.delete(key);
-            this.updateTimestamps(true);
+        parseArgument(key, keySchema);
+        if (this.#keyValueEntries.has(key)) {
+            this.#keyValueEntries.delete(key);
+            this.#updateTimestamps(true);
         }
     }
     toKeyValueStoreInfo() {
@@ -194,7 +170,7 @@ export class KeyValueStoreBackend extends BaseClient {
             modifiedAt: this.modifiedAt,
         };
     }
-    updateTimestamps(hasBeenModified) {
+    #updateTimestamps(hasBeenModified) {
         this.accessedAt = new Date();
         if (hasBeenModified) {
             this.modifiedAt = new Date();

package/memory-storage/resource-clients/request-queue.d.ts

@@ -22,6 +22,7 @@ export interface InternalRequest {
     json: string;
 }
 export declare class RequestQueueBackend extends BaseClient implements storage.RequestQueueBackend {
+    #private;
     name?: string;
     /**
      * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
@@ -33,48 +34,10 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
     modifiedAt: Date;
     handledRequestCount: number;
     pendingRequestCount: number;
-    /**
-     * Serializes every operation that reads-then-writes this backend's shared queue state — the
-     * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
-     * counts. Those mutations span `await` points, so without this mutex a concurrent operation could
-     * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
-     * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
-     * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
-     */
-    private readonly queueStateMutex;
-    private forefrontRequestIds;
-    /**
-     * IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
-     * "in progress" and will not be handed out again by {@link fetchNextRequest}.
-     *
-     * Unlike the file-system / platform clients, the in-memory queue lives entirely within a single
-     * process and is never shared with another consumer, so there is no need for an expiring,
-     * cross-process-visible lock — tracking in-progress requests in this set is enough.
-     */
-    private readonly inProgressRequestIds;
-    private readonly requests;
-    private readonly storageBackend;
     constructor(options: RequestQueueBackendOptions);
     getMetadata(): Promise<storage.RequestQueueInfo>;
     drop(): Promise<void>;
     purge(): Promise<void>;
-    private requestKeyIterator;
-    /**
-     * Scans the queue and returns the pending head — requests that are neither handled nor currently
-     * in progress — ordered by `orderNo`, deduplicated.
-     *
-     * When `detectInProgressRequests` is set, the result also carries an `hasInProgressRequests` flag
-     * telling whether any unhandled-but-in-progress request was skipped along the way. It lets
-     * {@link isFinished} distinguish "no work left at all" from "work remains, but it is currently being
-     * processed". Without it, a consumer with concurrency could consider the queue finished and shut the
-     * crawler down while it is still handling the last requests.
-     *
-     * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
-     * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
-     * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
-     * soon as the page is filled, keeping those calls O(head) instead of O(N).
-     */
-    private listPendingHead;
     fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;
     addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;
     getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;
@@ -89,8 +52,4 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
      */
     listItems(): Promise<storage.UpdateRequestSchema[]>;
     toRequestQueueInfo(): storage.RequestQueueInfo;
-    private updateTimestamps;
-    private _jsonToRequest;
-    private _createInternalRequest;
-    private _calculateOrderNo;
 }