@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/memory-storage/memory-storage.js

@@ -2,13 +2,15 @@ import { randomUUID } from 'node:crypto';
 import { DatasetBackend } from './resource-clients/dataset.js';
 import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
 import { RequestQueueBackend } from './resource-clients/request-queue.js';
+/** The alias the default (unnamed) storage is opened under. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
 export class MemoryStorageBackend {
     logger;
     /**
      * Unique per-instance cache partition key. Mirrors the way `FileSystemStorageBackend` partitions its
      * cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.
      */
-    instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
+    #instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
     keyValueStoreBackendCache = [];
     datasetBackendCache = [];
     requestQueueBackendCache = [];
@@ -20,24 +22,26 @@ export class MemoryStorageBackend {
      * cache partitions in the storage backend cache.
      */
     getStorageBackendCacheKey() {
-        return this.instanceCacheKey;
+        return this.#instanceCacheKey;
     }
-    static resolveStorageKey(options) {
-        const isAlias = 'alias' in options && !!options.alias;
-        const rawKey = isAlias ? options.alias : (options.name ?? options.id);
+    static #resolveStorageKey(options) {
+        // No identifier at all means the default storage, which is opened under the reserved alias —
+        // same rule as `resolveStorageIdentifier` in the storage frontends, so that a backend used
+        // directly lands on the very storage the frontends would have opened.
+        const alias = options.alias || (!options.id && !options.name ? DEFAULT_STORAGE_ALIAS : undefined);
+        // `alias` covers the identifier-less case, so one of the three is always set.
+        const rawKey = alias ?? options.name ?? options.id;
         // Normalize the internal __default__ alias to the user-facing 'default' name.
-        const cacheKey = rawKey === '__default__' ? 'default' : rawKey;
-        return { isAlias, cacheKey };
+        const cacheKey = rawKey === DEFAULT_STORAGE_ALIAS ? 'default' : rawKey;
+        return { isAlias: alias !== undefined, cacheKey };
     }
     async createDatasetBackend(options = {}) {
-        const { isAlias, cacheKey } = MemoryStorageBackend.resolveStorageKey(options);
-        if (cacheKey) {
-            const found = this.datasetBackendCache.find((store) => store.id === cacheKey ||
-                store.name?.toLowerCase() === cacheKey.toLowerCase() ||
-                store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
-            if (found) {
-                return found;
-            }
+        const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
+        const found = this.datasetBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
         }
         const newStore = new DatasetBackend({
             name: isAlias ? undefined : cacheKey,
@@ -48,14 +52,12 @@ export class MemoryStorageBackend {
         return newStore;
     }
     async createKeyValueStoreBackend(options = {}) {
-        const { isAlias, cacheKey } = MemoryStorageBackend.resolveStorageKey(options);
-        if (cacheKey) {
-            const found = this.keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
-                store.name?.toLowerCase() === cacheKey.toLowerCase() ||
-                store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
-            if (found) {
-                return found;
-            }
+        const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
+        const found = this.keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
         }
         const newStore = new KeyValueStoreBackend({
             name: isAlias ? undefined : cacheKey,
@@ -66,14 +68,12 @@ export class MemoryStorageBackend {
         return newStore;
     }
     async createRequestQueueBackend(options = {}) {
-        const { isAlias, cacheKey } = MemoryStorageBackend.resolveStorageKey(options);
-        if (cacheKey) {
-            const found = this.requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
-                queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
-                queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
-            if (found) {
-                return found;
-            }
+        const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
+        const found = this.requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
+            queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
         }
         const newStore = new RequestQueueBackend({
             name: isAlias ? undefined : cacheKey,
@@ -102,28 +102,22 @@ export class MemoryStorageBackend {
         return backends.some((store) => store.id === id);
     }
     /**
-     * Cleans up the default storages before the run starts. For the in-memory storage this simply
-     * resets the in-memory state of the cached default dataset, key-value store and request queue.
-     *
-     * As with `FileSystemStorageBackend`, the run's input (the `INPUT` key in the default key-value
-     * store) is preserved — only the rest of the default storages is cleared.
+     * Cleans up the run-scoped storages before the run starts. For the in-memory storage this simply
+     * resets the in-memory state of the cached backends.
      */
     async purge() {
-        // The run default is opened via `{ alias: '__default__' }`, which `resolveStorageKey`
-        // normalizes to `cacheKey === 'default'` (with `name === undefined`) — that is the clause
-        // that actually matches it. The `name === 'default'` clause additionally covers a store a user
-        // explicitly opened via `{ name: 'default' }`. (`'__default__'` never reaches `cacheKey`,
-        // as it is always normalized to `'default'` first, so it does not need to be checked here.)
+        // `#resolveStorageKey` leaves `name` unset for the default and alias-keyed storages, which is what
+        // marks them as run-scoped. `'default'` is the exception — it collapses onto the default storage.
+        const isRunScoped = (store) => store.name === undefined || store.name === 'default';
         const isDefault = (store) => store.name === 'default' || store.cacheKey === 'default';
-        const purgeDefaults = async (cache, purgeStore) => {
-            await Promise.all(cache.filter(isDefault).map(async (store) => purgeStore(store)));
+        const purgeRunScoped = async (cache, purgeStore) => {
+            await Promise.all(cache.filter(isRunScoped).map(async (store) => purgeStore(store)));
         };
         await Promise.all([
-            // Preserve the run input (INPUT) when purging the default key-value store, matching
-            // `FileSystemStorageBackend`.
-            purgeDefaults(this.keyValueStoreBackendCache, async (store) => store.purgeExceptInput()),
-            purgeDefaults(this.datasetBackendCache, async (store) => store.purge()),
-            purgeDefaults(this.requestQueueBackendCache, async (store) => store.purge()),
+            // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
+            purgeRunScoped(this.keyValueStoreBackendCache, async (store) => isDefault(store) ? store.purgeExceptInput() : store.purge()),
+            purgeRunScoped(this.datasetBackendCache, async (store) => store.purge()),
+            purgeRunScoped(this.requestQueueBackendCache, async (store) => store.purge()),
         ]);
     }
     /**

package/memory-storage/resource-clients/dataset.d.ts

@@ -14,6 +14,7 @@ export interface DatasetBackendOptions {
     storageBackend: MemoryStorageBackend;
 }
 export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends BaseClient implements storage.DatasetBackend<Data> {
+    #private;
     name?: string;
     /**
      * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
@@ -24,7 +25,6 @@ export declare class DatasetBackend<Data extends Dictionary = Dictionary> extend
     accessedAt: Date;
     modifiedAt: Date;
     itemCount: number;
-    private readonly datasetEntries;
     private readonly storageBackend;
     constructor(options: DatasetBackendOptions);
     getMetadata(): Promise<storage.DatasetInfo>;

package/memory-storage/resource-clients/dataset.js

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { BaseClient } from './common/base-client.js';
 /**
  * This is what API returns in the x-apify-pagination-limit
@@ -22,7 +22,8 @@ export class DatasetBackend extends BaseClient {
     accessedAt = new Date();
     modifiedAt = new Date();
     itemCount = 0;
-    datasetEntries = new Map();
+    #datasetEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
     storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
@@ -39,22 +40,16 @@ export class DatasetBackend extends BaseClient {
         if (storeIndex !== -1) {
             const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);
             oldBackend.itemCount = 0;
-            oldBackend.datasetEntries.clear();
+            oldBackend.#datasetEntries.clear();
         }
     }
     async purge() {
         this.itemCount = 0;
-        this.datasetEntries.clear();
+        this.#datasetEntries.clear();
         this.updateTimestamps(true);
     }
     getData(options = {}) {
-        const { desc, limit, offset } = s
-            .object({
-            desc: s.boolean().optional(),
-            limit: s.number().int().optional(),
-            offset: s.number().int().optional(),
-        })
-            .parse(options);
+        const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
         return this.getDataPage({
             desc,
             offset: offset ?? 0,
@@ -67,7 +62,7 @@ export class DatasetBackend extends BaseClient {
         const items = [];
         for (let idx = start; idx < end; idx++) {
             const entryNumber = this.generateLocalEntryName(idx);
-            items.push(this.datasetEntries.get(entryNumber));
+            items.push(this.#datasetEntries.get(entryNumber));
         }
         this.updateTimestamps(false);
         return {
@@ -82,7 +77,7 @@ export class DatasetBackend extends BaseClient {
     async pushData(items) {
         for (const entry of items) {
             const idx = this.generateLocalEntryName(++this.itemCount);
-            this.datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
+            this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
         }
         this.updateTimestamps(true);
     }

package/memory-storage/resource-clients/key-value-store.d.ts

@@ -19,6 +19,7 @@ export interface InternalKeyRecord {
     extension: string;
 }
 export declare class KeyValueStoreBackend extends BaseClient implements storage.KeyValueStoreBackend {
+    #private;
     name?: string;
     /**
      * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
@@ -28,7 +29,6 @@ export declare class KeyValueStoreBackend extends BaseClient implements storage.
     createdAt: Date;
     accessedAt: Date;
     modifiedAt: Date;
-    private readonly keyValueEntries;
     private readonly storageBackend;
     constructor(options: KeyValueStoreBackendOptions);
     getMetadata(): Promise<storage.KeyValueStoreInfo>;

package/memory-storage/resource-clients/key-value-store.js

@@ -1,9 +1,25 @@
 import { randomUUID } from 'node:crypto';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { isStream, toBuffer } from '../utils.js';
 import { BaseClient } from './common/base-client.js';
 import mime from 'mime-types';
 const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
+const keySchema = z.string();
+const inputRecordSchema = z.object({
+    key: z.string().min(1),
+    value: z.union([
+        z.null(),
+        z.string(),
+        z.number(),
+        z.instanceof(Buffer),
+        z.instanceof(ArrayBuffer),
+        schemas.typedArray,
+        // only checks the value is an actual object, not null, nor array
+        schemas.plainObject,
+    ]),
+    contentType: z.string().min(1).optional(),
+});
 /**
  * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
  * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
@@ -19,7 +35,8 @@ export class KeyValueStoreBackend extends BaseClient {
     createdAt = new Date();
     accessedAt = new Date();
     modifiedAt = new Date();
-    keyValueEntries = new Map();
+    #keyValueEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
     storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
@@ -35,11 +52,11 @@ export class KeyValueStoreBackend extends BaseClient {
         const storeIndex = this.storageBackend.keyValueStoreBackendCache.findIndex((store) => store.id === this.id);
         if (storeIndex !== -1) {
             const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);
-            oldBackend.keyValueEntries.clear();
+            oldBackend.#keyValueEntries.clear();
         }
     }
     async purge() {
-        this.keyValueEntries.clear();
+        this.#keyValueEntries.clear();
         this.updateTimestamps(true);
     }
     /**
@@ -49,23 +66,17 @@ export class KeyValueStoreBackend extends BaseClient {
      * preserve the bare `INPUT` key only.
      */
     async purgeExceptInput() {
-        for (const key of this.keyValueEntries.keys()) {
+        for (const key of this.#keyValueEntries.keys()) {
             if (key !== KEY_VALUE_STORE_INPUT_KEY) {
-                this.keyValueEntries.delete(key);
+                this.#keyValueEntries.delete(key);
             }
         }
         this.updateTimestamps(true);
     }
     async listKeys(options = {}) {
-        const { prefix, exclusiveStartKey, limit } = s
-            .object({
-            prefix: s.string().optional(),
-            exclusiveStartKey: s.string().optional(),
-            limit: s.number().int().greaterThan(0).optional(),
-        })
-            .parse(options);
+        const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
         const items = [];
-        for (const record of this.keyValueEntries.values()) {
+        for (const record of this.#keyValueEntries.values()) {
             const size = Buffer.byteLength(record.value);
             items.push({
                 key: record.key,
@@ -103,7 +114,7 @@ export class KeyValueStoreBackend extends BaseClient {
      * @param key The key of the record to generate the public URL for.
      */
     async getPublicUrl(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         return undefined;
     }
     /**
@@ -113,12 +124,12 @@ export class KeyValueStoreBackend extends BaseClient {
      * @returns `true` if the record exists, `false` if it does not.
      */
     async recordExists(key) {
-        s.string().parse(key);
-        return this.keyValueEntries.has(key);
+        parseArgument(key, keySchema);
+        return this.#keyValueEntries.has(key);
     }
     async getValue(key) {
-        s.string().parse(key);
-        const entry = this.keyValueEntries.get(key);
+        parseArgument(key, keySchema);
+        const entry = this.#keyValueEntries.get(key);
         if (!entry) {
             return undefined;
         }
@@ -135,20 +146,7 @@ export class KeyValueStoreBackend extends BaseClient {
         return record;
     }
     async setValue(record) {
-        s.object({
-            key: s.string().lengthGreaterThan(0),
-            value: s.union([
-                s.null(),
-                s.string(),
-                s.number(),
-                s.instance(Buffer),
-                s.instance(ArrayBuffer),
-                s.typedArray(),
-                // disabling validation will make shapeshift only check the object given is an actual object, not null, nor array
-                s.object({}).setValidationEnabled(false),
-            ]),
-            contentType: s.string().lengthGreaterThan(0).optional(),
-        }).parse(record);
+        parseArgument(record, inputRecordSchema);
         const { key } = record;
         let { value } = record;
         // The frontend (KeyValueStore codec) serializes the value and resolves its content type
@@ -169,19 +167,19 @@ export class KeyValueStoreBackend extends BaseClient {
         const normalizedValue = typeof value === 'string'
             ? Buffer.from(value, 'utf-8')
             : toBuffer(value);
-        const _record = {
+        const normalizedRecord = {
             extension,
             key,
             value: normalizedValue,
             contentType,
         };
-        this.keyValueEntries.set(key, _record);
+        this.#keyValueEntries.set(key, normalizedRecord);
         this.updateTimestamps(true);
     }
     async deleteValue(key) {
-        s.string().parse(key);
-        if (this.keyValueEntries.has(key)) {
-            this.keyValueEntries.delete(key);
+        parseArgument(key, keySchema);
+        if (this.#keyValueEntries.has(key)) {
+            this.#keyValueEntries.delete(key);
             this.updateTimestamps(true);
         }
     }

package/memory-storage/resource-clients/request-queue.d.ts

@@ -22,6 +22,7 @@ export interface InternalRequest {
     json: string;
 }
 export declare class RequestQueueBackend extends BaseClient implements storage.RequestQueueBackend {
+    #private;
     name?: string;
     /**
      * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
@@ -33,26 +34,6 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
     modifiedAt: Date;
     handledRequestCount: number;
     pendingRequestCount: number;
-    /**
-     * Serializes every operation that reads-then-writes this backend's shared queue state — the
-     * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
-     * counts. Those mutations span `await` points, so without this mutex a concurrent operation could
-     * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
-     * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
-     * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
-     */
-    private readonly queueStateMutex;
-    private forefrontRequestIds;
-    /**
-     * IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
-     * "in progress" and will not be handed out again by {@link fetchNextRequest}.
-     *
-     * Unlike the file-system / platform clients, the in-memory queue lives entirely within a single
-     * process and is never shared with another consumer, so there is no need for an expiring,
-     * cross-process-visible lock — tracking in-progress requests in this set is enough.
-     */
-    private readonly inProgressRequestIds;
-    private readonly requests;
     private readonly storageBackend;
     constructor(options: RequestQueueBackendOptions);
     getMetadata(): Promise<storage.RequestQueueInfo>;
@@ -90,7 +71,7 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
     listItems(): Promise<storage.UpdateRequestSchema[]>;
     toRequestQueueInfo(): storage.RequestQueueInfo;
     private updateTimestamps;
-    private _jsonToRequest;
-    private _createInternalRequest;
-    private _calculateOrderNo;
+    private jsonToRequest;
+    private createInternalRequest;
+    private calculateOrderNo;
 }