@crawlee/core 4.0.0-beta.134 → 4.0.0-beta.136

package/enqueue_links/shared.d.ts

@@ -2,7 +2,6 @@ import type { Awaitable, Dictionary } from '@crawlee/types';
 import { z } from 'zod';
 import type { RequestOptions } from '../request.js';
 import type { EnqueueStrategyOption } from './enqueue_links.js';
-export { tryAbsoluteURL } from '@crawlee/utils/internal';
 export interface UrlPatternObject {
     glob?: string;
     regexp?: RegExp;

package/enqueue_links/shared.js

@@ -2,7 +2,6 @@ import { URL } from 'node:url';
 import { Minimatch } from 'minimatch';
 import { z } from 'zod';
 import { schemas } from '../validators.js';
-export { tryAbsoluteURL } from '@crawlee/utils/internal';
 const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
 /**
  * To keep high performance when the same patterns are passed on every `enqueueLinks()` call,

package/index.d.ts

@@ -15,7 +15,7 @@ export * from './serialization.js';
 export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './memory-storage/index.js';
-export * from './validators.js';
+export { ArgumentValidationError, validators } from './validators.js';
 export * from './cookie_utils.js';
 export * from './http.js';
 export * from './recoverable_state.js';

package/index.js

@@ -15,7 +15,9 @@ export * from './serialization.js';
 export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './memory-storage/index.js';
-export * from './validators.js';
+// Not `export *`: the rest of the module re-exports `@crawlee/utils/internal` symbols, which carry no
+// semver guarantees and must not reach the public surface. Internal consumers import them directly.
+export { ArgumentValidationError, validators } from './validators.js';
 export * from './cookie_utils.js';
 export * from './http.js';
 export * from './recoverable_state.js';

package/memory-storage/memory-storage.d.ts

@@ -1,7 +1,5 @@
 import type * as storage from '@crawlee/types';
 import type { CrawleeLogger } from '@crawlee/types';
-import { DatasetBackend } from './resource-clients/dataset.js';
-import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
 import { RequestQueueBackend } from './resource-clients/request-queue.js';
 export interface MemoryStorageOptions {
     /**
@@ -12,15 +10,23 @@ export interface MemoryStorageOptions {
 export declare class MemoryStorageBackend implements storage.StorageBackend {
     #private;
     readonly logger?: CrawleeLogger;
-    readonly keyValueStoreBackendCache: KeyValueStoreBackend[];
-    readonly datasetBackendCache: DatasetBackend[];
-    readonly requestQueueBackendCache: RequestQueueBackend[];
     constructor(options?: MemoryStorageOptions);
     /**
      * Return a per-instance unique cache key so that distinct `MemoryStorageBackend` instances get separate
      * cache partitions in the storage backend cache.
      */
     getStorageBackendCacheKey(): string;
+    /**
+     * Evict a cached backend so that a dropped storage is no longer resolved by `createXBackend`,
+     * reported by `storageExists` or visited by `purge`. Returns whether the backend was cached, which
+     * tells the caller whether it still owns in-memory state worth clearing.
+     *
+     * The resource clients own their own entry's lifetime but must not reach into the caches directly.
+     * Because a client is only ever constructed by `createXBackend`, which caches it immediately, the
+     * entry matching `id` is always the caller itself.
+     * @internal
+     */
+    evictBackend(type: 'Dataset' | 'KeyValueStore' | 'RequestQueue', id: string): boolean;
     createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
     createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
     createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<RequestQueueBackend>;

package/memory-storage/memory-storage.js

@@ -11,9 +11,9 @@ export class MemoryStorageBackend {
      * cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.
      */
     #instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
-    keyValueStoreBackendCache = [];
-    datasetBackendCache = [];
-    requestQueueBackendCache = [];
+    #keyValueStoreBackendCache = [];
+    #datasetBackendCache = [];
+    #requestQueueBackendCache = [];
     constructor(options = {}) {
         this.logger = options.logger;
     }
@@ -24,6 +24,36 @@ export class MemoryStorageBackend {
     getStorageBackendCacheKey() {
         return this.#instanceCacheKey;
     }
+    /**
+     * Evict a cached backend so that a dropped storage is no longer resolved by `createXBackend`,
+     * reported by `storageExists` or visited by `purge`. Returns whether the backend was cached, which
+     * tells the caller whether it still owns in-memory state worth clearing.
+     *
+     * The resource clients own their own entry's lifetime but must not reach into the caches directly.
+     * Because a client is only ever constructed by `createXBackend`, which caches it immediately, the
+     * entry matching `id` is always the caller itself.
+     * @internal
+     */
+    evictBackend(type, id) {
+        let cache;
+        switch (type) {
+            case 'Dataset':
+                cache = this.#datasetBackendCache;
+                break;
+            case 'KeyValueStore':
+                cache = this.#keyValueStoreBackendCache;
+                break;
+            case 'RequestQueue':
+                cache = this.#requestQueueBackendCache;
+                break;
+        }
+        const index = cache.findIndex((entry) => entry.id === id);
+        if (index === -1) {
+            return false;
+        }
+        cache.splice(index, 1);
+        return true;
+    }
     static #resolveStorageKey(options) {
         // No identifier at all means the default storage, which is opened under the reserved alias —
         // same rule as `resolveStorageIdentifier` in the storage frontends, so that a backend used
@@ -37,7 +67,7 @@ export class MemoryStorageBackend {
     }
     async createDatasetBackend(options = {}) {
         const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
-        const found = this.datasetBackendCache.find((store) => store.id === cacheKey ||
+        const found = this.#datasetBackendCache.find((store) => store.id === cacheKey ||
             store.name?.toLowerCase() === cacheKey.toLowerCase() ||
             store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
         if (found) {
@@ -48,12 +78,12 @@ export class MemoryStorageBackend {
             cacheKey,
             storageBackend: this,
         });
-        this.datasetBackendCache.push(newStore);
+        this.#datasetBackendCache.push(newStore);
         return newStore;
     }
     async createKeyValueStoreBackend(options = {}) {
         const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
-        const found = this.keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
+        const found = this.#keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
             store.name?.toLowerCase() === cacheKey.toLowerCase() ||
             store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
         if (found) {
@@ -64,12 +94,12 @@ export class MemoryStorageBackend {
             cacheKey,
             storageBackend: this,
         });
-        this.keyValueStoreBackendCache.push(newStore);
+        this.#keyValueStoreBackendCache.push(newStore);
         return newStore;
     }
     async createRequestQueueBackend(options = {}) {
         const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
-        const found = this.requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
+        const found = this.#requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
             queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
             queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
         if (found) {
@@ -80,20 +110,20 @@ export class MemoryStorageBackend {
             cacheKey,
             storageBackend: this,
         });
-        this.requestQueueBackendCache.push(newStore);
+        this.#requestQueueBackendCache.push(newStore);
         return newStore;
     }
     async storageExists(id, type) {
         let backends;
         switch (type) {
             case 'Dataset':
-                backends = this.datasetBackendCache;
+                backends = this.#datasetBackendCache;
                 break;
             case 'KeyValueStore':
-                backends = this.keyValueStoreBackendCache;
+                backends = this.#keyValueStoreBackendCache;
                 break;
             case 'RequestQueue':
-                backends = this.requestQueueBackendCache;
+                backends = this.#requestQueueBackendCache;
                 break;
             default:
                 return false;
@@ -115,9 +145,9 @@ export class MemoryStorageBackend {
         };
         await Promise.all([
             // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
-            purgeRunScoped(this.keyValueStoreBackendCache, async (store) => isDefault(store) ? store.purgeExceptInput() : store.purge()),
-            purgeRunScoped(this.datasetBackendCache, async (store) => store.purge()),
-            purgeRunScoped(this.requestQueueBackendCache, async (store) => store.purge()),
+            purgeRunScoped(this.#keyValueStoreBackendCache, async (store) => isDefault(store) ? store.purgeExceptInput() : store.purge()),
+            purgeRunScoped(this.#datasetBackendCache, async (store) => store.purge()),
+            purgeRunScoped(this.#requestQueueBackendCache, async (store) => store.purge()),
         ]);
     }
     /**

package/memory-storage/resource-clients/dataset.js

@@ -36,11 +36,9 @@ export class DatasetBackend extends BaseClient {
         return this.toDatasetInfo();
     }
     async drop() {
-        const storeIndex = this.storageBackend.datasetBackendCache.findIndex((store) => store.id === this.id);
-        if (storeIndex !== -1) {
-            const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);
-            oldBackend.itemCount = 0;
-            oldBackend.#datasetEntries.clear();
+        if (this.storageBackend.evictBackend('Dataset', this.id)) {
+            this.itemCount = 0;
+            this.#datasetEntries.clear();
         }
     }
     async purge() {

package/memory-storage/resource-clients/key-value-store.js

@@ -49,10 +49,8 @@ export class KeyValueStoreBackend extends BaseClient {
         return this.toKeyValueStoreInfo();
     }
     async drop() {
-        const storeIndex = this.storageBackend.keyValueStoreBackendCache.findIndex((store) => store.id === this.id);
-        if (storeIndex !== -1) {
-            const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);
-            oldBackend.#keyValueEntries.clear();
+        if (this.storageBackend.evictBackend('KeyValueStore', this.id)) {
+            this.#keyValueEntries.clear();
         }
     }
     async purge() {

package/memory-storage/resource-clients/request-queue.js

@@ -55,16 +55,14 @@ export class RequestQueueBackend extends BaseClient {
         // removed, which `listPendingHead` would then dereference as `undefined`.
         await this.#queueStateMutex.wait();
         try {
-            const storeIndex = this.storageBackend.requestQueueBackendCache.findIndex((queue) => queue.id === this.id);
-            if (storeIndex !== -1) {
-                const [oldBackend] = this.storageBackend.requestQueueBackendCache.splice(storeIndex, 1);
-                oldBackend.pendingRequestCount = 0;
+            if (this.storageBackend.evictBackend('RequestQueue', this.id)) {
+                this.pendingRequestCount = 0;
                 // Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would
                 // leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head
                 // scan would resolve to a missing request and dereference.
-                oldBackend.#requests.clear();
-                oldBackend.#forefrontRequestIds = [];
-                oldBackend.#inProgressRequestIds.clear();
+                this.#requests.clear();
+                this.#forefrontRequestIds = [];
+                this.#inProgressRequestIds.clear();
             }
         }
         finally {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.134",
+    "version": "4.0.0-beta.136",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.134",
-        "@crawlee/http-client": "4.0.0-beta.134",
-        "@crawlee/types": "4.0.0-beta.134",
-        "@crawlee/utils": "4.0.0-beta.134",
+        "@crawlee/fs-storage": "4.0.0-beta.136",
+        "@crawlee/http-client": "4.0.0-beta.136",
+        "@crawlee/types": "4.0.0-beta.136",
+        "@crawlee/utils": "4.0.0-beta.136",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "0f269d5b24ec2829f3d44c19a35973fa20d95999"
+    "gitHead": "d425bb9841885b4b71b84ae6fb0bafbdc4a7e978"
 }