@crawlee/fs-storage 4.0.0-beta.104 → 4.0.0-beta.106

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/fs-storage",
-    "version": "4.0.0-beta.104",
+    "version": "4.0.0-beta.106",
     "description": "A file-system storage implementation of the Apify API",
     "engines": {
         "node": ">=22.0.0"
@@ -43,7 +43,7 @@
     },
     "dependencies": {
         "@crawlee/fs-storage-native": "0.1.5-beta.18",
-        "@crawlee/types": "4.0.0-beta.104",
+        "@crawlee/types": "4.0.0-beta.106",
         "@sapphire/shapeshift": "^4.0.0"
     },
     "lerna": {
@@ -53,5 +53,5 @@
             }
         }
     },
-    "gitHead": "1c761e689e56ac06b67fc4f1dc091f693e024d12"
+    "gitHead": "38186ba9ec1ab456b6c71ef9a9d45e675a9cd1e5"
 }

package/resource-clients/dataset.d.ts

@@ -21,10 +21,9 @@ export interface DatasetBackendOptions {
  * the `@crawlee/types` interfaces.
  */
 export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends CachedIdClient implements storage.DatasetBackend<Data> {
+    #private;
     readonly name?: string;
     readonly cacheKey: string;
-    private readonly nativeBackend;
-    private readonly logger?;
     constructor(options: DatasetBackendOptions);
     get datasetDirectory(): string;
     static create<Data extends Dictionary = Dictionary>(options: DatasetBackendOptions): Promise<DatasetBackend<Data>>;

package/resource-clients/dataset.js

@@ -19,17 +19,17 @@ const UNSUPPORTED_GET_DATA_OPTIONS = ['clean', 'fields', 'omit', 'skipHidden', '
 export class DatasetBackend extends CachedIdClient {
     name;
     cacheKey;
-    nativeBackend;
-    logger;
+    #nativeBackend;
+    #logger;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
-        this.nativeBackend = options.nativeBackend;
-        this.logger = options.logger;
+        this.#nativeBackend = options.nativeBackend;
+        this.#logger = options.logger;
     }
     get datasetDirectory() {
-        return this.nativeBackend.pathToDataset;
+        return this.#nativeBackend.pathToDataset;
     }
     static async create(options) {
         const backend = new DatasetBackend(options);
@@ -37,22 +37,22 @@ export class DatasetBackend extends CachedIdClient {
         return backend;
     }
     async getMetadata() {
-        return this.nativeBackend.getMetadata();
+        return this.#nativeBackend.getMetadata();
     }
     async drop() {
-        await this.nativeBackend.dropStorage();
+        await this.#nativeBackend.dropStorage();
     }
     async purge() {
-        await this.nativeBackend.purge();
+        await this.#nativeBackend.purge();
     }
     async pushData(items) {
-        await this.nativeBackend.pushData(items);
+        await this.#nativeBackend.pushData(items);
     }
     async getData(options = {}) {
         const passedOptions = options;
         const ignored = UNSUPPORTED_GET_DATA_OPTIONS.filter((key) => passedOptions[key] !== undefined);
         if (ignored.length > 0) {
-            this.logger?.warning?.(`getData() options [${ignored.join(', ')}] are not supported by the file-system dataset ` +
+            this.#logger?.warning?.(`getData() options [${ignored.join(', ')}] are not supported by the file-system dataset ` +
                 `and were ignored. Only "offset", "limit" and "desc" are honored.`);
         }
         const { desc, limit, offset } = s
@@ -62,7 +62,7 @@ export class DatasetBackend extends CachedIdClient {
             offset: s.number().int().optional(),
         })
             .parse(options);
-        const page = await this.nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);
+        const page = await this.#nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);
         return {
             count: page.count,
             desc: page.desc,

package/resource-clients/key-value-store.d.ts

@@ -22,9 +22,9 @@ export interface KeyValueStoreBackendOptions {
  * is the {@link KeyValueStore} frontend codec's job, not this backend's.
  */
 export declare class KeyValueStoreBackend extends CachedIdClient implements storage.KeyValueStoreBackend {
+    #private;
     readonly name?: string;
     readonly cacheKey: string;
-    private readonly nativeBackend;
     constructor(options: KeyValueStoreBackendOptions);
     get keyValueStoreDirectory(): string;
     static create(options: KeyValueStoreBackendOptions): Promise<KeyValueStoreBackend>;

package/resource-clients/key-value-store.js

@@ -46,15 +46,15 @@ const BARE_FILE_LOGICAL_KEYS = new Map(ALLOWED_BARE_FILES.flatMap((key) => BARE_
 export class KeyValueStoreBackend extends CachedIdClient {
     name;
     cacheKey;
-    nativeBackend;
+    #nativeBackend;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
-        this.nativeBackend = options.nativeBackend;
+        this.#nativeBackend = options.nativeBackend;
     }
     get keyValueStoreDirectory() {
-        return this.nativeBackend.pathToKvs;
+        return this.#nativeBackend.pathToKvs;
     }
     static async create(options) {
         const backend = new KeyValueStoreBackend(options);
@@ -62,13 +62,13 @@ export class KeyValueStoreBackend extends CachedIdClient {
         return backend;
     }
     async getMetadata() {
-        return this.nativeBackend.getMetadata();
+        return this.#nativeBackend.getMetadata();
     }
     async drop() {
-        await this.nativeBackend.dropStorage();
+        await this.#nativeBackend.dropStorage();
     }
     async purge() {
-        await this.nativeBackend.purge();
+        await this.#nativeBackend.purge();
     }
     /**
      * Remove every record from the store except the run input. Used by
@@ -79,7 +79,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
      * filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).
      */
     async purgeExceptInput() {
-        await this.nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));
+        await this.#nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));
     }
     async listKeys(options = {}) {
         const { prefix, exclusiveStartKey, limit } = s
@@ -94,7 +94,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         // everything it needs off the filesystem index — no per-file reads — so this stays cheap.
         // The native `listKeys` already returns a self-describing page (items + pagination cursors)
         // matching the `KeyValueStoreListKeysResult` contract, so we only post-process the items.
-        const page = await this.nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);
+        const page = await this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);
         const presentKeys = new Set(page.items.map((record) => record.key));
         // A bare value file is listed under its actual name (`INPUT.json`), which already round-trips
         // through `getValue`/`recordExists`. The only collision is a tracked record occupying the
@@ -130,7 +130,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         if (resolvedKey === undefined) {
             return undefined;
         }
-        return (await this.nativeBackend.getPublicUrl(resolvedKey)) ?? undefined;
+        return (await this.#nativeBackend.getPublicUrl(resolvedKey)) ?? undefined;
     }
     /**
      * Tests whether a record with the given key exists without retrieving its value.
@@ -146,8 +146,8 @@ export class KeyValueStoreBackend extends CachedIdClient {
         s.string().parse(key);
         const fallbacks = this.bareFallbacksFor(key);
         const record = fallbacks
-            ? await this.nativeBackend.resolveValue(key, fallbacks)
-            : await this.nativeBackend.getValue(key);
+            ? await this.#nativeBackend.resolveValue(key, fallbacks)
+            : await this.#nativeBackend.getValue(key);
         if (record) {
             return {
                 key: record.key,
@@ -183,7 +183,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         // consumes a Web `ReadableStream`, so convert the Node `Readable` we get from the frontend.
         if (isStream(value)) {
             const webStream = Readable.toWeb(value);
-            await this.nativeBackend.setValueStream(key, webStream, contentType);
+            await this.#nativeBackend.setValueStream(key, webStream, contentType);
             return;
         }
         // Normalize the remaining (already-serialized) value into a Buffer for the native client.
@@ -194,11 +194,11 @@ export class KeyValueStoreBackend extends CachedIdClient {
                 : ArrayBuffer.isView(value)
                     ? Buffer.from(value.buffer, value.byteOffset, value.byteLength)
                     : Buffer.from(value);
-        await this.nativeBackend.setValue(key, buffer, contentType);
+        await this.#nativeBackend.setValue(key, buffer, contentType);
     }
     async deleteValue(key) {
         s.string().parse(key);
-        await this.nativeBackend.deleteValue(key);
+        await this.#nativeBackend.deleteValue(key);
     }
     /**
      * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Every
@@ -211,9 +211,9 @@ export class KeyValueStoreBackend extends CachedIdClient {
     async resolveExistingKey(key) {
         const fallbacks = this.bareFallbacksFor(key);
         if (fallbacks) {
-            return ((await this.nativeBackend.resolveExistingKey(key, fallbacks.map(({ extension }) => extension))) ?? undefined);
+            return ((await this.#nativeBackend.resolveExistingKey(key, fallbacks.map(({ extension }) => extension))) ?? undefined);
         }
-        return (await this.nativeBackend.recordExists(key)) ? key : undefined;
+        return (await this.#nativeBackend.recordExists(key)) ? key : undefined;
     }
     /**
      * The native `resolveValue`/`resolveExistingKey` bare-file fallbacks to use for `key`, or

package/resource-clients/request-queue.d.ts

@@ -21,9 +21,9 @@ export interface RequestQueueBackendOptions {
  * This adapter forwards each operation and converts result shapes to the `@crawlee/types` interfaces.
  */
 export declare class RequestQueueBackend extends CachedIdClient implements storage.RequestQueueBackend {
+    #private;
     readonly name?: string;
     readonly cacheKey: string;
-    private readonly nativeBackend;
     constructor(options: RequestQueueBackendOptions);
     get requestQueueDirectory(): string;
     static create(options: RequestQueueBackendOptions): Promise<RequestQueueBackend>;

package/resource-clients/request-queue.js

@@ -40,15 +40,15 @@ const requestOptionsShape = s.object({
 export class RequestQueueBackend extends CachedIdClient {
     name;
     cacheKey;
-    nativeBackend;
+    #nativeBackend;
     constructor(options) {
         super();
         this.name = options.name;
         this.cacheKey = options.cacheKey;
-        this.nativeBackend = options.nativeBackend;
+        this.#nativeBackend = options.nativeBackend;
     }
     get requestQueueDirectory() {
-        return this.nativeBackend.pathToRq;
+        return this.#nativeBackend.pathToRq;
     }
     static async create(options) {
         const backend = new RequestQueueBackend(options);
@@ -60,21 +60,21 @@ export class RequestQueueBackend extends CachedIdClient {
      * available again.
      */
     async setExpectedRequestProcessingTimeSecs(secs) {
-        await this.nativeBackend.setExpectedRequestProcessingTime(secs);
+        await this.#nativeBackend.setExpectedRequestProcessingTime(secs);
     }
     async getMetadata() {
-        return this.nativeBackend.getMetadata();
+        return this.#nativeBackend.getMetadata();
     }
     async drop() {
-        await this.nativeBackend.dropStorage();
+        await this.#nativeBackend.dropStorage();
     }
     async purge() {
-        await this.nativeBackend.purge();
+        await this.#nativeBackend.purge();
     }
     async addBatchOfRequests(requests, options = {}) {
         batchRequestShapeWithoutId.parse(requests);
         requestOptionsShape.parse(options);
-        const response = await this.nativeBackend.addBatchOfRequests(requests.map((request) => plainifyRequest(request)), options.forefront ?? false);
+        const response = await this.#nativeBackend.addBatchOfRequests(requests.map((request) => plainifyRequest(request)), options.forefront ?? false);
         // `processedRequests` is structurally identical between the native and `storage` types, so it
         // passes through unchanged. `unprocessedRequests` only differs in that the native `method` is
         // a plain `string`, hence the cast to the narrower `AllowedHttpMethods` union.
@@ -88,25 +88,26 @@ export class RequestQueueBackend extends CachedIdClient {
         // The native client tags requests with an internal `orderNo`; it's harmless to leak, so we
         // hand the request back as-is rather than copying it just to drop one undeclared property.
         // The native client already returns `undefined` for a missing request, matching this contract.
-        return (await this.nativeBackend.getRequest(uniqueKey));
+        return (await this.#nativeBackend.getRequest(uniqueKey));
     }
     async fetchNextRequest() {
-        return (await this.nativeBackend.fetchNextRequest());
+        return (await this.#nativeBackend.fetchNextRequest());
     }
     async markRequestAsHandled(request) {
         requestShape.parse(request);
-        return (await this.nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;
+        return (await this.#nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;
     }
     async reclaimRequest(request, options = {}) {
         requestShape.parse(request);
         requestOptionsShape.parse(options);
-        return ((await this.nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ?? undefined);
+        return ((await this.#nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ??
+            undefined);
     }
     async isEmpty() {
-        return this.nativeBackend.isEmpty();
+        return this.#nativeBackend.isEmpty();
     }
     async isFinished() {
-        return this.nativeBackend.isFinished();
+        return this.#nativeBackend.isFinished();
     }
     /**
      * Persist the native client's in-memory state to disk. Called by
@@ -114,6 +115,6 @@ export class RequestQueueBackend extends CachedIdClient {
      * for the next consumer of the same on-disk queue.
      */
     async persistState() {
-        await this.nativeBackend.persistState();
+        await this.#nativeBackend.persistState();
     }
 }