@crawlee/fs-storage 4.0.0-beta.120 → 4.0.0-beta.122

package/file-system-storage.js

@@ -1,10 +1,24 @@
 import { opendir, readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
-import { s } from '@sapphire/shapeshift';
-import { FileSystemDatasetClient as NativeDatasetBackend, FileSystemKeyValueStoreClient as NativeKeyValueStoreBackend, FileSystemRequestQueueClient as NativeRequestQueueBackend, } from '@crawlee/fs-storage-native';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { DatasetBackend } from './resource-clients/dataset.js';
 import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
 import { RequestQueueBackend } from './resource-clients/request-queue.js';
+const fileSystemStorageOptionsSchema = z.object({
+    localDataDirectory: z.string(),
+    requestQueueAccess: z.enum(['single', 'shared']).default('single'),
+    logger: schemas.logger.optional(),
+});
+// The native package throws at load time on platforms without a published binary (e.g.
+// linux musl), and `@crawlee/core` imports this module eagerly via its service locator.
+// Load it lazily so merely importing `@crawlee/fs-storage` stays safe everywhere and the
+// native binding is only loaded when a file-system storage is actually used.
+let nativeModule;
+async function importNativeModule() {
+    nativeModule ??= import('@crawlee/fs-storage-native');
+    return nativeModule;
+}
 /** The alias `@crawlee/core` opens the default (unnamed) storage under. */
 const DEFAULT_STORAGE_ALIAS = '__default__';
 /** The directory the default storage lives in, one level below `datasets` / `key_value_stores` / etc. */
@@ -28,13 +42,10 @@ export class FileSystemStorageBackend {
     datasetBackendCache = [];
     requestQueueBackendCache = [];
     constructor(options) {
-        s.object({
-            localDataDirectory: s.string(),
-            requestQueueAccess: s.enum(['single', 'shared']).optional(),
-        }).parse(options);
-        this.logger = options.logger;
-        this.requestQueueAccess = options.requestQueueAccess ?? 'single';
-        this.localDataDirectory = options.localDataDirectory;
+        const { logger, requestQueueAccess, localDataDirectory } = parseArgument(options, fileSystemStorageOptionsSchema);
+        this.logger = logger;
+        this.requestQueueAccess = requestQueueAccess;
+        this.localDataDirectory = localDataDirectory;
         this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets');
         this.keyValueStoresDirectory = resolve(this.localDataDirectory, 'key_value_stores');
         this.requestQueuesDirectory = resolve(this.localDataDirectory, 'request_queues');
@@ -68,7 +79,7 @@ export class FileSystemStorageBackend {
         if (found) {
             return found;
         }
-        const nativeBackend = await NativeDatasetBackend.open(id, name, alias, this.localDataDirectory);
+        const nativeBackend = await (await importNativeModule()).FileSystemDatasetClient.open(id, name, alias, this.localDataDirectory);
         const newStore = await DatasetBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
             cacheKey,
@@ -86,7 +97,7 @@ export class FileSystemStorageBackend {
         if (found) {
             return found;
         }
-        const nativeBackend = await NativeKeyValueStoreBackend.open(id, name, alias, this.localDataDirectory);
+        const nativeBackend = await (await importNativeModule()).FileSystemKeyValueStoreClient.open(id, name, alias, this.localDataDirectory);
         const newStore = await KeyValueStoreBackend.create({
             name: alias ? undefined : (name ?? cacheKey),
             cacheKey,
@@ -104,7 +115,7 @@ export class FileSystemStorageBackend {
         if (found) {
             return found;
         }
-        const nativeBackend = await NativeRequestQueueBackend.open(id, name, alias, this.localDataDirectory, 
+        const nativeBackend = await (await importNativeModule()).FileSystemRequestQueueClient.open(id, name, alias, this.localDataDirectory, 
         // useTestClock — always real wall-clock outside of native tests.
         undefined, this.requestQueueAccess);
         const newStore = await RequestQueueBackend.create({

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/fs-storage",
-    "version": "4.0.0-beta.120",
+    "version": "4.0.0-beta.122",
     "description": "A file-system storage implementation of the Apify API",
     "engines": {
         "node": ">=22.0.0"
@@ -43,8 +43,9 @@
     },
     "dependencies": {
         "@crawlee/fs-storage-native": "0.1.5-beta.18",
-        "@crawlee/types": "4.0.0-beta.120",
-        "@sapphire/shapeshift": "^4.0.0"
+        "@crawlee/types": "4.0.0-beta.122",
+        "@crawlee/utils": "4.0.0-beta.122",
+        "zod": "^4.4.3"
     },
     "lerna": {
         "command": {
@@ -53,5 +54,5 @@
             }
         }
     },
-    "gitHead": "da5d427c4c1c9dcaea95b151e9c4c7310100885d"
+    "gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
 }

package/resource-clients/dataset.js

@@ -1,4 +1,4 @@
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { CachedIdClient } from './cached-id-client.js';
 /**
  * `getData` options accepted by the high-level `Dataset` frontend but not supported by the native
@@ -55,13 +55,7 @@ export class DatasetBackend extends CachedIdClient {
             this.#logger?.warning?.(`getData() options [${ignored.join(', ')}] are not supported by the file-system dataset ` +
                 `and were ignored. Only "offset", "limit" and "desc" are honored.`);
         }
-        const { desc, limit, offset } = s
-            .object({
-            desc: s.boolean().optional(),
-            limit: s.number().int().optional(),
-            offset: s.number().int().optional(),
-        })
-            .parse(options);
+        const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
         const page = await this.#nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);
         return {
             count: page.count,

package/resource-clients/key-value-store.js

@@ -1,5 +1,6 @@
 import { Readable } from 'node:stream';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { isStream } from '../utils.js';
 import { CachedIdClient } from './cached-id-client.js';
 /**
@@ -18,6 +19,20 @@ const BARE_FILE_FALLBACKS = [
     { extension: '.bin', contentType: '' },
 ];
 const ALLOWED_BARE_FILES = ['INPUT'];
+const keySchema = z.string();
+const inputRecordShape = z.object({
+    key: z.string().min(1),
+    value: z.union([
+        z.string(),
+        z.instanceof(Buffer),
+        z.instanceof(ArrayBuffer),
+        schemas.typedArray,
+        // A stream is an object; this only checks it is a non-null, non-array object
+        // (the stream guard in `setValue` does the real check).
+        schemas.plainObject,
+    ]),
+    contentType: z.string().min(1).optional(),
+});
 /**
  * The out-of-band ("bare") files to surface from the native `listKeys`, derived from
  * {@link ALLOWED_BARE_FILES} × {@link BARE_FILE_FALLBACKS}. Each native {@link ListBareFallback}
@@ -82,13 +97,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         await this.#nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));
     }
     async listKeys(options = {}) {
-        const { prefix, exclusiveStartKey, limit } = s
-            .object({
-            prefix: s.string().optional(),
-            exclusiveStartKey: s.string().optional(),
-            limit: s.number().int().greaterThan(0).optional(),
-        })
-            .parse(options);
+        const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
         // Pass the bare-file fallbacks so out-of-band value files (e.g. a hand-placed `INPUT.json`)
         // are enumerated alongside tracked records, under their actual on-disk name. The native reads
         // everything it needs off the filesystem index — no per-file reads — so this stays cheap.
@@ -122,7 +131,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
      * @param key The key of the record to generate the public URL for.
      */
     async getPublicUrl(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         // The native `getPublicUrl` stats the encoded path but does not probe bare-file extensions,
         // so we resolve the on-disk key first (handling e.g. `INPUT` -> `INPUT.json`) and normalize
         // the native `null`-for-missing result to the historical `undefined` contract.
@@ -139,11 +148,11 @@ export class KeyValueStoreBackend extends CachedIdClient {
      * @returns `true` if the record exists, `false` otherwise.
      */
     async recordExists(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         return (await this.resolveExistingKey(key)) !== undefined;
     }
     async getValue(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         const fallbacks = this.bareFallbacksFor(key);
         const record = fallbacks
             ? await this.#nativeBackend.resolveValue(key, fallbacks)
@@ -162,19 +171,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         // serialized it: non-bytes become a `string`, everything else is a `Buffer`/typed array or a
         // stream. So we only accept those shapes here — there is no JSON inference or `String(value)`
         // coercion left to do.
-        s.object({
-            key: s.string().lengthGreaterThan(0),
-            value: s.union([
-                s.string(),
-                s.instance(Buffer),
-                s.instance(ArrayBuffer),
-                s.typedArray(),
-                // A stream is an object; disabling validation makes shapeshift only check it is a
-                // non-null, non-array object (the stream guard below does the real check).
-                s.object({}).setValidationEnabled(false),
-            ]),
-            contentType: s.string().lengthGreaterThan(0).optional(),
-        }).parse(record);
+        parseArgument(record, inputRecordShape);
         const { key, value } = record;
         // The frontend resolves the content type before it reaches the backend; this backend is a plain
         // byte transport and does not infer content types.
@@ -197,7 +194,7 @@ export class KeyValueStoreBackend extends CachedIdClient {
         await this.#nativeBackend.setValue(key, buffer, contentType);
     }
     async deleteValue(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         await this.#nativeBackend.deleteValue(key);
     }
     /**

package/resource-clients/request-queue.js

@@ -1,5 +1,7 @@
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { CachedIdClient } from './cached-id-client.js';
+const uniqueKeySchema = z.string();
 /**
  * Convert a request (either a Crawlee `Request` instance or a plain schema object) into a plain object
  * whose properties are all enumerable.
@@ -15,21 +17,6 @@ import { CachedIdClient } from './cached-id-client.js';
 function plainifyRequest(request) {
     return JSON.parse(JSON.stringify(request));
 }
-const requestShape = s
-    .object({
-    id: s.string(),
-    url: s.string().url({ allowedProtocols: ['http:', 'https:'] }),
-    uniqueKey: s.string(),
-    method: s.string().optional(),
-    retryCount: s.number().int().optional(),
-    handledAt: s.union([s.string(), s.date().valid()]).optional(),
-})
-    .passthrough();
-const requestShapeWithoutId = requestShape.omit(['id']);
-const batchRequestShapeWithoutId = requestShapeWithoutId.array();
-const requestOptionsShape = s.object({
-    forefront: s.boolean().optional(),
-});
 /**
  * A file-system request queue backend backed by the native `@crawlee/fs-storage-native` Rust
  * extension.
@@ -72,8 +59,8 @@ export class RequestQueueBackend extends CachedIdClient {
         await this.#nativeBackend.purge();
     }
     async addBatchOfRequests(requests, options = {}) {
-        batchRequestShapeWithoutId.parse(requests);
-        requestOptionsShape.parse(options);
+        parseArgument(requests, schemas.storageRequestBatch);
+        parseArgument(options, schemas.requestQueueOperationOptions);
         const response = await this.#nativeBackend.addBatchOfRequests(requests.map((request) => plainifyRequest(request)), options.forefront ?? false);
         // `processedRequests` is structurally identical between the native and `storage` types, so it
         // passes through unchanged. `unprocessedRequests` only differs in that the native `method` is
@@ -84,7 +71,7 @@ export class RequestQueueBackend extends CachedIdClient {
         };
     }
     async getRequest(uniqueKey) {
-        s.string().parse(uniqueKey);
+        parseArgument(uniqueKey, uniqueKeySchema);
         // The native client tags requests with an internal `orderNo`; it's harmless to leak, so we
         // hand the request back as-is rather than copying it just to drop one undeclared property.
         // The native client already returns `undefined` for a missing request, matching this contract.
@@ -94,12 +81,12 @@ export class RequestQueueBackend extends CachedIdClient {
         return (await this.#nativeBackend.fetchNextRequest());
     }
     async markRequestAsHandled(request) {
-        requestShape.parse(request);
+        parseArgument(request, schemas.storageRequest);
         return (await this.#nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;
     }
     async reclaimRequest(request, options = {}) {
-        requestShape.parse(request);
-        requestOptionsShape.parse(options);
+        parseArgument(request, schemas.storageRequest);
+        parseArgument(options, schemas.requestQueueOperationOptions);
         return ((await this.#nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ??
             undefined);
     }