@crawlee/memory-storage 3.17.0 → 3.17.1-beta.0

package/cache-helpers.js

@@ -18,7 +18,7 @@ async function findOrCacheDatasetByPossibleId(client, entryNameOrId) {
     if (found) {
         return found;
     }
-    const datasetDir = (0, node_path_1.resolve)(client.datasetsDirectory, entryNameOrId);
+    const datasetDir = (0, utils_1.resolveWithinDirectory)(client.datasetsDirectory, entryNameOrId);
     try {
         // Check if directory exists
         await (0, promises_1.access)(datasetDir);
@@ -99,7 +99,7 @@ async function findOrCacheKeyValueStoreByPossibleId(client, entryNameOrId) {
     if (found) {
         return found;
     }
-    const keyValueStoreDir = (0, node_path_1.resolve)(client.keyValueStoresDirectory, entryNameOrId);
+    const keyValueStoreDir = (0, utils_1.resolveWithinDirectory)(client.keyValueStoresDirectory, entryNameOrId);
     try {
         // Check if directory exists
         await (0, promises_1.access)(keyValueStoreDir);
@@ -219,7 +219,7 @@ async function findRequestQueueByPossibleId(client, entryNameOrId) {
     if (found) {
         return found;
     }
-    const requestQueueDir = (0, node_path_1.resolve)(client.requestQueuesDirectory, entryNameOrId);
+    const requestQueueDir = (0, utils_1.resolveWithinDirectory)(client.requestQueuesDirectory, entryNameOrId);
     try {
         // Check if directory exists
         await (0, promises_1.access)(requestQueueDir);

package/fs/key-value-store/fs.js

@@ -60,7 +60,7 @@ class KeyValueFileSystemEntry {
         catch {
             try {
                 // Try without extension
-                file = await (0, promises_1.readFile)((0, node_path_1.resolve)(this.storeDirectory, this.rawRecord.key));
+                file = await (0, promises_1.readFile)((0, utils_1.resolveWithinDirectory)(this.storeDirectory, this.rawRecord.key));
                 utils_1.memoryStorageLog.warning([
                     `Key-value entry "${this.rawRecord.key}" for store ${(0, win32_1.basename)(this.storeDirectory)} does not have a file extension, assuming it as text.`,
                     'If you want to have correct interpretation of the file, you should add a file extension to the entry.',
@@ -88,8 +88,8 @@ class KeyValueFileSystemEntry {
         contentType && data.contentType && contentType.startsWith(data.contentType)
             ? data.key
             : `${data.key}.${data.extension}`;
-        this.filePath ?? (this.filePath = (0, node_path_1.resolve)(this.storeDirectory, fileName));
-        this.fileMetadataPath ?? (this.fileMetadataPath = (0, node_path_1.resolve)(this.storeDirectory, `${data.key}.__metadata__.json`));
+        this.filePath ?? (this.filePath = (0, utils_1.resolveWithinDirectory)(this.storeDirectory, fileName));
+        this.fileMetadataPath ?? (this.fileMetadataPath = (0, utils_1.resolveWithinDirectory)(this.storeDirectory, `${data.key}.__metadata__.json`));
         const { value, ...rest } = data;
         this.rawRecord = rest;
         try {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/memory-storage",
-    "version": "3.17.0",
+    "version": "3.17.1-beta.0",
     "description": "A simple in-memory storage implementation of the Apify API",
     "engines": {
         "node": ">= 16"
@@ -49,7 +49,7 @@
     },
     "dependencies": {
         "@apify/log": "^2.4.0",
-        "@crawlee/types": "3.17.0",
+        "@crawlee/types": "3.17.1-beta.0",
         "@sapphire/async-queue": "^1.5.0",
         "@sapphire/shapeshift": "^3.0.0",
         "content-type": "^1.0.4",
@@ -67,5 +67,5 @@
             }
         }
     },
-    "gitHead": "8f2663aedfd2280ca35b9185b4e287f65a2ce969"
+    "gitHead": "d76458ce1dc2c433988ea09a14e222b56e29bf06"
 }

package/resource-clients/dataset.js

@@ -4,7 +4,6 @@ exports.DatasetClient = void 0;
 /* eslint-disable import/no-duplicates */
 const node_crypto_1 = require("node:crypto");
 const promises_1 = require("node:fs/promises");
-const node_path_1 = require("node:path");
 const shapeshift_1 = require("@sapphire/shapeshift");
 const fs_extra_1 = require("fs-extra");
 const background_handler_1 = require("../background-handler");
@@ -75,7 +74,7 @@ class DatasetClient extends base_client_1.BaseClient {
             value: void 0
         });
         this.name = options.name;
-        this.datasetDirectory = (0, node_path_1.resolve)(options.baseStorageDirectory, this.name ?? this.id);
+        this.datasetDirectory = (0, utils_1.resolveWithinDirectory)(options.baseStorageDirectory, this.name ?? this.id);
         this.client = options.client;
     }
     async get() {
@@ -108,7 +107,7 @@ class DatasetClient extends base_client_1.BaseClient {
         }
         existingStoreById.name = parsed.name;
         const previousDir = existingStoreById.datasetDirectory;
-        existingStoreById.datasetDirectory = (0, node_path_1.resolve)(this.client.datasetsDirectory, parsed.name ?? existingStoreById.name ?? existingStoreById.id);
+        existingStoreById.datasetDirectory = (0, utils_1.resolveWithinDirectory)(this.client.datasetsDirectory, parsed.name ?? existingStoreById.name ?? existingStoreById.id);
         await (0, fs_extra_1.move)(previousDir, existingStoreById.datasetDirectory, { overwrite: true });
         // Update timestamps
         existingStoreById.updateTimestamps(true);

package/resource-clients/key-value-store.js

@@ -4,7 +4,6 @@ exports.KeyValueStoreClient = void 0;
 const tslib_1 = require("tslib");
 const node_crypto_1 = require("node:crypto");
 const promises_1 = require("node:fs/promises");
-const node_path_1 = require("node:path");
 const node_stream_1 = require("node:stream");
 const shapeshift_1 = require("@sapphire/shapeshift");
 const fs_extra_1 = require("fs-extra");
@@ -65,7 +64,7 @@ class KeyValueStoreClient extends base_client_1.BaseClient {
             value: void 0
         });
         this.name = options.name;
-        this.keyValueStoreDirectory = (0, node_path_1.resolve)(options.baseStorageDirectory, this.name ?? this.id);
+        this.keyValueStoreDirectory = (0, utils_1.resolveWithinDirectory)(options.baseStorageDirectory, this.name ?? this.id);
         this.client = options.client;
     }
     async get() {
@@ -98,7 +97,7 @@ class KeyValueStoreClient extends base_client_1.BaseClient {
         }
         existingStoreById.name = parsed.name;
         const previousDir = existingStoreById.keyValueStoreDirectory;
-        existingStoreById.keyValueStoreDirectory = (0, node_path_1.resolve)(this.client.keyValueStoresDirectory, parsed.name ?? existingStoreById.name ?? existingStoreById.id);
+        existingStoreById.keyValueStoreDirectory = (0, utils_1.resolveWithinDirectory)(this.client.keyValueStoresDirectory, parsed.name ?? existingStoreById.name ?? existingStoreById.id);
         await (0, fs_extra_1.move)(previousDir, existingStoreById.keyValueStoreDirectory, { overwrite: true });
         // Update timestamps
         existingStoreById.updateTimestamps(true);

package/resource-clients/request-queue.js

@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.RequestQueueClient = void 0;
 const node_crypto_1 = require("node:crypto");
 const promises_1 = require("node:fs/promises");
-const node_path_1 = require("node:path");
 const async_queue_1 = require("@sapphire/async-queue");
 const shapeshift_1 = require("@sapphire/shapeshift");
 const fs_extra_1 = require("fs-extra");
@@ -96,7 +95,7 @@ class RequestQueueClient extends base_client_1.BaseClient {
             value: void 0
         });
         this.name = options.name;
-        this.requestQueueDirectory = (0, node_path_1.resolve)(options.baseStorageDirectory, this.name ?? this.id);
+        this.requestQueueDirectory = (0, utils_1.resolveWithinDirectory)(options.baseStorageDirectory, this.name ?? this.id);
         this.client = options.client;
     }
     async getQueue() {
@@ -138,7 +137,7 @@ class RequestQueueClient extends base_client_1.BaseClient {
         }
         existingQueueById.name = parsed.name;
         const previousDir = existingQueueById.requestQueueDirectory;
-        existingQueueById.requestQueueDirectory = (0, node_path_1.resolve)(this.client.requestQueuesDirectory, parsed.name ?? existingQueueById.name ?? existingQueueById.id);
+        existingQueueById.requestQueueDirectory = (0, utils_1.resolveWithinDirectory)(this.client.requestQueuesDirectory, parsed.name ?? existingQueueById.name ?? existingQueueById.id);
         await (0, fs_extra_1.move)(previousDir, existingQueueById.requestQueueDirectory, { overwrite: true });
         // Update timestamps
         existingQueueById.updateTimestamps(true);

package/utils.d.ts

@@ -1,4 +1,10 @@
 import type * as storage from '@crawlee/types';
+/**
+ * Resolves `segment` against `baseDirectory` and ensures the result stays within `baseDirectory`.
+ * Storage names and record keys are used as filesystem path components, so a value containing `..`
+ * or an absolute path could otherwise escape the intended directory.
+ */
+export declare function resolveWithinDirectory(baseDirectory: string, segment: string): string;
 /**
  * Removes all properties with a null value
  * from the provided object.

package/utils.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.memoryStorageLog = void 0;
+exports.resolveWithinDirectory = resolveWithinDirectory;
 exports.purgeNullsFromObject = purgeNullsFromObject;
 exports.uniqueKeyToRequestId = uniqueKeyToRequestId;
 exports.isBuffer = isBuffer;
@@ -12,9 +13,24 @@ exports.createPaginatedEntryList = createPaginatedEntryList;
 exports.createLazyIterablePromise = createLazyIterablePromise;
 const tslib_1 = require("tslib");
 const node_crypto_1 = require("node:crypto");
+const node_path_1 = require("node:path");
 const shapeshift_1 = require("@sapphire/shapeshift");
 const log_1 = tslib_1.__importDefault(require("@apify/log"));
 const consts_1 = require("./consts");
+/**
+ * Resolves `segment` against `baseDirectory` and ensures the result stays within `baseDirectory`.
+ * Storage names and record keys are used as filesystem path components, so a value containing `..`
+ * or an absolute path could otherwise escape the intended directory.
+ */
+function resolveWithinDirectory(baseDirectory, segment) {
+    const base = (0, node_path_1.resolve)(baseDirectory);
+    const resolved = (0, node_path_1.resolve)(base, segment);
+    if (resolved !== base && !resolved.startsWith(`${base}${node_path_1.sep}`)) {
+        throw new Error(`"${segment}" is not allowed because it would resolve outside of the storage directory. ` +
+            `Storage names and record keys must not contain path traversal segments ("..") or absolute paths.`);
+    }
+    return resolved;
+}
 /**
  * Removes all properties with a null value
  * from the provided object.