@nestm/storage 0.1.0-alpha.6 → 0.1.0-alpha.7

package/dist/files-sdk/fs/index.js

@@ -1,5 +1,597 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants as fsConstants } from 'node:fs';
+import * as fsp from 'node:fs/promises';
+import path from 'node:path';
 import { fs } from 'files-sdk/fs';
+import { StorageError, StorageErrorCode } from '../../storage.error.js';
 import { createFilesSdkDriver, } from '../files-sdk.driver.js';
+const SIDECAR_SUFFIX = '.meta.json';
+const TEMP_SUFFIX = '.fls-part';
+const MAX_SIDECAR_BYTES = 1024 * 1024;
+function fsErrorCode(error) {
+    if (typeof error === 'object' && error !== null && 'code' in error) {
+        return typeof error.code === 'string' ? error.code : undefined;
+    }
+    return undefined;
+}
+function storageFsError(error, key, operation) {
+    if (error instanceof StorageError) {
+        return error;
+    }
+    const systemCode = fsErrorCode(error);
+    const code = systemCode === 'ENOENT' || systemCode === 'ENOTDIR'
+        ? StorageErrorCode.NOT_FOUND
+        : systemCode === 'EEXIST'
+            ? StorageErrorCode.CONFLICT
+            : systemCode === 'EACCES' || systemCode === 'EPERM'
+                ? StorageErrorCode.UNAUTHORIZED
+                : StorageErrorCode.PROVIDER;
+    return new StorageError(`Filesystem ${operation} failed for "${key}".`, {
+        cause: error,
+        code,
+        key,
+        operation,
+        permanent: code === StorageErrorCode.NOT_FOUND ||
+            code === StorageErrorCode.CONFLICT ||
+            code === StorageErrorCode.UNAUTHORIZED,
+    });
+}
+function invalidFsPath(key, message) {
+    throw new StorageError(`Invalid filesystem storage key: ${message}.`, {
+        code: StorageErrorCode.INVALID_ARGUMENT,
+        key,
+        permanent: true,
+    });
+}
+function strictSegments(key) {
+    if (key.length === 0 || key.includes('\0')) {
+        invalidFsPath(key, 'the key is empty or contains a null byte');
+    }
+    if (path.isAbsolute(key) || key.includes('\\')) {
+        invalidFsPath(key, 'only relative POSIX paths are accepted');
+    }
+    const segments = key.split('/');
+    if (segments.some((segment) => segment.length === 0 || segment === '.' || segment === '..')) {
+        invalidFsPath(key, 'empty, dot, and parent segments are not accepted');
+    }
+    if (segments.some((segment) => /[. ]$/u.test(segment))) {
+        invalidFsPath(key, 'segments ending in a dot or space are not accepted');
+    }
+    const leaf = segments.at(-1)?.toLowerCase() ?? '';
+    if (leaf.endsWith(SIDECAR_SUFFIX) || leaf.endsWith(TEMP_SUFFIX)) {
+        invalidFsPath(key, 'the final segment uses a reserved adapter suffix');
+    }
+    return segments;
+}
+function symlinkError(key) {
+    return new StorageError(`Filesystem storage key "${key}" crosses a symbolic link.`, {
+        code: StorageErrorCode.INVALID_ARGUMENT,
+        key,
+        permanent: true,
+    });
+}
+function unsupportedUpload(key, message) {
+    throw new StorageError(message, {
+        code: StorageErrorCode.NOT_SUPPORTED,
+        key,
+        operation: 'upload',
+        permanent: true,
+    });
+}
+function conflict(key) {
+    throw new StorageError(`Conditional mutation conflicted for "${key}".`, {
+        code: StorageErrorCode.CONFLICT,
+        key,
+        permanent: true,
+    });
+}
+function notFound(key) {
+    throw new StorageError(`Storage object "${key}" was not found.`, {
+        code: StorageErrorCode.NOT_FOUND,
+        key,
+        permanent: true,
+    });
+}
+function runtimeOf(options) {
+    const timeoutSignal = options.timeout === undefined || options.timeout <= 0
+        ? undefined
+        : AbortSignal.timeout(options.timeout);
+    return {
+        callerSignal: options.signal,
+        signal: options.signal === undefined
+            ? timeoutSignal
+            : timeoutSignal === undefined
+                ? options.signal
+                : AbortSignal.any([options.signal, timeoutSignal]),
+        timeoutSignal,
+    };
+}
+function assertActive(runtime, key) {
+    if (runtime.signal?.aborted !== true) {
+        return;
+    }
+    const timedOut = runtime.timeoutSignal?.aborted === true &&
+        runtime.callerSignal?.aborted !== true;
+    throw new StorageError(timedOut
+        ? `Filesystem mutation timed out for "${key}".`
+        : `Filesystem mutation was aborted for "${key}".`, {
+        aborted: true,
+        cause: runtime.signal.reason,
+        code: timedOut ? StorageErrorCode.TIMEOUT : StorageErrorCode.ABORTED,
+        key,
+        permanent: true,
+        timedOut,
+    });
+}
+async function ensureRoot(root, key) {
+    await fsp.mkdir(root, { mode: 0o700, recursive: true });
+    const stat = await fsp.lstat(root);
+    if (stat.isSymbolicLink()) {
+        throw symlinkError(key);
+    }
+    if (!stat.isDirectory()) {
+        invalidFsPath(key, 'the configured root is not a directory');
+    }
+}
+async function ensureParents(root, segments, key, create, runtime) {
+    await ensureRoot(root, key);
+    let current = root;
+    for (const segment of segments.slice(0, -1)) {
+        assertActive(runtime, key);
+        current = path.join(current, segment);
+        let stat;
+        try {
+            stat = await fsp.lstat(current);
+        }
+        catch (error) {
+            if (fsErrorCode(error) !== 'ENOENT') {
+                throw error;
+            }
+            if (!create) {
+                notFound(key);
+            }
+            try {
+                await fsp.mkdir(current, { mode: 0o700 });
+            }
+            catch (mkdirError) {
+                if (fsErrorCode(mkdirError) !== 'EEXIST') {
+                    throw mkdirError;
+                }
+            }
+            stat = await fsp.lstat(current);
+        }
+        if (stat.isSymbolicLink()) {
+            throw symlinkError(key);
+        }
+        if (!stat.isDirectory()) {
+            invalidFsPath(key, 'a parent segment is not a directory');
+        }
+    }
+    return path.join(root, ...segments);
+}
+async function regularFileOrMissing(target, key) {
+    try {
+        const stat = await fsp.lstat(target);
+        if (stat.isSymbolicLink()) {
+            throw symlinkError(key);
+        }
+        if (!stat.isFile()) {
+            invalidFsPath(key, 'an object path is not a regular file');
+        }
+        if (stat.nlink !== 1) {
+            invalidFsPath(key, 'hard-linked object files are not accepted');
+        }
+        return stat;
+    }
+    catch (error) {
+        if (fsErrorCode(error) === 'ENOENT') {
+            return undefined;
+        }
+        throw error;
+    }
+}
+async function assertSymlinkFreeReadPath(root, key) {
+    const segments = strictSegments(key);
+    let rootStat;
+    try {
+        rootStat = await fsp.lstat(root);
+    }
+    catch (error) {
+        if (fsErrorCode(error) === 'ENOENT') {
+            return;
+        }
+        throw error;
+    }
+    if (rootStat.isSymbolicLink()) {
+        throw symlinkError(key);
+    }
+    if (!rootStat.isDirectory()) {
+        invalidFsPath(key, 'the configured root is not a directory');
+    }
+    let current = root;
+    for (const [index, segment] of segments.entries()) {
+        current = path.join(current, segment);
+        let stat;
+        try {
+            stat = await fsp.lstat(current);
+        }
+        catch (error) {
+            if (fsErrorCode(error) === 'ENOENT') {
+                return;
+            }
+            throw error;
+        }
+        if (stat.isSymbolicLink()) {
+            throw symlinkError(key);
+        }
+        const leaf = index === segments.length - 1;
+        if ((!leaf && !stat.isDirectory()) || (leaf && !stat.isFile())) {
+            invalidFsPath(key, leaf
+                ? 'an object path is not a regular file'
+                : 'a parent segment is not a directory');
+        }
+        if (leaf && stat.nlink !== 1) {
+            invalidFsPath(key, 'hard-linked object files are not accepted');
+        }
+    }
+    const sidecar = await regularFileOrMissing(current + SIDECAR_SUFFIX, key);
+    if (sidecar?.isSymbolicLink()) {
+        throw symlinkError(key);
+    }
+}
+async function readSidecar(target, key) {
+    const handle = await fsp.open(target, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
+    try {
+        const stat = await handle.stat();
+        if (!stat.isFile() || stat.size > MAX_SIDECAR_BYTES) {
+            throw new StorageError(`Filesystem metadata is invalid for "${key}".`, {
+                code: StorageErrorCode.PROVIDER,
+                key,
+                permanent: true,
+            });
+        }
+        const parsed = JSON.parse(await handle.readFile('utf8'));
+        if (typeof parsed !== 'object' ||
+            parsed === null ||
+            !('etag' in parsed) ||
+            typeof parsed.etag !== 'string' ||
+            !('contentType' in parsed) ||
+            typeof parsed.contentType !== 'string' ||
+            !('lastModified' in parsed) ||
+            typeof parsed.lastModified !== 'number') {
+            throw new StorageError(`Filesystem metadata is invalid for "${key}".`, {
+                code: StorageErrorCode.PROVIDER,
+                key,
+                permanent: true,
+            });
+        }
+        return parsed;
+    }
+    finally {
+        await handle.close();
+    }
+}
+function bodyBytes(body) {
+    if (typeof body === 'string') {
+        return new TextEncoder().encode(body);
+    }
+    if (body instanceof Uint8Array) {
+        return body;
+    }
+    if (body instanceof ArrayBuffer) {
+        return new Uint8Array(body);
+    }
+    if (ArrayBuffer.isView(body)) {
+        return new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
+    }
+    return undefined;
+}
+function defaultContentType(body, override) {
+    if (override !== undefined) {
+        return override;
+    }
+    if (typeof body === 'string') {
+        return 'text/plain; charset=utf-8';
+    }
+    if (body instanceof Blob && body.type.length > 0) {
+        return body.type;
+    }
+    return 'application/octet-stream';
+}
+async function writeAll(handle, bytes) {
+    let offset = 0;
+    while (offset < bytes.byteLength) {
+        const { bytesWritten } = await handle.write(bytes, offset, bytes.byteLength - offset);
+        offset += bytesWritten;
+    }
+}
+function tempPath(directory) {
+    return path.join(directory, `.nestm-${randomUUID()}${TEMP_SUFFIX}`);
+}
+async function openTemp(directory) {
+    const target = tempPath(directory);
+    const handle = await fsp.open(target, fsConstants.O_CREAT |
+        fsConstants.O_EXCL |
+        fsConstants.O_WRONLY |
+        fsConstants.O_NOFOLLOW, 0o600);
+    return { handle, path: target };
+}
+async function removeTemp(target) {
+    if (target === undefined) {
+        return;
+    }
+    try {
+        await fsp.unlink(target);
+    }
+    catch (error) {
+        if (fsErrorCode(error) !== 'ENOENT') {
+            throw error;
+        }
+    }
+}
+async function stageBody(directory, key, body, options, runtime) {
+    const staged = await openTemp(directory);
+    const hash = createHash('sha256');
+    let size = 0;
+    const knownBytes = bodyBytes(body);
+    const knownSize = knownBytes?.byteLength ?? (body instanceof Blob ? body.size : undefined);
+    options.onProgress?.({
+        loaded: 0,
+        ...(knownSize !== undefined && { total: knownSize }),
+    });
+    try {
+        if (knownBytes !== undefined) {
+            assertActive(runtime, key);
+            await writeAll(staged.handle, knownBytes);
+            hash.update(knownBytes);
+            size = knownBytes.byteLength;
+            options.onProgress?.({ loaded: size, total: size });
+        }
+        else if (body instanceof Blob) {
+            const bytes = new Uint8Array(await body.arrayBuffer());
+            assertActive(runtime, key);
+            await writeAll(staged.handle, bytes);
+            hash.update(bytes);
+            size = bytes.byteLength;
+            options.onProgress?.({ loaded: size, total: size });
+        }
+        else {
+            const reader = body.getReader();
+            try {
+                while (true) {
+                    assertActive(runtime, key);
+                    const chunk = await reader.read();
+                    if (chunk.done) {
+                        break;
+                    }
+                    await writeAll(staged.handle, chunk.value);
+                    hash.update(chunk.value);
+                    size += chunk.value.byteLength;
+                    options.onProgress?.({ loaded: size });
+                }
+            }
+            catch (error) {
+                await reader.cancel(error).catch(() => undefined);
+                throw error;
+            }
+            finally {
+                reader.releaseLock();
+            }
+        }
+        await staged.handle.sync();
+        await staged.handle.close();
+        return {
+            etag: `"${hash.digest('hex')}"`,
+            path: staged.path,
+            size,
+        };
+    }
+    catch (error) {
+        await staged.handle.close().catch(() => undefined);
+        await removeTemp(staged.path).catch(() => undefined);
+        throw error;
+    }
+}
+async function stageSidecar(directory, sidecar) {
+    const staged = await openTemp(directory);
+    try {
+        await writeAll(staged.handle, new TextEncoder().encode(JSON.stringify(sidecar)));
+        await staged.handle.sync();
+        await staged.handle.close();
+        return staged.path;
+    }
+    catch (error) {
+        await staged.handle.close().catch(() => undefined);
+        await removeTemp(staged.path).catch(() => undefined);
+        throw error;
+    }
+}
+const CONDITIONAL_FS_TAILS = new Map();
+async function serializeFsKey(lockKey, operation) {
+    const previous = CONDITIONAL_FS_TAILS.get(lockKey) ?? Promise.resolve();
+    let release = () => undefined;
+    const gate = new Promise((resolve) => {
+        release = resolve;
+    });
+    const tail = previous.then(() => gate);
+    CONDITIONAL_FS_TAILS.set(lockKey, tail);
+    await previous;
+    try {
+        return await operation();
+    }
+    finally {
+        release();
+        if (CONDITIONAL_FS_TAILS.get(lockKey) === tail) {
+            CONDITIONAL_FS_TAILS.delete(lockKey);
+        }
+    }
+}
+/**
+ * Adds symlink-defended reads and process-local conditional mutations to a
+ * files-sdk fs adapter.
+ *
+ * The configured root must be dedicated to this driver: no other process and
+ * no unconditional filesystem writer may mutate it concurrently. Each call
+ * rejects symlinks in the existing key path and uses exclusive same-directory
+ * temporary files plus atomic renames, but Node has no portable `openat2`
+ * equivalent that could make those checks safe against a hostile concurrent
+ * tree rewrite. This is a CAS implementation for an exclusively-owned local
+ * workspace, not a host-filesystem sandbox. Body and metadata sidecars are two
+ * files, so a process crash between their renames can still require cleanup.
+ */
+export function withFsConditionalMutation(base) {
+    const root = path.resolve(base.root);
+    const download = base.download.bind(base);
+    const exists = base.exists.bind(base);
+    const head = base.head.bind(base);
+    const list = base.list.bind(base);
+    const serialize = (key, operation) => serializeFsKey(`${root}\0${key.normalize('NFC').toLowerCase()}`, operation);
+    return Object.assign(base, {
+        conditionalMutation: Object.freeze({
+            create: true,
+            delete: true,
+            etag: true,
+            replace: true,
+        }),
+        async download(key, options) {
+            await assertSymlinkFreeReadPath(root, key);
+            return download(key, options);
+        },
+        async exists(key, options) {
+            await assertSymlinkFreeReadPath(root, key);
+            return exists(key, options);
+        },
+        async head(key, options) {
+            await assertSymlinkFreeReadPath(root, key);
+            return head(key, options);
+        },
+        async list(options) {
+            const result = await list(options);
+            for (const item of result.items) {
+                await assertSymlinkFreeReadPath(root, item.key);
+            }
+            return result;
+        },
+        async deleteConditional(key, options) {
+            return serialize(key, async () => {
+                const runtime = runtimeOf(options);
+                try {
+                    const segments = strictSegments(key);
+                    const bodyPath = await ensureParents(root, segments, key, false, runtime);
+                    const sidecarPath = bodyPath + SIDECAR_SUFFIX;
+                    if ((await regularFileOrMissing(bodyPath, key)) === undefined ||
+                        (await regularFileOrMissing(sidecarPath, key)) === undefined) {
+                        notFound(key);
+                    }
+                    const sidecar = await readSidecar(sidecarPath, key);
+                    if (sidecar.etag !== options.condition.etag) {
+                        conflict(key);
+                    }
+                    assertActive(runtime, key);
+                    await ensureParents(root, segments, key, false, runtime);
+                    if ((await regularFileOrMissing(bodyPath, key)) === undefined ||
+                        (await regularFileOrMissing(sidecarPath, key)) === undefined ||
+                        (await readSidecar(sidecarPath, key)).etag !==
+                            options.condition.etag) {
+                        conflict(key);
+                    }
+                    await fsp.unlink(bodyPath);
+                    try {
+                        await fsp.unlink(sidecarPath);
+                    }
+                    catch (error) {
+                        throw storageFsError(error, key, 'delete');
+                    }
+                }
+                catch (error) {
+                    throw storageFsError(error, key, 'delete');
+                }
+            });
+        },
+        async uploadConditional(key, body, options) {
+            return serialize(key, async () => {
+                let bodyTemp;
+                let sidecarTemp;
+                const runtime = runtimeOf(options);
+                try {
+                    if (options.multipart !== undefined && options.multipart !== false) {
+                        unsupportedUpload(key, 'Conditional filesystem uploads do not support multipart mode.');
+                    }
+                    if (options.control !== undefined) {
+                        unsupportedUpload(key, 'Conditional filesystem uploads do not support resumable control.');
+                    }
+                    const segments = strictSegments(key);
+                    const bodyPath = await ensureParents(root, segments, key, true, runtime);
+                    const sidecarPath = bodyPath + SIDECAR_SUFFIX;
+                    const existingBody = await regularFileOrMissing(bodyPath, key);
+                    const existingSidecar = await regularFileOrMissing(sidecarPath, key);
+                    if (options.condition.type === 'create') {
+                        if (existingBody !== undefined || existingSidecar !== undefined) {
+                            conflict(key);
+                        }
+                    }
+                    else {
+                        if (existingBody === undefined || existingSidecar === undefined) {
+                            notFound(key);
+                        }
+                        if ((await readSidecar(sidecarPath, key)).etag !==
+                            options.condition.etag) {
+                            conflict(key);
+                        }
+                    }
+                    const staged = await stageBody(path.dirname(bodyPath), key, body, options, runtime);
+                    bodyTemp = staged.path;
+                    const lastModified = Date.now();
+                    const contentType = defaultContentType(body, options.contentType);
+                    sidecarTemp = await stageSidecar(path.dirname(bodyPath), {
+                        contentType,
+                        etag: staged.etag,
+                        lastModified,
+                        ...(options.cacheControl !== undefined && {
+                            cacheControl: options.cacheControl,
+                        }),
+                        ...(options.metadata !== undefined && {
+                            metadata: options.metadata,
+                        }),
+                    });
+                    assertActive(runtime, key);
+                    await ensureParents(root, segments, key, false, runtime);
+                    const currentBody = await regularFileOrMissing(bodyPath, key);
+                    const currentSidecar = await regularFileOrMissing(sidecarPath, key);
+                    if (options.condition.type === 'create') {
+                        if (currentBody !== undefined || currentSidecar !== undefined) {
+                            conflict(key);
+                        }
+                    }
+                    else if (currentBody === undefined ||
+                        currentSidecar === undefined ||
+                        (await readSidecar(sidecarPath, key)).etag !==
+                            options.condition.etag) {
+                        conflict(key);
+                    }
+                    await fsp.rename(bodyTemp, bodyPath);
+                    bodyTemp = undefined;
+                    await fsp.rename(sidecarTemp, sidecarPath);
+                    sidecarTemp = undefined;
+                    return {
+                        contentType,
+                        etag: staged.etag,
+                        key,
+                        lastModified: new Date(lastModified),
+                        size: staged.size,
+                    };
+                }
+                catch (error) {
+                    throw storageFsError(error, key, 'upload');
+                }
+                finally {
+                    await removeTemp(bodyTemp).catch(() => undefined);
+                    await removeTemp(sidecarTemp).catch(() => undefined);
+                }
+            });
+        },
+    });
+}
 /**
  * Creates the files-sdk filesystem driver from the storage package's own
  * dependency context. The adapter reaches only `node:fs`, so this entry point
@@ -11,10 +603,17 @@ import { createFilesSdkDriver, } from '../files-sdk.driver.js';
  * put. Sidecars never surface as keys: `list` and `search` skip them, and
  * uploading a key that ends in `.meta.json` fails closed rather than colliding
  * with one.
+ *
+ * Conditional mutations additionally require an exclusively-owned root. See
+ * {@link withFsConditionalMutation}; use an OS sandbox as well when running an
+ * agent that has direct shell or filesystem tools.
  */
 export function createFsStorageDriver(options) {
     const { adapter: adapterOptions, ...filesOptions } = options;
-    return createFilesSdkDriver({ ...filesOptions, adapter: fs(adapterOptions) });
+    return createFilesSdkDriver({
+        ...filesOptions,
+        adapter: withFsConditionalMutation(fs(adapterOptions)),
+    });
 }
 export { fs, mapFsError } from 'files-sdk/fs';
 //# sourceMappingURL=index.js.map