@ultimat3/storage 29.4.0 → 30.1.0

package/src/driver-local-list.ts

@@ -0,0 +1,75 @@
+// Single responsibility: one page of the local disk's listing — every object key under a prefix,
+// the `.meta/` sidecar tree kept out of the namespace, ordered, cut at the cursor, and a refused
+// scan reported rather than read as an empty disk (`sweepOrphans` trusts this listing).
+
+import { type Page, pageOf, renderFixShellArg } from '@ultimat3/core';
+import type { StorageListEntry } from './driver';
+import { isMissingFile } from './driver-local-remove';
+import { listFailed } from './errors';
+import { META_DIR } from './path';
+
+export interface LocalListInput {
+  readonly root: string;
+  readonly prefix: string;
+  /** The last key of the previous page; keys at or before it are skipped. */
+  readonly cursor: string | undefined;
+  /** A positive integer, already resolved by `resolveListLimit`. */
+  readonly limit: number;
+  /** Each key's listing view; `undefined` drops a key that went away mid-page. */
+  readonly head: (key: string) => Promise<StorageListEntry | undefined>;
+}
+
+export async function listLocalPage(input: LocalListInput): Promise<Page<StorageListEntry>> {
+  const { root, prefix, cursor, limit } = input;
+  const keys: string[] = [];
+  try {
+    // `dot: true`, and it is load-bearing: without it a glob matches no dot-prefixed entry, so
+    // every object whose key has one — `.hidden.txt`, `org/o1/pending/.x.png`, the
+    // `.metadata/a.json` `path.test.ts` pins as legal — was absent from the listing while
+    // `put`/`get`/`exists` handled it normally and the s3 listing returned it. `sweepOrphans`
+    // pages through `list()`, so those objects were swept as if they did not exist: a false
+    // erasure report by omission, which is what the classification below exists to prevent.
+    for await (const entry of new Bun.Glob('**/*').scan({
+      cwd: root,
+      onlyFiles: true,
+      dot: true,
+    })) {
+      const key = entry.replaceAll('\\', '/');
+      // The real filter now, not a second line of defence: the glob above yields the sidecar
+      // tree, and this is the one thing keeping `.meta/<key>.json` out of the object
+      // namespace. Folded like `assertSafeKey`'s reservation — `.META/` and `.meta/` are one
+      // directory on APFS and NTFS, and listing a key `get()` would refuse is the worse half.
+      if (key.toLowerCase().startsWith(`${META_DIR}/`)) continue;
+      if (!key.startsWith(prefix)) continue;
+      // The cursor IS the last key of the previous page — lexicographic order keeps it stable.
+      if (cursor !== undefined && key <= cursor) continue;
+      keys.push(key);
+    }
+  } catch (error) {
+    // A disk nobody has written to yet has no directory: an empty listing, not an error.
+    if (isMissingFile(error)) return pageOf([], null);
+    // Everything else is a refusal, and a bare `catch` reported all of them as "this disk is
+    // empty" — `EACCES` on the root, `ENOTDIR` on a root that is a file, an I/O error on the
+    // mount. `sweepOrphans` walks `list()`, so that swallow certified an unreadable prefix as
+    // having no orphans: the same false report `delete()`'s `.catch(() => undefined)` used to
+    // make, one call to the left.
+    throw listFailed(
+      'local',
+      prefix,
+      error,
+      `make the disk root readable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")}`,
+    );
+  }
+  keys.sort();
+  const page = keys.slice(0, limit);
+  const objects: StorageListEntry[] = [];
+  for (const key of page) {
+    const object = await input.head(key);
+    if (object !== undefined) objects.push(object);
+  }
+  // The cursor is minted only when a key PAST the page exists — `keys` is every match, so a
+  // full last page answers no cursor. `limit` is a positive integer, so a page with more after
+  // it always HAS a last key: the `?? null` is the type's, not a second condition. It used to
+  // be one, and `limit: 0` fell through it — an empty page reported as complete.
+  return pageOf(objects, keys.length > page.length ? (page.at(-1) ?? null) : null);
+}

package/src/driver-local-multipart.ts

@@ -0,0 +1,133 @@
+// Single responsibility: where the local disk keeps an open upload in parts — one directory per
+// session under `.meta/.multipart/`, which no key can name (`META_DIR` is reserved) and `list()`
+// never yields. A part's bytes land in a staged file renamed into place, so a dropped part is
+// absent rather than torn; its etag and size sit beside it.
+
+// why: Bun has no `mkdir`, no `readdir` and no recursive remove; a session is a directory.
+import { mkdir, readdir, rename, rm } from 'node:fs/promises';
+import type { UploadedPart } from './multipart';
+import type { PartStore, StagedSession } from './multipart-staged';
+import { META_DIR } from './path';
+
+/** Beside the write staging under `.meta/`, so one reserved segment covers both. */
+export const MULTIPART_DIR = `${META_DIR}/.multipart`;
+
+const PART_FILE = /^(\d{1,5})\.part$/;
+
+const isPart = (value: unknown): value is UploadedPart =>
+  typeof value === 'object' &&
+  value !== null &&
+  'partNumber' in value &&
+  Number.isSafeInteger(value.partNumber) &&
+  'etag' in value &&
+  typeof value.etag === 'string' &&
+  'size' in value &&
+  Number.isSafeInteger(value.size);
+
+const isSession = (value: unknown): value is StagedSession =>
+  typeof value === 'object' &&
+  value !== null &&
+  'key' in value &&
+  typeof value.key === 'string' &&
+  'contentType' in value &&
+  typeof value.contentType === 'string' &&
+  'initiated' in value &&
+  typeof value.initiated === 'string';
+
+/** A file that is absent or unreadable as JSON is no session: the directory is the disk's own. */
+const readJson = async (path: string): Promise<unknown> => {
+  const file = Bun.file(path);
+  if (!(await file.exists())) return undefined;
+  return file.json().catch(() => undefined);
+};
+
+/** Upload ids are the disk's own UUIDs; anything else never names a directory here. */
+const SESSION_ID = /^[A-Za-z0-9-]{1,64}$/;
+
+export function localPartStore(root: string): PartStore {
+  const dirOf = (uploadId: string): string | undefined =>
+    SESSION_ID.test(uploadId) ? `${root}/${MULTIPART_DIR}/${uploadId}` : undefined;
+  const sessionAt = async (uploadId: string): Promise<StagedSession | undefined> => {
+    const dir = dirOf(uploadId);
+    if (dir === undefined) return undefined;
+    const session = await readJson(`${dir}/session.json`);
+    return isSession(session) ? session : undefined;
+  };
+
+  return {
+    async open(uploadId, session): Promise<void> {
+      const dir = dirOf(uploadId) ?? '';
+      await mkdir(dir, { recursive: true });
+      await Bun.write(`${dir}/session.json`, JSON.stringify(session));
+    },
+
+    session: sessionAt,
+
+    async writePart(uploadId, part, bytes): Promise<void> {
+      const dir = dirOf(uploadId) ?? '';
+      const staged = `${dir}/${part.partNumber}.part.tmp`;
+      await Bun.write(staged, bytes);
+      await Bun.write(`${dir}/${part.partNumber}.json`, JSON.stringify(part));
+      await rename(staged, `${dir}/${part.partNumber}.part`);
+    },
+
+    async parts(uploadId): Promise<readonly UploadedPart[]> {
+      const dir = dirOf(uploadId);
+      if (dir === undefined) return [];
+      const names = await readdir(dir).catch(() => []);
+      const parts: UploadedPart[] = [];
+      for (const name of names) {
+        const number = PART_FILE.exec(name)?.[1];
+        if (number === undefined) continue;
+        const part = await readJson(`${dir}/${number}.json`);
+        if (isPart(part)) parts.push(part);
+      }
+      return parts.sort((a, b) => a.partNumber - b.partNumber);
+    },
+
+    read(uploadId, parts): ReadableStream<Uint8Array> {
+      const dir = dirOf(uploadId) ?? '';
+      let at = 0;
+      let current: ReadableStreamDefaultReader<Uint8Array> | undefined;
+      return new ReadableStream<Uint8Array>({
+        async pull(controller): Promise<void> {
+          for (;;) {
+            if (current === undefined) {
+              const part = parts[at];
+              if (part === undefined) {
+                controller.close();
+                return;
+              }
+              at += 1;
+              current = Bun.file(`${dir}/${part.partNumber}.part`).stream().getReader();
+            }
+            const next = await current.read();
+            if (!next.done) {
+              controller.enqueue(next.value);
+              return;
+            }
+            current = undefined;
+          }
+        },
+        async cancel(reason): Promise<void> {
+          await current?.cancel(reason);
+        },
+      });
+    },
+
+    async drop(uploadId): Promise<void> {
+      const dir = dirOf(uploadId);
+      if (dir !== undefined) await rm(dir, { recursive: true, force: true });
+    },
+
+    async sessions(): Promise<readonly (readonly [string, StagedSession])[]> {
+      const ids = await readdir(`${root}/${MULTIPART_DIR}`).catch(() => []);
+      const open: (readonly [string, StagedSession])[] = [];
+      for (const uploadId of ids) {
+        const session = await sessionAt(uploadId);
+        if (session !== undefined) open.push([uploadId, session]);
+      }
+      return open;
+    },
+  };
+}

package/src/driver-local-remove.ts

@@ -0,0 +1,39 @@
+// Single responsibility: how the local disk unlinks one file of an object, and which failures mean
+// "already gone". A refused unlink is reported, never swallowed: an erasure sweep that hears
+// "deleted" for bytes still on disk repeats the one lie it exists to prevent.
+
+import { renderFixShellArg, stringField } from '@ultimat3/core';
+import { deleteFailed } from './errors';
+
+/** `ENOENT` is the one delete failure that means "already in the desired state". */
+export const isMissingFile = (error: unknown): boolean => stringField(error, 'code') === 'ENOENT';
+
+/**
+ * A directory at an object's path is OTHER keys' parent (`c` beside `c/d`), never an object: no
+ * bytes are stored at the key, so deleting it is deleting an absent key. Asked only after an
+ * unlink failed, because the errno for "that is a directory" is `EISDIR` on Linux and `EPERM` on
+ * macOS — the stat is the portable answer.
+ */
+const isDirectory = async (path: string): Promise<boolean> =>
+  Bun.file(path)
+    .stat()
+    .then(
+      (stats) => stats.isDirectory(),
+      () => false,
+    );
+
+/** Removes one path under `root`, or reports WHY it could not — a swallowed refusal is a false erasure. */
+export async function removeIfPresent(root: string, path: string, key: string): Promise<void> {
+  try {
+    await Bun.file(path).delete();
+  } catch (error) {
+    if (isMissingFile(error) || (await isDirectory(path))) return;
+    throw deleteFailed(
+      'local',
+      key,
+      error,
+      // Screened: a key may carry `$(…)` and a root is configuration, and a fix: is pasted.
+      `make the disk root writable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")} && rm -f ${renderFixShellArg(path, "'<the object file>'")}`,
+    );
+  }
+}

package/src/driver-local-stream.ts

@@ -0,0 +1,104 @@
+// Single responsibility: the local disk's `putStream`: the body counted and hashed into one staged
+// file the commit renames into place, so any length costs one chunk of heap. Staged outside the key's
+// queue, so a slow sender does not hold the key's readers.
+
+// why: Bun has no `mkdir`; `Bun.write` makes parents but cannot open the incremental writer here.
+import { mkdir } from 'node:fs/promises';
+import { type Clock, isUltimateError, renderFixShellArg } from '@ultimat3/core';
+import { pumpBounded } from './byte-pump';
+import { DEFAULT_CONTENT_TYPE, type StorageObject } from './driver';
+import type { Sidecar } from './driver-local-sidecar';
+import { commitObject, STAGING_DIR } from './driver-local-write';
+import { putFailed } from './errors';
+import type { PutStreamOptions } from './put-stream';
+import { streamLimit } from './put-stream';
+
+export interface StagedStream {
+  readonly path: string;
+  readonly size: number;
+  /** `etagOf` over the same bytes, so a streamed object and a `put()` of it report one etag. */
+  readonly etag: string;
+}
+
+export interface LocalStreamInput extends StageStreamInput {
+  readonly clock: Clock;
+}
+
+/**
+ * Stage, then commit. `inTurn` is the disk's lock check in the key's turn, then `then`: asked before
+ * the transfer too, because a locked key should not cost the whole body to refuse.
+ */
+export async function putLocalStream(
+  input: LocalStreamInput,
+  options: PutStreamOptions,
+  inTurn: (then: () => Promise<void>) => Promise<void>,
+): Promise<StorageObject> {
+  const { root, key, disk, clock } = input;
+  await inTurn(async () => undefined);
+  const staged = await stageStream(input);
+  try {
+    const lastModified = clock.now();
+    const sidecar: Sidecar = {
+      contentType: options.contentType ?? DEFAULT_CONTENT_TYPE,
+      etag: staged.etag,
+      cacheControl: options.cacheControl,
+      metadata: options.metadata,
+      lastModified: lastModified.toISOString(),
+    };
+    const body = { stagedAt: staged.path };
+    await inTurn(() => commitObject({ root, key, disk, body, sidecar }));
+    return {
+      key,
+      size: staged.size,
+      contentType: sidecar.contentType,
+      etag: sidecar.etag,
+      lastModified,
+      ...(sidecar.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
+      ...(sidecar.metadata === undefined ? {} : { metadata: sidecar.metadata }),
+    };
+  } finally {
+    await discardStaged(staged.path);
+  }
+}
+
+export interface StageStreamInput {
+  readonly root: string;
+  readonly key: string;
+  readonly disk: string;
+  readonly body: ReadableStream<Uint8Array>;
+  readonly maxBytes: number;
+}
+
+/** A staged file gone, or never there. Safe after the commit renamed it away. */
+export const discardStaged = async (path: string): Promise<void> => {
+  await Bun.file(path)
+    .delete()
+    .catch(() => undefined);
+};
+
+/** Backpressure is the `flush`: awaiting it is what keeps a fast sender from filling the heap. */
+export async function stageStream(input: StageStreamInput): Promise<StagedStream> {
+  const { root, key, disk, body, maxBytes } = input;
+  const path = `${root}/${STAGING_DIR}/${crypto.randomUUID()}.incoming`;
+  const hasher = new Bun.CryptoHasher('sha256');
+  let sink: ReturnType<ReturnType<typeof Bun.file>['writer']> | undefined;
+  try {
+    await mkdir(`${root}/${STAGING_DIR}`, { recursive: true });
+    const open = Bun.file(path).writer();
+    sink = open;
+    const size = await pumpBounded(body, streamLimit('local', key, maxBytes), async (chunk) => {
+      hasher.update(chunk);
+      open.write(chunk);
+      await open.flush();
+    });
+    await open.end();
+    sink = undefined;
+    return { path, size, etag: hasher.digest('hex').slice(0, 32) };
+  } catch (error) {
+    await Promise.resolve(sink?.end()).catch(() => undefined);
+    await discardStaged(path);
+    if (isUltimateError(error)) throw error;
+    const at = renderFixShellArg(root, '<the disk root>');
+    throw putFailed(disk, key, error, `ls -ld ${at} && df -h ${at}`);
+  }
+}

package/src/driver-local-write.ts

@@ -5,13 +5,29 @@
 // why: Bun has no `rename` and no `mkdir` — `Bun.write` creates parents and replaces in place,
 // which is exactly the non-atomic write this file exists to stop. Delete both the day Bun ships them.
 import { mkdir, rename } from 'node:fs/promises';
-import { renderFixShellArg, stringField } from '@ultimat3/core';
+import { NotImplementedError, renderFixShellArg, stringField } from '@ultimat3/core';
+import type { PutOptions } from './driver';
 import type { Sidecar } from './driver-local-sidecar';
 import { isStorageError, keyConflict, putFailed } from './errors';
 import { META_DIR } from './path';
 
+/**
+ * A POSIX file is not encrypted at rest by this driver, and recording the request in the sidecar
+ * would answer a security review with a field the disk never honoured. Refused on the DEV disk
+ * too, and deliberately: a `put()` that succeeds locally and throws in production is a gap an app
+ * meets on the worst day, and both drivers refusing is one rule instead of two.
+ */
+export function refuseUnsupportedPut(putOptions?: PutOptions): void {
+  if (putOptions?.serverSideEncryption === undefined) return;
+  throw new NotImplementedError({
+    cause:
+      'server-side encryption on the local driver (it writes plain files under one root) is not implemented by this driver — drop serverSideEncryption from put() and encrypt the disk itself: an s3Driver over a bucket with a default KMS rule, or a LUKS/FileVault volume under root',
+    fix: 'put(key, body)',
+  });
+}
+
 /** Inside `.meta/`, so the listing's one filter already keeps a staged file out of the keys. */
-const STAGING_DIR = `${META_DIR}/.tmp`;
+export const STAGING_DIR = `${META_DIR}/.tmp`;
 
 export const sidecarPathOf = (key: string): string => `${META_DIR}/${key}.json`;
 /** Beside the sidecar, and never one itself: a sidecar's name ends in `.json`. */
@@ -79,8 +95,11 @@ export interface ObjectWrite {
   readonly key: string;
   /** The name the disk was REGISTERED under — what a refusal's `disk('…')` call must name. */
   readonly disk: string;
-  /** A `BunFile` is a `Blob`: a copy hands the source file over and no byte crosses the heap. */
-  readonly body: Uint8Array | Blob;
+  /**
+   * A `BunFile` is a `Blob`: a copy hands the source file over and no byte crosses the heap. A
+   * `stagedAt` is a file `putStream` already wrote under `STAGING_DIR`, renamed rather than copied.
+   */
+  readonly body: Uint8Array | Blob | { readonly stagedAt: string };
   /** `etag` MUST be the body's own: it is what the pending marker announces. */
   readonly sidecar: Sidecar;
 }
@@ -127,7 +146,11 @@ async function commit(write: ObjectWrite, steps: WriteSteps): Promise<void> {
     });
   };
   try {
-    await Bun.write(`${staged}.object`, write.body);
+    if (write.body instanceof Uint8Array || write.body instanceof Blob) {
+      await Bun.write(`${staged}.object`, write.body);
+    } else {
+      await rename(write.body.stagedAt, `${staged}.object`);
+    }
     await Bun.write(`${staged}.sidecar`, JSON.stringify(write.sidecar));
     await Bun.write(`${staged}.pending`, write.sidecar.etag);
     await settle('pending', pendingPath);