@ultimat3/storage 29.3.2 → 30.0.0

package/src/byte-pump.ts

@@ -0,0 +1,46 @@
+// Single responsibility: the one counting read of a body whose length is the sender's: `put()`
+// buffering a stream and `putStream` writing one both come through here. The caller says where the
+// chunks go and which refusal the limit raises; none of them decides when the body is too long.
+
+import { assert, describeValue } from '@ultimat3/core';
+
+export interface PumpLimit {
+  readonly driver: string;
+  readonly maxBytes: number;
+  /** The error to throw past `maxBytes`, given how far the body had got. */
+  readonly refuse: (total: number) => Error;
+}
+
+/**
+ * Every chunk handed to `sink` in order, and how many bytes they came to. Past the limit the sink
+ * sees nothing more. The reader is cancelled on every way out, so a refused body stops arriving
+ * rather than filling a socket buffer nobody will read. An already-closed stream answers that with
+ * a no-op.
+ */
+export async function pumpBounded(
+  body: ReadableStream<Uint8Array>,
+  limit: PumpLimit,
+  sink: (chunk: Uint8Array) => void | Promise<void>,
+): Promise<number> {
+  const reader = body.getReader();
+  let total = 0;
+  try {
+    for (;;) {
+      const next = await reader.read();
+      if (next.done) return total;
+      assert(
+        next.value instanceof Uint8Array,
+        `the stream handed to the ${limit.driver} disk yielded a chunk that is not a Uint8Array (${describeValue(next.value)})`,
+        'stream.pipeThrough(new TextEncoderStream())   # bytes, never strings',
+      );
+      total += next.value.byteLength;
+      if (total > limit.maxBytes) throw limit.refuse(total);
+      // Awaited only when the sink is async: a needless await is one more turn in which the stream
+      // pulls ahead, and a refused body must have pulled as little as possible.
+      const pending = sink(next.value);
+      if (pending !== undefined) await pending;
+    }
+  } finally {
+    await reader.cancel().catch(() => undefined);
+  }
+}

package/src/driver-local-list.ts

@@ -0,0 +1,75 @@
+// Single responsibility: one page of the local disk's listing — every object key under a prefix,
+// the `.meta/` sidecar tree kept out of the namespace, ordered, cut at the cursor, and a refused
+// scan reported rather than read as an empty disk (`sweepOrphans` trusts this listing).
+
+import { type Page, pageOf, renderFixShellArg } from '@ultimat3/core';
+import type { StorageListEntry } from './driver';
+import { isMissingFile } from './driver-local-remove';
+import { listFailed } from './errors';
+import { META_DIR } from './path';
+
+export interface LocalListInput {
+  readonly root: string;
+  readonly prefix: string;
+  /** The last key of the previous page; keys at or before it are skipped. */
+  readonly cursor: string | undefined;
+  /** A positive integer, already resolved by `resolveListLimit`. */
+  readonly limit: number;
+  /** Each key's listing view; `undefined` drops a key that went away mid-page. */
+  readonly head: (key: string) => Promise<StorageListEntry | undefined>;
+}
+
+export async function listLocalPage(input: LocalListInput): Promise<Page<StorageListEntry>> {
+  const { root, prefix, cursor, limit } = input;
+  const keys: string[] = [];
+  try {
+    // `dot: true`, and it is load-bearing: without it a glob matches no dot-prefixed entry, so
+    // every object whose key has one — `.hidden.txt`, `org/o1/pending/.x.png`, the
+    // `.metadata/a.json` `path.test.ts` pins as legal — was absent from the listing while
+    // `put`/`get`/`exists` handled it normally and the s3 listing returned it. `sweepOrphans`
+    // pages through `list()`, so those objects were swept as if they did not exist: a false
+    // erasure report by omission, which is what the classification below exists to prevent.
+    for await (const entry of new Bun.Glob('**/*').scan({
+      cwd: root,
+      onlyFiles: true,
+      dot: true,
+    })) {
+      const key = entry.replaceAll('\\', '/');
+      // The real filter now, not a second line of defence: the glob above yields the sidecar
+      // tree, and this is the one thing keeping `.meta/<key>.json` out of the object
+      // namespace. Folded like `assertSafeKey`'s reservation — `.META/` and `.meta/` are one
+      // directory on APFS and NTFS, and listing a key `get()` would refuse is the worse half.
+      if (key.toLowerCase().startsWith(`${META_DIR}/`)) continue;
+      if (!key.startsWith(prefix)) continue;
+      // The cursor IS the last key of the previous page — lexicographic order keeps it stable.
+      if (cursor !== undefined && key <= cursor) continue;
+      keys.push(key);
+    }
+  } catch (error) {
+    // A disk nobody has written to yet has no directory: an empty listing, not an error.
+    if (isMissingFile(error)) return pageOf([], null);
+    // Everything else is a refusal, and a bare `catch` reported all of them as "this disk is
+    // empty" — `EACCES` on the root, `ENOTDIR` on a root that is a file, an I/O error on the
+    // mount. `sweepOrphans` walks `list()`, so that swallow certified an unreadable prefix as
+    // having no orphans: the same false report `delete()`'s `.catch(() => undefined)` used to
+    // make, one call to the left.
+    throw listFailed(
+      'local',
+      prefix,
+      error,
+      `make the disk root readable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")}`,
+    );
+  }
+  keys.sort();
+  const page = keys.slice(0, limit);
+  const objects: StorageListEntry[] = [];
+  for (const key of page) {
+    const object = await input.head(key);
+    if (object !== undefined) objects.push(object);
+  }
+  // The cursor is minted only when a key PAST the page exists — `keys` is every match, so a
+  // full last page answers no cursor. `limit` is a positive integer, so a page with more after
+  // it always HAS a last key: the `?? null` is the type's, not a second condition. It used to
+  // be one, and `limit: 0` fell through it — an empty page reported as complete.
+  return pageOf(objects, keys.length > page.length ? (page.at(-1) ?? null) : null);
+}

package/src/driver-local-multipart.ts

@@ -0,0 +1,133 @@
+// Single responsibility: where the local disk keeps an open upload in parts — one directory per
+// session under `.meta/.multipart/`, which no key can name (`META_DIR` is reserved) and `list()`
+// never yields. A part's bytes land in a staged file renamed into place, so a dropped part is
+// absent rather than torn; its etag and size sit beside it.
+
+// why: Bun has no `mkdir`, no `readdir` and no recursive remove; a session is a directory.
+import { mkdir, readdir, rename, rm } from 'node:fs/promises';
+import type { UploadedPart } from './multipart';
+import type { PartStore, StagedSession } from './multipart-staged';
+import { META_DIR } from './path';
+
+/** Beside the write staging under `.meta/`, so one reserved segment covers both. */
+export const MULTIPART_DIR = `${META_DIR}/.multipart`;
+
+const PART_FILE = /^(\d{1,5})\.part$/;
+
+const isPart = (value: unknown): value is UploadedPart =>
+  typeof value === 'object' &&
+  value !== null &&
+  'partNumber' in value &&
+  Number.isSafeInteger(value.partNumber) &&
+  'etag' in value &&
+  typeof value.etag === 'string' &&
+  'size' in value &&
+  Number.isSafeInteger(value.size);
+
+const isSession = (value: unknown): value is StagedSession =>
+  typeof value === 'object' &&
+  value !== null &&
+  'key' in value &&
+  typeof value.key === 'string' &&
+  'contentType' in value &&
+  typeof value.contentType === 'string' &&
+  'initiated' in value &&
+  typeof value.initiated === 'string';
+
+/** A file that is absent or unreadable as JSON is no session: the directory is the disk's own. */
+const readJson = async (path: string): Promise<unknown> => {
+  const file = Bun.file(path);
+  if (!(await file.exists())) return undefined;
+  return file.json().catch(() => undefined);
+};
+
+/** Upload ids are the disk's own UUIDs; anything else never names a directory here. */
+const SESSION_ID = /^[A-Za-z0-9-]{1,64}$/;
+
+export function localPartStore(root: string): PartStore {
+  const dirOf = (uploadId: string): string | undefined =>
+    SESSION_ID.test(uploadId) ? `${root}/${MULTIPART_DIR}/${uploadId}` : undefined;
+  const sessionAt = async (uploadId: string): Promise<StagedSession | undefined> => {
+    const dir = dirOf(uploadId);
+    if (dir === undefined) return undefined;
+    const session = await readJson(`${dir}/session.json`);
+    return isSession(session) ? session : undefined;
+  };
+
+  return {
+    async open(uploadId, session): Promise<void> {
+      const dir = dirOf(uploadId) ?? '';
+      await mkdir(dir, { recursive: true });
+      await Bun.write(`${dir}/session.json`, JSON.stringify(session));
+    },
+
+    session: sessionAt,
+
+    async writePart(uploadId, part, bytes): Promise<void> {
+      const dir = dirOf(uploadId) ?? '';
+      const staged = `${dir}/${part.partNumber}.part.tmp`;
+      await Bun.write(staged, bytes);
+      await Bun.write(`${dir}/${part.partNumber}.json`, JSON.stringify(part));
+      await rename(staged, `${dir}/${part.partNumber}.part`);
+    },
+
+    async parts(uploadId): Promise<readonly UploadedPart[]> {
+      const dir = dirOf(uploadId);
+      if (dir === undefined) return [];
+      const names = await readdir(dir).catch(() => []);
+      const parts: UploadedPart[] = [];
+      for (const name of names) {
+        const number = PART_FILE.exec(name)?.[1];
+        if (number === undefined) continue;
+        const part = await readJson(`${dir}/${number}.json`);
+        if (isPart(part)) parts.push(part);
+      }
+      return parts.sort((a, b) => a.partNumber - b.partNumber);
+    },
+
+    read(uploadId, parts): ReadableStream<Uint8Array> {
+      const dir = dirOf(uploadId) ?? '';
+      let at = 0;
+      let current: ReadableStreamDefaultReader<Uint8Array> | undefined;
+      return new ReadableStream<Uint8Array>({
+        async pull(controller): Promise<void> {
+          for (;;) {
+            if (current === undefined) {
+              const part = parts[at];
+              if (part === undefined) {
+                controller.close();
+                return;
+              }
+              at += 1;
+              current = Bun.file(`${dir}/${part.partNumber}.part`).stream().getReader();
+            }
+            const next = await current.read();
+            if (!next.done) {
+              controller.enqueue(next.value);
+              return;
+            }
+            current = undefined;
+          }
+        },
+        async cancel(reason): Promise<void> {
+          await current?.cancel(reason);
+        },
+      });
+    },
+
+    async drop(uploadId): Promise<void> {
+      const dir = dirOf(uploadId);
+      if (dir !== undefined) await rm(dir, { recursive: true, force: true });
+    },
+
+    async sessions(): Promise<readonly (readonly [string, StagedSession])[]> {
+      const ids = await readdir(`${root}/${MULTIPART_DIR}`).catch(() => []);
+      const open: (readonly [string, StagedSession])[] = [];
+      for (const uploadId of ids) {
+        const session = await sessionAt(uploadId);
+        if (session !== undefined) open.push([uploadId, session]);
+      }
+      return open;
+    },
+  };
+}

package/src/driver-local-read.ts

@@ -61,6 +61,19 @@ export async function readObjectBytes(
     .then((buffer) => new Uint8Array(buffer), refusedRead(root, key, disk));
 }
 
+/** `readObjectBytes` for the inclusive window `[first, last]` — `Bun.file().slice()`, one read. */
+export async function readObjectRange(
+  root: string,
+  key: string,
+  disk: string,
+  [first, last]: readonly [number, number],
+): Promise<Uint8Array> {
+  return Bun.file(`${root}/${key}`)
+    .slice(first, last + 1)
+    .arrayBuffer()
+    .then((buffer) => new Uint8Array(buffer), refusedRead(root, key, disk));
+}
+
 /** `etagOf`, streamed: a copy or a `stat()` must settle a doubt without buffering the object. */
 export async function etagOfFile(root: string, key: string, disk = 'local'): Promise<string> {
   const hasher = new Bun.CryptoHasher('sha256');

package/src/driver-local-remove.ts

@@ -0,0 +1,39 @@
+// Single responsibility: how the local disk unlinks one file of an object, and which failures mean
+// "already gone". A refused unlink is reported, never swallowed: an erasure sweep that hears
+// "deleted" for bytes still on disk repeats the one lie it exists to prevent.
+
+import { renderFixShellArg, stringField } from '@ultimat3/core';
+import { deleteFailed } from './errors';
+
+/** `ENOENT` is the one delete failure that means "already in the desired state". */
+export const isMissingFile = (error: unknown): boolean => stringField(error, 'code') === 'ENOENT';
+
+/**
+ * A directory at an object's path is OTHER keys' parent (`c` beside `c/d`), never an object: no
+ * bytes are stored at the key, so deleting it is deleting an absent key. Asked only after an
+ * unlink failed, because the errno for "that is a directory" is `EISDIR` on Linux and `EPERM` on
+ * macOS — the stat is the portable answer.
+ */
+const isDirectory = async (path: string): Promise<boolean> =>
+  Bun.file(path)
+    .stat()
+    .then(
+      (stats) => stats.isDirectory(),
+      () => false,
+    );
+
+/** Removes one path under `root`, or reports WHY it could not — a swallowed refusal is a false erasure. */
+export async function removeIfPresent(root: string, path: string, key: string): Promise<void> {
+  try {
+    await Bun.file(path).delete();
+  } catch (error) {
+    if (isMissingFile(error) || (await isDirectory(path))) return;
+    throw deleteFailed(
+      'local',
+      key,
+      error,
+      // Screened: a key may carry `$(…)` and a root is configuration, and a fix: is pasted.
+      `make the disk root writable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")} && rm -f ${renderFixShellArg(path, "'<the object file>'")}`,
+    );
+  }
+}

package/src/driver-local-stream.ts

@@ -0,0 +1,104 @@
+// Single responsibility: the local disk's `putStream`: the body counted and hashed into one staged
+// file the commit renames into place, so any length costs one chunk of heap. Staged outside the key's
+// queue, so a slow sender does not hold the key's readers.
+
+// why: Bun has no `mkdir`; `Bun.write` makes parents but cannot open the incremental writer here.
+import { mkdir } from 'node:fs/promises';
+import { type Clock, isUltimateError, renderFixShellArg } from '@ultimat3/core';
+import { pumpBounded } from './byte-pump';
+import { DEFAULT_CONTENT_TYPE, type StorageObject } from './driver';
+import type { Sidecar } from './driver-local-sidecar';
+import { commitObject, STAGING_DIR } from './driver-local-write';
+import { putFailed } from './errors';
+import type { PutStreamOptions } from './put-stream';
+import { streamLimit } from './put-stream';
+
+export interface StagedStream {
+  readonly path: string;
+  readonly size: number;
+  /** `etagOf` over the same bytes, so a streamed object and a `put()` of it report one etag. */
+  readonly etag: string;
+}
+
+export interface LocalStreamInput extends StageStreamInput {
+  readonly clock: Clock;
+}
+
+/**
+ * Stage, then commit. `inTurn` is the disk's lock check in the key's turn, then `then`: asked before
+ * the transfer too, because a locked key should not cost the whole body to refuse.
+ */
+export async function putLocalStream(
+  input: LocalStreamInput,
+  options: PutStreamOptions,
+  inTurn: (then: () => Promise<void>) => Promise<void>,
+): Promise<StorageObject> {
+  const { root, key, disk, clock } = input;
+  await inTurn(async () => undefined);
+  const staged = await stageStream(input);
+  try {
+    const lastModified = clock.now();
+    const sidecar: Sidecar = {
+      contentType: options.contentType ?? DEFAULT_CONTENT_TYPE,
+      etag: staged.etag,
+      cacheControl: options.cacheControl,
+      metadata: options.metadata,
+      lastModified: lastModified.toISOString(),
+    };
+    const body = { stagedAt: staged.path };
+    await inTurn(() => commitObject({ root, key, disk, body, sidecar }));
+    return {
+      key,
+      size: staged.size,
+      contentType: sidecar.contentType,
+      etag: sidecar.etag,
+      lastModified,
+      ...(sidecar.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
+      ...(sidecar.metadata === undefined ? {} : { metadata: sidecar.metadata }),
+    };
+  } finally {
+    await discardStaged(staged.path);
+  }
+}
+
+export interface StageStreamInput {
+  readonly root: string;
+  readonly key: string;
+  readonly disk: string;
+  readonly body: ReadableStream<Uint8Array>;
+  readonly maxBytes: number;
+}
+
+/** A staged file gone, or never there. Safe after the commit renamed it away. */
+export const discardStaged = async (path: string): Promise<void> => {
+  await Bun.file(path)
+    .delete()
+    .catch(() => undefined);
+};
+
+/** Backpressure is the `flush`: awaiting it is what keeps a fast sender from filling the heap. */
+export async function stageStream(input: StageStreamInput): Promise<StagedStream> {
+  const { root, key, disk, body, maxBytes } = input;
+  const path = `${root}/${STAGING_DIR}/${crypto.randomUUID()}.incoming`;
+  const hasher = new Bun.CryptoHasher('sha256');
+  let sink: ReturnType<ReturnType<typeof Bun.file>['writer']> | undefined;
+  try {
+    await mkdir(`${root}/${STAGING_DIR}`, { recursive: true });
+    const open = Bun.file(path).writer();
+    sink = open;
+    const size = await pumpBounded(body, streamLimit('local', key, maxBytes), async (chunk) => {
+      hasher.update(chunk);
+      open.write(chunk);
+      await open.flush();
+    });
+    await open.end();
+    sink = undefined;
+    return { path, size, etag: hasher.digest('hex').slice(0, 32) };
+  } catch (error) {
+    await Promise.resolve(sink?.end()).catch(() => undefined);
+    await discardStaged(path);
+    if (isUltimateError(error)) throw error;
+    const at = renderFixShellArg(root, '<the disk root>');
+    throw putFailed(disk, key, error, `ls -ld ${at} && df -h ${at}`);
+  }
+}

package/src/driver-local-write.ts

@@ -5,13 +5,29 @@
 // why: Bun has no `rename` and no `mkdir` — `Bun.write` creates parents and replaces in place,
 // which is exactly the non-atomic write this file exists to stop. Delete both the day Bun ships them.
 import { mkdir, rename } from 'node:fs/promises';
-import { renderFixShellArg, stringField } from '@ultimat3/core';
+import { NotImplementedError, renderFixShellArg, stringField } from '@ultimat3/core';
+import type { PutOptions } from './driver';
 import type { Sidecar } from './driver-local-sidecar';
 import { isStorageError, keyConflict, putFailed } from './errors';
 import { META_DIR } from './path';
 
+/**
+ * A POSIX file is not encrypted at rest by this driver, and recording the request in the sidecar
+ * would answer a security review with a field the disk never honoured. Refused on the DEV disk
+ * too, and deliberately: a `put()` that succeeds locally and throws in production is a gap an app
+ * meets on the worst day, and both drivers refusing is one rule instead of two.
+ */
+export function refuseUnsupportedPut(putOptions?: PutOptions): void {
+  if (putOptions?.serverSideEncryption === undefined) return;
+  throw new NotImplementedError({
+    cause:
+      'server-side encryption on the local driver (it writes plain files under one root) is not implemented by this driver — drop serverSideEncryption from put() and encrypt the disk itself: an s3Driver over a bucket with a default KMS rule, or a LUKS/FileVault volume under root',
+    fix: 'put(key, body)',
+  });
+}
+
 /** Inside `.meta/`, so the listing's one filter already keeps a staged file out of the keys. */
-const STAGING_DIR = `${META_DIR}/.tmp`;
+export const STAGING_DIR = `${META_DIR}/.tmp`;
 
 export const sidecarPathOf = (key: string): string => `${META_DIR}/${key}.json`;
 /** Beside the sidecar, and never one itself: a sidecar's name ends in `.json`. */
@@ -79,8 +95,11 @@ export interface ObjectWrite {
   readonly key: string;
   /** The name the disk was REGISTERED under — what a refusal's `disk('…')` call must name. */
   readonly disk: string;
-  /** A `BunFile` is a `Blob`: a copy hands the source file over and no byte crosses the heap. */
-  readonly body: Uint8Array | Blob;
+  /**
+   * A `BunFile` is a `Blob`: a copy hands the source file over and no byte crosses the heap. A
+   * `stagedAt` is a file `putStream` already wrote under `STAGING_DIR`, renamed rather than copied.
+   */
+  readonly body: Uint8Array | Blob | { readonly stagedAt: string };
   /** `etag` MUST be the body's own: it is what the pending marker announces. */
   readonly sidecar: Sidecar;
 }
@@ -127,7 +146,11 @@ async function commit(write: ObjectWrite, steps: WriteSteps): Promise<void> {
     });
   };
   try {
-    await Bun.write(`${staged}.object`, write.body);
+    if (write.body instanceof Uint8Array || write.body instanceof Blob) {
+      await Bun.write(`${staged}.object`, write.body);
+    } else {
+      await rename(write.body.stagedAt, `${staged}.object`);
+    }
     await Bun.write(`${staged}.sidecar`, JSON.stringify(write.sidecar));
     await Bun.write(`${staged}.pending`, write.sidecar.etag);
     await settle('pending', pendingPath);