@ultimat3/storage 23.0.0 → 25.0.0

package/src/driver-local-read.ts

@@ -0,0 +1,114 @@
+// Single responsibility: what the local disk KNOWS about one object — its bytes file, its sidecar,
+// and the pending marker that says whether the two can be trusted as a pair. A write that died
+// between its renames leaves a sidecar describing bytes that never landed; this is where that is
+// read as "type unknown" instead of served as the truth.
+
+import { renderFixShellArg, stringField } from '@ultimat3/core';
+import type { StorageListEntry } from './driver';
+import { parseSidecar, type Sidecar } from './driver-local-sidecar';
+import { pendingPathOf, sidecarPathOf } from './driver-local-write';
+import { isStorageError, objectNotFound, readFailed } from './errors';
+
+/** The sidecar as recorded — trusted or not. The lock check reads it whole: a doubt never unlocks. */
+export async function readSidecar(root: string, key: string): Promise<Sidecar | undefined> {
+  const file = Bun.file(`${root}/${sidecarPathOf(key)}`);
+  if (!(await file.exists())) return undefined;
+  try {
+    const raw: unknown = await file.json();
+    return parseSidecar(raw);
+  } catch {
+    return undefined;
+  }
+}
+
+/** The etag a write in flight — or one that died — announced before it moved anything. */
+async function pendingEtag(root: string, key: string): Promise<string | undefined> {
+  const file = Bun.file(`${root}/${pendingPathOf(key)}`);
+  if (!(await file.exists())) return undefined;
+  return (await file.text().catch(() => '')).trim();
+}
+
+/**
+ * A read the OS refused, coded: `EACCES`, `EIO`, a directory where the bytes should be. A file
+ * that is simply gone — deleted between the `exists()` and the read — is "not found", as it
+ * would have been one call earlier.
+ */
+const refusedRead =
+  (root: string, key: string, disk: string) =>
+  (error: unknown): never => {
+    if (isStorageError(error)) throw error;
+    throw stringField(error, 'code') === 'ENOENT'
+      ? objectNotFound(disk, key)
+      : readFailed(
+          'local',
+          key,
+          error,
+          `ls -l ${renderFixShellArg(`${root}/${key}`, '<the object file>')}`,
+        );
+  };
+
+/**
+ * The whole object, buffered — `get()`'s one read. `disk` is the name `defineStorage` registered,
+ * because a not-found `fix:` is `disk('<it>').list(…)`; `local` only for a driver nobody named.
+ */
+export async function readObjectBytes(
+  root: string,
+  key: string,
+  disk = 'local',
+): Promise<Uint8Array> {
+  return Bun.file(`${root}/${key}`)
+    .arrayBuffer()
+    .then((buffer) => new Uint8Array(buffer), refusedRead(root, key, disk));
+}
+
+/** `etagOf`, streamed: a copy or a `stat()` must settle a doubt without buffering the object. */
+export async function etagOfFile(root: string, key: string, disk = 'local'): Promise<string> {
+  const hasher = new Bun.CryptoHasher('sha256');
+  try {
+    for await (const chunk of Bun.file(`${root}/${key}`).stream()) hasher.update(chunk);
+  } catch (error) {
+    return refusedRead(root, key, disk)(error);
+  }
+  return hasher.digest('hex').slice(0, 32);
+}
+
+/**
+ * One object's facts, or `undefined` when there are no bytes at the key — the bytes file is what
+ * makes an object exist, so a sidecar alone (a fresh write that died early) is nothing at all.
+ *
+ * The sidecar is TRUSTED unless the pending marker names its etag: that is the one state in which
+ * the sidecar is the new write's and the bytes may still be the old object's. Then, and for an
+ * object with no sidecar, the answer depends on `trueEtag`:
+ * - absent (a listing, which never reads bytes): no content type, `etag: ''` — "cannot know";
+ * - given: the bytes decide. A match keeps the sidecar; a mismatch drops every field it claimed
+ *   and reports the bytes' own etag.
+ *
+ * No path here answers a type or an etag the bytes were not checked against while in doubt.
+ */
+export async function headObject(
+  root: string,
+  key: string,
+  trueEtag?: () => string | Promise<string>,
+): Promise<StorageListEntry | undefined> {
+  const file = Bun.file(`${root}/${key}`);
+  if (!(await file.exists())) return undefined;
+  const recorded = await readSidecar(root, key);
+  const inDoubt = recorded !== undefined && (await pendingEtag(root, key)) === recorded.etag;
+  let sidecar = recorded;
+  let etag = recorded?.etag ?? '';
+  if (recorded === undefined || inDoubt) {
+    etag = trueEtag === undefined ? '' : await trueEtag();
+    if (recorded?.etag !== etag) sidecar = undefined;
+  }
+  return {
+    key,
+    size: file.size,
+    etag,
+    // The recorded instant when the sidecar is trusted, the file's mtime otherwise: one source per
+    // object, and the one `put()`/`copy()` returned — on the disk's injected clock.
+    lastModified: new Date(sidecar?.lastModified ?? file.lastModified),
+    ...(sidecar?.contentType === undefined ? {} : { contentType: sidecar.contentType }),
+    ...(sidecar?.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
+    ...(sidecar?.metadata === undefined ? {} : { metadata: sidecar.metadata }),
+  };
+}

package/src/driver-local-sidecar.ts

@@ -0,0 +1,96 @@
+// Single responsibility: the local disk's sidecar — the JSON file under `.meta/` holding what a
+// POSIX file has nowhere to keep (content type, etag, cache-control, user metadata, the object
+// lock), and the one parser that reads it back as untrusted bytes off a disk.
+
+import { isRetentionMode, type ObjectLock, type RetentionMode } from './object-lock';
+
+/** The lock as JSON holds it: the instant as an ISO string, the hold only when it is on. */
+export interface SidecarLock {
+  readonly retention?: { readonly mode: RetentionMode; readonly retainUntil: string } | undefined;
+  readonly legalHold?: true | undefined;
+}
+
+export interface Sidecar extends SidecarLock {
+  readonly contentType: string;
+  readonly etag: string;
+  readonly cacheControl?: string | undefined;
+  readonly metadata?: Readonly<Record<string, string>> | undefined;
+  /**
+   * The write's instant as ISO, read off the disk's injected `Clock` — what `put()`/`copy()`
+   * returned, so a later `stat`/`get`/`list` reports the same instant (the memory disk's rule).
+   * Absent on a sidecar written before it was recorded: the file's mtime answers instead.
+   */
+  readonly lastModified?: string | undefined;
+}
+
+/** `ObjectLock` → the fields a sidecar records. Nothing when nothing locks the object. */
+export function sidecarLock(lock: ObjectLock | undefined): SidecarLock {
+  if (lock === undefined) return {};
+  return {
+    ...(lock.retention === undefined
+      ? {}
+      : {
+          retention: {
+            mode: lock.retention.mode,
+            retainUntil: lock.retention.retainUntil.toISOString(),
+          },
+        }),
+    ...(lock.legalHold ? { legalHold: true } : {}),
+  };
+}
+
+/** The recorded lock as `ObjectLock`. A retention that does not parse is no retention. */
+export function lockOfSidecar(sidecar: SidecarLock | undefined): ObjectLock {
+  const recorded = sidecar?.retention;
+  return {
+    ...(recorded === undefined
+      ? {}
+      : { retention: { mode: recorded.mode, retainUntil: new Date(recorded.retainUntil) } }),
+    legalHold: sidecar?.legalHold === true,
+  };
+}
+
+function parseRetention(raw: unknown): SidecarLock['retention'] {
+  if (typeof raw !== 'object' || raw === null) return undefined;
+  const mode = (raw as Record<string, unknown>)['mode'];
+  const until = (raw as Record<string, unknown>)['retainUntil'];
+  if (!isRetentionMode(mode) || typeof until !== 'string') return undefined;
+  if (!Number.isFinite(Date.parse(until))) return undefined;
+  return { mode, retainUntil: until };
+}
+
+// `!Array.isArray` is the load-bearing clause, matching `isPlainObject` in
+// `@ultimat3/schema`'s `builder.ts`: `typeof [] === 'object'` and every value of `['a','b']` is a
+// string, so an array in the `metadata` slot was handed back through `head()`/`get()` as object
+// metadata — against a `Record<string, string>` every reader downstream is typed on.
+const isStringRecord = (value: unknown): value is Readonly<Record<string, string>> =>
+  typeof value === 'object' &&
+  value !== null &&
+  !Array.isArray(value) &&
+  Object.values(value as Record<string, unknown>).every((entry) => typeof entry === 'string');
+
+export function parseSidecar(raw: unknown): Sidecar | undefined {
+  if (typeof raw !== 'object' || raw === null) return undefined;
+  const record = raw as Record<string, unknown>;
+  const contentType = record['contentType'];
+  const etag = record['etag'];
+  if (typeof contentType !== 'string' || typeof etag !== 'string') return undefined;
+  // `put()` writes cacheControl/metadata into the same sidecar — dropping them here silently
+  // truncated what was just written, even though `Sidecar` itself declares both.
+  const cacheControl = record['cacheControl'];
+  const metadata = record['metadata'];
+  const retention = parseRetention(record['retention']);
+  const lastModified = record['lastModified'];
+  return {
+    contentType,
+    etag,
+    ...(typeof cacheControl === 'string' ? { cacheControl } : {}),
+    ...(isStringRecord(metadata) ? { metadata } : {}),
+    // An instant that does not parse is no instant — never an invalid `Date` handed to a reader.
+    ...(typeof lastModified === 'string' && Number.isFinite(Date.parse(lastModified))
+      ? { lastModified }
+      : {}),
+    ...(retention === undefined ? {} : { retention }),
+    ...(record['legalHold'] === true ? { legalHold: true } : {}),
+  };
+}

package/src/driver-local-write.ts

@@ -0,0 +1,185 @@
+// Single responsibility: how the local disk WRITES one object — the bytes, the sidecar and a
+// pending marker, staged under `.meta/.tmp/` and renamed into place in the one order whose every
+// crash point reads as absent, whole or untyped; refused by code when the disk says no.
+
+// why: Bun has no `rename` and no `mkdir` — `Bun.write` creates parents and replaces in place,
+// which is exactly the non-atomic write this file exists to stop. Delete both the day Bun ships them.
+import { mkdir, rename } from 'node:fs/promises';
+import { renderFixShellArg, stringField } from '@ultimat3/core';
+import type { Sidecar } from './driver-local-sidecar';
+import { isStorageError, keyConflict, putFailed } from './errors';
+import { META_DIR } from './path';
+
+/** Inside `.meta/`, so the listing's one filter already keeps a staged file out of the keys. */
+const STAGING_DIR = `${META_DIR}/.tmp`;
+
+export const sidecarPathOf = (key: string): string => `${META_DIR}/${key}.json`;
+/** Beside the sidecar, and never one itself: a sidecar's name ends in `.json`. */
+export const pendingPathOf = (key: string): string => `${META_DIR}/${key}.json.pending`;
+
+const codeOf = (error: unknown): string | undefined => stringField(error, 'code');
+/** `mkdir -p` through a file, a rename onto a directory, a write under a file. */
+const isPathConflict = (error: unknown): boolean =>
+  ['ENOTDIR', 'EEXIST', 'EISDIR', 'ENOTEMPTY'].includes(codeOf(error) ?? '');
+const isAbsent = (error: unknown): boolean => ['ENOENT', 'ENOTDIR'].includes(codeOf(error) ?? '');
+
+/** The object KEY a path inside the root stands for — a sidecar's path names its object. */
+const keyAt = (relative: string): string =>
+  relative.startsWith(`${META_DIR}/`)
+    ? relative.slice(META_DIR.length + 1).replace(/\.json(\.pending)?$/, '')
+    : relative;
+
+/** The first ancestor of `relative` that is a FILE: the stored object standing where a directory must be. */
+async function fileInTheWay(root: string, relative: string): Promise<string | undefined> {
+  const segments = relative.split('/').slice(0, -1);
+  for (let depth = 1; depth <= segments.length; depth += 1) {
+    const ancestor = segments.slice(0, depth).join('/');
+    if (await Bun.file(`${root}/${ancestor}`).exists()) return keyAt(ancestor);
+  }
+  // Nothing INSIDE the root is in the way: the root itself is the problem, and that is the
+  // disk refusing the write, not two keys colliding.
+  return undefined;
+}
+
+/**
+ * Make `relative` writable as a FILE, or refuse. Runs for both of an object's paths BEFORE either
+ * is touched: the sidecar tree collides where the object tree does not (`a` and `a.json/b`), and a
+ * put() that wrote its bytes and then failed on its sidecar left an object nobody had recorded.
+ */
+async function claim(write: ObjectWrite, relative: string): Promise<void> {
+  const { root, key } = write;
+  const target = `${root}/${relative}`;
+  try {
+    await mkdir(target.slice(0, target.lastIndexOf('/')), { recursive: true });
+  } catch (error) {
+    const blocking = isPathConflict(error) ? await fileInTheWay(root, relative) : undefined;
+    throw blocking === undefined ? error : keyConflict(write.disk, key, blocking);
+  }
+  let directory = false;
+  try {
+    directory = (await Bun.file(target).stat()).isDirectory();
+  } catch (error) {
+    if (!isAbsent(error)) throw error;
+  }
+  // The keys in the way live UNDER this path: `<key>/…`, or `<key>.json/…` in the sidecar tree.
+  const beneath = relative.startsWith(`${META_DIR}/`)
+    ? relative.slice(META_DIR.length + 1)
+    : relative;
+  if (directory) throw keyConflict(write.disk, key, `${beneath}/`);
+}
+
+const discard = async (path: string): Promise<void> => {
+  await Bun.file(path)
+    .delete()
+    .catch(() => undefined);
+};
+
+export interface ObjectWrite {
+  readonly root: string;
+  readonly key: string;
+  /** The name the disk was REGISTERED under — what a refusal's `disk('…')` call must name. */
+  readonly disk: string;
+  /** A `BunFile` is a `Blob`: a copy hands the source file over and no byte crosses the heap. */
+  readonly body: Uint8Array | Blob;
+  /** `etag` MUST be the body's own: it is what the pending marker announces. */
+  readonly sidecar: Sidecar;
+}
+
+/** The four mutations a commit makes, in order — the seam a crash test dies at. */
+export interface WriteSteps {
+  rename(from: string, to: string): Promise<void>;
+  unlink(path: string): Promise<void>;
+}
+
+const REAL_STEPS: WriteSteps = {
+  rename: (from, to) => rename(from, to),
+  unlink: async (path) => {
+    await Bun.file(path).delete();
+  },
+};
+
+/**
+ * Stage three files, then four steps: (1) rename the pending MARKER, which holds the new etag;
+ * (2) rename the sidecar; (3) rename the bytes; (4) remove the marker. Each rename replaces a
+ * directory entry whole, so no reader ever opens half a file.
+ *
+ * The order is chosen by what a crash leaves (`driver-local-crash.test.ts` dies at every step):
+ * - before (3), a FRESH key has no bytes file and so no object — absent, never half-present;
+ * - before (2), an overwrite still has the previous sidecar over the previous bytes — whole;
+ * - between (2) and (3), the new sidecar sits over the OLD bytes, and the marker naming that
+ *   sidecar's etag is what tells `headObject` not to believe it;
+ * - between (3) and (4) the pair is whole and still marked, so a read re-checks it and a listing
+ *   reports it untyped until the next put.
+ * Bytes-first has no such table: its torn overwrite is new bytes under the old type, undetectable.
+ */
+async function commit(write: ObjectWrite, steps: WriteSteps): Promise<void> {
+  const sidecarPath = sidecarPathOf(write.key);
+  const pendingPath = pendingPathOf(write.key);
+  await claim(write, write.key);
+  await claim(write, sidecarPath);
+  await claim(write, pendingPath);
+  const staged = `${write.root}/${STAGING_DIR}/${crypto.randomUUID()}`;
+  // A second writer can make the colliding key between `claim` and the rename; only a RENAME's
+  // refusal is that, never a failed staging write.
+  const settle = async (suffix: string, relative: string): Promise<void> => {
+    await steps.rename(`${staged}.${suffix}`, `${write.root}/${relative}`).catch((error) => {
+      throw isPathConflict(error) ? keyConflict(write.disk, write.key, write.key) : error;
+    });
+  };
+  try {
+    await Bun.write(`${staged}.object`, write.body);
+    await Bun.write(`${staged}.sidecar`, JSON.stringify(write.sidecar));
+    await Bun.write(`${staged}.pending`, write.sidecar.etag);
+    await settle('pending', pendingPath);
+    await settle('sidecar', sidecarPath);
+    await settle('object', write.key);
+  } catch (error) {
+    for (const suffix of ['object', 'sidecar', 'pending']) await discard(`${staged}.${suffix}`);
+    throw error;
+  }
+  // The object is COMMITTED: all three renames landed. A marker that will not clear is not a
+  // failed put — reporting one told the caller to retry a write every reader can already see.
+  // What it leaves is exactly the crash-at-step-4 state, which `headObject` re-checks.
+  await steps.unlink(`${write.root}/${pendingPath}`).catch(() => undefined);
+}
+
+/**
+ * The local write, coded. Whatever the filesystem refuses with — `EACCES`, `ENOSPC`, `EROFS`, a
+ * root that is a file — leaves here as `X_STORAGE_PUT_FAILED` carrying the errno, where it used to
+ * be the bare `Error` out of `Bun.write`. A refusal this package already named passes through.
+ */
+export async function commitObject(
+  write: ObjectWrite,
+  steps: WriteSteps = REAL_STEPS,
+): Promise<void> {
+  try {
+    await commit(write, steps);
+  } catch (error) {
+    if (isStorageError(error)) throw error;
+    const at = renderFixShellArg(write.root, '<the disk root>');
+    throw putFailed(write.disk, write.key, error, `ls -ld ${at} && df -h ${at}`);
+  }
+}
+
+/** Runs `task` after every earlier task queued under the same key has settled. */
+export type KeyedQueue = <T>(key: string, task: () => Promise<T>) => Promise<T>;
+
+/**
+ * One writer — or one reader — per key at a time, inside this process. An object is two files, so
+ * two unserialised `put()`s of one key interleaved their four writes and left one's bytes under the
+ * other's etag; a `get()` between a writer's two renames read the same mismatch.
+ */
+export function keyedQueue(): KeyedQueue {
+  const tails = new Map<string, Promise<unknown>>();
+  return async <T>(key: string, task: () => Promise<T>): Promise<T> => {
+    const result = (tails.get(key) ?? Promise.resolve()).then(task, task);
+    const tail = result.catch(() => undefined);
+    tails.set(key, tail);
+    try {
+      return await result;
+    } finally {
+      // Only the LAST task clears the entry, or a Map of every key ever written is a leak.
+      if (tails.get(key) === tail) tails.delete(key);
+    }
+  };
+}