@ultimat3/storage 1.2.0 → 2.0.0

package/src/attachment.ts

@@ -0,0 +1,213 @@
+// Single responsibility: where an entity's files live, and how one that was never attached is
+// found again. The upload happens BEFORE the row it belongs to exists, so it lands under a
+// `pending/` segment inside the org and is promoted once there is an id — which makes an orphan
+// a fact about the key rather than a join nobody runs. Every key here goes through `scopedKey`,
+// so the tenant prefix is a construction, not a check somebody remembered to write.
+
+import type { Clock } from '@ultimat3/core';
+import { renderThrowable, systemClock } from '@ultimat3/core';
+import type { ListPage, StorageDriver, StorageListEntry, StorageObject } from './driver';
+import { orgMismatch, quarantined } from './errors';
+import { isWithinOrg, orgPrefix, scopedKey } from './path';
+
+/** The one segment an unattached upload lives under. `sweepOrphans` reads only this prefix. */
+export const PENDING_SEGMENT = 'pending';
+
+/**
+ * One more segment inside `pending/`, and the whole of Ultimate's content-scanning mechanism.
+ *
+ * Magic-byte sniffing closes stored XSS; it is not malware scanning, and `application/zip` is
+ * deliberately accepted as the OOXML container, so a macro-laden `.docx` passes `validateUpload`
+ * exactly as a clean one does. Scanning is the APP's — a scanner is a business decision with a
+ * vendor, a licence and a latency budget (axiom 8) — so what ships is the place to put one: an
+ * upload granted with `quarantine: true` lands under this prefix, `promoteAttachment` REFUSES a
+ * key that is still there (`X_STORAGE_QUARANTINED`), and the app's scan job calls
+ * `releaseQuarantine` on a verdict of clean. Inside `pending/` on purpose: an upload nobody ever
+ * scanned is still an orphan, so `sweepOrphans` collects it with no second prefix to walk.
+ */
+export const QUARANTINE_SEGMENT = 'quarantine';
+
+export interface AttachmentTarget {
+  /** Entity name as the app declares it — `post`, `user`. Exactly one key segment. */
+  readonly entity: string;
+  readonly id: string;
+  /** The field on that entity the file belongs to — `avatar`, `attachments`. */
+  readonly field: string;
+}
+
+// Deliberately narrow. A filename is client-supplied, and an extension is decoration: the stored
+// content type is the SNIFFED one (`validateUpload`), so anything longer or stranger is dropped
+// rather than sanitised — sanitising is what turns `evil.png.html` into a key that looks safe.
+const EXTENSION = /^\.[a-z0-9]{1,12}$/;
+
+export function uploadExtension(filename: string): string {
+  const cut = Math.max(filename.lastIndexOf('/'), filename.lastIndexOf('\\'));
+  const base = filename.slice(cut + 1);
+  const dot = base.lastIndexOf('.');
+  const extension = dot <= 0 ? '' : base.slice(dot).toLowerCase();
+  return EXTENSION.test(extension) ? extension : '';
+}
+
+/**
+ * The final key segment: an opaque id plus the surviving extension. The id — never the client's
+ * filename — is what makes the key unguessable and collision-free; the original name belongs in
+ * the entity row, where it can be displayed without ever being a path.
+ */
+export function uploadName(uploadId: string, filename: string): string {
+  return `${uploadId}${uploadExtension(filename)}`;
+}
+
+/** `org/<orgId>/pending/<name>` — an upload with no row behind it yet. */
+export function pendingKey(orgId: string, name: string): string {
+  return scopedKey(orgId, PENDING_SEGMENT, name);
+}
+
+export function pendingPrefix(orgId: string): string {
+  return `${orgPrefix(orgId)}${PENDING_SEGMENT}/`;
+}
+
+/** `org/<orgId>/<entity>/<id>/<field>/` — every file one field of one row owns. */
+export function attachmentPrefix(orgId: string, target: AttachmentTarget): string {
+  return `${scopedKey(orgId, target.entity, target.id, target.field)}/`;
+}
+
+export function attachmentKey(orgId: string, target: AttachmentTarget, name: string): string {
+  return scopedKey(orgId, target.entity, target.id, target.field, name);
+}
+
+/** `org/<orgId>/pending/quarantine/<name>` — uploaded, validated, and not yet cleared to use. */
+export function quarantineKey(orgId: string, name: string): string {
+  return scopedKey(orgId, PENDING_SEGMENT, QUARANTINE_SEGMENT, name);
+}
+
+export function quarantinePrefix(orgId: string): string {
+  return `${pendingPrefix(orgId)}${QUARANTINE_SEGMENT}/`;
+}
+
+export const isPendingKey = (key: string, orgId: string): boolean =>
+  key.startsWith(pendingPrefix(orgId));
+
+/** True while nothing has cleared the key. `promoteAttachment` refuses exactly this. */
+export const isQuarantinedKey = (key: string, orgId: string): boolean =>
+  key.startsWith(quarantinePrefix(orgId));
+
+export interface ReleaseQuarantineInput {
+  readonly disk: StorageDriver;
+  /** A key under `quarantinePrefix(orgId)`. */
+  readonly key: string;
+  readonly orgId: string;
+}
+
+/**
+ * The app's scan said clean: move the object out of quarantine and onto the ordinary `pending/`
+ * key, which `promoteAttachment` will accept. Copy then delete, for the reason promotion does —
+ * a delete that ran first loses the bytes on a failed write.
+ *
+ * Releasing a key that is not quarantined returns it untouched rather than copying it onto
+ * itself: a scan job that retries after a crash must not be a second round trip, and a `pending/`
+ * key is already the released state.
+ */
+export async function releaseQuarantine(input: ReleaseQuarantineInput): Promise<string> {
+  if (!isWithinOrg(input.key, input.orgId)) throw orgMismatch(input.key, input.orgId);
+  if (!isQuarantinedKey(input.key, input.orgId)) return input.key;
+  const name = input.key.slice(input.key.lastIndexOf('/') + 1);
+  const released = pendingKey(input.orgId, name);
+  await input.disk.copy(input.key, released);
+  await input.disk.delete(input.key);
+  return released;
+}
+
+export interface PromoteAttachmentInput {
+  readonly disk: StorageDriver;
+  /** A key `grantUpload` minted with no `target`. */
+  readonly key: string;
+  readonly orgId: string;
+  readonly target: AttachmentTarget;
+}
+
+/**
+ * Move a pending upload onto the row that now exists. Copy first, delete second: a delete that
+ * ran first would lose the bytes on a failed write, and the sweep would have collected them
+ * anyway. Re-promoting an already-promoted key is `X_STORAGE_NOT_FOUND` from the copy, never a
+ * silent no-op that leaves the caller believing a file is attached.
+ *
+ * The copy is `disk.copy`, not `get` + `put`: promotion used to download the whole object into
+ * this process and upload it again, so attaching a 500MB file moved a gigabyte through the pod
+ * for a rename. `copy` never touches the app's heap on either driver.
+ */
+export async function promoteAttachment(input: PromoteAttachmentInput): Promise<StorageObject> {
+  if (!isWithinOrg(input.key, input.orgId)) throw orgMismatch(input.key, input.orgId);
+  if (isQuarantinedKey(input.key, input.orgId)) throw quarantined(input.key, input.orgId);
+  const name = input.key.slice(input.key.lastIndexOf('/') + 1);
+  const attached = attachmentKey(input.orgId, input.target, name);
+  const object = await input.disk.copy(input.key, attached);
+  await input.disk.delete(input.key);
+  return object;
+}
+
+export interface SweepOrphansInput {
+  readonly disk: StorageDriver;
+  readonly orgId: string;
+  /** Age past which an unattached upload is an orphan. Longer than any form can stay open. */
+  readonly olderThanMs: number;
+  readonly clock?: Clock | undefined;
+  /**
+   * Return `true` to spare a key the app can still account for. Absent spares nothing, because
+   * a pending key by definition has no row pointing at it — override only when the app parks
+   * a reference somewhere the prefix cannot express.
+   */
+  readonly keep?: ((object: StorageListEntry) => boolean | Promise<boolean>) | undefined;
+}
+
+/** One key the sweep tried to delete and could not, with the disk's own words for why. */
+export interface SweepFailure {
+  readonly key: string;
+  readonly reason: string;
+}
+
+/**
+ * Two lists, because one cannot be wrong. A sweep that answered a single array of "deleted" keys
+ * put every refusal in it: a GDPR erasure over 200 objects against a bucket whose policy had lost
+ * `s3:DeleteObject` returned all 200 as deleted, and the compliance report said the data was gone.
+ */
+export interface SweepResult {
+  readonly deleted: readonly string[];
+  readonly failed: readonly SweepFailure[];
+}
+
+/**
+ * Delete every unattached upload past the window, and answer with what went and what did not.
+ * Scoped to one org and to the `pending/` prefix — quarantine included, since it lives inside
+ * that prefix: a sweep that could reach an attached key is a job that deletes production data
+ * the first time an app forgets to promote something.
+ *
+ * A refusal is recorded and the sweep continues. Stopping at the first one would leave the caller
+ * unable to distinguish "one key is stuck" from "this whole disk denies deletes", and a caller
+ * with `failed` non-empty already knows not to report the batch as erased.
+ */
+export async function sweepOrphans(input: SweepOrphansInput): Promise<SweepResult> {
+  const clock = input.clock ?? systemClock;
+  const cutoff = clock.now().getTime() - input.olderThanMs;
+  const prefix = pendingPrefix(input.orgId);
+  const deleted: string[] = [];
+  const failed: SweepFailure[] = [];
+  let cursor: string | undefined;
+  do {
+    const page: ListPage = await input.disk.list({
+      prefix,
+      ...(cursor === undefined ? {} : { cursor }),
+    });
+    for (const object of page.objects) {
+      if (object.lastModified.getTime() > cutoff) continue;
+      if ((await input.keep?.(object)) === true) continue;
+      try {
+        await input.disk.delete(object.key);
+        deleted.push(object.key);
+      } catch (error) {
+        failed.push({ key: object.key, reason: renderThrowable(error) });
+      }
+    }
+    cursor = page.truncated ? page.cursor : undefined;
+  } while (cursor !== undefined);
+  return { deleted, failed };
+}

package/src/driver-local.ts

@@ -3,7 +3,7 @@
 // Content type, etag and user metadata live in a sidecar under `.meta/`: a POSIX file has
 // nowhere to keep them, and `get` must round-trip exactly what `put` was handed.
 
-import { type Clock, systemClock } from '@ultimat3/core';
+import { type Clock, isLocal, resolveEnvironment, stringField, systemClock } from '@ultimat3/core';
 import {
   DEFAULT_CONTENT_TYPE,
   DEFAULT_LIST_LIMIT,
@@ -14,18 +14,49 @@ import {
   type SignedUrlOptions,
   type StorageBody,
   type StorageDriver,
+  type StorageListEntry,
   type StorageObject,
   type StorageRead,
   sha256Base64,
   toBytes,
 } from './driver';
-import { checksumMismatch, objectNotFound } from './errors';
-import { assertSafeKey } from './path';
+import {
+  checksumMismatch,
+  deleteFailed,
+  listFailed,
+  objectNotFound,
+  signingSecretMissing,
+  storageNotImplemented,
+} from './errors';
+import { assertSafeKey, META_DIR } from './path';
 import { buildSignedUrl } from './signed-url';
+import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
 
-const META_DIR = '.meta';
 const DRIVER_NAME = 'local';
 
+/**
+ * The dev-only fallback signing key. A literal, not a per-process random one, so a restart does
+ * not invalidate every URL `x dev` handed out — and published in this repo, which is exactly why
+ * `localDriver` refuses to use it outside a development or test environment.
+ */
+export const DEV_SIGNING_SECRET = 'ultimate-dev-signing-secret';
+
+/** The env key production must set. Named once, read by the driver and by the predicate below. */
+export const STORAGE_SIGNING_SECRET_KEY = 'STORAGE_SIGNING_SECRET';
+
+/**
+ * True while a local disk built without an explicit `signingSecret` would sign with the shipped
+ * development key — `x doctor` reports it, exactly as it reports `usesDevCursorSecret()`.
+ *
+ * Reads the environment, not a driver instance: this is the same question `x doctor` asks about
+ * the cursor secret, and a disk handed an explicit `signingSecret` in `app.config.ts` never
+ * consults the variable at all.
+ */
+export function usesDevStorageSecret(): boolean {
+  const configured = process.env[STORAGE_SIGNING_SECRET_KEY];
+  return configured === undefined || configured === '' || configured === DEV_SIGNING_SECRET;
+}
+
 export interface LocalDriverOptions {
   /** Directory the disk owns outright. Created on first write. */
   readonly root: string;
@@ -34,6 +65,13 @@ export interface LocalDriverOptions {
   /** Route prefix the dev server serves signed URLs from. */
   readonly baseUrl?: string | undefined;
   readonly clock?: Clock | undefined;
+  /**
+   * Ceiling on ONE server-side `put()`, because `put()` buffers the whole body. Defaults to the
+   * upload policy's ceiling — the same number for the same fact. The dev disk enforces it for
+   * the same reason production does: a limit an app only meets in production is a limit it
+   * discovers by being OOM-killed.
+   */
+  readonly maxPutBytes?: number | undefined;
 }
 
 interface Sidecar {
@@ -43,23 +81,70 @@ interface Sidecar {
   readonly metadata?: Readonly<Record<string, string>> | undefined;
 }
 
+// `!Array.isArray` is the load-bearing clause, matching `isPlainObject` in
+// `@ultimat3/schema`'s `builder.ts`: `typeof [] === 'object'` and every value of `['a','b']` is a
+// string, so an array in the `metadata` slot was handed back through `head()`/`get()` as object
+// metadata — against a `Record<string, string>` every reader downstream is typed on.
+const isStringRecord = (value: unknown): value is Readonly<Record<string, string>> =>
+  typeof value === 'object' &&
+  value !== null &&
+  !Array.isArray(value) &&
+  Object.values(value as Record<string, unknown>).every((entry) => typeof entry === 'string');
+
 function parseSidecar(raw: unknown): Sidecar | undefined {
   if (typeof raw !== 'object' || raw === null) return undefined;
   const record = raw as Record<string, unknown>;
   const contentType = record['contentType'];
   const etag = record['etag'];
   if (typeof contentType !== 'string' || typeof etag !== 'string') return undefined;
-  return { contentType, etag };
+  // `put()` writes cacheControl/metadata into the same sidecar (below) — dropping them here
+  // silently truncated what was just written, even though `Sidecar` itself declares both.
+  const cacheControl = record['cacheControl'];
+  const metadata = record['metadata'];
+  return {
+    contentType,
+    etag,
+    ...(typeof cacheControl === 'string' ? { cacheControl } : {}),
+    ...(isStringRecord(metadata) ? { metadata } : {}),
+  };
+}
+
+/**
+ * A POSIX file is not encrypted at rest by this driver, and recording the request in the sidecar
+ * would answer a security review with a field the disk never honoured. Refused on the DEV disk
+ * too, and deliberately: a `put()` that succeeds locally and throws in production is a gap an app
+ * meets on the worst day, and both drivers refusing is one rule instead of two.
+ */
+function refuseUnsupportedPut(putOptions?: PutOptions): void {
+  if (putOptions?.serverSideEncryption === undefined) return;
+  throw storageNotImplemented(
+    'server-side encryption on the local driver (it writes plain files under one root)',
+    'drop serverSideEncryption from put(), and encrypt the disk itself — an s3Driver over a bucket with a default KMS rule, or a LUKS/FileVault volume under `root`',
+  );
 }
 
+/** `ENOENT` is the one delete failure that means "already in the desired state". */
+const isMissingFile = (error: unknown): boolean => stringField(error, 'code') === 'ENOENT';
+
 export function localDriver(options: LocalDriverOptions): StorageDriver {
   const root = options.root.replace(/\/+$/, '');
+  const maxPutBytes = options.maxPutBytes ?? DEFAULT_MAX_UPLOAD_BYTES;
   const clock = options.clock ?? systemClock;
   const baseUrl = options.baseUrl ?? `/_storage/${DRIVER_NAME}`;
-  // A dev disk must work with zero config; a production disk that forgets the secret still
-  // gets a *shared* secret, never a per-process random one that breaks on restart.
-  const secret =
-    options.signingSecret ?? process.env['STORAGE_SIGNING_SECRET'] ?? 'ultimate-dev-signing-secret';
+  // A dev disk must work with zero config. Outside development the fallback is refused rather
+  // than used: the literal is published, so signing with it hands every reader the power to mint
+  // a PUT for any key with any size and type limit — which `acceptSignedUpload` then trusts over
+  // the app's own `uploadPolicy`. Refused HERE, at construction, so the boot fails rather than
+  // the first upload.
+  // The published literal counts as no secret at all, whichever way it arrives: an env var or an
+  // `app.config.ts` that pasted it in signs exactly as weakly as the fallback does.
+  const supplied = options.signingSecret ?? process.env[STORAGE_SIGNING_SECRET_KEY];
+  const configured =
+    supplied === undefined || supplied === '' || supplied === DEV_SIGNING_SECRET
+      ? undefined
+      : supplied;
+  if (configured === undefined && !isLocal()) throw signingSecretMissing(resolveEnvironment());
+  const secret = configured ?? DEV_SIGNING_SECRET;
 
   const filePath = (key: string): string => `${root}/${key}`;
   const metaPath = (key: string): string => `${root}/${META_DIR}/${key}.json`;
@@ -75,27 +160,55 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
     }
   };
 
-  const head = async (key: string): Promise<StorageObject | undefined> => {
+  // No `contentType` fallback: the sidecar is the only thing that knows, so a missing one means
+  // this driver does not know either — exactly what the s3 driver's `list()` reports. `get()`
+  // fills the default below, because a `StorageObject` promises a type and a read has one.
+  //
+  // `hash` is the ONLY thing that reads the object's bytes, and it defaults off. The etag used to
+  // be computed unconditionally when the sidecar was missing, under a comment saying "`list()`
+  // must not read every file it lists" — which is exactly what `list()` then did, one whole
+  // object at a time, sequentially, for every sidecar-less key on the disk (a `put()` that died
+  // between its two writes leaves one). `copy()` inherited it too, so a copy documented as never
+  // routing bytes through the heap buffered the whole source. A listing that cannot know an etag
+  // reports `''`, which is what the s3 listing already answers for a provider that returns none.
+  const head = async (key: string, hash = false): Promise<StorageListEntry | undefined> => {
     const file = Bun.file(filePath(key));
     if (!(await file.exists())) return undefined;
     const sidecar = await readSidecar(key);
-    // Only hash when the sidecar is gone — `list()` must not read every file it lists.
-    const etag = sidecar?.etag ?? etagOf(new Uint8Array(await file.arrayBuffer()));
+    const etag = sidecar?.etag ?? (hash ? etagOf(new Uint8Array(await file.arrayBuffer())) : '');
     return {
       key,
       size: file.size,
-      contentType: sidecar?.contentType ?? DEFAULT_CONTENT_TYPE,
       etag,
       lastModified: new Date(file.lastModified),
+      ...(sidecar?.contentType === undefined ? {} : { contentType: sidecar.contentType }),
+      ...(sidecar?.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
+      ...(sidecar?.metadata === undefined ? {} : { metadata: sidecar.metadata }),
     };
   };
 
+  /** Removes one path, or reports WHY it could not — a swallowed refusal is a false erasure. */
+  const removeIfPresent = async (path: string, key: string): Promise<void> => {
+    try {
+      await Bun.file(path).delete();
+    } catch (error) {
+      if (isMissingFile(error)) return;
+      throw deleteFailed(
+        DRIVER_NAME,
+        key,
+        error,
+        `make the disk root writable by this process, then retry: ls -ld ${root} && rm -f ${path}`,
+      );
+    }
+  };
+
   return {
     name: DRIVER_NAME,
 
     async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
       const safe = assertSafeKey(key);
-      const bytes = await toBytes(body);
+      refuseUnsupportedPut(putOptions);
+      const bytes = await toBytes(body, { driver: DRIVER_NAME, key: safe, maxBytes: maxPutBytes });
       const claimed = putOptions?.checksum;
       if (claimed !== undefined) {
         const actual = sha256Base64(bytes);
@@ -115,15 +228,26 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
         contentType: sidecar.contentType,
         etag: sidecar.etag,
         lastModified: clock.now(),
+        ...(sidecar.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
+        ...(sidecar.metadata === undefined ? {} : { metadata: sidecar.metadata }),
       };
     },
 
     async get(key: string): Promise<StorageRead> {
       const safe = assertSafeKey(key);
-      const object = await head(safe);
-      if (object === undefined) throw objectNotFound(DRIVER_NAME, safe);
+      const entry = await head(safe);
+      if (entry === undefined) throw objectNotFound(DRIVER_NAME, safe);
       const bytes = new Uint8Array(await Bun.file(filePath(safe)).arrayBuffer());
-      return { object, bytes };
+      return {
+        object: {
+          ...entry,
+          contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE,
+          // Hashed HERE and not inside `head`, so a sidecar-less object is read exactly once: a
+          // `get()` already holds every byte, and `head(key, true)` would have read them again.
+          etag: entry.etag === '' ? etagOf(bytes) : entry.etag,
+        },
+        bytes,
+      };
     },
 
     async stream(key: string): Promise<ReadableStream<Uint8Array>> {
@@ -133,15 +257,38 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
       return file.stream();
     },
 
+    /** A real file copy — `Bun.write` from a `BunFile` never routes the bytes through the heap. */
+    async copy(from: string, to: string): Promise<StorageObject> {
+      const source = assertSafeKey(from);
+      const destination = assertSafeKey(to);
+      // `hash: true` — the destination gets a sidecar, and a sidecar carrying `etag: ''` is a
+      // durable lie every later `get()` of the copy would trust. The read is bounded to the one
+      // case the source has no sidecar of its own; the common path still touches no bytes.
+      const entry = await head(source, true);
+      if (entry === undefined) throw objectNotFound(DRIVER_NAME, source);
+      await Bun.write(filePath(destination), Bun.file(filePath(source)));
+      const sidecar: Sidecar = {
+        contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE,
+        etag: entry.etag,
+        cacheControl: entry.cacheControl,
+        metadata: entry.metadata,
+      };
+      await Bun.write(metaPath(destination), JSON.stringify(sidecar));
+      return {
+        ...entry,
+        key: destination,
+        contentType: sidecar.contentType,
+        lastModified: clock.now(),
+      };
+    },
+
     async delete(key: string): Promise<void> {
       const safe = assertSafeKey(key);
-      // Idempotent by contract: a missing key is already in the desired state.
-      await Bun.file(filePath(safe))
-        .delete()
-        .catch(() => undefined);
-      await Bun.file(metaPath(safe))
-        .delete()
-        .catch(() => undefined);
+      // Idempotent by contract: a missing key is already in the desired state. A REFUSED unlink
+      // is not — a read-only mount or a root this process cannot write reports the bytes gone
+      // when they are still on disk, which is the one lie an erasure sweep must never repeat.
+      await removeIfPresent(filePath(safe), safe);
+      await removeIfPresent(metaPath(safe), safe);
     },
 
     async exists(key: string): Promise<boolean> {
@@ -162,13 +309,24 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
           if (cursor !== undefined && key <= cursor) continue;
           keys.push(key);
         }
-      } catch {
+      } catch (error) {
         // A disk nobody has written to yet has no directory: an empty listing, not an error.
-        return { objects: [], truncated: false };
+        if (isMissingFile(error)) return { objects: [], truncated: false };
+        // Everything else is a refusal, and a bare `catch` reported all of them as "this disk is
+        // empty" — `EACCES` on the root, `ENOTDIR` on a root that is a file, an I/O error on the
+        // mount. `sweepOrphans` walks `list()`, so that swallow certified an unreadable prefix as
+        // having no orphans: the same false report `delete()`'s `.catch(() => undefined)` used to
+        // make, one call to the left.
+        throw listFailed(
+          DRIVER_NAME,
+          prefix,
+          error,
+          `make the disk root readable by this process, then retry: ls -ld ${root}`,
+        );
       }
       keys.sort();
       const page = keys.slice(0, limit);
-      const objects: StorageObject[] = [];
+      const objects: StorageListEntry[] = [];
       for (const key of page) {
         const object = await head(key);
         if (object !== undefined) objects.push(object);