@tokenoftrust/storefront-runner 4.13.1 → 4.15.0

package/apps/storefront/src/config/kvTenantResolver.ts

@@ -17,7 +17,8 @@
  */
 import type { TenantConfig, TenantResolver } from "@tot/public-runtime";
 import { withPlatformPolicy } from "@/config/tenants";
-import { TENANT_RECORD_PREFIX } from "@/config/storedTenantRecord";
+import { TENANT_RECORD_PREFIX, getStoredTenantRecord } from "@/config/storedTenantRecord";
+import { mergeStoredTenantRecord } from "@tot/public-runtime/tenant-record-merge";
 
 const DOMAIN_PREFIX = "domain:";
 const TENANT_PREFIX = TENANT_RECORD_PREFIX;
@@ -76,16 +77,25 @@ export class KvTenantResolver implements TenantResolver {
 }
 
 /**
- * Register (or update) a tenant in KV. Idempotent: writes the config blob and a
- * domain->id index entry for each claimed hostname. Pass `makeDefault` to point
- * the unknown-host fallback at this tenant. Call this from onboarding/migration.
+ * Register (or update) a tenant in KV. Idempotent: merges the config into the stored
+ * `tenant:<id>` record (provisioned keys win, stored keys the config does not carry survive;
+ * see `@tot/public-runtime/tenant-record-merge`) and writes a domain->id index entry for each
+ * claimed hostname. The stored record is read uncached. A stored record that is not a JSON
+ * object throws before any write. Pass `makeDefault` to point the unknown-host fallback at
+ * this tenant. Call this from onboarding/migration.
  */
 export async function putTenant(
   kv: KVNamespace,
   config: TenantConfig,
   opts: { makeDefault?: boolean } = {},
 ): Promise<void> {
-  await kv.put(TENANT_PREFIX + config.tenant_id, JSON.stringify(config));
+  const key = TENANT_PREFIX + config.tenant_id;
+  const record = mergeStoredTenantRecord(
+    key,
+    await getStoredTenantRecord(kv, config.tenant_id),
+    config as unknown as Record<string, unknown>,
+  );
+  await kv.put(key, record);
   for (const domain of config.domains) {
     const host = cleanHost(domain);
     if (host) await kv.put(DOMAIN_PREFIX + host, config.tenant_id);

package/apps/storefront/src/lib/assets/durableR2.ts

@@ -7,7 +7,6 @@
  * Order, always: write the references (version manifest already in R2, or the holds), wait while a
  * bundle-GC sweep holds its lease, then stage and verify every object those references name.
  */
-import { sha256Hex } from "@tot/public-runtime";
 import { PINNED_ASSET_MEDIA_TYPE_BY_EXTENSION, sealedMediaTypeAllowed } from "@tot/public-runtime/hosted-image-media-types";
 import {
   durableAssetsStatusKey,
@@ -21,6 +20,7 @@ import {
 import { GC_LEASE_KEY, awaitGcQuiescence } from "@tot/public-runtime/retention-holds";
 import { authorizeAssetReference, type AssetRegistryStore, type AssetWrite } from "@tot/public-runtime/asset-registry";
 import { imageLimitsFor, type ImageTier } from "@tot/public-runtime/image-limits";
+import { incrementalSha256 } from "@tot/public-runtime/incremental-sha256";
 
 const MAX_MANIFEST = 1024 * 1024;
 const POOL_ENTRY_RE = /^([0-9a-f]{64})(\.[A-Za-z0-9]+)$/;
@@ -30,13 +30,17 @@ export interface R2ObjectLike {
   size: number;
   body?: ReadableStream<Uint8Array> | null;
   arrayBuffer?(): Promise<ArrayBuffer>;
+  /** The sha256 R2 verified when the object was put with one (`put(..., { sha256 })`). */
+  checksums?: { sha256?: ArrayBuffer };
 }
 
 export interface R2BucketLike {
   get(key: string): Promise<R2ObjectLike | null>;
-  put?(key: string, body: Uint8Array, options: {
+  put?(key: string, body: Uint8Array | ReadableStream<Uint8Array>, options: {
     httpMetadata: { contentType: string; cacheControl: string };
     onlyIf?: Headers;
+    /** R2 hashes the body as it arrives and refuses the put, writing nothing, when it differs. */
+    sha256?: string;
   }): Promise<unknown>;
 }
 
@@ -90,27 +94,136 @@ export async function readBounded(bucket: R2BucketLike, key: string, limit: numb
   return bytes;
 }
 
+
+function toHex(buffer: ArrayBuffer): string {
+  return [...new Uint8Array(buffer)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+
+/**
+ * Size and sha256 of an object of at most `limit` bytes, never holding it whole: R2's own verified
+ * checksum when the object was put with one, else its body hashed as it streams. Null when absent,
+ * larger than `limit`, or its body is not the size R2 reports.
+ */
+export async function digestBounded(bucket: Pick<R2BucketLike, "get">, key: string, limit: number): Promise<{ size: number; sha256: string } | null> {
+  const object = await bucket.get(key);
+  if (!object) return null;
+  if (!Number.isSafeInteger(object.size) || object.size < 0 || object.size > limit) {
+    await object.body?.cancel().catch(() => {});
+    return null;
+  }
+  const checksum = object.checksums?.sha256;
+  if (checksum && checksum.byteLength === 32) {
+    await object.body?.cancel().catch(() => {});
+    return { size: object.size, sha256: toHex(checksum) };
+  }
+  if (!object.body) {
+    // Only an in-memory test double lacks a body; R2 always streams one.
+    if (!object.arrayBuffer) return null;
+    const whole = new Uint8Array(await object.arrayBuffer());
+    if (whole.byteLength !== object.size) return null;
+    const hash = incrementalSha256();
+    await hash.update(whole);
+    return { size: object.size, sha256: await hash.hex() };
+  }
+  const hash = incrementalSha256();
+  const reader = object.body.getReader();
+  let size = 0;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      size += value.byteLength;
+      if (size > object.size || size > limit) {
+        await reader.cancel().catch(() => {});
+        return null;
+      }
+      await hash.update(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  return size === object.size ? { size, sha256: await hash.hex() } : null;
+}
+
+/**
+ * `body` passed through unchanged, erroring the stream on an overrun or a short end: a copy can never
+ * be stored at a size other than the one verified. workerd's `FixedLengthStream` then gives R2 the
+ * known length a streamed put requires.
+ */
+function exactLength(body: ReadableStream<Uint8Array>, length: number): ReadableStream<Uint8Array> {
+  let seen = 0;
+  const counted = body.pipeThrough(new TransformStream<Uint8Array, Uint8Array>({
+    transform(chunk, controller) {
+      seen += chunk.byteLength;
+      if (seen > length) throw new R2StagingError("source-overrun");
+      controller.enqueue(chunk);
+    },
+    flush() {
+      if (seen !== length) throw new R2StagingError("source-short");
+    },
+  }));
+  const Fixed = (globalThis as { FixedLengthStream?: new (length: number) => TransformStream<Uint8Array, Uint8Array> }).FixedLengthStream;
+  return typeof Fixed === "function" ? counted.pipeThrough(new Fixed(length)) : counted;
+}
+
+/** A verified object to copy as a stream: its size, and a fresh read of its body. */
+export interface StreamSource {
+  size: number;
+  open(): Promise<ReadableStream<Uint8Array>>;
+}
+
+/**
+ * The object at `key` as a {@link StreamSource}, once its size and sha256 (hashed as a stream) are
+ * `address` within `limit`; null otherwise. Nothing is held: the copy re-reads the body.
+ */
+export async function verifiedObjectSource(bucket: Pick<R2BucketLike, "get">, key: string, address: string, limit: number): Promise<StreamSource | null> {
+  const digest = await digestBounded(bucket, key, limit);
+  if (!digest || digest.size < 1 || digest.sha256 !== address) return null;
+  return {
+    size: digest.size,
+    async open() {
+      const object = await bucket.get(key);
+      if (!object || object.size !== digest.size) {
+        await object?.body?.cancel().catch(() => {});
+        throw new R2StagingError("original-missing");
+      }
+      if (object.body) return object.body;
+      // Only an in-memory test double lacks a body; R2 always streams one.
+      if (!object.arrayBuffer) throw new R2StagingError("original-missing");
+      const whole = new Uint8Array(await object.arrayBuffer());
+      return new ReadableStream<Uint8Array>({ start(controller) { controller.enqueue(whole); controller.close(); } });
+    },
+  };
+}
+
 /** Wait while a bundle-GC sweep holds its lease in `bucket`. */
 export function awaitR2Quiescence(bucket: R2BucketLike, waiter: QuiescenceWaiter = awaitGcQuiescence): Promise<{ waited: boolean }> {
   return waiter({ readLease: () => readBounded(bucket, GC_LEASE_KEY, 64 * 1024) });
 }
 
-/** Present with exactly `bytes`, writing it create-only when absent; refuses a mismatched object. */
-export async function ensurePoolObject(bucket: R2BucketLike, key: string, bytes: Uint8Array, mediaType: string): Promise<void> {
+/**
+ * Present with exactly the source's bytes, writing it create-only when absent; refuses a mismatched
+ * object. A {@link StreamSource} is streamed into R2 with its sha256, so R2 refuses (writes nothing)
+ * on any other body; the existing object and the read-back are hashed as streams, never held whole.
+ */
+export async function ensurePoolObject(bucket: R2BucketLike, key: string, source: Uint8Array | StreamSource, mediaType: string): Promise<void> {
   const address = POOL_ENTRY_RE.exec(key.replace(/^assets\//, ""))?.[1];
   if (!address) throw new R2StagingError("pool-key");
-  // Bounded by the bytes being staged: an object of any other size cannot hold this address.
-  const existing = await readBounded(bucket, key, bytes.byteLength);
-  if (existing && await sha256Hex(existing) !== address) throw new R2StagingError("store-mismatch");
+  const size = source instanceof Uint8Array ? source.byteLength : source.size;
+  // Bounded by the size being staged: an object of any other size cannot hold this address.
+  const existing = await digestBounded(bucket, key, size);
+  if (existing && (existing.sha256 !== address || existing.size !== size)) throw new R2StagingError("store-mismatch");
   if (!existing) {
     if (!bucket.put) throw new R2StagingError("store-unwritable");
-    await bucket.put(key, bytes, {
+    await bucket.put(key, source instanceof Uint8Array ? source : exactLength(await source.open(), size), {
       httpMetadata: { contentType: mediaType, cacheControl: IMMUTABLE },
       onlyIf: new Headers({ "If-None-Match": "*" }),
+      sha256: address,
     });
   }
-  const readback = await readBounded(bucket, key, bytes.byteLength);
-  if (!readback || readback.byteLength !== bytes.byteLength || await sha256Hex(readback) !== address) {
+  const readback = await digestBounded(bucket, key, size);
+  if (!readback || readback.size !== size || readback.sha256 !== address) {
     throw new R2StagingError("store-mismatch");
   }
 }
@@ -132,13 +245,14 @@ function mediaTypeOf(entry: string): string {
   return mediaType;
 }
 
-/** The exact captured bytes for `<sha256><ext>` from the tenant's capture store, within its tier. */
-async function capturedBytes(captureBucket: R2BucketLike, tenant: string, entry: string, imageTier: ImageTier | undefined): Promise<Uint8Array> {
+/** The tenant's captured `<sha256><ext>`, verified as a stream within its tier; never held whole. */
+async function capturedSource(captureBucket: R2BucketLike, tenant: string, entry: string, imageTier: ImageTier | undefined): Promise<StreamSource> {
   const address = POOL_ENTRY_RE.exec(entry)?.[1];
   if (!address) throw new R2StagingError("pool-key");
-  const bytes = await readBounded(captureBucket, `original-images/${tenant}/objects/${address}`, imageLimitsFor(imageTier).maxEncodedBytes);
-  if (!bytes || bytes.byteLength < 1 || await sha256Hex(bytes) !== address) throw new R2StagingError("original-missing");
-  return bytes;
+  const source = await verifiedObjectSource(captureBucket, `original-images/${tenant}/objects/${address}`, address,
+    imageLimitsFor(imageTier).maxEncodedBytes);
+  if (!source) throw new R2StagingError("original-missing");
+  return source;
 }
 
 /**
@@ -151,7 +265,8 @@ async function capturedBytes(captureBucket: R2BucketLike, tenant: string, entry:
  * tenant's own capture store (which registers it); otherwise nothing is staged and the go-live
  * refuses `asset-unauthorized`.
  *
- * Originals are read one at a time, within the tenant's image tier, and never held across entries.
+ * Originals are hashed and copied as streams, one at a time, within the tenant's image tier: none is
+ * ever held whole.
  */
 export async function stageCapturedOriginalsR2(input: {
   tenant: string;
@@ -167,11 +282,11 @@ export async function stageCapturedOriginalsR2(input: {
   const unauthorized: string[] = [];
   for (const entry of input.entries) {
     if (authorizeAssetReference([registry], input.tenant, entry).ok) continue;
-    const own = await capturedBytes(input.captureBucket, input.tenant, entry, input.imageTier).catch((error: unknown) => {
+    const own = await capturedSource(input.captureBucket, input.tenant, entry, input.imageTier).catch((error: unknown) => {
       if (error instanceof R2StagingError) return null;
       throw error;
     });
-    if (own) owned.set(entry, own.byteLength);
+    if (own) owned.set(entry, own.size);
     else unauthorized.push(entry);
   }
   if (input.registry && owned.size > 0) {
@@ -184,10 +299,10 @@ export async function stageCapturedOriginalsR2(input: {
   let staged = 0;
   for (const entry of input.entries) {
     const key = `assets/${entry}`;
-    const present = await readBounded(input.publicBucket, key, imageLimitsFor(input.imageTier).maxEncodedBytes);
+    const present = await digestBounded(input.publicBucket, key, imageLimitsFor(input.imageTier).maxEncodedBytes);
     const address = POOL_ENTRY_RE.exec(entry)?.[1];
-    if (present && address && await sha256Hex(present) === address) continue;
-    await ensurePoolObject(input.publicBucket, key, await capturedBytes(input.captureBucket, input.tenant, entry, input.imageTier), mediaTypeOf(entry));
+    if (present && address && present.sha256 === address) continue;
+    await ensurePoolObject(input.publicBucket, key, await capturedSource(input.captureBucket, input.tenant, entry, input.imageTier), mediaTypeOf(entry));
     staged++;
   }
   return staged;
@@ -281,7 +396,7 @@ export async function designateDurableR2(input: {
   for (const item of prepared) {
     await createHold(input.publicBucket, item.holdKey, item.holdBody);
     await ensurePoolObject(input.publicBucket, `assets/${item.entry}`,
-      await capturedBytes(input.captureBucket, input.tenant, item.entry, input.imageTier), item.mediaType);
+      await capturedSource(input.captureBucket, input.tenant, item.entry, input.imageTier), item.mediaType);
   }
   await recordDurableStatusKv(input.kv, input.tenant, { revision: input.revision, at: now().toISOString(), declared,
     designated: prepared.map((item) => ({ entry: item.entry, name: item.image.name, source: item.image.source, verifiedStores: ["r2"] })) });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tokenoftrust/storefront-runner",
-  "version": "4.13.1",
+  "version": "4.15.0",
   "license": "SEE LICENSE IN LICENSE",
   "description": "World-shareable storefront runner: multi-tenant renderer on Astro/Cloudflare. No control plane.",
   "packageManager": "pnpm@11.9.0",

package/packages/public-runtime/package.json

@@ -24,7 +24,9 @@
     "./asset-registry": "./src/asset-registry.ts",
     "./responsive-renditions": "./src/responsive-renditions.ts",
     "./svg-sanitize": "./src/svg-sanitize.ts",
+    "./incremental-sha256": "./src/incremental-sha256.ts",
     "./image-limits": "./src/image-limits.mjs",
+    "./tenant-record-merge": "./src/tenant-record-merge.mjs",
     "./image-release-gate": "./src/image-release-gate.ts",
     "./pool-asset-headers": "./src/pool-asset-headers.ts",
     "./catalog-files": "./src/catalog-files.ts",

package/packages/public-runtime/src/incremental-sha256.ts

@@ -0,0 +1,40 @@
+/**
+ * An incremental sha256 that holds one chunk at a time: workerd's `crypto.DigestStream`, else (Node,
+ * where the tests run) `node:crypto` through `process.getBuiltinModule`, never a static import, so
+ * this module stays safe to bundle into the Worker.
+ */
+type DigestStreamClass = new (algorithm: string) => WritableStream<Uint8Array> & { digest: Promise<ArrayBuffer> };
+type NodeHash = { update(chunk: Uint8Array): unknown; digest(encoding: "hex"): string };
+
+export type IncrementalSha256 = { update(chunk: Uint8Array): Promise<void>; hex(): Promise<string> };
+
+function toHex(buffer: ArrayBuffer): string {
+  return [...new Uint8Array(buffer)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+export function incrementalSha256(): IncrementalSha256 {
+  const Digest = (globalThis.crypto as unknown as { DigestStream?: DigestStreamClass } | undefined)?.DigestStream;
+  if (typeof Digest === "function") {
+    const stream = new Digest("SHA-256");
+    const writer = stream.getWriter();
+    return {
+      update: (chunk) => writer.write(chunk),
+      async hex() {
+        await writer.close();
+        return toHex(await stream.digest);
+      },
+    };
+  }
+  const nodeCrypto = (globalThis as { process?: { getBuiltinModule?(id: string): unknown } }).process
+    ?.getBuiltinModule?.("node:crypto") as { createHash(algorithm: string): NodeHash } | undefined;
+  if (!nodeCrypto) throw new Error("digest-unavailable");
+  const hash = nodeCrypto.createHash("sha256");
+  return {
+    async update(chunk) {
+      hash.update(chunk);
+    },
+    async hex() {
+      return hash.digest("hex");
+    },
+  };
+}

package/packages/public-runtime/src/tenant-record-merge.d.mts

@@ -0,0 +1,9 @@
+export declare function mergeTenantRecord(
+  stored: Record<string, unknown>,
+  provisioned: Record<string, unknown>,
+): Record<string, unknown>;
+export declare function mergeStoredTenantRecord(
+  key: string,
+  storedRaw: string | null | undefined,
+  provisioned: Record<string, unknown>,
+): string;

package/packages/public-runtime/src/tenant-record-merge.mjs

@@ -0,0 +1,68 @@
+/**
+ * Merge a provisioned tenant config into the `tenant:<id>` KV record already stored.
+ *
+ * Provisioners carry only the fields they own. The stored record also holds fields written by
+ * other paths (`storefront.mode`, `addOns.largeImages`), so a provisioner must read the stored
+ * record, merge, and write the result; it never replaces the record wholesale.
+ *
+ * Contract:
+ * - the provisioned config wins for every key it carries;
+ * - plain objects merge recursively; arrays and scalars from the provisioned config replace;
+ * - keys the provisioned config does not carry are preserved;
+ * - `__proto__`, `constructor` and `prototype` keys are dropped at every level;
+ * - no stored record: the provisioned config is written as given;
+ * - a stored value that is not a JSON object (unparseable, array, scalar, empty string) throws,
+ *   naming the key, and the caller writes nothing.
+ *
+ * KV has no transaction. A provisioner that reads, merges and writes while the admin add-ons
+ * switch writes the same record can still lose one of the two writes; the merge narrows that
+ * window to one read-to-write interval and cannot close it.
+ */
+
+const UNSAFE_KEYS = new Set(["__proto__", "constructor", "prototype"]);
+
+/** @param {unknown} value @returns {value is Record<string, unknown>} */
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/**
+ * @param {Record<string, unknown>} stored
+ * @param {Record<string, unknown>} provisioned
+ * @returns {Record<string, unknown>}
+ */
+export function mergeTenantRecord(stored, provisioned) {
+  /** @type {Record<string, unknown>} */
+  const out = {};
+  for (const [key, value] of Object.entries(stored)) {
+    if (UNSAFE_KEYS.has(key)) continue;
+    out[key] = value;
+  }
+  for (const [key, value] of Object.entries(provisioned)) {
+    if (UNSAFE_KEYS.has(key)) continue;
+    const existing = out[key];
+    out[key] = isPlainObject(value) && isPlainObject(existing) ? mergeTenantRecord(existing, value) : value;
+  }
+  return out;
+}
+
+/**
+ * The JSON to write at `key` for `provisioned`, given the raw stored value (`null` when absent).
+ * @param {string} key
+ * @param {string | null | undefined} storedRaw
+ * @param {Record<string, unknown>} provisioned
+ * @returns {string}
+ */
+export function mergeStoredTenantRecord(key, storedRaw, provisioned) {
+  if (storedRaw === null || storedRaw === undefined) return JSON.stringify(provisioned);
+  let stored;
+  try {
+    stored = JSON.parse(storedRaw);
+  } catch {
+    throw new Error(`stored KV record ${key} is not valid JSON; refusing to overwrite it`);
+  }
+  if (!isPlainObject(stored)) {
+    throw new Error(`stored KV record ${key} is not a JSON object; refusing to overwrite it`);
+  }
+  return JSON.stringify(mergeTenantRecord(stored, provisioned));
+}