@tokenoftrust/storefront-runner 1.4.1 → 2.0.0

package/packages/public-runtime/src/static-bundle.ts

@@ -0,0 +1,357 @@
+/**
+ * Digest-addressed static-bundle serving (ADR 0012 —
+ * `docs/architecture/decisions/0012-instant-static-rollback-pointer-substrate.md`),
+ * dedup-aware (per-file content-addressed asset dedup,
+ * `docs/epics/proposed/storefront-instant-static-rollback.md` Addendum §A).
+ *
+ * Reads an already-materialized, already-staged static bundle
+ * (`scripts/publish/materialize-tenant.mjs` output, staged by
+ * `scripts/publish/stage-bundle.mjs` to `versions/<treeDigest>/…` PLUS the
+ * shared `assets/<sha256>` pool) straight out of R2 and serves the bytes
+ * VERBATIM — no re-render, no per-request CSP-nonce middleware (the bundle
+ * carries one pinned nonce for its whole tree; see the ADR's "Consequences"
+ * section). This is the read half: the write path (what PUTs a bundle into this
+ * R2 bucket) is `scripts/publish/lib/r2-runner.mjs`, which stages every
+ * `versions/<digest>/` bundle at candidate-prebuild time. Every static-publish
+ * tenant's production reads are served through here.
+ *
+ * DIGEST CONTRACT (RECONCILED, T14 — ADR 0012): the caller resolves "what to
+ * serve" via `resolveChannelDigest(get, tenantId, "live")` — the `channelDigests`
+ * companion to the `live` channel pointer, NOT `resolveChannelVersion` (which
+ * returns the versionId / Gitea sha, a DIFFERENT namespace than the treeDigest).
+ * `gatedPromote` (go-live) and `rollbackCandidateAtomic` (rollback) write the
+ * staged bundle's real `treeDigest` to `channelDigests.live` in the SAME CAS flip
+ * that moves the versionId, so this module gets the exact R2 key prefix
+ * `versions/<treeDigest>/`. A static tenant whose `live` has a versionId but no
+ * digest companion yet resolves to `null` here and fails closed (see below)
+ * rather than serving the wrong bytes under the versionId.
+ *
+ * T6 KEY RESOLUTION (`kind: "route"` vs `kind: "asset"`, `scripts/publish/lib/
+ * asset-pool.mjs`'s classification, carried verbatim in `manifest.files`): the
+ * manifest is now read on EVERY request (not just HTML, as before T6) because
+ * it's the ONLY source of truth for which R2 key a bundle-relative path
+ * resolves to — `kind: "route"` files still live at `versions/<digest>/<rel>`
+ * (rendered fresh per version), `kind: "asset"` files live at the SHARED,
+ * content-addressed `assets/<sha256><ext>` pool (`staticBundleAssetPoolKey`,
+ * kept in lockstep with `scripts/publish/lib/asset-pool.mjs`'s `assetPoolKey`
+ * — that module can't be imported here, this is the Worker bundle, that one
+ * runs under plain node). A `rel` absent from `manifest.files` (a genuinely
+ * unexpected path, or a pre-T6 manifest with no `files` classification) falls
+ * back to the OLD `versions/<digest>/<rel>` lookup — fail-closed on a miss
+ * either way, never a guess.
+ */
+import { assetContentType } from "./tenant-assets.js";
+
+// ---------------------------------------------------------------------------
+// R2 seam — same structural subset tenant-assets.ts uses (no workers-types dep)
+// ---------------------------------------------------------------------------
+
+export interface StaticBundleR2ObjectLike {
+  body: ReadableStream<Uint8Array> | null;
+  arrayBuffer(): Promise<ArrayBuffer>;
+  size?: number;
+}
+
+export interface StaticBundleR2BucketLike {
+  get(key: string): Promise<StaticBundleR2ObjectLike | null>;
+}
+
+// ---------------------------------------------------------------------------
+// Path mapping — request pathname -> bundle-relative object path
+// ---------------------------------------------------------------------------
+
+/**
+ * Reject empty, absolute-outside-bundle, or traversal-bearing relative paths
+ * (mirrors `tenant-assets.ts`'s `isSafeRel`, duplicated here to keep this
+ * module import-independent of the assets pointer model).
+ */
+function isSafeRel(rel: string): boolean {
+  if (!rel) return false;
+  return !rel.split("/").some((seg) => seg === ".." || seg === ".");
+}
+
+/**
+ * Map a request pathname to the bundle-relative file the materializer wrote
+ * for it — the SAME transform `scripts/publish/lib/materialize.mjs`'s
+ * `bundlePathForRoute` (routes) and `assetFilePath` (assets) apply, ported
+ * here as one pure function since `scripts/` is not importable from the
+ * Worker bundle:
+ *   "/"            -> "index.html"
+ *   "/about"       -> "about/index.html"   (directory-index for pretty URLs)
+ *   "/a/b/"        -> "a/b/index.html"
+ *   "/sitemap.xml" -> "sitemap.xml"        (a real file extension: as-is)
+ *   "/_astro/x.js" -> "_astro/x.js"        (asset: as-is)
+ * Returns null for an unsafe path (traversal, decode failure).
+ */
+export function staticBundleRelPath(pathname: string): string | null {
+  let decoded: string;
+  try {
+    decoded = decodeURIComponent(pathname);
+  } catch {
+    return null;
+  }
+  const clean = decoded.replace(/^\/+/, "").replace(/\/+$/, "");
+  if (clean === "") return "index.html";
+  if (!isSafeRel(clean)) return null;
+  const last = clean.split("/").pop() || "";
+  if (/\.[a-z0-9]+$/i.test(last)) return clean;
+  return `${clean}/index.html`;
+}
+
+/** Full R2 object key for a `route`-kind bundle-relative path at a given digest. */
+export function staticBundleObjectKey(digest: string, rel: string): string {
+  return `versions/${digest}/${rel}`;
+}
+
+/** R2 key for a digest's bundle manifest (materialize-tenant.mjs's `manifest.json`). */
+export function staticBundleManifestKey(digest: string): string {
+  return `versions/${digest}/manifest.json`;
+}
+
+/**
+ * T6: the shared content-addressed pool key for one asset's hash — MUST stay
+ * in lockstep with `scripts/publish/lib/asset-pool.mjs`'s `assetPoolKey`
+ * (same extension-preserving `assets/<sha256hex><ext>` shape; that module
+ * can't be imported here since this is the Worker bundle and that one runs
+ * under plain node — both sides' tests assert the exact key shape).
+ */
+export function staticBundleAssetPoolKey(hash: string, rel: string): string {
+  const dot = rel.lastIndexOf(".");
+  const slash = rel.lastIndexOf("/");
+  const ext = dot > slash ? rel.slice(dot) : "";
+  return `assets/${hash}${ext}`;
+}
+
+/**
+ * Resolve the R2 key to fetch for a `rel` bundle path, using the manifest's
+ * T6 file classification: `kind: "asset"` -> the shared pool key (by content
+ * hash, NOT the version); anything else (`kind: "route"`, or `rel` absent
+ * from `files` entirely — a pre-T6 manifest, or a genuinely unlisted path) ->
+ * the per-version `versions/<digest>/<rel>` key, same as before T6.
+ */
+export function resolveStaticBundleObjectKey(
+  digest: string,
+  rel: string,
+  files: StaticBundleManifest["files"] | undefined,
+): string {
+  const entry = files?.[rel];
+  if (entry?.kind === "asset" && entry.hash) {
+    return staticBundleAssetPoolKey(entry.hash, rel);
+  }
+  return staticBundleObjectKey(digest, rel);
+}
+
+// ---------------------------------------------------------------------------
+// Manifest (CSP pinned-nonce header) — read once per hit for an HTML response
+// ---------------------------------------------------------------------------
+
+/** One `manifest.files` entry (T6, `scripts/publish/lib/asset-pool.mjs`'s `classifyBundleFiles` shape). */
+export interface StaticBundleFileEntry {
+  hash: string;
+  kind: "route" | "asset";
+}
+
+/** The slice of `materialize-tenant.mjs`'s manifest this read path needs. */
+export interface StaticBundleManifest {
+  treeDigest: string;
+  pinnedNonce: string;
+  csp: { headerName: string; value: string } | null;
+  /** T6 classification: relPath -> {hash, kind}. Absent on a pre-T6 manifest — every lookup then falls back to the old versions/<digest>/<rel> key (see `resolveStaticBundleObjectKey`). */
+  files?: Record<string, StaticBundleFileEntry>;
+}
+
+function parseManifest(raw: string): StaticBundleManifest | null {
+  try {
+    const m = JSON.parse(raw) as Partial<StaticBundleManifest>;
+    if (typeof m.treeDigest !== "string" || typeof m.pinnedNonce !== "string") return null;
+    const csp =
+      m.csp &&
+      typeof (m.csp as { headerName?: unknown }).headerName === "string" &&
+      typeof (m.csp as { value?: unknown }).value === "string"
+        ? (m.csp as { headerName: string; value: string })
+        : null;
+    let files: StaticBundleManifest["files"];
+    if (m.files && typeof m.files === "object") {
+      files = {};
+      for (const [rel, entry] of Object.entries(m.files as Record<string, unknown>)) {
+        const e = entry as Partial<StaticBundleFileEntry> | null;
+        if (
+          e &&
+          typeof e.hash === "string" &&
+          (e.kind === "route" || e.kind === "asset")
+        ) {
+          files[rel] = { hash: e.hash, kind: e.kind };
+        }
+      }
+    }
+    return { treeDigest: m.treeDigest, pinnedNonce: m.pinnedNonce, csp, files };
+  } catch {
+    return null;
+  }
+}
+
+/** Read+parse a digest's `manifest.json` from R2. Null on any miss/malformed data. */
+export async function readStaticBundleManifest(
+  bucket: StaticBundleR2BucketLike,
+  digest: string,
+): Promise<StaticBundleManifest | null> {
+  const object = await bucket.get(staticBundleManifestKey(digest));
+  if (!object) return null;
+  const text = new TextDecoder().decode(await object.arrayBuffer());
+  return parseManifest(text);
+}
+
+// ---------------------------------------------------------------------------
+// Serve — resolve digest, stream the object from R2, fail closed on a miss
+// ---------------------------------------------------------------------------
+
+export interface ServeStaticBundleDeps {
+  bucket: StaticBundleR2BucketLike;
+  /**
+   * The digest to serve, already resolved by the caller (typically
+   * `resolveChannelVersion(get, tenantId, "live")` — see the module
+   * docstring's DIGEST CONTRACT). `null` means "pointer-serving is on for
+   * this tenant but nothing is live yet" — fails closed, same as a missing
+   * object, per the ADR: this path is only ever reached for a confirmed
+   * pointer-serving tenant, so an unresolved digest is itself a fault, not a
+   * "fall through to SSR" signal.
+   */
+  digest: string | null;
+  /** Best-effort diagnostic sink (never throws, never blocks the response). */
+  onFailClosed?: (detail: string) => void;
+}
+
+/** Structured detail for the one fail-closed shape this module can produce. */
+export class StaticBundleUnavailableError extends Error {
+  readonly code = "static-bundle-unavailable" as const;
+  constructor(detail: string) {
+    super(detail);
+    this.name = "StaticBundleUnavailableError";
+  }
+}
+
+function failClosed(deps: ServeStaticBundleDeps, detail: string): Response {
+  try {
+    deps.onFailClosed?.(detail);
+  } catch {
+    /* diagnostic sink must never break the response path */
+  }
+  // 503, not a 200 masquerading as content and not a 404 (which would read as
+  // "this route doesn't exist" — it does, the bundle is just unreachable) and
+  // never an SSR fallthrough: this route is ONLY reached for a tenant
+  // confirmed to be on the pointer-serving model, so an unreadable bundle is
+  // an infra fault to surface (verify/monitoring), not a content gap.
+  return new Response("Static bundle unavailable", {
+    status: 503,
+    headers: {
+      "cache-control": "no-store",
+      "x-tot-static-bundle": "fail-closed",
+    },
+  });
+}
+
+/** Cache-Control for a `route`-kind file (HTML/xml/txt — rendered fresh per version, stable non-digest url). */
+const ROUTE_CACHE_CONTROL = "public, max-age=0, s-maxage=300, stale-while-revalidate=86400";
+/** Cache-Control for an `asset`-kind file — the pool key IS the content hash, so caching forever is always correct (epic §A). */
+const ASSET_CACHE_CONTROL = "public, max-age=31536000, immutable";
+
+/**
+ * Serve `pathname` from the digest-addressed static bundle in R2.
+ *
+ * Returns `null` — the ONE "fall through to SSR" signal — when the manifest's
+ * `files` map proves `pathname` is not part of this bundle's static surface:
+ * app surfaces (/admin, /cockpit, checkout, …) share the tenant's host but are
+ * never materialized into a bundle, so the caller must render them. Only a
+ * manifest WITH `files` can prove non-membership; a pre-T6 manifest keeps the
+ * fetch-then-fail-closed path.
+ *
+ * Every OTHER outcome (hit, unsafe path, unresolved digest, missing manifest,
+ * missing object for a LISTED file) terminates the request here rather than
+ * falling through — a staged digest's listed route must never silently degrade
+ * into SSR (wrong/divergent bytes), per the ADR's fail-closed guidance.
+ *
+ * T6: the manifest is now read FIRST, on every request (not just HTML, as
+ * before T6) — it's the only source of truth for whether `rel` resolves to
+ * the per-version `versions/<digest>/<rel>` key or the shared content-addressed
+ * `assets/<sha256>` pool key (`resolveStaticBundleObjectKey`). A missing
+ * manifest fails closed the same way a missing object always has: the
+ * manifest is deliberately the LAST thing `stage-bundle.mjs` writes for a
+ * digest (see `s3-publish.mjs`'s `buildManifestPromoteStep`), so its absence
+ * means this digest was never fully staged — serving from it would be wrong
+ * before T6 too, this just makes the fail-closed check unconditional.
+ *
+ * On a hit, serves the R2 bytes VERBATIM with a Cache-Control keyed by the
+ * manifest's file `kind` (long-lived immutable for the shared `asset` pool —
+ * safe because the key itself is the content hash; short/revalidating for a
+ * `route` doc, rendered fresh per version at a STABLE, non-digest url) and,
+ * for an HTML response, the bundle's PINNED CSP header read back from the
+ * same manifest read — never the Worker's per-request nonce middleware, per
+ * the ADR's CSP-consequences note.
+ */
+export async function serveStaticBundle(
+  deps: ServeStaticBundleDeps,
+  pathname: string,
+): Promise<Response | null> {
+  const { bucket, digest } = deps;
+  if (!digest) {
+    return failClosed(deps, "pointer-serving tenant has no resolved live digest");
+  }
+
+  const rel = staticBundleRelPath(pathname);
+  if (!rel) {
+    return failClosed(deps, `unsafe/unparseable path: ${JSON.stringify(pathname)}`);
+  }
+
+  const manifest = await readStaticBundleManifest(bucket, digest);
+  if (!manifest) {
+    return failClosed(deps, `missing/unreadable manifest for digest ${digest}`);
+  }
+
+  const entry = manifest.files?.[rel];
+  if (manifest.files && !entry) {
+    // Provably not in this bundle's static surface — an app surface (/admin,
+    // /cockpit, …) or a route this version never materialized. SSR owns it.
+    return null;
+  }
+  const objectKey = resolveStaticBundleObjectKey(digest, rel, manifest.files);
+  const object = await bucket.get(objectKey);
+  if (!object) {
+    return failClosed(deps, `missing R2 object ${objectKey}`);
+  }
+
+  const isHtml = rel.endsWith(".html");
+  // Cache tier: use the manifest's `kind` classification when present.
+  // A manifest with no `files` at all falls back to the ORIGINAL
+  // isHtml-based heuristic — every non-html path used to be treated as a
+  // long-cacheable asset, which this preserves exactly for an old bundle
+  // this Worker version might still encounter mid-rollout.
+  const cacheControl = entry
+    ? entry.kind === "asset"
+      ? ASSET_CACHE_CONTROL
+      : ROUTE_CACHE_CONTROL
+    : isHtml
+      ? ROUTE_CACHE_CONTROL
+      : ASSET_CACHE_CONTROL;
+  const headers: Record<string, string> = {
+    "content-type": assetContentType(rel),
+    // Cache-key-by-digest lives in the caller's edge-cache layer for `route`
+    // keys (versions/<digest>/<rel> is digest-scoped — a pointer flip is a
+    // clean cache miss on the NEW digest's keys, never a stale serve of the
+    // old one); `asset` keys are ADDITIONALLY digest-independent (content-
+    // addressed by hash), so they're safe to cache forever downstream too.
+    // This header governs downstream/browser caching of the STABLE URL:
+    "cache-control": cacheControl,
+    "x-tot-static-bundle": "hit",
+    "x-tot-static-bundle-digest": digest,
+  };
+
+  if (isHtml && manifest.csp) {
+    headers[manifest.csp.headerName] = manifest.csp.value;
+  }
+
+  if (typeof object.size === "number") {
+    headers["content-length"] = String(object.size);
+  }
+  const body: BodyInit = object.body ?? (await object.arrayBuffer());
+  return new Response(body, { status: 200, headers });
+}

package/packages/public-runtime/src/tenant-assets.ts

@@ -1,5 +1,5 @@
 /**
- * Versioned tenant static-asset store (decision `assets-r2-versioned`).
+ * Versioned tenant static-asset store.
  *
  * Tenant static assets (`tenants/<id>/public/**` — logo, CSS, ~36 binaries) are
  * served from **R2, version-scoped**, replacing the build-time
@@ -275,6 +275,46 @@ export interface IngestResult {
   count: number;
 }
 
+const R2_PUT_MAX_ATTEMPTS = 3;
+const R2_PUT_RETRY_DELAYS_MS = [100, 400] as const;
+
+function errorField(error: unknown, field: string): unknown {
+  if (typeof error !== "object" || error === null || !(field in error)) return undefined;
+  return (error as Record<string, unknown>)[field];
+}
+
+function isTransientR2PutError(error: unknown): boolean {
+  const code = String(errorField(error, "code") ?? "");
+  const status = Number(errorField(error, "status") ?? errorField(error, "statusCode"));
+  const message = error instanceof Error ? error.message : String(error);
+  return (
+    code === "10001" ||
+    code === "InternalError" ||
+    (Number.isFinite(status) && status >= 500 && status <= 599) ||
+    /\b10001\b|internal error|temporarily unavailable|service unavailable|timed? out/i.test(message)
+  );
+}
+
+function waitForR2Retry(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function putAssetWithRetry(
+  bucket: R2BucketLike,
+  key: string,
+  bytes: Uint8Array,
+): Promise<void> {
+  for (let attempt = 1; attempt <= R2_PUT_MAX_ATTEMPTS; attempt++) {
+    try {
+      await bucket.put(key, bytes);
+      return;
+    } catch (error) {
+      if (attempt === R2_PUT_MAX_ATTEMPTS || !isTransientR2PutError(error)) throw error;
+      await waitForR2Retry(R2_PUT_RETRY_DELAYS_MS[attempt - 1]!);
+    }
+  }
+}
+
 /**
  * Write a version's public assets to R2 under `tenants/<id>/<versionId>/<rel>`.
  * Content-addressed by the (caller-supplied or computed) snapshot id → immutable
@@ -294,7 +334,9 @@ export async function ingestAssetVersion(
   const keys: string[] = [];
   for (const a of assets) {
     const key = tenantAssetKey(tenantId, vid, a.rel);
-    await bucket.put(key, a.bytes);
+    // The key and bytes are immutable for this version, so retrying an ambiguous
+    // transient R2 failure is idempotent even if the first write landed server-side.
+    await putAssetWithRetry(bucket, key, a.bytes);
     keys.push(key);
   }
   return { versionId: vid, keys, count: keys.length };
@@ -411,6 +453,16 @@ export interface ServeTenantAssetDeps {
   kvGet: (key: string) => Promise<string | null>;
   /** Channel to serve; default "live". Pass `locals.customizationEnv`. */
   channel?: CustomizationEnv;
+  /**
+   * Explicit snapshot versionId that OVERRIDES the channel→version pointer. Set for
+   * a CANDIDATE preview session: a candidate's assets are ingested to R2 keyed by
+   * the candidate versionId but NEVER pointed to by the shared test/live channel
+   * (the no-channel-flip invariant), so without this the candidate's NEW assets are
+   * materialized-but-unreachable. The caller must have authorized the viewer for
+   * THIS tenant (the versionId is a content hash, tenant-scoped by the R2 key) —
+   * see the `/tenants/<id>` route's preview-token check.
+   */
+  versionId?: string;
   /** Request `If-None-Match` header, for conditional GET (304). */
   ifNoneMatch?: string | null;
 }
@@ -434,8 +486,13 @@ export async function serveTenantAsset(
   if (!parsed) return null;
   const { tenantId, rel } = parsed;
 
-  const pointer = await readAssetsPointer({ get: deps.kvGet }, tenantId);
-  const versionId = resolveAssetVersion(pointer, deps.channel ?? "live");
+  // A candidate-preview override pins the snapshot directly; otherwise resolve the
+  // channel→version pointer (live/test) as usual.
+  let versionId = deps.versionId ?? null;
+  if (!versionId) {
+    const pointer = await readAssetsPointer({ get: deps.kvGet }, tenantId);
+    versionId = resolveAssetVersion(pointer, deps.channel ?? "live");
+  }
   if (!versionId) return null; // non-versioned tenant → fall through
 
   const etag = `"${versionId}"`;
@@ -477,3 +534,85 @@ function matchesEtag(ifNoneMatch: string, etag: string): boolean {
     .split(",")
     .some((t) => t.trim() === "*" || norm(t) === target);
 }
+
+// ---------------------------------------------------------------------------
+// Verify — fetch-confirm a version's app-asset subtree actually materialized
+// ---------------------------------------------------------------------------
+
+/** Outcome of {@link verifyMaterializedAssets}. */
+export interface AssetVerificationResult {
+  /** App-asset objects expected to serve at this version. */
+  expected: number;
+  /** How many were confirmed present + non-empty in R2. */
+  verified: number;
+  /** The served relative paths (or artifact paths) that did NOT materialize. */
+  missing: string[];
+  /** Set when the verification itself could not run (fail-closed, not a pass). */
+  error?: string;
+}
+
+/** How many R2 existence probes run at once — matches the runtime's ~6/host reality. */
+const VERIFY_PROBE_POOL = 6;
+
+/**
+ * Confirm every app-asset artifact path is present + non-empty in R2 at
+ * `versionId` — the EXACT object `serveTenantAsset` streams for `/tenants/<id>/<rel>`.
+ * Positive, fail-closed evidence for the version-scoped asset lane: an asset whose
+ * path can't normalize to a served rel, or is absent/empty in R2, is reported
+ * `missing`; a systemic fault (the bucket throwing on every probe) reports the whole
+ * subtree missing with an `error` rather than a vacuous pass. Checking the object at
+ * the published `versionId` (not an HTTP self-fetch) is channel-independent: the
+ * served route may resolve a different channel→version pointer per request, but a
+ * version's bytes are fixed. `bucket` needs only `get`; the body is never read.
+ */
+export async function verifyMaterializedAssets(
+  bucket: Pick<R2BucketLike, "get">,
+  tenantId: string,
+  versionId: string,
+  appAssetArtifactPaths: readonly string[],
+): Promise<AssetVerificationResult> {
+  const expected = appAssetArtifactPaths.length;
+  if (expected === 0) return { expected: 0, verified: 0, missing: [] };
+
+  try {
+    const missing: string[] = [];
+    let verified = 0;
+    let next = 0;
+    const worker = async (): Promise<void> => {
+      while (next < appAssetArtifactPaths.length) {
+        const artifactPath = appAssetArtifactPaths[next++]!;
+        const rel = toPublicRel(tenantId, artifactPath);
+        // An app asset whose path doesn't normalize to a served rel can't serve —
+        // count it missing (fail-closed), labelled by its artifact path.
+        if (!rel) {
+          missing.push(artifactPath);
+          continue;
+        }
+        let present = false;
+        try {
+          const obj = await bucket.get(tenantAssetKey(tenantId, versionId, rel));
+          // Present when the object exists and (when the store reports size) is non-empty.
+          present = obj != null && (obj.size == null || obj.size > 0);
+        } catch {
+          present = false;
+        }
+        if (present) verified++;
+        else missing.push(rel);
+      }
+    };
+    const width = Math.max(1, Math.min(VERIFY_PROBE_POOL, appAssetArtifactPaths.length));
+    await Promise.all(Array.from({ length: width }, worker));
+    // Stable order aids diffing/readability — pool completion order is nondeterministic.
+    missing.sort();
+    return { expected, verified, missing };
+  } catch (err) {
+    // Systemic failure (e.g. the bucket seam throwing on every call): fail closed —
+    // report the whole subtree as unverified rather than claiming a pass.
+    return {
+      expected,
+      verified: 0,
+      missing: [...appAssetArtifactPaths],
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}