@tokenoftrust/storefront-runner 2.2.91 → 2.2.93

package/agency-kit/tools/lib/catalog-model/files.d.mts

@@ -46,3 +46,11 @@ export function readCatalogFiles(
   entries: Iterable<{ path: string; raw: string | null }>,
   store: CatalogStore,
 ): { products: CatalogRecord[]; collections: CatalogRecord[] };
+
+export const CATALOG_LISTING_VERSION: number;
+export const LISTING_DESCRIPTION_CHARS: number;
+export function catalogListingProjection(entries: Iterable<{ path: string; raw: string | null }>): string | null;
+export function readCatalogListing(
+  raw: string,
+  store: CatalogStore,
+): { products: CatalogRecord[]; collections: CatalogRecord[] };

package/agency-kit/tools/lib/catalog-model/files.mjs

@@ -261,3 +261,96 @@ export function readCatalogFiles(entries, { tenantScope, inventoryProvider }) {
   if (failures.length > 0) throw new CatalogFileError(failures);
   return { products, collections };
 }
+
+// --- The listing projection ------------------------------------------------------------------
+//
+// A large catalog cannot be read file by file on every request where each read is a separate
+// storage call (the hosted preview). The projection is one derived document built once per
+// reconciled version: every product as a listing needs it (no `description_html`, no
+// `metafields`, `description_text` cut to LISTING_DESCRIPTION_CHARS) plus every collection whole,
+// or, when any file fails validation, the failures. A product page reads its full file.
+
+/** Version of the projection document; a reader refuses any other. */
+export const CATALOG_LISTING_VERSION = 1;
+/** Characters of `description_text` a listing keeps: cards show 180, search weighs the rest lightly. */
+export const LISTING_DESCRIPTION_CHARS = 400;
+
+/** @param {Record<string, any>} record */
+function listingProduct(record) {
+  const { description_html: _html, metafields: _metafields, ...rest } = record;
+  return {
+    ...rest,
+    description_text:
+      typeof rest.description_text === "string"
+        ? rest.description_text.slice(0, LISTING_DESCRIPTION_CHARS)
+        : rest.description_text,
+  };
+}
+
+/**
+ * Build the listing projection from a store's catalog files, validating each one exactly as the
+ * reader does. Records are kept as their files hold them (identity and availability are stamped
+ * when the projection is read). Returns the document's text, or null when there are no catalog
+ * files.
+ *
+ * @param {Iterable<{ path: string, raw: string | null }>} entries `content/`-relative paths
+ * @returns {string | null}
+ */
+export function catalogListingProjection(entries) {
+  const products = [];
+  const collections = [];
+  const failures = [];
+  let any = false;
+  for (const { path, raw } of entries) {
+    if (classifyCatalogPath(path) === null) continue;
+    any = true;
+    if (raw === null) {
+      failures.push({ path, issues: ["listed in the catalog but unreadable"] });
+      continue;
+    }
+    const parsed = parse(path, raw, "", { available: true });
+    if (parsed.issues.length > 0) {
+      failures.push({ path, issues: parsed.issues });
+      continue;
+    }
+    const doc = JSON.parse(raw);
+    if (parsed.kind === "product") products.push(listingProduct(catalogFileRecord("product", doc)));
+    else collections.push(doc);
+  }
+  if (!any) return null;
+  return JSON.stringify(
+    failures.length > 0
+      ? { version: CATALOG_LISTING_VERSION, failures }
+      : { version: CATALOG_LISTING_VERSION, products, collections },
+  );
+}
+
+/**
+ * Read a listing projection for the store being read, records stamped exactly as file reads
+ * are. Throws `CatalogFileError` with the failures the projection recorded, or when the document
+ * is not a projection this module wrote.
+ *
+ * @param {string} raw
+ * @param {{ tenantScope: string, inventoryProvider?: string }} store
+ * @returns {{ products: object[], collections: object[] }}
+ */
+export function readCatalogListing(raw, { tenantScope, inventoryProvider }) {
+  const availability = declaredAvailability(inventoryProvider);
+  /** @type {any} */
+  let doc = null;
+  try {
+    doc = JSON.parse(raw);
+  } catch {
+    doc = null;
+  }
+  if (!isObject(doc) || doc.version !== CATALOG_LISTING_VERSION) {
+    throw new CatalogFileError([
+      { path: "catalog/", issues: ["the catalog listing projection is unreadable; reconcile the preview again"] },
+    ]);
+  }
+  if (Array.isArray(doc.failures)) throw new CatalogFileError(doc.failures);
+  return {
+    products: doc.products.map((/** @type {unknown} */ p) => stamp("product", p, tenantScope, availability)),
+    collections: doc.collections.map((/** @type {unknown} */ c) => stamp("collection", c, tenantScope, availability)),
+  };
+}

package/apps/storefront/src/lib/storyblok/provider.ts

@@ -22,10 +22,13 @@ import type { ChromeConfig } from "../chrome/model.js";
 import type { CustomizationEnv, KvGet, TenantConfig } from "@tot/public-runtime";
 import {
   contentArtifactPath,
+  derivedArtifactPath,
   listPublishedArtifactPaths,
   listVersionArtifactPaths,
   readPublishedArtifact,
   readVersionArtifact,
+  resolvePublishedArtifactKey,
+  resolveVersionArtifactKey,
   tenantDirRelative,
   tenantDirSegment,
 } from "@tot/public-runtime";
@@ -86,6 +89,17 @@ export interface ContentProvider {
    * providers), so a caller that reads many artifacts in one request budgets against it.
    */
   readonly perRequestArtifactReads?: boolean;
+  /**
+   * A document the reconcile derived for the selected version (`derivedArtifactPath`), as its
+   * immutable KV key plus a reader, or null when the version has none. The key names
+   * content-addressed bytes, so a caller may keep what it parsed from them under it.
+   */
+  derivedArtifact?(name: string): Promise<DerivedArtifact | null>;
+}
+
+export interface DerivedArtifact {
+  key: string;
+  read(): Promise<string | null>;
 }
 
 // ---------------------------------------------------------------------------
@@ -695,6 +709,14 @@ export class PublishedContentProvider implements ContentProvider {
     return this.publishedRaw(relative);
   }
 
+  async derivedArtifact(name: string): Promise<DerivedArtifact | null> {
+    const get = this.cust.get;
+    const key = await resolvePublishedArtifactKey(
+      get, this.tenantId, this.cust.env, derivedArtifactPath(this.tenantId, name),
+    );
+    return key ? { key, read: () => get(key) } : null;
+  }
+
   async listArtifactPaths(prefix = ""): Promise<string[] | null> {
     const root = contentArtifactPath(this.tenantId, "");
     const paths = await listPublishedArtifactPaths(
@@ -801,6 +823,14 @@ export class VersionPinnedContentProvider implements ContentProvider {
   async listArtifactPaths(prefix = ""): Promise<string[] | null> {
     return listVersionArtifactPaths(this.get, this.tenantId, this.versionId, prefix);
   }
+
+  async derivedArtifact(name: string): Promise<DerivedArtifact | null> {
+    const get = this.get;
+    const key = await resolveVersionArtifactKey(
+      get, this.tenantId, this.versionId, derivedArtifactPath(this.tenantId, name),
+    );
+    return key ? { key, read: () => get(key) } : null;
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -865,6 +895,12 @@ export class DevDraftContentProvider implements ContentProvider {
   listArtifactPaths(prefix?: string): Promise<string[] | null> {
     return this.base.listArtifactPaths(prefix);
   }
+  get perRequestArtifactReads(): boolean | undefined {
+    return this.base.perRequestArtifactReads;
+  }
+  derivedArtifact(name: string): Promise<DerivedArtifact | null> {
+    return this.base.derivedArtifact?.(name) ?? Promise.resolve(null);
+  }
 }
 
 const PLACEHOLDER_TOKEN_RE = /PLACEHOLDER|^$/i;

package/apps/storefront/src/lib/tot/ToTClient.ts

@@ -46,6 +46,8 @@ export class FixtureToTClient implements ToTClient {
     private readonly collections: CatalogCollection[],
     private readonly scope: string,
     private readonly reviews: Record<string, Review[]> = {},
+    /** The full record for a product page, when the products above are listing records. */
+    private readonly detail?: (handle: string) => Promise<CatalogProduct | null>,
   ) {}
 
   async getReviews(handle: string): Promise<Review[]> {
@@ -75,6 +77,7 @@ export class FixtureToTClient implements ToTClient {
 
   async getProductByHandle(handle: string): Promise<CatalogProduct | null> {
     // NOTE: resolves drafts too (preview/direct links). Callers gate visibility.
+    if (this.detail) return this.detail(handle);
     return this.scoped().find((p) => p.handle === handle) ?? null;
   }
 
@@ -214,7 +217,9 @@ export async function createToTClient(
   return new DeferredToTClient(async () => {
     const files = await readTenantCatalogFiles(tenant, content);
     if (files) {
-      return new FixtureToTClient(files.products, files.collections, tenant.tot_data_scope);
+      return new FixtureToTClient(
+        files.products, files.collections, tenant.tot_data_scope, {}, files.productDetail,
+      );
     }
     const { products, collections, reviews } = await loadFixtures(tenant.tot_data_scope);
     return new FixtureToTClient(products, collections, tenant.tot_data_scope, reviews);

package/apps/storefront/src/lib/tot/tenantCatalog.ts

@@ -4,7 +4,9 @@
  * uses (`ContentProvider.readArtifact` / `listArtifactPaths`), so it follows the store's
  * `.tot/config.json` content mapping and whichever version the request selected:
  *
- *   - hosted preview (Worker): the reconciled version manifest in KV, one read per file;
+ *   - hosted preview (Worker): the reconciled version in KV. Listings, search and the
+ *     sitemap read the version's derived listing projection (one read, parsed once per
+ *     isolate and version); a product page reads that product's full file;
  *   - static publish (both CDNs) and `tot dev`: the store's checkout as materialized
  *     into the build (`LocalContentProvider`'s build-time globs).
  *
@@ -13,31 +15,41 @@
  * silently missing.
  */
 import type { CatalogCollection, CatalogProduct, TenantConfig } from "@tot/public-runtime";
+import { CATALOG_LISTING_ARTIFACT } from "@tot/public-runtime";
 import {
   CATALOG_ARTIFACT_PREFIX,
+  CATALOG_PRODUCTS_PREFIX,
+  CATALOG_HANDLE_RE,
   CatalogFileError,
+  readCatalogFile,
   readCatalogFiles,
+  readCatalogListing,
   type CatalogStore,
 } from "@tot/public-runtime/catalog-files";
 import { LocalContentProvider, type ContentProvider } from "../storyblok/provider.js";
 
 export type CatalogArtifactSource = Pick<
   ContentProvider,
-  "readArtifact" | "listArtifactPaths" | "perRequestArtifactReads"
+  "readArtifact" | "listArtifactPaths" | "perRequestArtifactReads" | "derivedArtifact"
 >;
 
 export interface TenantCatalogFiles {
   products: CatalogProduct[];
   collections: CatalogCollection[];
+  /**
+   * The full record for a product page, when `products` are listing records (the listing
+   * projection drops `description_html` and `metafields`). Absent: `products` are full.
+   */
+  productDetail?: (handle: string) => Promise<CatalogProduct | null>;
 }
 
 /**
- * The most catalog files one request may read from a per-request source. A Worker
- * invocation may make 1,000 calls to KV and other bindings, and each artifact read is
- * one of them; the rest of the budget is left to the page's other reads. Measured
- * against a 2,959-product, 357-collection catalog (3,316 files, 44.6 MB at 13.4 KB per
- * product, ~290 ms to validate): that store cannot be read file by file per request on
- * the hosted preview, while `tot dev` and the static publish read it from the checkout.
+ * The most catalog files one request may read from a per-request source that has no listing
+ * projection (a version reconciled before projections existed). Each artifact read is its own
+ * KV call, and the rest of the page needs its own. Measured 2026-10-06 on giantvapes.com's
+ * catalog (2,959 products, 357 collections, 27.2 MB of files): read file by file it is 3,316
+ * calls and ~40 MB of parsed heap per request; its listing projection is one 11.4 MB read
+ * (1.55 MB gzipped), 18 MB of heap, parsed in ~20 ms once per isolate and version.
  */
 export const CATALOG_REQUEST_READ_BUDGET = 800;
 
@@ -94,10 +106,32 @@ function readBuildCatalog(
   return { products, collections };
 }
 
+/** Listing projections parsed in this isolate, by immutable KV key and the store reading them. */
+const parsedListings = new Map<string, { products: CatalogProduct[]; collections: CatalogCollection[] }>();
+const PARSED_LISTINGS_KEPT = 2;
+
+async function readListing(
+  listing: { key: string; read(): Promise<string | null> },
+  store: CatalogStore,
+): Promise<{ products: CatalogProduct[]; collections: CatalogCollection[] }> {
+  const cacheKey = `${listing.key}\0${store.tenantScope}\0${store.inventoryProvider ?? ""}`;
+  const hit = parsedListings.get(cacheKey);
+  if (hit) return hit;
+  const raw = await listing.read();
+  if (raw === null) {
+    throw new CatalogFileError([{ path: "catalog/", issues: ["the catalog listing projection is unreadable; reconcile the preview again"] }]);
+  }
+  const parsed = readCatalogListing(raw, store);
+  parsedListings.set(cacheKey, parsed);
+  while (parsedListings.size > PARSED_LISTINGS_KEPT) parsedListings.delete(parsedListings.keys().next().value!);
+  return parsed;
+}
+
 /**
  * The store's catalog files, or null when it has none. Throws `CatalogFileError` naming
- * every file that fails validation, and refuses a per-request source holding more files
- * than `CATALOG_REQUEST_READ_BUDGET` before reading any of them.
+ * every file that fails validation. A per-request source serves its version's listing
+ * projection; one without a projection reads its files only up to
+ * `CATALOG_REQUEST_READ_BUDGET`, refusing above it before reading any of them.
  */
 export async function readTenantCatalogFiles(
   tenant: TenantConfig,
@@ -106,19 +140,32 @@ export async function readTenantCatalogFiles(
   const { source, paths } = await catalogPaths(tenant, content);
   if (paths.length === 0) return null;
 
+  const store: CatalogStore = {
+    tenantScope: tenant.tot_data_scope,
+    inventoryProvider: tenant.inventory?.provider,
+  };
+
+  const listing = await source.derivedArtifact?.(CATALOG_LISTING_ARTIFACT);
+  if (listing) {
+    const { products, collections } = await readListing(listing, store);
+    const productDetail = async (handle: string): Promise<CatalogProduct | null> => {
+      if (!CATALOG_HANDLE_RE.test(handle)) return null;
+      const path = `${CATALOG_PRODUCTS_PREFIX}${handle}.json`;
+      const raw = await source.readArtifact(path);
+      if (raw === null) return null;
+      const read = readCatalogFile(path, raw, store);
+      return read.kind === "product" ? read.record : null;
+    };
+    return { products, collections, productDetail };
+  }
+
   if (source.perRequestArtifactReads && paths.length > CATALOG_REQUEST_READ_BUDGET) {
     throw new Error(
-      `catalog: ${tenant.tenant_id} has ${paths.length} files in content/catalog/, and the hosted ` +
-        `preview reads at most ${CATALOG_REQUEST_READ_BUDGET} catalog files per request (one read ` +
-        `each, inside the Worker's per-request limit). \`tot dev\` and the published site render ` +
-        `this catalog from the checkout.`,
+      `catalog: ${tenant.tenant_id} has ${paths.length} files in content/catalog/ and this preview ` +
+        `version has no catalog listing; reconcile the preview again to build it.`,
     );
   }
 
-  const store: CatalogStore = {
-    tenantScope: tenant.tot_data_scope,
-    inventoryProvider: tenant.inventory?.provider,
-  };
   const entries = await Promise.all(paths.map(async (path) => ({ path, raw: await source.readArtifact(path) })));
   return source.perRequestArtifactReads
     ? readCatalogFiles(entries, store)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tokenoftrust/storefront-runner",
-  "version": "2.2.91",
+  "version": "2.2.93",
   "license": "SEE LICENSE IN LICENSE",
   "description": "World-shareable storefront runner: multi-tenant renderer on Astro/Cloudflare. No control plane.",
   "packageManager": "pnpm@11.9.0",

package/packages/public-runtime/src/catalog-files.ts

@@ -10,6 +10,7 @@
 import {
   readCatalogFile as readKitCatalogFile,
   readCatalogFiles as readKitCatalogFiles,
+  readCatalogListing as readKitCatalogListing,
   type CatalogStore,
 } from "../../../agency-kit/tools/lib/catalog-model/files.mjs";
 import type { CatalogCollection, CatalogProduct } from "./product.js";
@@ -22,6 +23,7 @@ export {
   CATALOG_PRODUCTS_PREFIX,
   CatalogFileError,
   catalogFileIssues,
+  catalogListingProjection,
   classifyCatalogPath,
   declaredAvailability,
   planCatalogFiles,
@@ -49,3 +51,18 @@ export function readCatalogFiles(
     collections: CatalogCollection[];
   };
 }
+
+/**
+ * The derived listing projection's catalog: listing records (no `description_html`, no
+ * `metafields`, `description_text` shortened) for every product, collections whole. Throws
+ * `CatalogFileError` with the failures the reconcile recorded.
+ */
+export function readCatalogListing(
+  raw: string,
+  store: CatalogStore,
+): { products: CatalogProduct[]; collections: CatalogCollection[] } {
+  return readKitCatalogListing(raw, store) as unknown as {
+    products: CatalogProduct[];
+    collections: CatalogCollection[];
+  };
+}

package/packages/public-runtime/src/customization-runtime.ts

@@ -67,6 +67,18 @@ export function contentArtifactPath(tenantId: string, relative: string): string
   return `tenants/${tenantId}/content/${relative}`;
 }
 
+/**
+ * Canonical artifact path for a document the reconcile DERIVES from a tenant's files (never one
+ * the tenant writes): `tenants/<id>/.derived/<name>`, outside `content/` so no repo file can
+ * collide with it. Recorded in the same immutable site version as the files it derives from.
+ */
+export function derivedArtifactPath(tenantId: string, name: string): string {
+  return `tenants/${tenantId}/.derived/${name}`;
+}
+
+/** The derived catalog listing projection (`catalogListingProjection` in the kit's catalog module). */
+export const CATALOG_LISTING_ARTIFACT = "catalog-listing.json";
+
 async function readPointer(
   get: KvGet,
   tenantId: string,
@@ -270,6 +282,21 @@ export async function readPublishedArtifact(
   tenantId: string,
   env: CustomizationEnv,
   path: string,
+): Promise<string | null> {
+  const key = await resolvePublishedArtifactKey(get, tenantId, env, path);
+  return key ? get(key) : null;
+}
+
+/**
+ * The KV key holding the published artifact for (tenant, env, path), resolved exactly as
+ * {@link readPublishedArtifact} resolves it, or null. The key names immutable, content-addressed
+ * bytes, so a caller may cache what it parsed from them under the key.
+ */
+export async function resolvePublishedArtifactKey(
+  get: KvGet,
+  tenantId: string,
+  env: CustomizationEnv,
+  path: string,
 ): Promise<string | null> {
   const pointer = await readPointer(get, tenantId);
   if (!pointer) return null;
@@ -277,13 +304,25 @@ export async function readPublishedArtifact(
   const channel = channelFromEnv(env);
   if (pointer.channels?.[channel]) {
     // Versioned model engaged for this channel — authoritative.
-    return readVersionedArtifact(get, tenantId, channel, path);
+    const { versionId } = await resolveServableChannelVersion(get, tenantId, channel);
+    return versionId ? resolveVersionArtifactKey(get, tenantId, versionId, path) : null;
   }
 
   // Legacy per-path selection.
   const versionId = pointer?.[env]?.[path];
-  if (!versionId) return null;
-  return get(customizationArtifactKey(tenantId, versionId));
+  return versionId ? customizationArtifactKey(tenantId, versionId) : null;
+}
+
+/** The KV key of one artifact in an immutable site version, or null when the version lacks it. */
+export async function resolveVersionArtifactKey(
+  get: KvGet,
+  tenantId: string,
+  versionId: string,
+  path: string,
+): Promise<string | null> {
+  const manifest = await readSiteVersion(get, tenantId, versionId);
+  const contentHash = manifest?.artifacts?.[path];
+  return contentHash ? customizationArtifactKey(tenantId, contentHash) : null;
 }
 
 /**
@@ -299,10 +338,8 @@ export async function readVersionArtifact(
   versionId: string,
   path: string,
 ): Promise<string | null> {
-  const manifest = await readSiteVersion(get, tenantId, versionId);
-  const contentHash = manifest?.artifacts?.[path];
-  if (!contentHash) return null;
-  return get(customizationArtifactKey(tenantId, contentHash));
+  const key = await resolveVersionArtifactKey(get, tenantId, versionId, path);
+  return key ? get(key) : null;
 }
 
 /**

package/apps/storefront/src/lib/compliance/rawComplianceNotices.ts

@@ -1,72 +0,0 @@
-/**
- * Force-injectable compliance markup for the raw-HTML serve path.
- *
- * `rawHtmlResponse` (rawMarketingHtml.ts) bypasses Astro's Layout, so these are
- * the platform-owned anchors that CAN be guaranteed on a raw document
- * independent of its content: the FDA nicotine warning, the shipping/
- * jurisdiction notice, and the excise-tax disclosure. Each mirrors the exact
- * markup its Layout.astro sibling renders (NicotineWarning.astro "bar" variant,
- * ShippingRestrictionNotice.astro "footer" variant via ComplianceNotice.astro,
- * and Layout's own excise-tax <p>) so a raw and a block-composed page present
- * these anchors identically. See `rawHtmlPolicy` for which anchors this does
- * and doesn't cover (notably: NOT `age-gate`).
- */
-
-function escapeHtml(value: string): string {
-  return value
-    .replace(/&/g, "&amp;")
-    .replace(/</g, "&lt;")
-    .replace(/>/g, "&gt;")
-    .replace(/"/g, "&quot;")
-    .replace(/'/g, "&#39;");
-}
-
-const NICOTINE_WARNING_TEXT =
-  "WARNING: This product contains nicotine. Nicotine is an addictive chemical.";
-
-/** Mirrors NicotineWarning.astro's "bar" variant. */
-export function nicotineWarningBarHtml(): string {
-  return (
-    '<div class="border-b-2 border-[var(--color-text)] bg-[var(--color-text)] text-[var(--color-bg)]" role="note" aria-label="Health warning" data-compliance="nicotine-warning">' +
-    '<p class="container-prose py-2 text-center text-[0.72rem] font-semibold uppercase tracking-[0.06em] sm:text-xs">' +
-    NICOTINE_WARNING_TEXT +
-    "</p></div>"
-  );
-}
-
-/** Mirrors ShippingRestrictionNotice.astro's "footer" variant (via ComplianceNotice.astro). Empty when `text` is absent — same as the Astro component. */
-export function shippingRestrictionNoticeHtml(text: string | undefined): string {
-  if (!text) return "";
-  return (
-    '<div class="border-t border-[var(--color-border)] bg-[var(--color-bg)]" data-compliance="shipping-restriction">' +
-    '<p class="container-prose py-3 text-center text-sm text-[var(--color-muted)]" role="note" aria-label="Shipping restriction notice">' +
-    escapeHtml(text) +
-    "</p></div>"
-  );
-}
-
-/** Mirrors Layout.astro's excise-tax disclosure <p>. */
-export function exciseTaxNoticeHtml(): string {
-  return (
-    '<p class="border-t border-[var(--color-border)] bg-[var(--color-bg)] py-2.5 text-center text-xs text-[var(--color-muted)]" data-capability="excise-tax">' +
-    "Applicable excise tax is calculated at checkout.</p>"
-  );
-}
-
-export interface RawComplianceChromeInput {
-  nicotineWarning?: boolean;
-  shippingRestriction?: string;
-  exciseTaxEnabled?: boolean;
-}
-
-/** The markup to inject at `rawHtmlResponse`'s body-start boundary. */
-export function renderRawComplianceTop(input: RawComplianceChromeInput): string {
-  return input.nicotineWarning ? nicotineWarningBarHtml() : "";
-}
-
-/** The markup to inject at `rawHtmlResponse`'s body-deferred boundary. */
-export function renderRawComplianceBottom(input: RawComplianceChromeInput): string {
-  let out = shippingRestrictionNoticeHtml(input.shippingRestriction);
-  if (input.exciseTaxEnabled) out += exciseTaxNoticeHtml();
-  return out;
-}