@tokenoftrust/storefront-runner 2.0.2 → 2.0.3

package/scripts/tenant/materialize-tenant-content.mjs

@@ -0,0 +1,533 @@
+#!/usr/bin/env node
+/**
+ * Materialize a Gitea-hosted tenant's content into the monorepo working tree.
+ *
+ * Real tenants live in per-tenant Gitea repos (git.tokenoftrust.com/storefront/
+ * <repo>) — the SOURCE OF TRUTH. The monorepo's `tenants/<id>/` dirs for those
+ * tenants are gitignored, disposable LOCAL/PER-TENANT build input (the same
+ * stance as apps/storefront/public/tenants/, see .gitignore). The shared
+ * Storefront Worker reads published tenant revisions from KV/R2 and does NOT run
+ * this first. Local tenant development and a named per-tenant static publish do.
+ * Committed-fixture tenants (home, sample-store.e2e.test) are NOT materialized
+ * — they're tracked in git and this script refuses to touch them.
+ *
+ * Which tenants are Gitea-classified lives in `tenants/.gitea-tenants.json`
+ * (the same manifest the CI reintroduction guard reads —
+ * scripts/tenant/check-no-tracked-real-tenants.mjs).
+ *
+ * What it does per tenant (idempotent; pinnable to a ref):
+ *   1. `git clone` the tenant repo to a temp dir and check out `--ref` (default:
+ *      the repo's default branch). Full clone — tenant repos are small and a SHA
+ *      ref can't be shallow-fetched portably.
+ *   2. Read the repo's own `.tot/config.json` mappings (workspace → monorepo
+ *      artifact path, e.g. `content/` → `tenants/<id>/content/`); fall back to
+ *      the canonical shape (theme.json, content/, public/) when absent.
+ *   3. Copy each mapped workspace path into the monorepo target, REPLACING the
+ *      target tree (stale files from a previous materialize are removed). Every
+ *      target must resolve under `tenants/<id>/` — anything else is refused.
+ *   4. Copy `.tot/config.json` itself (the CLI + validate flows read it) and
+ *      stamp `tenants/<id>/.materialized.json` with { repo, ref, sha, at } so
+ *      `--check` and humans can see what's populated.
+ *
+ * Modes:
+ *   node scripts/tenant/materialize-tenant-content.mjs <tenant> [--ref <sha|branch>]
+ *   node scripts/tenant/materialize-tenant-content.mjs <tenant> --remote <cloneUrl>
+ *       # e.g. the single-use gitRemote minted by the MCP `tenant_checkout` tool
+ *   node scripts/tenant/materialize-tenant-content.mjs --all          # every manifest tenant
+ *   node scripts/tenant/materialize-tenant-content.mjs --check        # verify populated; no network
+ *   node scripts/tenant/materialize-tenant-content.mjs --check-pinned [--expect-ref <sha>] [--allow-unpinned]
+ *       # stricter deploy gate: verify populated AND materialized from Gitea at a
+ *       # PINNED ref (.materialized.json present, pinned, on the expected ref); no network
+ *
+ * Auth (never logged; tokens are redacted from every error):
+ *   TENANT_GIT_BASE   clone base. Default https://git.tokenoftrust.com/storefront
+ *   TENANT_GIT_USER / TENANT_GIT_TOKEN
+ *                     basic-auth pair injected into the clone URL (CI: a
+ *                     read-only Gitea token; local dev can instead rely on an
+ *                     ambient credential helper and set neither). For a HUMAN
+ *                     running this manually (e.g. handing an agent a scoped
+ *                     token instead of ambient creds), mint a short-lived
+ *                     per-repo read-only TENANT_GIT_TOKEN with
+ *                     scripts/tenant/mint-read-token.sh <tenant> — it never
+ *                     exposes the app's MCP client-assertion signing key to
+ *                     whoever consumes the output.
+ */
+
+import {
+  cpSync,
+  existsSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { execFileSync } from "node:child_process";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve, sep } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const repoRoot = resolve(here, "..", "..");
+
+export const MANIFEST_PATH = "tenants/.gitea-tenants.json";
+const TENANT_ID_RE = /^[a-z0-9][a-z0-9.-]{1,60}[a-z0-9]$/;
+const DEFAULT_GIT_BASE = "https://git.tokenoftrust.com/storefront";
+
+/** Load the Gitea-tenant manifest. Returns { tenants: { <id>: { repo } } }. */
+export function loadManifest(root = repoRoot) {
+  const p = join(root, MANIFEST_PATH);
+  if (!existsSync(p)) {
+    throw new Error(`${MANIFEST_PATH} not found — no Gitea-classified tenants declared`);
+  }
+  const parsed = JSON.parse(readFileSync(p, "utf8"));
+  if (!parsed || typeof parsed.tenants !== "object" || parsed.tenants === null) {
+    throw new Error(`${MANIFEST_PATH}: expected a "tenants" object`);
+  }
+  return parsed;
+}
+
+/** Redact any secret substrings (tokens) from a message before it can escape. */
+export function redact(message, secrets) {
+  let out = String(message);
+  for (const s of secrets) if (s) out = out.split(s).join("***REDACTED***");
+  return out;
+}
+
+/**
+ * Build the authenticated clone URL. The token rides ONLY in the returned URL
+ * (passed to git, never printed); callers must route errors through `redact`.
+ */
+export function buildCloneUrl({ base, repo, user, token }) {
+  const clean = base.replace(/\/+$/, "");
+  if (!token) return `${clean}/${repo}.git`;
+  const u = new URL(`${clean}/${repo}.git`);
+  u.username = user || "token";
+  u.password = token;
+  return u.toString();
+}
+
+/**
+ * Resolve a mapping's monorepo target and REFUSE anything that escapes
+ * `tenants/<id>/` — a hostile/typo'd `.tot/config.json` must not be able to
+ * write elsewhere in the tree (compare reconcile's manifest deny-by-default).
+ */
+export function resolveTargetPath(root, tenant, repoPath) {
+  const target = resolve(root, repoPath);
+  const bound = resolve(root, "tenants", tenant) + sep;
+  if (!target.startsWith(bound) && target !== resolve(root, "tenants", tenant)) {
+    throw new Error(
+      `mapping target "${repoPath}" escapes tenants/${tenant}/ — refusing`,
+    );
+  }
+  return target;
+}
+
+/** The canonical mapping fallback (matches buildTotCheckoutConfig). */
+function canonicalMappings(tenant) {
+  return [
+    { workspace: "theme.json", repo: `tenants/${tenant}/theme.json`, kind: "file" },
+    { workspace: "content/", repo: `tenants/${tenant}/content/`, kind: "tree" },
+    { workspace: "public/", repo: `tenants/${tenant}/public/`, kind: "tree" },
+  ];
+}
+
+/**
+ * Copy one mapping from the checked-out tenant repo into the monorepo tree.
+ * `tree` targets are replaced wholesale so a removed file doesn't linger from a
+ * previous materialize. Missing OPTIONAL workspace paths are skipped (a tenant
+ * without public/ is fine); the reconcile-side required-artifact rules (theme +
+ * home present) are enforced by validate/reconcile, not here.
+ */
+export function copyMapping({ checkoutDir, root, tenant, mapping, log = (/** @type {string} */ _m) => {} }) {
+  const src = join(checkoutDir, mapping.workspace.replace(/\/+$/, ""));
+  const target = resolveTargetPath(root, tenant, mapping.repo.replace(/\/+$/, ""));
+  if (!existsSync(src)) {
+    log(`  ~ ${mapping.workspace} absent in repo — skipped`);
+    return { copied: false, target };
+  }
+  if (mapping.kind === "tree" && existsSync(target)) {
+    rmSync(target, { recursive: true, force: true });
+  }
+  cpSync(src, target, { recursive: true });
+  log(`  + ${mapping.workspace} → ${mapping.repo}`);
+  return { copied: true, target };
+}
+
+/**
+ * Materialize one tenant. `gitBase` may be a file:// URL (tests) or an
+ * authenticated https URL base. Returns { tenant, ref, sha, mappingsCopied }.
+ */
+export function materializeTenant({
+  tenant,
+  repo = tenant,
+  ref = null,
+  root = repoRoot,
+  gitBase = DEFAULT_GIT_BASE,
+  /** Full clone URL override (e.g. a tenant_checkout-minted single-use remote). */
+  remote = null,
+  user = process.env.TENANT_GIT_USER,
+  token = process.env.TENANT_GIT_TOKEN,
+  log = (/** @type {string} */ _m) => {},
+}) {
+  if (!TENANT_ID_RE.test(tenant)) {
+    throw new Error(`invalid tenant id "${tenant}"`);
+  }
+  const manifest = loadManifest(root);
+  if (!manifest.tenants[tenant]) {
+    throw new Error(
+      `"${tenant}" is not a Gitea-classified tenant (${MANIFEST_PATH}) — its ` +
+        `tenants/ dir is committed, not materialized. Refusing to overwrite it.`,
+    );
+  }
+  // A full remote URL may itself carry a credential — treat it as a secret too.
+  // (scp-style remotes like git@host:path aren't URL-parseable; they carry no
+  // inline password, so an unparseable remote contributes no secret.)
+  let remoteSecret = null;
+  if (remote) {
+    try {
+      remoteSecret = new URL(remote).password || null;
+    } catch {
+      remoteSecret = null;
+    }
+  }
+  const secrets = [token, remoteSecret].filter(Boolean);
+  const cloneUrl = remote ?? buildCloneUrl({ base: gitBase, repo, user, token });
+  const work = mkdtempSync(join(tmpdir(), `materialize-${tenant}-`));
+  const git = (args, cwd = work) =>
+    execFileSync("git", args, { cwd, stdio: ["ignore", "pipe", "pipe"] })
+      .toString()
+      .trim();
+  try {
+    try {
+      git(["clone", "--quiet", cloneUrl, work], tmpdir());
+    } catch (e) {
+      throw new Error(
+        `clone of ${gitBase.startsWith("file:") ? repo : `${gitBase}/${repo}`} failed: ` +
+          redact(e.stderr?.toString() || e.message, secrets),
+      );
+    }
+    if (ref) {
+      try {
+        git(["checkout", "--quiet", ref]);
+      } catch {
+        // A SHA not on the default branch (e.g. a merged-then-advanced tip)
+        // may need an explicit fetch before checkout.
+        git(["fetch", "--quiet", "origin", ref]);
+        git(["checkout", "--quiet", "FETCH_HEAD"]);
+      }
+    } else {
+      // No --ref = the repo's DEFAULT branch, which for tenant repos is often
+      // days behind the promoted/preview content. Loud, unmissable, but not fatal —
+      // local iteration against the default branch is legitimate; publishes are
+      // separately guarded by ci-www-publish.sh's PUBLISH_REF requirement.
+      console.warn(
+        `[materialize-tenant-content] WARNING: no --ref given — materializing ${tenant} from the ` +
+          `repo's DEFAULT branch, which may be stale. Pin a commit (--ref <sha|branch>) for anything ` +
+          `feeding a build or publish.`,
+      );
+    }
+    const sha = git(["rev-parse", "HEAD"]);
+
+    // Mappings: the repo's own .tot/config.json wins; canonical shape otherwise.
+    let mappings = canonicalMappings(tenant);
+    const cfgPath = join(work, ".tot", "config.json");
+    if (existsSync(cfgPath)) {
+      const cfg = JSON.parse(readFileSync(cfgPath, "utf8"));
+      if (Array.isArray(cfg.mappings) && cfg.mappings.length > 0) {
+        mappings = cfg.mappings;
+      }
+    }
+
+    let mappingsCopied = 0;
+    for (const m of mappings) {
+      if (typeof m?.repo !== "string" || (m.kind !== "file" && m.kind !== "tree")) {
+        throw new Error(
+          `.tot/config.json: malformed mapping for "${m?.workspace ?? "?"}" ` +
+            `(need string repo + kind file|tree)`,
+        );
+      }
+      const { copied } = copyMapping({ checkoutDir: work, root, tenant, mapping: m, log });
+      if (copied) mappingsCopied++;
+    }
+
+    // The checkout config itself + the provenance stamp.
+    if (existsSync(cfgPath)) {
+      const cfgTarget = resolveTargetPath(root, tenant, `tenants/${tenant}/.tot/config.json`);
+      cpSync(cfgPath, cfgTarget);
+    }
+    // The assets-r2-versioned opt-in marker:
+    // copy-tenant-assets.mjs reads this from the MATERIALIZED tree, not the repo
+    // directly, so it must ride along here or the marker a tenant commits in
+    // Gitea would silently never take effect on a real build.
+    const versionedMarkerSrc = join(work, ".tot", "assets-versioned");
+    if (existsSync(versionedMarkerSrc)) {
+      const versionedMarkerTarget = resolveTargetPath(
+        root,
+        tenant,
+        `tenants/${tenant}/.tot/assets-versioned`,
+      );
+      cpSync(versionedMarkerSrc, versionedMarkerTarget);
+      log(`  + .tot/assets-versioned → tenants/${tenant}/.tot/assets-versioned`);
+    }
+    const stamp = resolveTargetPath(root, tenant, `tenants/${tenant}/.materialized.json`);
+    writeFileSync(
+      stamp,
+      JSON.stringify(
+        { tenant, repo, ref: ref ?? "(default branch)", sha, at: new Date().toISOString() },
+        null,
+        2,
+      ) + "\n",
+    );
+    log(`  ✓ ${tenant} materialized @ ${sha.slice(0, 10)} (${mappingsCopied} mapping(s))`);
+    return { tenant, ref, sha, mappingsCopied };
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+}
+
+/**
+ * `--check`: every manifest tenant dir must exist and carry content (at least a
+ * theme or content/ dir — the build globs would otherwise silently render an
+ * empty tenant). Returns { ok, missing: string[] }. No network.
+ */
+export function checkMaterialized(root = repoRoot) {
+  const manifest = loadManifest(root);
+  const missing = [];
+  for (const tenant of Object.keys(manifest.tenants)) {
+    const dir = join(root, "tenants", tenant);
+    const populated =
+      existsSync(join(dir, "theme.json")) ||
+      existsSync(join(dir, "content")) ||
+      existsSync(join(dir, "content.json"));
+    if (!populated) missing.push(tenant);
+  }
+  return { ok: missing.length === 0, missing };
+}
+
+/**
+ * `--check-pinned`: the stricter DEPLOY gate. Beyond presence (checkMaterialized),
+ * every manifest tenant must carry a `.materialized.json` provenance stamp proving
+ * it was materialized from Gitea at a PINNED ref — mirroring ci-www-publish.sh's
+ * PUBLISH_REF contract so a Worker deploy can never bake a stale local-disk tree
+ * (local tenants/ is a disposable cache, never a source of truth). No network.
+ * Returns { ok, problems: [{ tenant, reason }] } where
+ * reason ∈ "missing" | "unpinned" | "off-ref":
+ *   missing   no stamp / unpopulated dir
+ *   unpinned  stamp exists but ref == "(default branch)" (materialized off the
+ *             tenant repo's default branch, which may be stale). Tolerated only
+ *             with allowUnpinned.
+ *   off-ref   expectRef given and the stamp's sha/ref doesn't match it.
+ *
+ * @param {string} [root]
+ * @param {object} [opts]
+ * @param {string|null} [opts.expectRef]     assert the stamp is this exact commit (sha or ref;
+ *                                           short-sha prefix matches, like git)
+ * @param {boolean} [opts.allowUnpinned]     tolerate a default-branch (unpinned) materialize
+ * @param {string[]|null} [opts.onlyTenants]  restrict the check to these tenant ids (a per-tenant
+ *                                            STATIC publish only ships one tenant's routes, so it
+ *                                            need only guard that tenant; null ⇒ every Gitea tenant
+ *                                            for explicit operator diagnostics)
+ */
+export function checkPinned(
+  root = repoRoot,
+  { expectRef = null, allowUnpinned = false, onlyTenants = null } = {},
+) {
+  const manifest = loadManifest(root);
+  const problems = [];
+  const scope = onlyTenants ? new Set(onlyTenants) : null;
+  for (const tenant of Object.keys(manifest.tenants)) {
+    if (scope && !scope.has(tenant)) continue;
+    const dir = join(root, "tenants", tenant);
+    const populated =
+      existsSync(join(dir, "theme.json")) ||
+      existsSync(join(dir, "content")) ||
+      existsSync(join(dir, "content.json"));
+    const stampPath = join(dir, ".materialized.json");
+    if (!populated) {
+      // Nothing on disk at all — bakes NOTHING (vs "missing": populated content
+      // of unknown provenance). A framework-only build may tolerate this; a
+      // deploy never should.
+      problems.push({ tenant, reason: "absent" });
+      continue;
+    }
+    if (!existsSync(stampPath)) {
+      problems.push({ tenant, reason: "missing" });
+      continue;
+    }
+    let stamp;
+    try {
+      stamp = JSON.parse(readFileSync(stampPath, "utf8"));
+    } catch {
+      problems.push({ tenant, reason: "missing" });
+      continue;
+    }
+    const ref = typeof stamp.ref === "string" ? stamp.ref : "";
+    const sha = typeof stamp.sha === "string" ? stamp.sha : "";
+    const pinned = ref !== "" && ref !== "(default branch)";
+    if (!pinned && !allowUnpinned) {
+      problems.push({ tenant, reason: "unpinned" });
+      continue;
+    }
+    if (expectRef) {
+      const shaMatch =
+        sha.length >= 7 && (sha.startsWith(expectRef) || expectRef.startsWith(sha));
+      const refMatch = ref !== "" && ref === expectRef;
+      if (!shaMatch && !refMatch) {
+        problems.push({ tenant, reason: "off-ref" });
+      }
+    }
+  }
+  return { ok: problems.length === 0, problems };
+}
+
+/** Human-readable refusal for a failed `checkPinned` (shared by the CLI + the
+ * build-time guard). `problems` is checkPinned().problems; `expectRef` is echoed
+ * into the off-ref explanation when set. */
+export function describePinnedProblems(problems, /** @type {string|null} */ expectRef = null) {
+  const how = {
+    absent: "not materialized (no tenant content on disk at all)",
+    missing: "not materialized (populated but no tenants/<id>/.materialized.json)",
+    unpinned:
+      "materialized from the tenant repo's DEFAULT branch (no pinned ref) — may be stale",
+    "off-ref": `materialized at a different ref than expected (${expectRef})`,
+  };
+  return (
+    "REFUSING: tenant content is not materialized from Gitea at a pinned ref.\n" +
+    problems.map(({ tenant, reason }) => `  ✗ ${tenant}: ${how[reason]}`).join("\n") +
+    "\n\nLocal tenants/ is a disposable cache, never a source of truth. Re-materialize\n" +
+    "at the promoted commit:\n" +
+    "  node scripts/tenant/materialize-tenant-content.mjs <tenant> --ref <sha>\n" +
+    "(--allow-unpinned deliberately tolerates a default-branch materialize)."
+  );
+}
+
+/**
+ * Throw if any Gitea-classified tenant is unmaterialized/unpinned/off-ref — the
+ * intrinsic build gate. Called from the storefront's `astro:build:start` hook so
+ * `astro build` refuses to bake empty/stale tenant content on its own, not only
+ * when routed through deploy.sh. No network.
+ *
+ * Skips silently when the manifest is absent (a pruned standalone runner tree
+ * declares no Gitea tenants — nothing to guard).
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.root]           monorepo root (default: this repo)
+ * @param {string|null} [opts.expectRef] assert every stamp is this exact commit
+ * @param {boolean} [opts.allowUnpinned] tolerate a default-branch materialize
+ * @param {string[]|null} [opts.onlyTenants] restrict to these tenant ids (per-tenant static
+ *                                           publish); null ⇒ every Gitea tenant (operator check)
+ */
+export function assertPinnedMaterialized({
+  root = repoRoot,
+  expectRef = null,
+  allowUnpinned = false,
+  onlyTenants = null,
+} = {}) {
+  if (!existsSync(join(root, MANIFEST_PATH))) return;
+  const { ok, problems } = checkPinned(root, { expectRef, allowUnpinned, onlyTenants });
+  if (!ok) throw new Error(describePinnedProblems(problems, expectRef));
+}
+
+function parseArgs(argv) {
+  const a = {
+    tenant: null,
+    ref: null,
+    remote: null,
+    all: false,
+    check: false,
+    checkPinned: false,
+    expectRef: null,
+    allowUnpinned: false,
+    help: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const t = argv[i];
+    if (t === "--ref") a.ref = argv[++i];
+    else if (t === "--remote") a.remote = argv[++i];
+    else if (t === "--all") a.all = true;
+    else if (t === "--check") a.check = true;
+    else if (t === "--check-pinned") a.checkPinned = true;
+    else if (t === "--expect-ref") a.expectRef = argv[++i];
+    else if (t === "--allow-unpinned") a.allowUnpinned = true;
+    else if (t === "--help" || t === "-h") a.help = true;
+    else if (!t.startsWith("--") && !a.tenant) a.tenant = t;
+    else throw new Error(`unknown argument: ${t}`);
+  }
+  if (a.remote && a.all) throw new Error("--remote targets ONE tenant; not valid with --all");
+  return a;
+}
+
+const USAGE = `materialize-tenant-content — populate tenants/<id>/ from its Gitea repo
+
+  node scripts/tenant/materialize-tenant-content.mjs <tenant> [--ref <sha|branch>] [--remote <cloneUrl>]
+  node scripts/tenant/materialize-tenant-content.mjs --all
+  node scripts/tenant/materialize-tenant-content.mjs --check
+  node scripts/tenant/materialize-tenant-content.mjs --check-pinned [--expect-ref <sha>] [--allow-unpinned]
+
+  env: TENANT_GIT_BASE (default ${DEFAULT_GIT_BASE}), TENANT_GIT_USER, TENANT_GIT_TOKEN`;
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help) {
+    console.log(USAGE);
+    return;
+  }
+  const gitBase = process.env.TENANT_GIT_BASE || DEFAULT_GIT_BASE;
+  if (args.check) {
+    const { ok, missing } = checkMaterialized();
+    if (!ok) {
+      console.error(
+        `NOT MATERIALIZED: ${missing.join(", ")}\n` +
+          `Run: node scripts/tenant/materialize-tenant-content.mjs --all`,
+      );
+      process.exit(1);
+    }
+    console.log("All Gitea-classified tenants are materialized.");
+    return;
+  }
+  if (args.checkPinned) {
+    const { ok, problems } = checkPinned(repoRoot, {
+      expectRef: args.expectRef,
+      allowUnpinned: args.allowUnpinned,
+    });
+    if (!ok) {
+      console.error(describePinnedProblems(problems, args.expectRef));
+      process.exit(1);
+    }
+    console.log("All Gitea-classified tenants materialized from Gitea at a pinned ref.");
+    return;
+  }
+  const manifest = loadManifest();
+  const targets = args.all
+    ? Object.keys(manifest.tenants)
+    : args.tenant
+      ? [args.tenant]
+      : null;
+  if (!targets) {
+    console.error(USAGE);
+    process.exit(2);
+  }
+  for (const tenant of targets) {
+    const repo = manifest.tenants[tenant]?.repo ?? tenant;
+    console.log(`Materializing ${tenant} (repo ${repo})…`);
+    materializeTenant({
+      tenant,
+      repo,
+      ref: args.ref,
+      remote: args.remote,
+      gitBase,
+      log: (s) => console.log(s),
+    });
+  }
+}
+
+const invokedDirectly =
+  process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (invokedDirectly) {
+  try {
+    main();
+  } catch (e) {
+    console.error("ERROR:", redact(String(e?.message || e), [process.env.TENANT_GIT_TOKEN]));
+    process.exit(1);
+  }
+}