@decocms/blocks-cli 7.0.0

package/scripts/lib/blocks-dedupe.ts

@@ -0,0 +1,142 @@
+/**
+ * Pure helpers used by `generate-blocks.ts` to choose between multiple files
+ * that decode to the same logical CMS block key.
+ *
+ * Background: the live decofile snapshot lives under `.deco/blocks/`, with
+ * one file per block. The filename is `encodeURIComponent(<rawProdKey>) +
+ * ".json"`. The Deco admin sometimes serves URL-encoded keys (e.g.
+ * `pages-Home%20-%20LB-618509`), so a single block can land on disk with
+ * different filenames depending on which writer produced it:
+ *
+ *   - The `deco-sync-bot` (CI) encodes the raw prod key as-is, producing
+ *     `pages-Home%2520-%2520LB-618509.json` (two decode passes back to the
+ *     literal key).
+ *   - The legacy manual `sync-decofile.ts` decoded keys to literal first,
+ *     so it wrote `pages-Home%20-%20LB-618509.json` (one decode pass).
+ *
+ * Both files decode to the same logical key, so the block generator must
+ * pick one. Picking by file size is wrong (a shrunk live page gives a
+ * smaller JSON than the stale older snapshot, so size silently prefers
+ * stale); picking by mtime alone is wrong (fresh `git clone` writes all
+ * files with the clone time and erases temporal ordering).
+ */
+
+export interface Candidate {
+  file: string;
+  passes: number;
+  mtimeMs: number;
+  hasPath: boolean;
+  parsed: unknown;
+}
+
+/**
+ * Repeatedly URL-decode the basename of `filename` until no `%` sequence
+ * remains. Returns the fully-decoded canonical key plus the number of
+ * decode rounds it took. Higher pass count = the writer encoded a key
+ * that itself contained `%XX` sequences = bot scheme. See module-level
+ * comment for why this matters.
+ */
+export function decodeBlockNameWithPasses(filename: string): {
+  name: string;
+  passes: number;
+} {
+  let name = filename.replace(/\.json$/, "");
+  let passes = 0;
+  while (name.includes("%")) {
+    try {
+      const next = decodeURIComponent(name);
+      if (next === name) break;
+      name = next;
+      passes++;
+    } catch {
+      break;
+    }
+  }
+  return { name, passes };
+}
+
+export function decodeBlockName(filename: string): string {
+  return decodeBlockNameWithPasses(filename).name;
+}
+
+/**
+ * Decode the filename stem exactly once — matches the Deno runtime's
+ * `parseBlockId`. Use this for the decofile key so that the CMS editor's
+ * `encodeURIComponent(blockKey)` round-trips back to the original filename.
+ */
+export function singleDecodeBlockName(filename: string): string {
+  const stem = filename.replace(/\.json$/, "");
+  try {
+    return decodeURIComponent(stem);
+  } catch {
+    return stem;
+  }
+}
+
+/**
+ * Tie-break two candidates that decode to the same key. Priority:
+ *   1. Block has a non-null `path`     — beats zombie/orphan entries.
+ *   2. More decode passes              — bot's "encode raw prod key"
+ *                                        scheme wins over legacy
+ *                                        "decode-then-encode" leftovers
+ *                                        when prod uses URL-encoded keys
+ *                                        (the only case that collides).
+ *   3. Newer mtime                     — last-write-wins for same scheme.
+ *   4. Lexicographic filename          — deterministic last resort.
+ */
+export function pickWinner(a: Candidate, b: Candidate): Candidate {
+  if (a.hasPath !== b.hasPath) return a.hasPath ? a : b;
+  if (a.passes !== b.passes) return a.passes > b.passes ? a : b;
+  if (a.mtimeMs !== b.mtimeMs) return a.mtimeMs > b.mtimeMs ? a : b;
+  return a.file < b.file ? a : b;
+}
+
+/** True iff a parsed block JSON looks like a live page (non-empty `.path`). */
+export function blockHasPath(parsed: unknown): boolean {
+  return (
+    typeof parsed === "object" &&
+    parsed !== null &&
+    "path" in parsed &&
+    typeof (parsed as { path?: unknown }).path === "string" &&
+    (parsed as { path: string }).path.length > 0
+  );
+}
+
+export interface CollisionRecord {
+  key: string;
+  files: string[];
+  winner: string;
+}
+
+export interface MergeResult {
+  winners: Record<string, Candidate>;
+  collisions: CollisionRecord[];
+}
+
+/**
+ * Reduce a list of candidates into one winner per decoded key, recording
+ * every collision so the caller can surface it as a build warning.
+ */
+export function mergeCandidates(
+  candidates: Array<{ candidate: Candidate; key: string }>,
+): MergeResult {
+  const winners: Record<string, Candidate> = {};
+  // Track every file that decoded to a given key so three-way (and beyond)
+  // collisions don't lose the eventual winner from the file list.
+  const filesByKey: Record<string, string[]> = {};
+  for (const { candidate, key } of candidates) {
+    if (!filesByKey[key]) filesByKey[key] = [];
+    const list = filesByKey[key];
+    if (!list.includes(candidate.file)) list.push(candidate.file);
+
+    const existing = winners[key];
+    winners[key] = existing ? pickWinner(existing, candidate) : candidate;
+  }
+
+  const collisions: CollisionRecord[] = [];
+  for (const [key, files] of Object.entries(filesByKey)) {
+    if (files.length < 2) continue;
+    collisions.push({ key, files, winner: winners[key].file });
+  }
+  return { winners, collisions };
+}

package/scripts/lib/cf-kv-rest.ts

@@ -0,0 +1,78 @@
+/**
+ * Minimal Cloudflare KV REST API client for CI.
+ *
+ * CI has no Worker KV binding, so the fast-deploy sync/migrate scripts write to
+ * KV over the REST API instead. Only the two operations the scripts need —
+ * single-key GET and PUT — are implemented.
+ *
+ * Auth/config via env (read by the scripts, passed to `createKvRestClient`):
+ *   - CF_ACCOUNT_ID       Cloudflare account id
+ *   - CF_KV_NAMESPACE_ID  target KV namespace id
+ *   - CF_API_TOKEN        API token with "Workers KV Storage:Edit"
+ *
+ * `fetch` is injectable so the client is unit-testable without network.
+ */
+
+export interface KvRestConfig {
+  accountId: string;
+  namespaceId: string;
+  token: string;
+  /** Injectable for tests; defaults to global fetch. */
+  fetchImpl?: typeof fetch;
+  /** Override API base (tests). Defaults to the Cloudflare API. */
+  baseUrl?: string;
+}
+
+export interface KvRestClient {
+  get(key: string): Promise<string | null>;
+  put(key: string, value: string): Promise<void>;
+}
+
+const DEFAULT_BASE = "https://api.cloudflare.com/client/v4";
+
+/** Resolve the three required env vars or throw a clear error. */
+export function kvConfigFromEnv(env: NodeJS.ProcessEnv = process.env): Omit<KvRestConfig, "fetchImpl" | "baseUrl"> {
+  const accountId = env.CF_ACCOUNT_ID;
+  const namespaceId = env.CF_KV_NAMESPACE_ID;
+  const token = env.CF_API_TOKEN;
+  const missing = [
+    !accountId && "CF_ACCOUNT_ID",
+    !namespaceId && "CF_KV_NAMESPACE_ID",
+    !token && "CF_API_TOKEN",
+  ].filter(Boolean);
+  if (missing.length) {
+    throw new Error(`missing required env var(s): ${missing.join(", ")}`);
+  }
+  return { accountId: accountId!, namespaceId: namespaceId!, token: token! };
+}
+
+export function createKvRestClient(config: KvRestConfig): KvRestClient {
+  const fetchImpl = config.fetchImpl ?? fetch;
+  const base = config.baseUrl ?? DEFAULT_BASE;
+  const root = `${base}/accounts/${config.accountId}/storage/kv/namespaces/${config.namespaceId}`;
+  const authHeaders = { Authorization: `Bearer ${config.token}` };
+
+  return {
+    async get(key) {
+      const res = await fetchImpl(`${root}/values/${encodeURIComponent(key)}`, {
+        headers: authHeaders,
+      });
+      if (res.status === 404) return null;
+      if (!res.ok) {
+        throw new Error(`KV GET ${key} failed: ${res.status} ${await res.text()}`);
+      }
+      return res.text();
+    },
+
+    async put(key, value) {
+      const res = await fetchImpl(`${root}/values/${encodeURIComponent(key)}`, {
+        method: "PUT",
+        headers: { ...authHeaders, "Content-Type": "text/plain" },
+        body: value,
+      });
+      if (!res.ok) {
+        throw new Error(`KV PUT ${key} failed: ${res.status} ${await res.text()}`);
+      }
+    },
+  };
+}

package/scripts/lib/jsonc.ts

@@ -0,0 +1,122 @@
+/**
+ * Shared JSONC helpers.
+ *
+ * Vendored mini-parser used by the observability codemod
+ * (`migrate-to-cf-observability.ts`) and the audit
+ * (`audit-observability-config.ts`). Kept here so a single bugfix
+ * lands in both call sites.
+ *
+ * - `stripJsoncComments` removes line + block comments while:
+ *   - preserving quoted strings (handles `\"` and `\\` escapes),
+ *   - preserving newlines (so JSON.parse error line numbers stay
+ *     aligned with the original source file).
+ *
+ * - `stripJsoncTrailingCommas` removes trailing commas before `}` and
+ *   `]`. JSONC allows them; vanilla `JSON.parse` does not. Real
+ *   `wrangler.jsonc` files commonly have trailing commas — audits that
+ *   call `JSON.parse` directly fail surprisingly otherwise.
+ *
+ * - `parseJsonc` is the convenience wrapper: strip comments + trailing
+ *   commas, then `JSON.parse`. Throws on malformed input.
+ */
+export function stripJsoncComments(src: string): string {
+  let out = "";
+  let i = 0;
+  let inString = false;
+  let stringQuote = "";
+  while (i < src.length) {
+    const ch = src[i];
+    const next = src[i + 1];
+    if (inString) {
+      out += ch;
+      if (ch === "\\" && i + 1 < src.length) {
+        out += next;
+        i += 2;
+        continue;
+      }
+      if (ch === stringQuote) {
+        inString = false;
+      }
+      i++;
+      continue;
+    }
+    if (ch === '"' || ch === "'") {
+      inString = true;
+      stringQuote = ch;
+      out += ch;
+      i++;
+      continue;
+    }
+    if (ch === "/" && next === "/") {
+      while (i < src.length && src[i] !== "\n") i++;
+      continue;
+    }
+    if (ch === "/" && next === "*") {
+      i += 2;
+      while (i < src.length - 1 && !(src[i] === "*" && src[i + 1] === "/")) {
+        if (src[i] === "\n") out += "\n";
+        i++;
+      }
+      i += 2;
+      continue;
+    }
+    out += ch;
+    i++;
+  }
+  return out;
+}
+
+/**
+ * Strip trailing commas from a JSONC string — the comma between the
+ * last value and its closing `}` or `]`. String-aware so commas inside
+ * string literals are preserved verbatim.
+ */
+export function stripJsoncTrailingCommas(src: string): string {
+  let out = "";
+  let i = 0;
+  let inString = false;
+  let stringQuote = "";
+  while (i < src.length) {
+    const ch = src[i];
+    if (inString) {
+      out += ch;
+      if (ch === "\\" && i + 1 < src.length) {
+        out += src[i + 1];
+        i += 2;
+        continue;
+      }
+      if (ch === stringQuote) inString = false;
+      i++;
+      continue;
+    }
+    if (ch === '"' || ch === "'") {
+      inString = true;
+      stringQuote = ch;
+      out += ch;
+      i++;
+      continue;
+    }
+    if (ch === ",") {
+      // Look ahead through whitespace for the next non-space character.
+      let j = i + 1;
+      while (j < src.length && /\s/.test(src[j])) j++;
+      if (j < src.length && (src[j] === "}" || src[j] === "]")) {
+        // Drop the comma, keep the whitespace so line numbers stay aligned.
+        i++;
+        continue;
+      }
+    }
+    out += ch;
+    i++;
+  }
+  return out;
+}
+
+/**
+ * Parse a JSONC string. Strips comments + trailing commas before
+ * handing to `JSON.parse`. Throws the same `SyntaxError` as JSON.parse
+ * for inputs that remain malformed after stripping.
+ */
+export function parseJsonc<T = unknown>(src: string): T {
+  return JSON.parse(stripJsoncTrailingCommas(stripJsoncComments(src))) as T;
+}

package/scripts/lib/kv-snapshot.ts

@@ -0,0 +1,51 @@
+/**
+ * Shared fast-deploy snapshot helpers for the CI scripts.
+ *
+ * Both `migrate-blocks-to-kv.ts` and `sync-blocks-to-kv.ts` write the same two
+ * keys (`decofile:current` + `index:revision`). The revision is computed with
+ * the SAME `computeRevision` the runtime uses (`src/cms/blockSource.ts`) so a
+ * hydrating isolate computes a matching revision and the poller doesn't loop.
+ */
+
+import { computeRevision, KV_KEYS } from "@decocms/blocks/cms";
+import type { KvRestClient } from "./cf-kv-rest";
+
+export interface Snapshot {
+  /** Serialized decofile written to `decofile:current`. */
+  snapshot: string;
+  /** DJB2 revision written to `index:revision`. */
+  revision: string;
+  /** Block count, for logging. */
+  count: number;
+}
+
+export function buildSnapshot(blocks: Record<string, unknown>): Snapshot {
+  return {
+    snapshot: JSON.stringify(blocks),
+    revision: computeRevision(blocks),
+    count: Object.keys(blocks).length,
+  };
+}
+
+/** Write the snapshot + revision to KV. Snapshot first, then revision, so a
+ * poller never sees a new revision pointing at an old snapshot. */
+export async function writeSnapshotToKv(client: KvRestClient, snap: Snapshot): Promise<void> {
+  await client.put(KV_KEYS.SNAPSHOT, snap.snapshot);
+  await client.put(KV_KEYS.REVISION, snap.revision);
+}
+
+/** Read both keys back and confirm the revision matches what we wrote. */
+export async function verifySnapshotInKv(
+  client: KvRestClient,
+  expectedRevision: string,
+): Promise<{ ok: boolean; reason?: string }> {
+  const [snapshot, revision] = await Promise.all([
+    client.get(KV_KEYS.SNAPSHOT),
+    client.get(KV_KEYS.REVISION),
+  ]);
+  if (snapshot === null) return { ok: false, reason: `${KV_KEYS.SNAPSHOT} missing` };
+  if (revision !== expectedRevision) {
+    return { ok: false, reason: `${KV_KEYS.REVISION} is "${revision}", expected "${expectedRevision}"` };
+  }
+  return { ok: true };
+}

package/scripts/lib/read-decofile.ts

@@ -0,0 +1,70 @@
+/**
+ * Read a `.deco/blocks/*.json` directory into a single decofile map.
+ *
+ * This is the same merge the block generator performs (filename → decoded
+ * block key, with collision tie-breaking via `pickWinner`), factored out so
+ * the fast-deploy CI scripts (`migrate-blocks-to-kv.ts`, `sync-blocks-to-kv.ts`)
+ * produce a decofile byte-identical to the bundled `blocks.gen` snapshot.
+ *
+ * Reuses `scripts/lib/blocks-dedupe.ts`.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import {
+  blockHasPath,
+  type Candidate,
+  type CollisionRecord,
+  decodeBlockNameWithPasses,
+  mergeCandidates,
+} from "./blocks-dedupe";
+
+export interface ReadDecofileResult {
+  /** Decoded block key → parsed block JSON. */
+  blocks: Record<string, unknown>;
+  /** Filename collisions (same decoded key from multiple files). */
+  collisions: CollisionRecord[];
+}
+
+/**
+ * Read and merge every `*.json` under `blocksDir` into a decofile map.
+ * Throws if the directory does not exist. Unparseable files are skipped
+ * (logged unless `silent`).
+ */
+export function readDecofileFromDir(blocksDir: string, opts: { silent?: boolean } = {}): ReadDecofileResult {
+  if (!fs.existsSync(blocksDir)) {
+    throw new Error(`blocks directory not found: ${blocksDir}`);
+  }
+
+  const files = fs.readdirSync(blocksDir).filter((f) => f.endsWith(".json"));
+  const candidatesWithKeys: Array<{ candidate: Candidate; key: string }> = [];
+
+  for (const file of files) {
+    const { name, passes } = decodeBlockNameWithPasses(file);
+    const fp = path.join(blocksDir, file);
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(fs.readFileSync(fp, "utf-8"));
+    } catch (e) {
+      if (!opts.silent) console.warn(`Failed to parse ${file}:`, e);
+      continue;
+    }
+    candidatesWithKeys.push({
+      key: name,
+      candidate: {
+        file,
+        passes,
+        mtimeMs: fs.statSync(fp).mtimeMs,
+        hasPath: blockHasPath(parsed),
+        parsed,
+      },
+    });
+  }
+
+  const { winners, collisions } = mergeCandidates(candidatesWithKeys);
+  const blocks: Record<string, unknown> = {};
+  for (const [key, candidate] of Object.entries(winners)) {
+    blocks[key] = candidate.parsed;
+  }
+  return { blocks, collisions };
+}

package/scripts/lib/sync-helpers.ts

@@ -0,0 +1,44 @@
+/**
+ * Pure helpers for `sync-blocks-to-kv.ts` — kept separate from the CLI so they
+ * can be unit-tested without executing the script's `main()`.
+ */
+
+import { decodeBlockName } from "./blocks-dedupe";
+
+/**
+ * Parse `git diff --name-only` output into the set of changed block files that
+ * live under `blocksDir`. Paths are POSIX (git always emits forward slashes).
+ */
+export function changedBlockFiles(gitOutput: string, blocksDir: string): string[] {
+  const prefix = blocksDir.replace(/\\/g, "/").replace(/\/$/, "");
+  return gitOutput
+    .split("\n")
+    .map((l) => l.trim())
+    .filter(Boolean)
+    .filter((l) => l.startsWith(`${prefix}/`) && l.endsWith(".json"));
+}
+
+/** Map changed block-file paths to their decoded block keys. */
+export function changedBlockKeys(files: string[]): string[] {
+  return files.map((f) => decodeBlockName(f.split("/").pop() ?? f));
+}
+
+/**
+ * Derive the page paths to purge from the changed block keys. Only page blocks
+ * carrying a string `path` contribute; "/" is always included so the home page
+ * (and anything edge-cached under the static profile) is refreshed.
+ */
+export function purgePathsForChangedKeys(
+  blocks: Record<string, unknown>,
+  changedKeys: string[],
+): string[] {
+  const paths = new Set<string>(["/"]);
+  for (const key of changedKeys) {
+    const block = blocks[key];
+    if (block && typeof block === "object" && !Array.isArray(block)) {
+      const p = (block as { path?: unknown }).path;
+      if (typeof p === "string" && p.length > 0) paths.add(p);
+    }
+  }
+  return [...paths];
+}