@stelstone/server 0.26.0

package/src/adapters/github-content.mjs

@@ -0,0 +1,577 @@
+import { createGitHubApi } from "./github-api.mjs";
+import { sanitize, safeFileName, sortPages, buildDuplicateData, listScheduledDue, commitMsg, buildListEntry, isEntryFile } from "./_shared.mjs";
+
+/**
+ * GitHub Contents API adapter — serverless content backend.
+ *
+ * Every writePage/createPage/deletePage is an immediate commit to GitHub.
+ * No git binary required. Safe to run on Vercel, Cloudflare Workers, etc.
+ *
+ * With `draftBranch` set, those commits land on the draft branch instead of
+ * the deploy branch: saving stops publishing, and Publish (whole-site or a
+ * single record) moves files onto the deploy branch — the same
+ * save-then-publish model as the fs backend.
+ *
+ * Listing strategy: each collection keeps a `_index.json` manifest that
+ * contains only the listing metadata (id, slug, lang, title, file, meta).
+ * listPages fetches this single file — 1 subrequest regardless of size.
+ * On first call (no index yet) it bootstraps via GraphQL and persists the
+ * index. All write operations keep the index in sync.
+ *
+ * Config (cms.config.mjs):
+ *   content: {
+ *     provider: "github",
+ *     githubTokenEnv: "GITHUB_TOKEN",   // PAT with repo scope
+ *     owner: "your-org",
+ *     repo:  "your-site",
+ *     branch: "main",
+ *     pagesDir: "src/pages-data",
+ *     commitMessage: (ts) => `Content updated ${ts}`,
+ *     list: {
+ *       strategy: "index",       // "index" (default) — only built-in strategy
+ *       rebuild: "build",        // "build" (default) | "lazy"
+ *       indexFile: "_index.json", // manifest filename
+ *       resolve: undefined,      // optional async (collection, { sortConfig }) => entries[]|null
+ *     },
+ *   }
+ *
+ * @param {Object} opts
+ * @param {string} opts.token
+ * @param {string} opts.owner
+ * @param {string} opts.repo
+ * @param {string} opts.branch
+ * @param {string} [opts.draftBranch]  Opt-in draft workflow. When set, every
+ *        CMS read and write targets this branch instead of `branch`; Publish
+ *        moves the record's file onto `branch` with a git-data commit. Without
+ *        it, behaviour is the original: saving commits straight to `branch`,
+ *        i.e. saving IS publishing. The gap this closes: with commit-on-save
+ *        to the deploy branch, editing a LIVE page put it in production the
+ *        moment it was saved — there was no "work on it, publish when ready".
+ * @param {string} opts.pagesDir
+ * @param {(ts: string) => string} opts.commitMessage
+ * @param {Object} [opts.list]  Listing strategy config (see above)
+ * @returns {import('./types.mjs').ContentAdapter}
+ */
+export function createGitHubContent({ token, owner, repo, branch, draftBranch, pagesDir, commitMessage, list = {}, collections = {} }) {
+  const listConfig = {
+    strategy: "index",
+    rebuild: "build",
+    indexFile: "_index.json",
+    resolve: undefined,
+    ...list,
+  };
+  const { apiGet, apiPut, apiDelete, apiPost, apiPatch, graphql } = createGitHubApi({ token, owner, repo });
+  // SHA cache: avoid extra GET before every PUT. Invalidated on write.
+  const shaCache = new Map();
+
+  // The branch the CMS works on. In draft mode that is the draft branch and
+  // `branch` remains the published/deploy branch; otherwise they are one and
+  // the adapter behaves exactly as before this option existed.
+  const draftMode = !!draftBranch && draftBranch !== branch;
+  const workBranch = draftMode ? draftBranch : branch;
+
+  // Lazily create the draft branch off the published branch. Checked once per
+  // process; a 404 on the ref is the "first ever use" case, not an error.
+  let draftBranchReady = !draftMode;
+  async function ensureDraftBranch() {
+    if (draftBranchReady) return;
+    const ref = await apiGet(`/git/ref/heads/${draftBranch}`);
+    if (!ref) {
+      const base = await apiGet(`/git/ref/heads/${branch}`);
+      if (!base) throw new Error(`Published branch "${branch}" does not exist`);
+      await apiPost(`/git/refs`, { ref: `refs/heads/${draftBranch}`, sha: base.object.sha });
+    }
+    draftBranchReady = true;
+  }
+
+  function contentPath(collection, file) {
+    return `${pagesDir}/${collection}/${file}`;
+  }
+
+  function indexFilePath(collection) {
+    return `${pagesDir}/${collection}/${listConfig.indexFile}`;
+  }
+
+  async function getFileSha(path) {
+    if (shaCache.has(path)) return shaCache.get(path);
+    const data = await apiGet(`/contents/${path}?ref=${workBranch}`);
+    if (!data || Array.isArray(data)) return null;
+    shaCache.set(path, data.sha);
+    return data.sha;
+  }
+
+  function decodeContent(data) {
+    const raw = Buffer.from(data.content, "base64").toString("utf-8");
+    return JSON.parse(raw);
+  }
+
+  function encodeContent(obj) {
+    return Buffer.from(JSON.stringify(obj, null, 2), "utf-8").toString("base64");
+  }
+
+  /** Build the listing record stored in _index.json for a single entry. */
+  function buildIndexEntry(collection, file, data) {
+    return buildListEntry(collections[collection], collection, file, data);
+  }
+
+  /** Read _index.json for a collection; returns { entries: [] } if absent. */
+  async function readIndex(collection) {
+    try {
+      const path = indexFilePath(collection);
+      const data = await apiGet(`/contents/${path}?ref=${workBranch}`);
+      if (!data || Array.isArray(data)) return { entries: [] };
+      shaCache.set(path, data.sha);
+      return decodeContent(data);
+    } catch {
+      return { entries: [] };
+    }
+  }
+
+  /** Persist an updated index object to _index.json (single commit). */
+  async function writeIndex(collection, index) {
+    const path = indexFilePath(collection);
+    const sha = shaCache.get(path) ?? await getFileSha(path);
+    const result = await apiPut(`/contents/${path}`, {
+      message: commitMsg(commitMessage),
+      content: encodeContent(index),
+      branch: workBranch,
+      ...(sha ? { sha } : {}),
+    });
+    shaCache.set(path, result.content?.sha);
+  }
+
+  /** Add or replace one entry in _index.json, then persist. */
+  async function upsertIndexEntry(collection, file, data) {
+    const index = await readIndex(collection);
+    const entry = buildIndexEntry(collection, file, data);
+    const pos = index.entries.findIndex((e) => e.file === file);
+    if (pos >= 0) index.entries[pos] = entry;
+    else index.entries.push(entry);
+    await writeIndex(collection, index);
+  }
+
+  /** Remove one entry from _index.json, then persist. */
+  async function removeIndexEntry(collection, file) {
+    const index = await readIndex(collection);
+    index.entries = index.entries.filter((e) => e.file !== file);
+    await writeIndex(collection, index);
+  }
+
+  // GraphQL query for lazy bootstrap: fetch all blob text in one request.
+  const BOOTSTRAP_QUERY = `
+    query($owner: String!, $repo: String!, $expr: String!) {
+      repository(owner: $owner, name: $repo) {
+        object(expression: $expr) {
+          ... on Tree {
+            entries {
+              name
+              oid
+              type
+              object {
+                ... on Blob {
+                  text
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  `;
+
+  return {
+    async listCollections() {
+      await ensureDraftBranch();
+      const items = await apiGet(`/contents/${pagesDir}?ref=${workBranch}`);
+      if (!items || !Array.isArray(items)) return [];
+      const dirs = items.filter((i) => i.type === "dir");
+      return Promise.all(
+        dirs.map(async (d) => {
+          const files = await apiGet(`/contents/${pagesDir}/${d.name}?ref=${workBranch}`);
+          const count = Array.isArray(files)
+            ? files.filter((f) => isEntryFile(f.name) && f.name !== listConfig.indexFile).length
+            : 0;
+          return { name: d.name, count };
+        }),
+      );
+    },
+
+    async listPages(collection, sortConfig = null) {
+      await ensureDraftBranch();
+      const safeCollection = sanitize(collection);
+
+      // 1. Custom resolver takes full control — nothing else runs.
+      if (typeof listConfig.resolve === "function") {
+        return await listConfig.resolve(collection, { sortConfig });
+      }
+
+      // 2. strategy "index": try the pre-built manifest first (1 subrequest).
+      const idxPath = indexFilePath(safeCollection);
+      try {
+        const data = await apiGet(`/contents/${idxPath}?ref=${workBranch}`);
+        if (data && !Array.isArray(data)) {
+          shaCache.set(idxPath, data.sha);
+          const index = decodeContent(data);
+          if (Array.isArray(index.entries)) {
+            return sortPages(index.entries, sortConfig);
+          }
+        }
+      } catch {
+        // Non-404 error reading index — fall through to rebuild logic.
+      }
+
+      // Index absent: behaviour depends on rebuild strategy.
+      if (listConfig.rebuild === "lazy") {
+        // Bootstrap: one GraphQL call fetching all blob text, then persist.
+        // On GraphQL failure, throw — never fall back to REST N+1.
+        const expression = `${workBranch}:${pagesDir}/${safeCollection}`;
+        const gqlData = await graphql(BOOTSTRAP_QUERY, { owner, repo, expr: expression });
+
+        const treeObj = gqlData?.repository?.object;
+        if (!treeObj) return null;
+
+        const pages = [];
+        for (const entry of treeObj.entries || []) {
+          if (entry.type !== "blob" || !entry.name.endsWith(".json") || entry.name === listConfig.indexFile) continue;
+          shaCache.set(`${pagesDir}/${safeCollection}/${entry.name}`, entry.oid);
+          let data;
+          try { data = JSON.parse(entry.object.text); } catch { continue; }
+          pages.push(buildIndexEntry(collection, entry.name, data));
+        }
+
+        // Persist for all future calls.
+        try {
+          await writeIndex(safeCollection, { entries: pages });
+        } catch (err) {
+          console.warn(`[listPages] Could not write ${listConfig.indexFile}:`, err?.message);
+        }
+
+        return sortPages(pages, sortConfig);
+      }
+
+      // rebuild === "build" (default): never fetch all bodies at runtime.
+      // Check whether the collection directory actually exists.
+      const items = await apiGet(`/contents/${pagesDir}/${safeCollection}?ref=${workBranch}`);
+      if (!Array.isArray(items)) return null; // 404 or non-dir → unknown collection
+
+      // Collection exists but index hasn't been built yet.
+      console.warn(
+        `[listPages] No ${listConfig.indexFile} found for collection "${safeCollection}". ` +
+        `Run \`stelstone build-index\` to generate it.`,
+      );
+      return [];
+    },
+
+    async readPage(collection, file) {
+      await ensureDraftBranch();
+      if (!safeFileName(file)) return null;
+      const path = contentPath(collection, sanitize(file));
+      const data = await apiGet(`/contents/${path}?ref=${workBranch}`);
+      if (!data || Array.isArray(data)) return null;
+      shaCache.set(path, data.sha);
+      return decodeContent(data);
+    },
+
+    /** Version token for optimistic concurrency — the blob SHA GitHub already keeps. */
+    async versionOf(collection, file) {
+      if (!safeFileName(file)) return null;
+      return getFileSha(contentPath(collection, sanitize(file)));
+    },
+
+    /**
+     * @param {Object} [opts]
+     * @param {string} [opts.expectedVersion] The blob SHA the caller last read.
+     *        Passing it through means GitHub itself enforces the precondition:
+     *        it rejects a PUT whose sha is not current. Re-reading the sha at
+     *        write time — which is what this did before — discarded that
+     *        guarantee and silently overwrote a concurrent edit.
+     */
+    async writePage(collection, file, data, { expectedVersion } = {}) {
+      await ensureDraftBranch();
+      const path = contentPath(collection, sanitize(file));
+      const sha = expectedVersion ?? (await getFileSha(path));
+      let result;
+      try {
+        result = await apiPut(`/contents/${path}`, {
+          message: commitMsg(commitMessage),
+          content: encodeContent(data),
+          branch: workBranch,
+          ...(sha ? { sha } : {}),
+        });
+      } catch (err) {
+        // GitHub answers 409 when the supplied sha is stale.
+        if (expectedVersion && (err.upstreamStatus === 409 || err.status === 409)) {
+          shaCache.delete(path);
+          const conflict = new Error("This entry changed since you loaded it.");
+          conflict.status = 412;
+          throw conflict;
+        }
+        throw err;
+      }
+      shaCache.set(path, result.content?.sha);
+      await upsertIndexEntry(collection, file, data);
+    },
+
+    async createPage(collection, data) {
+      await ensureDraftBranch();
+      const slug = data.slug || data.id || `new-${Date.now()}`;
+      if (!sanitize(slug) || sanitize(slug) !== String(slug)) {
+        // Reject rather than silently stripping characters — otherwise the
+        // file name and data.slug would diverge (e.g. "ürünler" → "rnler").
+        const err = new Error(
+          `Slug "${slug}" contains unsupported characters — use only a-z, 0-9, dots, dashes, underscores.`,
+        );
+        err.status = 400;
+        throw err;
+      }
+      const fileName = `${sanitize(data.lang || "en")}-${sanitize(slug)}.json`;
+      const path = contentPath(collection, fileName);
+      if (await getFileSha(path)) {
+        const err = new Error(
+          `An entry already exists for "${data.lang || "en"}/${slug}". Change the slug or language.`,
+        );
+        err.status = 409;
+        throw err;
+      }
+      const result = await apiPut(`/contents/${path}`, {
+        message: commitMsg(commitMessage),
+        content: encodeContent(data),
+        branch: workBranch,
+      });
+      shaCache.set(path, result.content?.sha);
+      await upsertIndexEntry(collection, fileName, data);
+      return { file: fileName };
+    },
+
+    async deletePage(collection, file) {
+      await ensureDraftBranch();
+      if (!safeFileName(file)) return null;
+      const path = contentPath(collection, sanitize(file));
+      const sha = await getFileSha(path);
+      if (!sha) return false;
+      await apiDelete(`/contents/${path}`, { message: commitMsg(commitMessage), sha, branch: workBranch });
+      shaCache.delete(path);
+      await removeIndexEntry(collection, file);
+      return true;
+    },
+
+    async duplicatePage(collection, file) {
+      if (!safeFileName(file)) return null;
+      const src = await this.readPage(collection, file);
+      if (!src) return null;
+      const { data, fileName } = buildDuplicateData(src);
+      return this.createPage(collection, data);
+    },
+
+    // In draft mode saving and publishing are separate steps, exactly like
+    // the fs backend. Without a draft branch, writes commit straight to the
+    // deploy branch — saving IS publishing — and the per-entry publish route
+    // answers 501 instead of pretending.
+    capabilities: draftMode
+      ? { deferredPublish: true, perEntryPublish: true }
+      : { deferredPublish: false, perEntryPublish: false },
+
+    async pendingChanges() {
+      if (!draftMode) return { hasChanges: false, changedFiles: 0, files: [] };
+      await ensureDraftBranch();
+      // One compare call: which files differ between published and draft?
+      // (GitHub caps the file list at 300 — orders of magnitude above any
+      // real collection here; still, say so rather than rely on it silently.)
+      const cmp = await apiGet(`/compare/${branch}...${draftBranch}`);
+      const files = (cmp?.files ?? [])
+        .filter((f) => f.filename.startsWith(`${pagesDir}/`))
+        // Index manifests live on the draft branch only — the site build
+        // ignores files starting with "_", so they are never published.
+        .filter((f) => !f.filename.endsWith(`/${listConfig.indexFile}`))
+        .map((f) => {
+          const rel = f.filename.slice(pagesDir.length + 1);
+          const slash = rel.indexOf("/");
+          return slash > 0
+            ? { path: f.filename, collection: rel.slice(0, slash), file: rel.slice(slash + 1), status: f.status }
+            : { path: f.filename, status: f.status };
+        });
+      return { hasChanges: files.length > 0, changedFiles: files.length, files };
+    },
+
+    /**
+     * Draft mode: move the draft branch's version of the chosen files onto
+     * the published branch with one git-data commit. Blobs already live in
+     * the repo (they were committed to the draft branch), so the tree
+     * references them by SHA — nothing is re-uploaded. A `removed` status
+     * becomes a tree entry with `sha: null`, which is how git-data deletes.
+     *
+     * @param {string} [message]
+     * @param {{ entries?: {collection: string, file: string}[] }} [opts]
+     */
+    async publish(message, { entries } = {}) {
+      if (!draftMode) {
+        // Writes are committed instantly; trigger is external (Netlify webhook on push)
+        return { ok: true, message: "All changes are already committed to GitHub" };
+      }
+      await ensureDraftBranch();
+      const pending = await this.pendingChanges();
+      if (!pending.hasChanges) return { ok: false, message: "No changes to publish" };
+
+      const scoped = Array.isArray(entries) && entries.length > 0;
+      let targets = pending.files;
+      if (scoped) {
+        const wanted = new Set(
+          entries.map((e) => contentPath(sanitize(e.collection), sanitize(e.file))),
+        );
+        targets = pending.files.filter((f) => wanted.has(f.path));
+        if (!targets.length) return { ok: false, message: "No changes to publish" };
+      }
+
+      const treeItems = await Promise.all(
+        targets.map(async (f) => {
+          if (f.status === "removed") return { path: f.path, mode: "100644", type: "blob", sha: null };
+          const blob = await apiGet(`/contents/${f.path}?ref=${draftBranch}`);
+          return { path: f.path, mode: "100644", type: "blob", sha: blob.sha };
+        }),
+      );
+
+      // The ref update races with concurrent pushes to the published branch;
+      // retry once from a fresh base, as writeBatch does.
+      let lastErr;
+      for (let attempt = 0; attempt < 2; attempt++) {
+        try {
+          const refData = await apiGet(`/git/ref/heads/${branch}`);
+          const baseCommitSha = refData.object.sha;
+          const baseCommit = await apiGet(`/git/commits/${baseCommitSha}`);
+          const timestamp = new Date().toISOString().replace("T", " ").slice(0, 19);
+          const msg = message || commitMsg(commitMessage) || `cms: publish ${timestamp}`;
+          const newTree = await apiPost(`/git/trees`, { base_tree: baseCommit.tree.sha, tree: treeItems });
+          const newCommit = await apiPost(`/git/commits`, {
+            message: msg,
+            tree: newTree.sha,
+            parents: [baseCommitSha],
+          });
+          await apiPatch(`/git/refs/heads/${branch}`, { sha: newCommit.sha });
+          const shortSha = newCommit.sha.slice(0, 7);
+          return {
+            ok: true,
+            message: `Published ${targets.length} file(s) to ${branch} (${shortSha})`,
+            sha: newCommit.sha,
+            shortSha,
+            branch,
+            scope: scoped ? "entries" : "all",
+          };
+        } catch (err) {
+          lastErr = err;
+        }
+      }
+      throw lastErr;
+    },
+
+    async listHistory(collection, file) {
+      if (!safeFileName(file)) return null;
+      const path = contentPath(collection, sanitize(file));
+      const commits = await apiGet(`/commits?path=${encodeURIComponent(path)}&per_page=20&sha=${workBranch}`);
+      if (!Array.isArray(commits)) return [];
+      return commits.map((c) => ({
+        ts: c.sha,
+        label: c.commit.message,
+        date: c.commit.author?.date,
+        author: c.commit.author?.name || c.commit.author?.email || "",
+        size: null,
+      }));
+    },
+
+    async restoreHistory(collection, file, commitSha) {
+      if (!safeFileName(file)) return null;
+      const path = contentPath(collection, sanitize(file));
+      const fileData = await apiGet(`/contents/${path}?ref=${commitSha}`);
+      if (!fileData || Array.isArray(fileData)) return false;
+      const data = decodeContent(fileData);
+      await this.writePage(collection, file, data);
+      return true;
+    },
+
+    async writeBatch(items, message) {
+      await ensureDraftBranch();
+      if (!items.length) return { ok: true, commitCount: 0 };
+
+      // The ref update (step 7) races with concurrent pushes: if the branch
+      // moved after we read it, GitHub rejects the non-fast-forward PATCH.
+      // Retry the whole flow once from a fresh base before giving up.
+      let lastErr;
+      for (let attempt = 0; attempt < 2; attempt++) {
+        try {
+          return await this._writeBatchOnce(items, message);
+        } catch (err) {
+          lastErr = err;
+        }
+      }
+      throw lastErr;
+    },
+
+    async _writeBatchOnce(items, message) {
+      // 1. Get current ref → base commit SHA
+      const refData = await apiGet(`/git/ref/heads/${workBranch}`);
+      const baseCommitSha = refData.object.sha;
+
+      // 2. Get base commit → base tree SHA
+      const baseCommit = await apiGet(`/git/commits/${baseCommitSha}`);
+      const baseTreeSha = baseCommit.tree.sha;
+
+      // 3. Create one blob per content file
+      const treeItems = await Promise.all(
+        items.map(async ({ collection, file, data }) => {
+          const filePath = contentPath(sanitize(collection), sanitize(file));
+          const content = Buffer.from(JSON.stringify(data, null, 2), "utf-8").toString("base64");
+          const blob = await apiPost(`/git/blobs`, { content, encoding: "base64" });
+          return { path: filePath, mode: "100644", type: "blob", sha: blob.sha };
+        }),
+      );
+
+      // 4. Compute updated _index.json for each affected collection and add
+      //    them to the same tree so the index stays in sync atomically.
+      const byCollection = {};
+      for (const item of items) {
+        const col = sanitize(item.collection ?? item.data?.collection ?? "");
+        if (!col) continue;
+        (byCollection[col] ??= []).push(item);
+      }
+      for (const [col, colItems] of Object.entries(byCollection)) {
+        const index = await readIndex(col);
+        for (const { file, data } of colItems) {
+          const entry = buildIndexEntry(col, file, data);
+          const pos = index.entries.findIndex((e) => e.file === file);
+          if (pos >= 0) index.entries[pos] = entry;
+          else index.entries.push(entry);
+        }
+        const idxContent = Buffer.from(JSON.stringify(index, null, 2), "utf-8").toString("base64");
+        const idxBlob = await apiPost(`/git/blobs`, { content: idxContent, encoding: "base64" });
+        treeItems.push({ path: indexFilePath(col), mode: "100644", type: "blob", sha: idxBlob.sha });
+      }
+
+      // 5. Create tree
+      const newTree = await apiPost(`/git/trees`, { base_tree: baseTreeSha, tree: treeItems });
+
+      // 6. Create commit
+      const msg = message || commitMsg(commitMessage, "Batch content update");
+      const newCommit = await apiPost(`/git/commits`, {
+        message: msg,
+        tree: newTree.sha,
+        parents: [baseCommitSha],
+      });
+
+      // 7. Update ref
+      await apiPatch(`/git/refs/heads/${workBranch}`, { sha: newCommit.sha });
+
+      // 8. Invalidate shaCache for all written paths
+      for (const { collection, file } of items) {
+        shaCache.delete(contentPath(sanitize(collection), sanitize(file)));
+      }
+      for (const col of Object.keys(byCollection)) {
+        shaCache.delete(indexFilePath(col));
+      }
+
+      return { ok: true, sha: newCommit.sha, commitCount: 1 };
+    },
+
+    async listScheduled() {
+      return listScheduledDue(this);
+    },
+  };
+}