@crombolo/server 0.0.0-stage → 0.9.1

package/src/git-github.ts

@@ -0,0 +1,171 @@
+/**
+ * GitHub, over its REST API.
+ *
+ * A commit is built the long way — blobs, a tree, a commit, then moving a ref —
+ * rather than with the contents API, because that writes one file per request and
+ * a schema change is usually several. Built this way the whole change is one
+ * commit, which is what a reviewer wants to see, and a partly-applied change is
+ * not a state that can happen.
+ *
+ * Needs a token with `contents: write`, and `pull_requests: write` to raise a PR.
+ * A fine-grained personal access token or a GitHub App installation token both
+ * work; the token is the CMS's own identity, so its commits are attributable to
+ * the CMS rather than to whoever was signed in.
+ */
+import {
+  diffFiles,
+  type GitDiff,
+  type GitProvider,
+  type PushResult,
+  type RepositoryFile,
+} from './git.ts'
+
+export interface GitHubOptions {
+  /** `owner/repo`. */
+  repository: string
+  token: string
+  /** The branch to compare against and commit to. Default `main`. */
+  branch?: string
+  /** For GitHub Enterprise; default is github.com's API. */
+  api?: string
+  /** The commit author; defaults to a bot identity naming the site. */
+  author?: { name: string; email: string }
+}
+
+interface TreeEntry {
+  path: string
+  mode: '100644'
+  type: 'blob'
+  sha: string
+}
+
+export function gitHub(options: GitHubOptions): GitProvider {
+  const api = (options.api ?? 'https://api.github.com').replace(/\/$/, '')
+  const branch = options.branch ?? 'main'
+  const base = `${api}/repos/${options.repository}`
+  const author = options.author ?? { name: 'crombolo', email: 'crombolo@users.noreply.github.com' }
+
+  const call = async <T>(path: string, init: RequestInit = {}): Promise<T> => {
+    const response = await fetch(`${base}${path}`, {
+      ...init,
+      headers: {
+        accept: 'application/vnd.github+json',
+        authorization: `Bearer ${options.token}`,
+        'x-github-api-version': '2022-11-28',
+        ...(init.body ? { 'content-type': 'application/json' } : {}),
+        ...init.headers,
+      },
+    })
+    if (!response.ok) {
+      const detail = (await response.text()).slice(0, 300)
+      throw new Error(
+        `GitHub ${init.method ?? 'GET'} ${path} failed (${response.status}): ${detail}`,
+      )
+    }
+    return (await response.json()) as T
+  }
+
+  return {
+    description: `github:${options.repository}`,
+    branch,
+
+    async read(prefix) {
+      const ref = await call<{ object: { sha: string } }>(`/git/ref/heads/${branch}`)
+      const head = ref.object.sha
+      // One recursive call for the whole tree beats walking it a directory at a time.
+      const tree = await call<{ tree: { path: string; type: string; sha: string }[] }>(
+        `/git/trees/${head}?recursive=1`,
+      )
+      const wanted = tree.tree.filter(
+        (entry) => entry.type === 'blob' && entry.path.startsWith(`${prefix}/`),
+      )
+      const files: RepositoryFile[] = []
+      for (const entry of wanted) {
+        const blob = await call<{ content: string; encoding: string }>(`/git/blobs/${entry.sha}`)
+        const contents =
+          blob.encoding === 'base64'
+            ? new TextDecoder().decode(
+                Uint8Array.from(atob(blob.content.replace(/\n/g, '')), (c) => c.charCodeAt(0)),
+              )
+            : blob.content
+        files.push({ path: entry.path, contents })
+      }
+      return { head, files }
+    },
+
+    async commit({ prefix, files, message, pullRequest }) {
+      const ref = await call<{ object: { sha: string } }>(`/git/ref/heads/${branch}`)
+      const head = ref.object.sha
+      const headCommit = await call<{ tree: { sha: string } }>(`/git/commits/${head}`)
+
+      const entries: TreeEntry[] = []
+      for (const file of files) {
+        const blob = await call<{ sha: string }>('/git/blobs', {
+          method: 'POST',
+          body: JSON.stringify({ content: file.contents, encoding: 'utf-8' }),
+        })
+        entries.push({ path: file.path, mode: '100644', type: 'blob', sha: blob.sha })
+      }
+
+      // Files the site no longer has are deleted by giving them a null sha, which
+      // is how the tree API spells "remove this path".
+      const existing = await call<{ tree: { path: string; type: string }[] }>(
+        `/git/trees/${head}?recursive=1`,
+      )
+      const keeping = new Set(files.map((file) => file.path))
+      const removals = existing.tree
+        .filter((entry) => entry.type === 'blob' && entry.path.startsWith(`${prefix}/`))
+        .filter((entry) => !keeping.has(entry.path))
+        .map((entry) => ({ path: entry.path, mode: '100644', type: 'blob', sha: null }))
+
+      const tree = await call<{ sha: string }>('/git/trees', {
+        method: 'POST',
+        body: JSON.stringify({ base_tree: headCommit.tree.sha, tree: [...entries, ...removals] }),
+      })
+      const commit = await call<{ sha: string }>('/git/commits', {
+        method: 'POST',
+        body: JSON.stringify({ message, tree: tree.sha, parents: [head], author }),
+      })
+
+      if (!pullRequest) {
+        await call(`/git/refs/heads/${branch}`, {
+          method: 'PATCH',
+          body: JSON.stringify({ sha: commit.sha }),
+        })
+        return { branch, commit: commit.sha }
+      }
+
+      const topic = `crombolo/schema-${commit.sha.slice(0, 8)}`
+      await call('/git/refs', {
+        method: 'POST',
+        body: JSON.stringify({ ref: `refs/heads/${topic}`, sha: commit.sha }),
+      })
+      const raised = await call<{ number: number; html_url: string }>('/pulls', {
+        method: 'POST',
+        body: JSON.stringify({
+          title: pullRequest.title,
+          body: pullRequest.body,
+          head: topic,
+          base: branch,
+        }),
+      })
+      return {
+        branch: topic,
+        commit: commit.sha,
+        pullRequest: { number: raised.number, url: raised.html_url },
+      }
+    },
+  }
+}
+
+/** The live files against the repository, for the backoffice to show before sending. */
+export async function diffAgainst(
+  provider: GitProvider,
+  prefix: string,
+  local: readonly RepositoryFile[],
+): Promise<GitDiff> {
+  const { head, files } = await provider.read(prefix)
+  return { branch: provider.branch, head, changes: diffFiles(files, local) }
+}
+
+export type { PushResult }

package/src/git-routes.ts

@@ -0,0 +1,154 @@
+/**
+ * The backoffice's view of source control: what the live schema has that the
+ * repository does not, and a way to send it.
+ *
+ * Mounted only when `git` is configured. Every route needs a signed-in user, and
+ * sending is a deliberate act — nothing here happens on a timer, because a commit
+ * nobody asked for is a commit nobody reviewed.
+ */
+import { existsSync } from 'node:fs'
+import { readdir, readFile } from 'node:fs/promises'
+import { join, relative, sep } from 'node:path'
+import type { Principal } from '@crombolo/api-management'
+import { diffFiles, type GitProvider, type RepositoryFile } from './git.ts'
+import { logger } from './logging.ts'
+
+const log = logger('Git')
+
+export interface GitConfig {
+  provider: GitProvider
+  /**
+   * Where the schema lives in the repository. The live directory is a cache when a
+   * schema store is configured, so its path is not the repository's.
+   */
+  schemaPath?: string
+  /**
+   * Raise a pull request rather than committing to the branch. On by default: a
+   * schema change arriving as a PR is reviewable, and one arriving as a commit on
+   * main is not.
+   */
+  pullRequest?: boolean
+}
+
+export interface GitHost {
+  handle(request: Request, pathname: string, principal: Principal): Promise<Response | undefined>
+}
+
+export interface GitHostOptions {
+  config: GitConfig
+  /** The directory the live schema is loaded from. */
+  schemaDir: string
+  siteName: string
+}
+
+/** Every `.toml` under `dir`, as repository paths. */
+async function localSchema(dir: string, prefix: string): Promise<RepositoryFile[]> {
+  if (!existsSync(dir)) return []
+  const files: RepositoryFile[] = []
+  const walk = async (at: string): Promise<void> => {
+    for (const entry of await readdir(at, { withFileTypes: true })) {
+      const path = join(at, entry.name)
+      if (entry.isDirectory()) await walk(path)
+      else if (entry.name.toLowerCase().endsWith('.toml')) {
+        files.push({
+          path: `${prefix}/${relative(dir, path).split(sep).join('/')}`,
+          contents: await readFile(path, 'utf8'),
+        })
+      }
+    }
+  }
+  await walk(dir)
+  return files.sort((a, b) => a.path.localeCompare(b.path))
+}
+
+export function createGitHost(options: GitHostOptions): GitHost {
+  const prefix = (options.config.schemaPath ?? 'schema').replace(/^\/+|\/+$/g, '')
+  const provider = options.config.provider
+
+  return {
+    async handle(request, pathname, principal) {
+      if (pathname === '/status' && request.method === 'GET') {
+        return Response.json({
+          provider: provider.description,
+          branch: provider.branch,
+          schemaPath: prefix,
+          raisesPullRequest: options.config.pullRequest !== false,
+        })
+      }
+
+      if (pathname === '/diff' && request.method === 'GET') {
+        const local = await localSchema(options.schemaDir, prefix)
+        const { head, files } = await provider.read(prefix)
+        return Response.json({
+          branch: provider.branch,
+          head,
+          changes: diffFiles(files, local),
+        })
+      }
+
+      if (pathname === '/commit' && request.method === 'POST') {
+        const body = (await request.json().catch(() => ({}))) as {
+          message?: unknown
+          description?: unknown
+        }
+        const local = await localSchema(options.schemaDir, prefix)
+        const { files } = await provider.read(prefix)
+        const changes = diffFiles(files, local)
+        if (changes.length === 0) {
+          return Response.json(
+            { error: 'The repository already matches this site’s schema.' },
+            { status: 409 },
+          )
+        }
+
+        const message =
+          typeof body.message === 'string' && body.message.trim() !== ''
+            ? body.message.trim()
+            : `Schema changes from ${options.siteName}`
+        const description =
+          typeof body.description === 'string' ? body.description : summarise(changes)
+
+        try {
+          const result = await provider.commit({
+            prefix,
+            files: local,
+            message,
+            pullRequest:
+              options.config.pullRequest === false
+                ? undefined
+                : { title: message, body: description },
+          })
+          log.info('{user} sent {count} schema change(s) to {provider} as {what}', {
+            user: principal.userName,
+            count: changes.length,
+            provider: provider.description,
+            what: result.pullRequest
+              ? `PR #${result.pullRequest.number}`
+              : `commit ${result.commit.slice(0, 8)}`,
+          })
+          return Response.json(result)
+        } catch (error) {
+          log.error('Could not send schema to {provider}: {message}', {
+            provider: provider.description,
+            message: (error as Error).message,
+          })
+          return Response.json({ error: (error as Error).message }, { status: 502 })
+        }
+      }
+
+      return undefined
+    },
+  }
+}
+
+/** A plain description of the change, which becomes the pull request's body. */
+function summarise(changes: readonly { path: string; status: string }[]): string {
+  const lines = changes.map((change) => `- ${change.status}: ${change.path}`)
+  return [
+    'Schema changed in the backoffice and sent from there.',
+    '',
+    ...lines,
+    '',
+    'Review as you would any schema change: these files are what the next deploy syncs.',
+  ].join('\n')
+}

package/src/git.ts

@@ -0,0 +1,88 @@
+/**
+ * Getting schema changes made on a running site back into source control,
+ * without an editor ever meeting git.
+ *
+ * Optional, like everything else here: absent from the configuration there is no
+ * route and no credentials anywhere. Configured, the backoffice can show what the
+ * live schema has that the repository does not, and send it — as a commit on a
+ * branch, or as a pull request for somebody to review.
+ *
+ * Over the provider's HTTP API rather than a `git` binary, because the CMS image
+ * has no git and should not grow one, and because a shallow clone per diff to
+ * compare a handful of TOML files would be absurd. `GitProvider` is the seam;
+ * GitHub is the implementation that ships.
+ */
+
+export interface RepositoryFile {
+  /** Repository-relative, e.g. `schema/content-types/home-page.toml`. */
+  path: string
+  contents: string
+}
+
+export type FileChange =
+  | { path: string; status: 'added'; local: string; remote: undefined }
+  | { path: string; status: 'modified'; local: string; remote: string }
+  | { path: string; status: 'removed'; local: undefined; remote: string }
+
+export interface GitDiff {
+  branch: string
+  /** The commit the comparison was made against. */
+  head: string
+  changes: FileChange[]
+}
+
+export interface PushResult {
+  /** The branch the commit landed on. */
+  branch: string
+  commit: string
+  /** Present when a pull request was opened. */
+  pullRequest?: { number: number; url: string }
+}
+
+export interface GitProvider {
+  /** Named in logs and in the backoffice, e.g. `github:kineco-au/site`. */
+  readonly description: string
+  /** The branch changes are compared against and sent to. */
+  readonly branch: string
+  /** Every file under `prefix`, at the head of `branch`. */
+  read(prefix: string): Promise<{ head: string; files: RepositoryFile[] }>
+  /** Commits `files`, replacing what is under `prefix`, and optionally raises a PR. */
+  commit(input: {
+    prefix: string
+    files: readonly RepositoryFile[]
+    message: string
+    /** Raise a pull request instead of committing straight to `branch`. */
+    pullRequest?: { title: string; body: string }
+  }): Promise<PushResult>
+}
+
+/** What `git status` would say, comparing the repository against what is on disk now. */
+export function diffFiles(
+  remote: readonly RepositoryFile[],
+  local: readonly RepositoryFile[],
+): FileChange[] {
+  const byPath = new Map(remote.map((file) => [file.path, file.contents]))
+  const changes: FileChange[] = []
+
+  for (const file of local) {
+    const previous = byPath.get(file.path)
+    if (previous === undefined) {
+      changes.push({ path: file.path, status: 'added', local: file.contents, remote: undefined })
+    } else if (previous !== file.contents) {
+      changes.push({
+        path: file.path,
+        status: 'modified',
+        local: file.contents,
+        remote: previous,
+      })
+    }
+  }
+
+  const here = new Set(local.map((file) => file.path))
+  for (const file of remote) {
+    if (here.has(file.path)) continue
+    changes.push({ path: file.path, status: 'removed', local: undefined, remote: file.contents })
+  }
+
+  return changes.sort((a, b) => a.path.localeCompare(b.path))
+}