@codepress/design-kit 0.8.0 → 0.9.0

package/src/storage/bucket.mjs

@@ -0,0 +1,237 @@
+/**
+ * `s3` and `r2` storage: your own AWS S3 or Cloudflare R2 bucket, through
+ * the `aws` CLI (`aws s3 cp`). Credentials come from the environment the
+ * CLI reads (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY).
+ *
+ * Layout, under the optional prefix:
+ *
+ *     <prefix>/<branch>/<sha>.zip     each kit
+ *     <prefix>/<branch>/latest.json   {sha, created_at, size_bytes, key, manifest}
+ *
+ * An upload writes the zip, then latest.json, so latest.json never names a
+ * zip that is not there yet. `r2` is `s3` with the endpoint
+ * `https://<accountId>.r2.cloudflarestorage.com` and the region `auto`.
+ */
+import { execFile } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+import { ConfigError } from "../config.mjs";
+import {
+  StorageError,
+  checkSha,
+  installKit,
+  isKit,
+  kitCacheDir,
+} from "./common.mjs";
+import { kitSummary } from "./codepress.mjs";
+import { zipDirectory } from "./zip.mjs";
+
+/** The endpoint of a Cloudflare R2 account. */
+export function r2Endpoint(accountId) {
+  return `https://${accountId}.r2.cloudflarestorage.com`;
+}
+
+/** Runs `aws` with `args`; resolves its stdout, rejects with its stderr. */
+export function runAws(args, { env = process.env } = {}) {
+  return new Promise((resolvePromise, reject) => {
+    execFile(
+      "aws",
+      args,
+      { env, encoding: "buffer", maxBuffer: 512 * 1024 * 1024 },
+      (error, stdout, stderr) => {
+        if (error) {
+          error.stderr = stderr?.toString("utf8") ?? "";
+          reject(error);
+        } else resolvePromise(stdout);
+      },
+    );
+  });
+}
+
+const NOT_FOUND = /\(404\)|NoSuchKey|Not Found|does not exist/i;
+
+/**
+ * The `s3` or `r2` store. `storage` is the config's `storage` object;
+ * `aws` runs the CLI (tests pass their own, or put a stub on PATH).
+ */
+export function bucketStorage(
+  storage,
+  { env = process.env, aws = runAws, now = () => new Date() } = {},
+) {
+  const r2 = storage.type === "r2";
+  const endpoint = r2 ? r2Endpoint(storage.accountId) : storage.endpoint;
+  const region = r2 ? "auto" : storage.region;
+  const common = [
+    ...(endpoint ? ["--endpoint-url", endpoint] : []),
+    ...(region ? ["--region", region] : []),
+  ];
+  const prefix = String(storage.prefix ?? "").replace(/^\/+|\/+$/g, "");
+  const key = (branch, name) =>
+    [prefix, branch, name].filter(Boolean).join("/");
+  const url = (path) => `s3://${storage.bucket}/${path}`;
+  const label = r2 ? "R2" : "S3";
+
+  const call = async (args) => {
+    try {
+      return await aws([...args, ...common], { env });
+    } catch (error) {
+      if (error?.code === "ENOENT")
+        throw new StorageError(
+          "aws_cli_missing",
+          `${storage.type} storage uses the aws CLI, and there is no aws on PATH. Install it (https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html); GitHub's ubuntu runners have it already.`,
+        );
+      throw error;
+    }
+  };
+  const needBranch = (branch) => {
+    if (!branch)
+      throw new ConfigError(
+        "--branch <name> is needed: GITHUB_REF_NAME is not set and the checkout's origin/HEAD names no default branch",
+      );
+    return branch;
+  };
+  const failed = (what, error) =>
+    new StorageError(
+      "kit_storage_unavailable",
+      `${what} in ${label} bucket ${storage.bucket} failed: ${String(
+        error?.stderr || error?.message || error,
+      )
+        .trim()
+        .split("\n")
+        .pop()}`,
+    );
+
+  const readLatest = async (branch) => {
+    let out;
+    try {
+      out = await call(["s3", "cp", url(key(branch, "latest.json")), "-"]);
+    } catch (error) {
+      if (error instanceof StorageError) throw error;
+      if (NOT_FOUND.test(String(error?.stderr ?? ""))) return null;
+      throw failed(`Reading ${key(branch, "latest.json")}`, error);
+    }
+    try {
+      const latest = JSON.parse(out.toString("utf8"));
+      if (!latest?.sha) throw new Error("no sha");
+      return latest;
+    } catch {
+      throw new StorageError(
+        "kit_storage_unavailable",
+        `${key(branch, "latest.json")} is not a kit pointer`,
+      );
+    }
+  };
+
+  return {
+    type: storage.type,
+    async latest({ branch } = {}) {
+      const latest = await readLatest(needBranch(branch));
+      return latest
+        ? { sha: latest.sha, createdAt: latest.created_at, branch }
+        : null;
+    },
+    async fetch({ branch, out } = {}) {
+      const name = needBranch(branch);
+      const latest = await readLatest(name);
+      if (!latest)
+        throw new StorageError(
+          "no_kit",
+          `no kit on ${name} in ${label} bucket ${storage.bucket}`,
+        );
+      checkSha(latest.sha, `${key(name, "latest.json")}`);
+      const path = resolve(out ?? kitCacheDir(latest.sha, env));
+      const result = { path, sha: latest.sha, branch: name };
+      if (isKit(path)) return { ...result, cached: true };
+      const scratch = mkdtempSync(
+        join(env.TMPDIR || tmpdir(), "design-kit-zip-"),
+      );
+      try {
+        const zipFile = join(scratch, "kit.zip");
+        try {
+          await call([
+            "s3",
+            "cp",
+            url(latest.key ?? key(name, `${latest.sha}.zip`)),
+            zipFile,
+          ]);
+        } catch (error) {
+          if (error instanceof StorageError) throw error;
+          throw failed(`Downloading ${latest.sha}.zip`, error);
+        }
+        installKit(readFileSync(zipFile), path);
+      } finally {
+        rmSync(scratch, { recursive: true, force: true });
+      }
+      return { ...result, cached: false };
+    },
+    async upload({ dir, branch, sha } = {}) {
+      const name = needBranch(branch);
+      const manifest = kitSummary(dir);
+      const kitSha = sha ?? manifest.sha;
+      if (!kitSha)
+        throw new ConfigError(
+          "--sha <commit> is needed: GITHUB_SHA is not set and the kit's manifest names no commit",
+        );
+      checkSha(kitSha, sha ? "--sha (or GITHUB_SHA)" : "the kit's manifest");
+      const scratch = mkdtempSync(
+        join(env.TMPDIR || tmpdir(), "design-kit-zip-"),
+      );
+      try {
+        const zip = zipDirectory(dir);
+        const zipFile = join(scratch, `${kitSha}.zip`);
+        writeFileSync(zipFile, zip);
+        const zipKey = key(name, `${kitSha}.zip`);
+        try {
+          await call([
+            "s3",
+            "cp",
+            zipFile,
+            url(zipKey),
+            "--content-type",
+            "application/zip",
+            "--only-show-errors",
+          ]);
+        } catch (error) {
+          if (error instanceof StorageError) throw error;
+          throw failed(`Uploading ${zipKey}`, error);
+        }
+        const pointer = join(scratch, "latest.json");
+        writeFileSync(
+          pointer,
+          `${JSON.stringify({
+            sha: kitSha,
+            branch: name,
+            created_at: now().toISOString(),
+            size_bytes: zip.length,
+            key: zipKey,
+            manifest,
+          })}\n`,
+        );
+        try {
+          await call([
+            "s3",
+            "cp",
+            pointer,
+            url(key(name, "latest.json")),
+            "--content-type",
+            "application/json",
+            "--only-show-errors",
+          ]);
+        } catch (error) {
+          if (error instanceof StorageError) throw error;
+          throw failed(`Writing ${key(name, "latest.json")}`, error);
+        }
+        return {
+          sha: kitSha,
+          branch: name,
+          sizeBytes: zip.length,
+          key: zipKey,
+        };
+      } finally {
+        rmSync(scratch, { recursive: true, force: true });
+      }
+    },
+  };
+}

package/src/storage/codepress.mjs

@@ -0,0 +1,363 @@
+/**
+ * `codepress` storage: CodePress keeps the newest kit of each branch.
+ *
+ * - Upload (GitHub Actions only): the workflow's OIDC token, requested with
+ *   the CodePress address as its audience, signs in; the repository holds
+ *   no secret. `POST /v1/design-kits/uploads` with the zip's size and a
+ *   summary of the kit answers a presigned PUT, the zip goes straight to
+ *   storage with exactly the headers it names (its `Content-Length` must be
+ *   the declared size), and `POST /v1/design-kits/uploads/{id}/complete`
+ *   makes it the branch's kit.
+ * - Reads: `GET /v1/design-kits/repos/{owner}/{repo}/latest?branch=` with a
+ *   CodePress API key (CODEPRESS_API_KEY, or a file named by --token-file),
+ *   which answers the kit's commit and a short-lived download URL.
+ *
+ * A refusal keeps the backend's name for it (`forbidden`, `no_kit`,
+ * `repo_not_connected`, …); a store that cannot be reached is
+ * `kit_storage_unavailable`.
+ */
+import { readFileSync } from "node:fs";
+import http from "node:http";
+import https from "node:https";
+import { join, resolve } from "node:path";
+
+import { ConfigError } from "../config.mjs";
+import {
+  StorageError,
+  checkSha,
+  installKit,
+  isKit,
+  kitCacheDir,
+} from "./common.mjs";
+import { zipDirectory } from "./zip.mjs";
+
+export const DEFAULT_URL = "https://codepress.dev";
+export const DEFAULT_API_URL = "https://api.codepress.dev/v1";
+const UNAVAILABLE = "kit_storage_unavailable";
+/** How long one request may take before the store counts as unreachable. */
+const REQUEST_TIMEOUT_MS = Number(
+  process.env.DESIGN_KIT_REQUEST_TIMEOUT_MS || 120_000,
+);
+const MANIFEST_BYTES = 4096;
+
+/**
+ * The API base for a storage config: `apiUrl` when set; the CodePress API
+ * for the default address; else `<url>/v1`.
+ */
+export function apiBase(storage = {}) {
+  if (storage.apiUrl) return storage.apiUrl.replace(/\/+$/, "");
+  const url = (storage.url ?? DEFAULT_URL).replace(/\/+$/, "");
+  return url === DEFAULT_URL ? DEFAULT_API_URL : `${url}/v1`;
+}
+
+/**
+ * One HTTP request: `{status, headers, body}` (body a Buffer). Headers are
+ * sent exactly as given, so a presigned PUT carries its signed
+ * Content-Length. A failure to connect, or no answer within the timeout, is
+ * `kit_storage_unavailable`.
+ */
+export function request(url, { method = "GET", headers = {}, body } = {}) {
+  return new Promise((resolvePromise, reject) => {
+    let parsed;
+    try {
+      parsed = new URL(url);
+    } catch {
+      reject(new StorageError(UNAVAILABLE, `not a URL: ${url}`));
+      return;
+    }
+    const client = parsed.protocol === "http:" ? http : https;
+    const req = client.request(
+      parsed,
+      { method, headers, timeout: REQUEST_TIMEOUT_MS },
+      (res) => {
+        const chunks = [];
+        res.on("data", (chunk) => chunks.push(chunk));
+        res.on("end", () =>
+          resolvePromise({
+            status: res.statusCode ?? 0,
+            headers: res.headers,
+            body: Buffer.concat(chunks),
+          }),
+        );
+        res.on("error", (error) =>
+          reject(new StorageError(UNAVAILABLE, error.message)),
+        );
+      },
+    );
+    req.on("timeout", () =>
+      req.destroy(new Error(`no answer from ${parsed.host} in time`)),
+    );
+    req.on("error", (error) =>
+      reject(
+        new StorageError(
+          UNAVAILABLE,
+          `could not reach ${parsed.host} (${error.code ?? error.message})`,
+        ),
+      ),
+    );
+    if (body !== undefined) req.end(body);
+    else req.end();
+  });
+}
+
+/** The backend's named error for a non-2xx answer. */
+function refusal(response, what) {
+  let payload = null;
+  try {
+    payload = JSON.parse(response.body.toString("utf8"));
+  } catch {
+    // Not JSON: a proxy or gateway answered.
+  }
+  const reason = payload?.reason ?? payload?.code;
+  if (typeof reason === "string" && reason)
+    return new StorageError(
+      reason,
+      `${what} was refused (${response.status}): ${payload.detail ?? reason}`,
+    );
+  if (response.status >= 500 || response.status === 0)
+    return new StorageError(
+      UNAVAILABLE,
+      `${what} failed: CodePress answered ${response.status}`,
+    );
+  const named = { 401: "unauthorized", 403: "forbidden", 404: "not_found" };
+  const detail =
+    typeof payload?.detail === "string" ? `: ${payload.detail}` : "";
+  return new StorageError(
+    named[response.status] ?? `http_${response.status}`,
+    `${what} was refused (${response.status})${detail}`,
+  );
+}
+
+async function json(url, options, what) {
+  const response = await request(url, options);
+  if (response.status < 200 || response.status >= 300)
+    throw refusal(response, what);
+  try {
+    return JSON.parse(response.body.toString("utf8"));
+  } catch {
+    throw new StorageError(UNAVAILABLE, `${what}: the answer is not JSON`);
+  }
+}
+
+/**
+ * The read credential: CODEPRESS_API_KEY, or the token in `tokenFile`. The
+ * file may hold the token alone, or `KEY=value` lines (an env file, as
+ * CodePress Terminal's `create_backend_access_token_file` writes): then
+ * CODEPRESS_API_KEY's line, or the only line.
+ */
+export function readToken({ tokenFile, env = process.env } = {}) {
+  if (tokenFile) {
+    let text;
+    try {
+      text = readFileSync(tokenFile, "utf8");
+    } catch (error) {
+      throw new ConfigError(
+        `--token-file ${tokenFile}: could not read it (${error.code ?? error.message})`,
+      );
+    }
+    const lines = text
+      .split(/\r?\n/)
+      .map((line) => line.trim())
+      .filter((line) => line && !line.startsWith("#"));
+    const assignments = lines
+      .map((line) =>
+        line.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/),
+      )
+      .filter(Boolean);
+    const unquote = (value) => value.replace(/^(['"])(.*)\1$/, "$2");
+    if (assignments.length) {
+      const named = assignments.find(([, key]) => key === "CODEPRESS_API_KEY");
+      const chosen =
+        named ?? (assignments.length === 1 ? assignments[0] : undefined);
+      if (!chosen)
+        throw new ConfigError(
+          `--token-file ${tokenFile}: it has several KEY=value lines and none is CODEPRESS_API_KEY`,
+        );
+      return unquote(chosen[2].trim());
+    }
+    if (lines.length === 1) return lines[0];
+    throw new ConfigError(`--token-file ${tokenFile}: it holds no token`);
+  }
+  if (env.CODEPRESS_API_KEY) return env.CODEPRESS_API_KEY.trim();
+  throw new StorageError(
+    "no_credentials",
+    "codepress storage needs a CodePress API key to read kits. Set CODEPRESS_API_KEY, or pass --token-file <path> (a file holding the key alone, or a KEY=value env file with a CODEPRESS_API_KEY line)",
+  );
+}
+
+/**
+ * The credential a read sends: an API key (CODEPRESS_API_KEY or
+ * --token-file) when there is one; otherwise, inside a GitHub Actions job
+ * with `id-token: write`, the repository's own OIDC token for `audience`
+ * (the backend reads it as that repository, #22530). Neither is
+ * `no_credentials`.
+ */
+export async function readCredential({
+  tokenFile,
+  env = process.env,
+  audience,
+}) {
+  if (
+    !tokenFile &&
+    !env.CODEPRESS_API_KEY &&
+    env.ACTIONS_ID_TOKEN_REQUEST_URL &&
+    env.ACTIONS_ID_TOKEN_REQUEST_TOKEN
+  )
+    return oidcToken(audience, env);
+  return readToken({ tokenFile, env });
+}
+
+/**
+ * The workflow's GitHub OIDC token for `audience`. Needs
+ * ACTIONS_ID_TOKEN_REQUEST_URL and ACTIONS_ID_TOKEN_REQUEST_TOKEN, which
+ * GitHub sets only in a job with `permissions: id-token: write`.
+ */
+export async function oidcToken(audience, env = process.env) {
+  const url = env.ACTIONS_ID_TOKEN_REQUEST_URL;
+  const token = env.ACTIONS_ID_TOKEN_REQUEST_TOKEN;
+  if (!url || !token)
+    throw new StorageError(
+      "oidc_unavailable",
+      "design-kit upload with codepress storage signs in with the GitHub Actions OIDC token, and ACTIONS_ID_TOKEN_REQUEST_URL is not set. Run it in a GitHub Actions job with `permissions: id-token: write` (the workflow design-kit init writes does).",
+    );
+  const separator = url.includes("?") ? "&" : "?";
+  const body = await json(
+    `${url}${separator}audience=${encodeURIComponent(audience)}`,
+    {
+      headers: { Authorization: `bearer ${token}`, Accept: "application/json" },
+    },
+    "Requesting the GitHub OIDC token",
+  );
+  if (!body?.value)
+    throw new StorageError("oidc_unavailable", "GitHub answered no OIDC token");
+  return body.value;
+}
+
+/** The kit summary sent with an upload: at most 4 KB. */
+export function kitSummary(dir) {
+  let manifest = {};
+  try {
+    manifest = JSON.parse(readFileSync(join(dir, "manifest.json"), "utf8"));
+  } catch (error) {
+    throw new ConfigError(
+      `${dir} is not a kit: no readable manifest.json (${error.code ?? error.message})`,
+    );
+  }
+  const summary = {};
+  for (const key of ["kit_version", "sha", "built_at", "counts"])
+    if (manifest[key] !== undefined) summary[key] = manifest[key];
+  if (Buffer.byteLength(JSON.stringify(summary)) > MANIFEST_BYTES)
+    delete summary.counts;
+  return summary;
+}
+
+/**
+ * The `codepress` store. `storage` is the config's `storage` object; `repo`
+ * is `owner/name` (reads only).
+ */
+export function codepressStorage(
+  storage = {},
+  { repo, env = process.env, tokenFile } = {},
+) {
+  const audience = (storage.url ?? DEFAULT_URL).replace(/\/+$/, "");
+  const api = apiBase(storage);
+  const readLatest = async (branch) => {
+    if (!repo)
+      throw new ConfigError(
+        "--repo <owner/name> is needed: no GITHUB_REPOSITORY and no github.com origin remote",
+      );
+    const token = await readCredential({ tokenFile, env, audience });
+    const [owner, name] = repo.split("/");
+    const query = branch ? `?branch=${encodeURIComponent(branch)}` : "";
+    const response = await request(
+      `${api}/design-kits/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/latest${query}`,
+      {
+        headers: {
+          Authorization: `Bearer ${token}`,
+          Accept: "application/json",
+        },
+      },
+    );
+    if (response.status < 200 || response.status >= 300) {
+      const error = refusal(response, `Reading the newest kit of ${repo}`);
+      if (error.code === "no_kit") return null;
+      throw error;
+    }
+    try {
+      return JSON.parse(response.body.toString("utf8"));
+    } catch {
+      throw new StorageError(UNAVAILABLE, "the latest kit answer is not JSON");
+    }
+  };
+  return {
+    type: "codepress",
+    async latest({ branch } = {}) {
+      const kit = await readLatest(branch);
+      if (!kit) return null;
+      return {
+        sha: kit.commit_sha,
+        createdAt: kit.created_at,
+        branch: kit.branch,
+        sizeBytes: kit.size_bytes,
+      };
+    },
+    async fetch({ branch, out } = {}) {
+      const kit = await readLatest(branch);
+      if (!kit)
+        throw new StorageError(
+          "no_kit",
+          `CodePress has no kit for ${repo}${branch ? ` on ${branch}` : " on its default branch"}`,
+        );
+      checkSha(kit.commit_sha, "CodePress");
+      const path = resolve(out ?? kitCacheDir(kit.commit_sha, env));
+      const result = { path, sha: kit.commit_sha, branch: kit.branch };
+      if (isKit(path)) return { ...result, cached: true };
+      const response = await request(kit.download_url);
+      if (response.status < 200 || response.status >= 300)
+        throw refusal(response, "Downloading the kit");
+      installKit(response.body, path);
+      return { ...result, cached: false };
+    },
+    async upload({ dir } = {}) {
+      const manifest = kitSummary(dir);
+      const token = await oidcToken(audience, env);
+      const zip = zipDirectory(dir);
+      const auth = {
+        Authorization: `Bearer ${token}`,
+        Accept: "application/json",
+        "Content-Type": "application/json",
+      };
+      const started = await json(
+        `${api}/design-kits/uploads`,
+        {
+          method: "POST",
+          headers: auth,
+          body: JSON.stringify({ size_bytes: zip.length, manifest }),
+        },
+        "Starting the upload",
+      );
+      const target = started.upload ?? {};
+      const put = await request(target.url, {
+        method: target.method ?? "PUT",
+        headers: { ...target.headers },
+        body: zip,
+      });
+      if (put.status < 200 || put.status >= 300)
+        throw new StorageError(
+          "upload_failed",
+          `storage refused the zip (${put.status}): ${put.body.toString("utf8").slice(0, 300)}`,
+        );
+      const done = await json(
+        `${api}/design-kits/uploads/${encodeURIComponent(started.upload_id)}/complete`,
+        { method: "POST", headers: auth, body: "{}" },
+        "Completing the upload",
+      );
+      return {
+        sha: done.commit_sha ?? started.commit_sha,
+        branch: done.branch ?? started.branch,
+        sizeBytes: zip.length,
+        uploadId: started.upload_id,
+      };
+    },
+  };
+}