eve-hub 0.4.3 → 0.5.0

package/artifacts/core.mjs

@@ -1,48 +1,22 @@
 // Artifacts: a versioned static site a bot publishes and keeps current, such
 // as a change explainer. The core is type-agnostic. A type (artifacts/types/)
-// builds the site and its card from a source, and says whether a published
-// version is still current.
+// builds the site's files and its card from a source, in the bot's sandbox
+// (where the repo is), and says whether a published version is still current.
 //
-// The session that publishes an artifact owns it. Its record lives in that
-// session's state (the tools keep it in defineState("hub.artifacts")), and
-// every deployment also carries its own artifact.json with the full history.
-// The tool result carries the record, so the registry folds it into its
-// read-only index (GET /api/artifacts) and the thread's card.
-//
-// Publishing a version:
-//   1. the type builds the site, its card fields, and a thumbnail
-//   2. artifact.json (record + history) is written into the site
-//   3. `vercel deploy --prebuilt` to the hub-artifacts project, tagged with
-//      --meta artifact=<slug> --meta version=<n>
-//   4. the stable alias <slug>.<domain> moves to it, unless another session
-//      published a newer version meanwhile
-//   5. the site goes to the registry, which serves it at /artifacts/<id>/v<n>/
-//      for the hub's inline preview (no Vercel sign-in)
-//   6. deployments older than the last KEEP versions are removed
-import { execFile } from "node:child_process";
-import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { homedir, tmpdir } from "node:os";
-import { join } from "node:path";
+// The hub owns the record and the deploy (registry/artifacts.mjs). Publishing
+// a version:
+//   1. the type builds the site: { files: [{ file, data }], card, meta }
+//   2. each file goes to the hub: PUT /api/artifacts/:id/files/:sha
+//   3. POST /api/artifacts/:id/v/:n creates version n (the hub refuses with
+//      409 when another session got there first), deploys it and moves the
+//      stable link; its answer is the record.
+import { createHash } from "node:crypto";
 import changeExplainer from "./types/change-explainer.mjs";
-
 import { registryFetch } from "../registry-fetch.mjs";
-const SCOPE = process.env.HUB_ARTIFACTS_SCOPE ?? "vercel-internal-playground";
-const PROJECT = process.env.HUB_ARTIFACTS_PROJECT ?? "hub-artifacts";
-const DOMAIN = process.env.HUB_ARTIFACTS_DOMAIN ?? "playground-vercel.tools";
-const WORK = process.env.HUB_ARTIFACTS_WORK ?? join(process.env.HUB_HOME ?? join(homedir(), ".hub"), "artifacts");
-export const KEEP = 5;
 
 export const TYPES = { "change-explainer": changeExplainer };
-export const STATE_NAME = "hub.artifacts";
-
 export const artifactId = (type, key) => `${type}:${key}`;
-// A DNS label (at most 63 characters): the alias host and the --meta tag.
-export function slugOf(id) {
-  const slug = id.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
-  return slug.length <= 63 ? slug : slug.slice(0, 63).replace(/-+$/, "");
-}
-export const aliasOf = (id) => `${slugOf(id)}.${DOMAIN}`;
-export const previewPath = (id, version) => `/artifacts/${encodeURIComponent(id)}/v${version}/`;
+const UPLOADS_AT_ONCE = 8;
 
 export function typeOf(name) {
   const type = TYPES[name];
@@ -50,172 +24,49 @@ export function typeOf(name) {
   return type;
 }
 
-export function run(cmd, args, { cwd, input, timeoutMs = 600_000 } = {}) {
-  return new Promise((resolve, reject) => {
-    const child = execFile(cmd, args, { cwd, maxBuffer: 64 << 20, timeout: timeoutMs, env: { ...process.env, FORCE_COLOR: "0" } }, (error, stdout, stderr) => {
-      if (error) {
-        const tail = String(stderr || stdout || "").trim().split("\n").slice(-6).join("\n");
-        reject(new Error(`${cmd} ${args[0] ?? ""} failed: ${tail || error.message}`));
-      } else resolve(String(stdout));
-    });
-    if (input !== undefined) child.stdin.end(input);
-    else child.stdin.end();
-  });
+async function hub(path, init, fetchImpl) {
+  const res = await registryFetch(path, init, { fetchImpl });
+  const body = await res.json().catch(() => ({}));
+  if (!res.ok) throw new Error(`${init?.method ?? "GET"} ${path.split("?")[0]}: ${body.error ?? res.status}`);
+  return body;
 }
 
-// The registry's copy of an artifact (owner, latest version), or null.
+// The hub's record of an artifact, or null when it has none.
 export async function registryRecord(id, fetchImpl = fetch) {
-  const res = await registryFetch(`/api/artifacts/${encodeURIComponent(id)}`, {}, { fetchImpl }).catch(() => null);
-  if (!res?.ok) return null;
-  return res.json().catch(() => null);
-}
-
-// The newest record this session can see: its own state or, for a fork or
-// another session picking an artifact up, the registry's index.
-export async function latestRecord(id, stateRecords, fetchImpl = fetch) {
-  const mine = stateRecords?.[id] ?? null;
-  const hub = await registryRecord(id, fetchImpl);
-  if (!hub) return mine;
-  if (!mine) return stripOwner(hub);
-  return (hub.version ?? 0) > (mine.version ?? 0) ? stripOwner(hub) : mine;
-}
-const stripOwner = ({ owner, ...record }) => record;
-
-function projectLink() {
-  const dir = join(WORK, "vercel-link");
-  const file = join(dir, ".vercel", "project.json");
-  return { dir, file };
-}
-
-async function ensureProject() {
-  const { dir, file } = projectLink();
-  if (existsSync(file)) return file;
-  mkdirSync(dir, { recursive: true });
-  await run("vercel", ["link", "--yes", "--project", PROJECT, "--scope", SCOPE], { cwd: dir });
-  if (!existsSync(file)) throw new Error(`vercel link didn't write ${file}`);
-  return file;
-}
-
-// A Build Output API v3 tree around the site: static files, no SPA fallback
-// (artifact sites route with the hash), the target pinned to preview.
-function stage(site, slug, version) {
-  const dir = mkdtempSync(join(tmpdir(), `artifact-${slug}-v${version}-`));
-  const output = join(dir, ".vercel", "output");
-  cpSync(site, join(output, "static"), { recursive: true });
-  writeFileSync(
-    join(output, "config.json"),
-    JSON.stringify({
-      version: 3,
-      routes: [{ src: "/(.*)", headers: { "x-robots-tag": "noindex", "referrer-policy": "no-referrer" }, continue: true }, { handle: "filesystem" }],
-    }),
-  );
-  writeFileSync(join(output, "builds.json"), JSON.stringify({ target: "preview", builds: [] }));
-  return dir;
-}
-
-async function deploy(site, slug, version) {
-  const link = await ensureProject();
-  const dir = stage(site, slug, version);
-  try {
-    cpSync(link, join(dir, ".vercel", "project.json"));
-    const out = await run("vercel", ["deploy", "--prebuilt", "--target", "preview", "--scope", SCOPE, "--yes", "--meta", `artifact=${slug}`, "--meta", `version=${version}`], { cwd: dir });
-    const url = out.trim().split(/\s+/).reverse().find((w) => /^https:\/\/\S+$/.test(w));
-    if (!url) throw new Error(`vercel deploy printed no URL: ${out.slice(-300)}`);
-    return url;
-  } finally {
-    rmSync(dir, { recursive: true, force: true });
-  }
-}
-
-async function uploadBundle(site, id, version, fetchImpl) {
-  const tar = await new Promise((resolve, reject) =>
-    execFile("tar", ["-czf", "-", "-C", site, "."], { encoding: "buffer", maxBuffer: 1 << 30 }, (error, stdout) => (error ? reject(error) : resolve(stdout))),
-  );
-  const res = await registryFetch(`/api/artifacts/${encodeURIComponent(id)}/v/${version}/bundle`, {
-    method: "POST",
-    headers: { "content-type": "application/gzip" },
-    body: tar,
-    signal: AbortSignal.timeout(120_000),
-  }, { fetchImpl });
-  if (!res.ok) throw new Error(`preview upload failed: ${(await res.json().catch(() => ({}))).error ?? res.status}`);
-  return previewPath(id, version);
-}
-
-async function uploadThumbnail(path, fetchImpl) {
-  if (!path || !existsSync(path)) return undefined;
-  const res = await registryFetch(`/api/uploads`, {
-    method: "POST",
-    headers: { "content-type": "application/octet-stream", "x-filename": "thumbnail.png" },
-    body: readFileSync(path),
-    signal: AbortSignal.timeout(30_000),
-  }, { fetchImpl }).catch(() => null);
-  if (!res?.ok) return undefined;
-  return (await res.json()).id;
-}
-
-// Removes deployments older than the last KEEP versions; best effort.
-async function prune(history) {
-  const old = history.slice(0, Math.max(0, history.length - KEEP)).filter((h) => h.url && !h.removed);
-  for (const h of old) {
-    await run("vercel", ["rm", h.url, "--yes", "--scope", SCOPE], { timeoutMs: 60_000 }).then(
-      () => (h.removed = true),
-      () => {},
-    );
-  }
+  const res = await registryFetch(`/api/artifacts/${encodeURIComponent(id)}`, {}, { fetchImpl });
+  if (res.status === 404) return null;
+  if (!res.ok) throw new Error(`GET /api/artifacts/${id}: ${(await res.json().catch(() => ({}))).error ?? res.status}`);
+  return res.json();
 }
 
 /**
- * Publishes the next version of `type:key` from `source`. `prior` is the
- * newest record this session knows (latestRecord). Returns the new record.
- * `deps` swaps the side effects out in tests.
+ * Publishes the next version of `type:key` from `source` and returns the
+ * hub's record. `sandbox` is the session's (ctx.getSandbox()).
  */
-export async function publish({ type: typeName, key, source }, prior, deps = {}) {
+export async function publish({ type: typeName, key, source }, { sandbox, sessionId, fetch: fetchImpl = fetch }) {
   const type = typeOf(typeName);
   const id = artifactId(typeName, key);
-  const slug = slugOf(id);
-  const fetchImpl = deps.fetch ?? fetch;
+  const at = `/api/artifacts/${encodeURIComponent(id)}`;
+  const prior = await registryRecord(id, fetchImpl);
   const version = (prior?.version ?? 0) + 1;
-  const work = mkdtempSync(join(tmpdir(), `artifact-build-${slug}-`));
-  try {
-    const built = await (deps.build ?? type.build)(source, { work, id, version });
-    const now = new Date().toISOString();
-    const history = [...(prior?.history ?? []), { version, at: now, ...(built.meta?.head ? { head: built.meta.head } : {}) }];
-    const record = {
-      id,
-      type: typeName,
-      key,
-      version,
-      status: "fresh",
-      card: { ...built.card, updatedAt: now },
-      source,
-      meta: built.meta ?? {},
-      history,
-      updatedAt: now,
-    };
-    writeFileSync(join(built.site, "artifact.json"), JSON.stringify(record, null, 2));
-    const url = await (deps.deploy ?? deploy)(built.site, slug, version);
-    history.at(-1).url = url;
-    record.url = url;
-    // Another session may have moved the alias on while this one built.
-    const current = await registryRecord(id, fetchImpl);
-    if ((current?.version ?? 0) >= version) {
-      throw new Error(`${id} v${current.version} was published by ${current.owner?.bot ?? "another session"} while this one built v${version}; run artifact_status and publish again`);
-    }
-    const alias = aliasOf(id);
-    await (deps.alias ?? ((u, a) => run("vercel", ["alias", "set", u, a, "--scope", SCOPE])))(url, alias);
-    record.alias = `https://${alias}`;
-    record.preview = await (deps.uploadBundle ?? uploadBundle)(built.site, id, version, fetchImpl);
-    const thumbnail = await (deps.uploadThumbnail ?? uploadThumbnail)(built.thumbnail, fetchImpl);
-    if (thumbnail) record.card.thumbnail = thumbnail;
-    await (deps.prune ?? prune)(history);
-    return record;
-  } finally {
-    rmSync(work, { recursive: true, force: true });
+  const built = await type.build(source, { sandbox, id, version });
+  const files = built.files.map(({ file, data }) => ({ file, data, sha: createHash("sha1").update(data).digest("hex"), size: data.length }));
+  const unique = [...new Map(files.map((f) => [f.sha, f])).values()];
+  for (let i = 0; i < unique.length; i += UPLOADS_AT_ONCE) {
+    await Promise.all(unique.slice(i, i + UPLOADS_AT_ONCE).map((f) =>
+      hub(`${at}/files/${f.sha}`, { method: "PUT", headers: { "content-type": "application/octet-stream" }, body: f.data, signal: AbortSignal.timeout(120_000) }, fetchImpl),
+    ));
   }
+  return hub(`${at}/v/${version}`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ sessionId, type: typeName, key, source, card: built.card, meta: built.meta ?? {}, files: files.map(({ file, sha, size }) => ({ file, sha, size })) }),
+    signal: AbortSignal.timeout(600_000),
+  }, fetchImpl);
 }
 
 /** What the type says about a published record: { status, reasons[] }. */
-export async function status(record) {
+export async function status(record, { sandbox }) {
   const type = typeOf(record.type);
-  return type.status ? type.status(record) : { status: record.status ?? "fresh", reasons: [] };
+  return type.status ? type.status(record, { sandbox }) : { status: record.status ?? "fresh", reasons: [] };
 }

package/artifacts/types/change-explainer.mjs

@@ -3,51 +3,24 @@
 // overview.json, sequence.json) and a read-only checkout of the repo.
 //
 // source: { fixture: <fixture dir>, repo: <git checkout>, scip?: <index.scip>, branch?: <ref the change lives on> }
-import { execFile } from "node:child_process";
-import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
-import { hubPath } from "../../hub-path.mjs";
+import { randomUUID } from "node:crypto";
+import { posix as path } from "node:path";
+import { appFiles, scripts } from "./explainer-assets.gen.mjs";
 
-// packages/explainer, resolved per call: eve bundles this module into the
-// workspace's module cache, so import.meta.dirname isn't in the hub checkout.
-const explainerDir = () => hubPath(join("packages", "explainer"), { probe: join("scripts", "build-data.mjs") });
-
-// packages/explainer has its own lockfile, outside the hub's pnpm workspace,
-// so the hub's install doesn't cover it. Say so up front rather than failing
-// halfway through build-data on a missing import.
-function explainerReady() {
-  const dir = explainerDir();
-  if (!existsSync(join(dir, "node_modules", "@sourcegraph", "scip-typescript"))) {
-    throw new Error(`the explainer's dependencies aren't installed: run \`pnpm install --ignore-workspace --frozen-lockfile\` in ${dir}`);
-  }
-  return dir;
+const quote = (value) => "'" + String(value).replaceAll("'", "'\\''") + "'";
+async function run(sandbox, args) {
+  const result = await sandbox.run({ command: args.map(quote).join(" ") });
+  if (result.exitCode !== 0) throw new Error(`${args[0]} failed: ${result.stderr || result.stdout}`);
+  return result.stdout.trim();
 }
-
-const node = (script, args, dir = explainerDir()) => {
-  return new Promise((ok, fail) =>
-    execFile(process.execPath, [join(dir, "scripts", script), ...args], { cwd: dir, maxBuffer: 64 << 20, timeout: 900_000 }, (error, stdout, stderr) =>
-      error ? fail(new Error(`${script} failed: ${String(stderr || error.message).trim().split("\n").slice(-6).join("\n")}`)) : ok(String(stdout)),
-    ),
-  );
-};
-
-const git = (repo, ...args) =>
-  new Promise((ok, fail) => execFile("git", ["-C", repo, ...args], (error, stdout) => (error ? fail(error) : ok(String(stdout).trim()))));
-
-const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
+const git = (sandbox, repo, ...args) => run(sandbox, ["git", "-C", repo, ...args]);
+const readJson = async (sandbox, file) => JSON.parse(await sandbox.readTextFile({ path: file }));
 
 export function firstSentence(text = "") {
   const sentence = String(text).trim().split(/(?<=[.!?])\s+(?=[A-Z"'`(])/, 1)[0] ?? "";
   return sentence.length > 220 ? `${sentence.slice(0, 219).replace(/\s+\S*$/, "")}…` : sentence;
 }
 
-function checkSource(source) {
-  if (!source || typeof source !== "object") throw new Error("change-explainer source must be { fixture, repo, scip?, branch? }");
-  for (const key of ["fixture", "repo"]) if (typeof source[key] !== "string" || !existsSync(source[key])) throw new Error(`change-explainer source.${key}: no such path ${JSON.stringify(source[key])}`);
-  if (!existsSync(join(source.fixture, "fixture.json"))) throw new Error(`no fixture.json in ${source.fixture}`);
-  if (source.scip && !existsSync(source.scip)) throw new Error(`change-explainer source.scip: no such file ${source.scip}`);
-}
-
 // Card fields from the fixture: what the change is, and which commits.
 export function cardOf(fixture, overview) {
   const short = (sha) => String(sha ?? "").slice(0, 9);
@@ -60,41 +33,44 @@ export function cardOf(fixture, overview) {
   };
 }
 
-async function build(source, { work }) {
-  checkSource(source);
-  const fixture = readJson(join(source.fixture, "fixture.json"));
-  const overviewPath = join(source.fixture, "overview.json");
-  const overview = existsSync(overviewPath) ? readJson(overviewPath) : null;
-  const data = join(work, "data");
-  const site = join(work, "site");
-  const dir = explainerReady();
-  await node("build-data.mjs", ["--fixture", source.fixture, "--repo", source.repo, ...(source.scip ? ["--scip", source.scip] : []), "--out", data], dir);
-  await node("build.mjs", ["--out", site, "--data", data, "--fixture", fixture.id], dir);
-  const thumbnail = join(work, "thumbnail.png");
-  await node("thumbnail.mjs", [site, thumbnail], dir).catch(() => {});
-  const [base, head] = await Promise.all([git(source.repo, "rev-parse", `${fixture.base}^{commit}`), git(source.repo, "rev-parse", `${fixture.head}^{commit}`)]);
-  return {
-    site,
-    thumbnail: existsSync(thumbnail) ? thumbnail : undefined,
-    card: cardOf(fixture, overview),
-    meta: { repo: fixture.repo, base, head, prs: (fixture.prs ?? []).map((p) => p.number) },
-  };
+async function build(source, { sandbox }) {
+  const work = `/tmp/hub-explainer/${randomUUID()}`;
+  try {
+    await run(sandbox, ["mkdir", "-p", work]);
+    for (const [name, text] of Object.entries(scripts)) await sandbox.writeTextFile({ path: `${work}/${name}.mjs`, content: text });
+    const fixture = await readJson(sandbox, path.join(source.fixture, "fixture.json"));
+    await run(sandbox, ["node", `${work}/lint-fixture.mjs`, source.fixture]);
+    const data = `${work}/data`;
+    await run(sandbox, ["node", `${work}/build-data.mjs`, "--fixture", source.fixture, "--repo", source.repo, ...(source.scip ? ["--scip", source.scip] : []), "--out", data]);
+    const overview = await readJson(sandbox, path.join(source.fixture, "overview.json"));
+    const [base, head] = await Promise.all([git(sandbox, source.repo, "rev-parse", `${fixture.base}^{commit}`), git(sandbox, source.repo, "rev-parse", `${fixture.head}^{commit}`)]);
+    const files = appFiles.map(({ file, base64 }) => {
+      let bytes = Buffer.from(base64, "base64");
+      if (file === "index.html") bytes = Buffer.from(bytes.toString("utf8").replace("</title>", `</title>\n    <meta name="explainer-fixture" content="${fixture.id.replace(/[^\w.-]/g, "")}" />`));
+      return { file, data: bytes };
+    });
+    const listing = await run(sandbox, ["find", data, "-type", "f", "-print0"]);
+    for (const file of listing.split("\0").filter(Boolean)) files.push({ file: `data/${file.slice(data.length + 1)}`, data: await sandbox.readBinaryFile({ path: file }) });
+    return { files, card: cardOf(fixture, overview), meta: { repo: fixture.repo, base, head, prs: (fixture.prs ?? []).map((p) => p.number) } };
+  } finally {
+    await sandbox.removePath({ path: work, recursive: true, force: true });
+  }
 }
 
 // Whether the published version still describes the change. This checks the
 // pinned range against the fixture and, with source.branch, the branch head;
 // which parts went stale (snippet hashes, re-anchoring) is the next step.
-async function status(record) {
+async function status(record, { sandbox }) {
   const { source = {}, meta = {} } = record;
   const reasons = [];
   if (record.status === "merged" || record.status === "closed") return { status: record.status, reasons };
   try {
-    const fixture = readJson(join(source.fixture, "fixture.json"));
-    const [base, head] = await Promise.all([git(source.repo, "rev-parse", `${fixture.base}^{commit}`), git(source.repo, "rev-parse", `${fixture.head}^{commit}`)]);
+    const fixture = await readJson(sandbox, path.join(source.fixture, "fixture.json"));
+    const [base, head] = await Promise.all([git(sandbox, source.repo, "rev-parse", `${fixture.base}^{commit}`), git(sandbox, source.repo, "rev-parse", `${fixture.head}^{commit}`)]);
     if (base !== meta.base) reasons.push(`the fixture's base is now ${base.slice(0, 9)} (published at ${String(meta.base).slice(0, 9)})`);
     if (head !== meta.head) reasons.push(`the fixture's head is now ${head.slice(0, 9)} (published at ${String(meta.head).slice(0, 9)})`);
     if (source.branch) {
-      const tip = await git(source.repo, "rev-parse", `${source.branch}^{commit}`);
+      const tip = await git(sandbox, source.repo, "rev-parse", `${source.branch}^{commit}`);
       if (tip !== meta.head) reasons.push(`${source.branch} is at ${tip.slice(0, 9)}, ahead of the published head ${String(meta.head).slice(0, 9)}`);
     }
   } catch (error) {