@valbuild/cli 0.132.1 → 0.134.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@valbuild/cli",
   "private": false,
-  "version": "0.132.1",
+  "version": "0.134.0",
   "description": "Val CLI tools",
   "repository": {
     "type": "git",
@@ -24,11 +24,13 @@
     "jszip": "^3.10.1",
     "meow": "^9.0.0",
     "picocolors": "^1.1.1",
-    "@valbuild/core": "0.130.0",
+    "zod": "^4.4.3",
+    "zod-validation-error": "^5.0.0",
+    "@valbuild/core": "0.134.0",
     "@valbuild/eslint-plugin": "0.125.0",
-    "@valbuild/language-server": "0.132.1",
-    "@valbuild/server": "0.132.1",
-    "@valbuild/shared": "0.130.0"
+    "@valbuild/language-server": "0.134.0",
+    "@valbuild/shared": "0.134.0",
+    "@valbuild/server": "0.134.0"
   },
   "peerDependencies": {
     "prettier": "*",

package/src/cli.ts

@@ -9,6 +9,7 @@ import { login } from "./login";
 import { lsp } from "./lsp";
 import { debug } from "./debug";
 import { deleteUnappliablePatches } from "./deleteUnappliablePatches";
+import { publish } from "./publish";
 
 async function main(): Promise<void> {
   const { input, flags, showHelp } = meow(
@@ -24,6 +25,7 @@ async function main(): Promise<void> {
         login
         files
         connect
+        publish
         versions
         lsp
         debug
@@ -48,6 +50,32 @@ async function main(): Promise<void> {
       Options:
         --root [root], -r [root] Set project root directory (default process.cwd())
 
+      Command: publish
+      Description: publish this project's build through content.val.build.
+        Declares the build's artifacts, uploads the ones content does not already hold,
+        and has content verify it by building and rendering a canary before the site
+        changes. Authenticates with VAL_PROJECT_TOKEN, or with the "val login" token in
+        .val/pat.json (which needs the project, as "<org>/<project>", in val.config or
+        VAL_PROJECT). Never as a flag: an argument is visible to anyone who can list
+        processes, and it is kept in shell history and in every CI log.
+      Options:
+        --root [root], -r [root] Set project root directory (default process.cwd())
+        --artifacts [dir]        The built artifacts (default <root>/.val/publish). The path
+                                 of each file under it is its artifact key: server, client,
+                                 css, rsc, layer, or a path under chunk/server, chunk/client,
+                                 chunk/rsc, asset, public
+        --commit [sha]           The commit this build is of (default VAL_GIT_COMMIT,
+                                 else GITHUB_SHA, else git HEAD)
+        --branch [name]          The branch content commits saves to (default VAL_GIT_BRANCH,
+                                 else GITHUB_REF_NAME, else the current git branch)
+        --layer-rev [rev]        Name the dependency layer content already holds, when this
+                                 build does not send one
+        --build-hash [hash]      Identify the build (default: a hash of its artifacts).
+                                 Declaring the same one twice resumes that publish
+        --links-own-css          The app links its own CSS (--no-links-own-css for the
+                                 opposite; omit it when the build did not say)
+        --dry-run                Verify, and stop before the site changes
+
       Command: list-unused-files
       Description: EXPERIMENTAL.
         List files that are in the configured files directory (files.directory, default public/val) but not in use by any Val module.
@@ -112,6 +140,18 @@ async function main(): Promise<void> {
         out: {
           type: "string",
         },
+        artifacts: {
+          type: "string",
+        },
+        layerRev: {
+          type: "string",
+        },
+        buildHash: {
+          type: "string",
+        },
+        linksOwnCss: {
+          type: "boolean",
+        },
         commit: {
           type: "string",
         },
@@ -181,6 +221,17 @@ async function main(): Promise<void> {
         yes: flags.yes,
         verbose: flags.verbose,
       });
+    case "publish":
+      return publish({
+        root: flags.root,
+        artifacts: flags.artifacts,
+        commit: flags.commit,
+        branch: flags.branch,
+        layerRev: flags.layerRev,
+        buildHash: flags.buildHash,
+        linksOwnCss: flags.linksOwnCss,
+        dryRun: flags.dryRun,
+      });
     case "login":
       return login({
         root: flags.root,

package/src/debug/context.ts

@@ -120,8 +120,7 @@ export async function createDebugContext(options: {
       serverOps: new ValOpsHttp(
         contentUrl,
         project,
-        commit,
-        branch,
+        { commit, branch },
         auth,
         valModules,
         { root: config.root, config },

package/src/publish/artifacts.ts

@@ -0,0 +1,194 @@
+import crypto from "crypto";
+import fs from "fs";
+import path from "path";
+import zlib from "zlib";
+import { DeclaredArtifact } from "./protocol";
+
+/**
+ * Where an artifact's bytes are, and what it is called on the wire.
+ *
+ * The key namespace is flat and is content's: `server`, `client`, `css`,
+ * `rsc`, `layer`, and paths under `chunk/server/`, `chunk/client/`,
+ * `chunk/rsc/`, `asset/` and `public/`. Content validates it and answers with
+ * every problem at once, so this does not re-implement the rules - a second,
+ * slightly different copy of them here is how a CLI comes to refuse a publish
+ * the service would have taken.
+ */
+export type Artifact = DeclaredArtifact & { file: string };
+
+export type CollectedArtifacts = {
+  artifacts: Artifact[];
+  totalBytes: number;
+  skipped: string[];
+};
+
+/**
+ * The artifacts to publish, read from a directory laid out by key.
+ *
+ * The path under the directory IS the artifact key: a file at `public/app.css`
+ * is the artifact `public/app.css`, and `server` is the server bundle. Nothing
+ * is renamed on the way - an asset is addressed by the path the built code
+ * imports it at, so a normalised name produces a bundle whose imports resolve
+ * to nothing, at runtime, in the isolate.
+ *
+ * **This is the seam with the build.** `val publish` uploads artifacts; it does
+ * not produce them. Whatever builds the project - today the platform's own
+ * builder, which is where the wire, rolldown and dependency-layer steps live -
+ * writes this directory, and this reads it.
+ */
+export async function collectArtifacts(
+  dir: string,
+): Promise<CollectedArtifacts> {
+  const artifacts: Artifact[] = [];
+  const skipped: string[] = [];
+  let totalBytes = 0;
+
+  const walk = async (current: string): Promise<void> => {
+    const entries = await fs.promises.readdir(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const absolute = path.join(current, entry.name);
+      const key = path.relative(dir, absolute).split(path.sep).join("/");
+      if (entry.isDirectory()) {
+        await walk(absolute);
+        continue;
+      }
+      if (entry.isSymbolicLink()) {
+        // A symlinked file is published as its bytes; a symlinked directory is
+        // not walked, because one usually comes with a cycle, and a publisher
+        // that hangs is worse than one that says what it left out.
+        const stat = await fs.promises.stat(absolute).catch(() => null);
+        if (stat === null) {
+          skipped.push(`${key} (broken symlink)`);
+          continue;
+        }
+        if (!stat.isFile()) {
+          skipped.push(
+            `${key} (symlink to a ${stat.isDirectory() ? "directory" : "special file"})`,
+          );
+          continue;
+        }
+      } else if (!entry.isFile()) {
+        skipped.push(`${key} (not a regular file)`);
+        continue;
+      }
+      const { sha256, bytes } = await hashFile(absolute);
+      artifacts.push({ key, sha256, bytes, file: absolute });
+      totalBytes += bytes;
+    }
+  };
+
+  await walk(dir);
+  // Sorted so two publishes of one build declare the same list in the same
+  // order, which is what makes the build hash below reproducible.
+  artifacts.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+  return { artifacts, totalBytes, skipped };
+}
+
+async function hashFile(
+  absolute: string,
+): Promise<{ sha256: string; bytes: number }> {
+  const hash = crypto.createHash("sha256");
+  let bytes = 0;
+  // Streamed: an artifact may be 128 MB, and reading one into a Buffer to hash
+  // it is a needless way to run a CI runner out of memory.
+  const stream = fs.createReadStream(absolute);
+  await new Promise<void>((resolve, reject) => {
+    stream.on("data", (chunk) => {
+      hash.update(chunk);
+      bytes += chunk.length;
+    });
+    stream.on("end", resolve);
+    stream.on("error", reject);
+  });
+  return { sha256: hash.digest("hex"), bytes };
+}
+
+/**
+ * The build's own hash, when the build did not say.
+ *
+ * Content uses it for idempotency - declaring the same `buildHash` twice
+ * returns the same publish, which is what makes a retried CI job resume rather
+ * than start again - so it has to be a function of the build and nothing else.
+ * Every artifact's key and hash, in order: two runs of one build agree, and
+ * any change to any artifact is a different publish.
+ */
+export function buildHashOf(artifacts: Artifact[]): string {
+  const hash = crypto.createHash("sha256");
+  for (const artifact of artifacts) {
+    hash.update(`${artifact.key} ${artifact.sha256}\n`);
+  }
+  return hash.digest("hex");
+}
+
+/**
+ * Which dependency layer this build was built against.
+ *
+ * A layer is sent or named, never neither: the loader reads the head's layer
+ * by rev, and a publish that declares no layer at all loses every dependency
+ * on the next render. When the layer is being uploaded, its rev is inside it -
+ * it is gzipped JSON with a `rev` - so there is nothing to ask the caller for.
+ * When it is not, the build knows which stored layer it used, and says so with
+ * `--layer-rev`.
+ */
+export async function layerRevOf(
+  artifacts: Artifact[],
+): Promise<
+  | { status: "ok"; layerRev: string | null }
+  | { status: "error"; message: string }
+> {
+  const layer = artifacts.find((artifact) => artifact.key === "layer");
+  if (!layer) {
+    return { status: "ok", layerRev: null };
+  }
+  const bytes = await fs.promises.readFile(layer.file);
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(zlib.gunzipSync(bytes).toString("utf-8"));
+  } catch (err) {
+    return {
+      status: "error",
+      message:
+        `The layer artifact is not gzipped JSON: ${
+          err instanceof Error ? err.message : String(err)
+        }\n\n` +
+        "A layer is `{ rev, worker, browser, ... }`, gzipped. Pass --layer-rev to\n" +
+        "name a layer content already holds instead of sending one.",
+    };
+  }
+  const rev =
+    typeof parsed === "object" && parsed !== null
+      ? Reflect.get(parsed, "rev")
+      : undefined;
+  if (typeof rev !== "string" || rev === "") {
+    return {
+      status: "error",
+      message:
+        "The layer artifact has no rev, and a layer that is sent has to say which\n" +
+        "it is: that is the name the loader stores and later reuses it by.",
+    };
+  }
+  return { status: "ok", layerRev: rev };
+}
+
+/** Where the artifacts are, if the caller did not say. */
+export const DEFAULT_ARTIFACTS_DIR = ".val/publish";
+
+export function resolveArtifactsDir(options: {
+  root: string;
+  dir?: string;
+}): { status: "ok"; dir: string } | { status: "error"; message: string } {
+  const named = options.dir ?? DEFAULT_ARTIFACTS_DIR;
+  const dir = path.resolve(options.root, named);
+  if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
+    return {
+      status: "error",
+      message:
+        `No artifacts to publish: ${dir} is not a directory.\n\n` +
+        "Build the project first, then point at what it produced:\n\n" +
+        "    npx val publish --artifacts <directory>\n\n" +
+        "The path of each file under it is its artifact key - `server`, `client`,\n" +
+        "`public/...`, `chunk/client/...`, `layer`.",
+    };
+  }
+  return { status: "ok", dir };
+}

package/src/publish/client.ts

@@ -0,0 +1,177 @@
+import fs from "fs";
+import { getJson, postJson } from "./contentHost";
+import {
+  ArtifactsResponse,
+  DeclareBody,
+  DeclareResponse,
+  PromoteResponse,
+  StatusResponse,
+  UploadSlot,
+  VerifyResponse,
+  parseArtifacts,
+  parseDeclare,
+  parsePromote,
+  parseStatus,
+  parseVerify,
+} from "./protocol";
+
+export type PublishClient = {
+  declare(body: DeclareBody): Promise<DeclareResponse>;
+  confirmArtifacts(publishId: string): Promise<ArtifactsResponse>;
+  verify(publishId: string): Promise<VerifyResponse>;
+  promote(publishId: string): Promise<PromoteResponse>;
+  status(publishId: string): Promise<StatusResponse>;
+  upload(slot: UploadSlot, file: string): Promise<void>;
+};
+
+export function createPublishClient(options: {
+  host: string;
+  token: string;
+  fetchImpl?: typeof fetch;
+}): PublishClient {
+  const { host, token } = options;
+  const fetchImpl = options.fetchImpl ?? fetch;
+  // Every call, and nothing else: not a cookie, not the project's api key, not
+  // a personal access token.
+  const headers = { Authorization: `Bearer ${token}` };
+  const publishUrl = (publishId: string, step?: string) =>
+    `${host}/v1/publish/${encodeURIComponent(publishId)}${step ? `/${step}` : ""}`;
+
+  return {
+    declare: async (body) =>
+      parseDeclare(
+        await postJson({ url: `${host}/v1/publish`, headers, body, fetchImpl }),
+      ),
+    confirmArtifacts: async (publishId) =>
+      parseArtifacts(
+        await postJson({
+          url: publishUrl(publishId, "artifacts"),
+          headers,
+          fetchImpl,
+        }),
+      ),
+    verify: async (publishId) =>
+      parseVerify(
+        await postJson({
+          url: publishUrl(publishId, "verify"),
+          headers,
+          fetchImpl,
+        }),
+      ),
+    promote: async (publishId) =>
+      parsePromote(
+        await postJson({
+          url: publishUrl(publishId, "promote"),
+          headers,
+          fetchImpl,
+        }),
+      ),
+    status: async (publishId) =>
+      parseStatus(
+        await getJson({ url: publishUrl(publishId), headers, fetchImpl }),
+      ),
+    upload: (slot, file) => putArtifact(slot, file, fetchImpl),
+  };
+}
+
+/** Object storage refused the bytes, or could not be reached. */
+export class UploadError extends Error {
+  readonly statusCode: number;
+  readonly key: string;
+  constructor(statusCode: number, key: string, message: string) {
+    super(message);
+    this.name = "UploadError";
+    this.statusCode = statusCode;
+    this.key = key;
+  }
+}
+
+/**
+ * A slot's permission to write has run out.
+ *
+ * Not a failed publish: declaring again mints fresh slots and the publish
+ * carries on from where it was, which is why this is its own kind of failure
+ * rather than an error message.
+ */
+export function isExpiredSlot(err: UploadError): boolean {
+  return err.statusCode === 403 || err.statusCode === 401;
+}
+
+async function putArtifact(
+  slot: UploadSlot,
+  file: string,
+  fetchImpl: typeof fetch,
+): Promise<void> {
+  /*
+   * Read, rather than streamed.
+   *
+   * `ContentLength` is signed into the slot's URL, so the store rejects a body
+   * of a different size - and a streamed body goes out chunked, which is a
+   * different size as far as the signature is concerned. An artifact is capped
+   * at 128 MB by the API, which is the bound this trades against.
+   */
+  const body = await fs.promises.readFile(file);
+  let res: Response;
+  try {
+    res = await fetchImpl(slot.url, {
+      method: slot.method,
+      headers: slot.headers,
+      body,
+    });
+  } catch (err) {
+    throw new UploadError(
+      0,
+      slot.key,
+      `${slot.key}: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+  if (!res.ok) {
+    // The store answers XML, and a whole error document in a CI log buries the
+    // line that matters. The status is the actionable half.
+    throw new UploadError(
+      res.status,
+      slot.key,
+      `${slot.key}: object storage answered ${res.status} ${res.statusText}`,
+    );
+  }
+}
+
+/**
+ * Run `worker` over `items`, `limit` at a time.
+ *
+ * A project can have hundreds of small artifacts, so one at a time is minutes
+ * of latency and all at once is hundreds of open sockets on a CI runner. The
+ * first failure stops the pool: there is no point uploading the rest of a
+ * publish that is not going to be promoted.
+ */
+export async function pool<T>(
+  items: T[],
+  limit: number,
+  worker: (item: T) => Promise<void>,
+): Promise<void> {
+  let next = 0;
+  let failure: unknown = undefined;
+  const run = async (): Promise<void> => {
+    for (;;) {
+      if (failure !== undefined) {
+        return;
+      }
+      const index = next++;
+      if (index >= items.length) {
+        return;
+      }
+      try {
+        await worker(items[index]);
+      } catch (err) {
+        failure = err;
+        return;
+      }
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, () => run()),
+  );
+  if (failure !== undefined) {
+    throw failure;
+  }
+}