@valbuild/cli 0.133.0 → 0.134.1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@valbuild/cli",
   "private": false,
-  "version": "0.133.0",
+  "version": "0.134.1",
   "description": "Val CLI tools",
   "repository": {
     "type": "git",
@@ -24,11 +24,13 @@
     "jszip": "^3.10.1",
     "meow": "^9.0.0",
     "picocolors": "^1.1.1",
+    "zod": "^4.4.3",
+    "zod-validation-error": "^5.0.0",
     "@valbuild/eslint-plugin": "0.125.0",
-    "@valbuild/language-server": "0.133.0",
-    "@valbuild/server": "0.133.0",
-    "@valbuild/shared": "0.133.0",
-    "@valbuild/core": "0.130.0"
+    "@valbuild/core": "0.134.0",
+    "@valbuild/language-server": "0.134.1",
+    "@valbuild/server": "0.134.1",
+    "@valbuild/shared": "0.134.1"
   },
   "peerDependencies": {
     "prettier": "*",

package/src/cli.ts

@@ -9,6 +9,7 @@ import { login } from "./login";
 import { lsp } from "./lsp";
 import { debug } from "./debug";
 import { deleteUnappliablePatches } from "./deleteUnappliablePatches";
+import { publish } from "./publish";
 
 async function main(): Promise<void> {
   const { input, flags, showHelp } = meow(
@@ -24,6 +25,7 @@ async function main(): Promise<void> {
         login
         files
         connect
+        publish
         versions
         lsp
         debug
@@ -48,6 +50,32 @@ async function main(): Promise<void> {
       Options:
         --root [root], -r [root] Set project root directory (default process.cwd())
 
+      Command: publish
+      Description: publish this project's build through content.val.build.
+        Declares the build's artifacts, uploads the ones content does not already hold,
+        and has content verify it by building and rendering a canary before the site
+        changes. Authenticates with VAL_PROJECT_TOKEN, or with the "val login" token in
+        .val/pat.json (which needs the project, as "<org>/<project>", in val.config or
+        VAL_PROJECT). Never as a flag: an argument is visible to anyone who can list
+        processes, and it is kept in shell history and in every CI log.
+      Options:
+        --root [root], -r [root] Set project root directory (default process.cwd())
+        --artifacts [dir]        The built artifacts (default <root>/.val/publish). The path
+                                 of each file under it is its artifact key: server, client,
+                                 css, rsc, layer, or a path under chunk/server, chunk/client,
+                                 chunk/rsc, asset, public
+        --commit [sha]           The commit this build is of (default VAL_GIT_COMMIT,
+                                 else GITHUB_SHA, else git HEAD)
+        --branch [name]          The branch content commits saves to (default VAL_GIT_BRANCH,
+                                 else GITHUB_REF_NAME, else the current git branch)
+        --layer-rev [rev]        Name the dependency layer content already holds, when this
+                                 build does not send one
+        --build-hash [hash]      Identify the build (default: a hash of its artifacts).
+                                 Declaring the same one twice resumes that publish
+        --links-own-css          The app links its own CSS (--no-links-own-css for the
+                                 opposite; omit it when the build did not say)
+        --dry-run                Verify, and stop before the site changes
+
       Command: list-unused-files
       Description: EXPERIMENTAL.
         List files that are in the configured files directory (files.directory, default public/val) but not in use by any Val module.
@@ -112,6 +140,18 @@ async function main(): Promise<void> {
         out: {
           type: "string",
         },
+        artifacts: {
+          type: "string",
+        },
+        layerRev: {
+          type: "string",
+        },
+        buildHash: {
+          type: "string",
+        },
+        linksOwnCss: {
+          type: "boolean",
+        },
         commit: {
           type: "string",
         },
@@ -181,6 +221,17 @@ async function main(): Promise<void> {
         yes: flags.yes,
         verbose: flags.verbose,
       });
+    case "publish":
+      return publish({
+        root: flags.root,
+        artifacts: flags.artifacts,
+        commit: flags.commit,
+        branch: flags.branch,
+        layerRev: flags.layerRev,
+        buildHash: flags.buildHash,
+        linksOwnCss: flags.linksOwnCss,
+        dryRun: flags.dryRun,
+      });
     case "login":
       return login({
         root: flags.root,

package/src/publish/artifacts.ts

@@ -0,0 +1,194 @@
+import crypto from "crypto";
+import fs from "fs";
+import path from "path";
+import zlib from "zlib";
+import { DeclaredArtifact } from "./protocol";
+
+/**
+ * Where an artifact's bytes are, and what it is called on the wire.
+ *
+ * The key namespace is flat and is content's: `server`, `client`, `css`,
+ * `rsc`, `layer`, and paths under `chunk/server/`, `chunk/client/`,
+ * `chunk/rsc/`, `asset/` and `public/`. Content validates it and answers with
+ * every problem at once, so this does not re-implement the rules - a second,
+ * slightly different copy of them here is how a CLI comes to refuse a publish
+ * the service would have taken.
+ */
+export type Artifact = DeclaredArtifact & { file: string };
+
+export type CollectedArtifacts = {
+  artifacts: Artifact[];
+  totalBytes: number;
+  skipped: string[];
+};
+
+/**
+ * The artifacts to publish, read from a directory laid out by key.
+ *
+ * The path under the directory IS the artifact key: a file at `public/app.css`
+ * is the artifact `public/app.css`, and `server` is the server bundle. Nothing
+ * is renamed on the way - an asset is addressed by the path the built code
+ * imports it at, so a normalised name produces a bundle whose imports resolve
+ * to nothing, at runtime, in the isolate.
+ *
+ * **This is the seam with the build.** `val publish` uploads artifacts; it does
+ * not produce them. Whatever builds the project - today the platform's own
+ * builder, which is where the wire, rolldown and dependency-layer steps live -
+ * writes this directory, and this reads it.
+ */
+export async function collectArtifacts(
+  dir: string,
+): Promise<CollectedArtifacts> {
+  const artifacts: Artifact[] = [];
+  const skipped: string[] = [];
+  let totalBytes = 0;
+
+  const walk = async (current: string): Promise<void> => {
+    const entries = await fs.promises.readdir(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const absolute = path.join(current, entry.name);
+      const key = path.relative(dir, absolute).split(path.sep).join("/");
+      if (entry.isDirectory()) {
+        await walk(absolute);
+        continue;
+      }
+      if (entry.isSymbolicLink()) {
+        // A symlinked file is published as its bytes; a symlinked directory is
+        // not walked, because one usually comes with a cycle, and a publisher
+        // that hangs is worse than one that says what it left out.
+        const stat = await fs.promises.stat(absolute).catch(() => null);
+        if (stat === null) {
+          skipped.push(`${key} (broken symlink)`);
+          continue;
+        }
+        if (!stat.isFile()) {
+          skipped.push(
+            `${key} (symlink to a ${stat.isDirectory() ? "directory" : "special file"})`,
+          );
+          continue;
+        }
+      } else if (!entry.isFile()) {
+        skipped.push(`${key} (not a regular file)`);
+        continue;
+      }
+      const { sha256, bytes } = await hashFile(absolute);
+      artifacts.push({ key, sha256, bytes, file: absolute });
+      totalBytes += bytes;
+    }
+  };
+
+  await walk(dir);
+  // Sorted so two publishes of one build declare the same list in the same
+  // order, which is what makes the build hash below reproducible.
+  artifacts.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+  return { artifacts, totalBytes, skipped };
+}
+
+async function hashFile(
+  absolute: string,
+): Promise<{ sha256: string; bytes: number }> {
+  const hash = crypto.createHash("sha256");
+  let bytes = 0;
+  // Streamed: an artifact may be 128 MB, and reading one into a Buffer to hash
+  // it is a needless way to run a CI runner out of memory.
+  const stream = fs.createReadStream(absolute);
+  await new Promise<void>((resolve, reject) => {
+    stream.on("data", (chunk) => {
+      hash.update(chunk);
+      bytes += chunk.length;
+    });
+    stream.on("end", resolve);
+    stream.on("error", reject);
+  });
+  return { sha256: hash.digest("hex"), bytes };
+}
+
+/**
+ * The build's own hash, when the build did not say.
+ *
+ * Content uses it for idempotency - declaring the same `buildHash` twice
+ * returns the same publish, which is what makes a retried CI job resume rather
+ * than start again - so it has to be a function of the build and nothing else.
+ * Every artifact's key and hash, in order: two runs of one build agree, and
+ * any change to any artifact is a different publish.
+ */
+export function buildHashOf(artifacts: Artifact[]): string {
+  const hash = crypto.createHash("sha256");
+  for (const artifact of artifacts) {
+    hash.update(`${artifact.key} ${artifact.sha256}\n`);
+  }
+  return hash.digest("hex");
+}
+
+/**
+ * Which dependency layer this build was built against.
+ *
+ * A layer is sent or named, never neither: the loader reads the head's layer
+ * by rev, and a publish that declares no layer at all loses every dependency
+ * on the next render. When the layer is being uploaded, its rev is inside it -
+ * it is gzipped JSON with a `rev` - so there is nothing to ask the caller for.
+ * When it is not, the build knows which stored layer it used, and says so with
+ * `--layer-rev`.
+ */
+export async function layerRevOf(
+  artifacts: Artifact[],
+): Promise<
+  | { status: "ok"; layerRev: string | null }
+  | { status: "error"; message: string }
+> {
+  const layer = artifacts.find((artifact) => artifact.key === "layer");
+  if (!layer) {
+    return { status: "ok", layerRev: null };
+  }
+  const bytes = await fs.promises.readFile(layer.file);
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(zlib.gunzipSync(bytes).toString("utf-8"));
+  } catch (err) {
+    return {
+      status: "error",
+      message:
+        `The layer artifact is not gzipped JSON: ${
+          err instanceof Error ? err.message : String(err)
+        }\n\n` +
+        "A layer is `{ rev, worker, browser, ... }`, gzipped. Pass --layer-rev to\n" +
+        "name a layer content already holds instead of sending one.",
+    };
+  }
+  const rev =
+    typeof parsed === "object" && parsed !== null
+      ? Reflect.get(parsed, "rev")
+      : undefined;
+  if (typeof rev !== "string" || rev === "") {
+    return {
+      status: "error",
+      message:
+        "The layer artifact has no rev, and a layer that is sent has to say which\n" +
+        "it is: that is the name the loader stores and later reuses it by.",
+    };
+  }
+  return { status: "ok", layerRev: rev };
+}
+
+/** Where the artifacts are, if the caller did not say. */
+export const DEFAULT_ARTIFACTS_DIR = ".val/publish";
+
+export function resolveArtifactsDir(options: {
+  root: string;
+  dir?: string;
+}): { status: "ok"; dir: string } | { status: "error"; message: string } {
+  const named = options.dir ?? DEFAULT_ARTIFACTS_DIR;
+  const dir = path.resolve(options.root, named);
+  if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
+    return {
+      status: "error",
+      message:
+        `No artifacts to publish: ${dir} is not a directory.\n\n` +
+        "Build the project first, then point at what it produced:\n\n" +
+        "    npx val publish --artifacts <directory>\n\n" +
+        "The path of each file under it is its artifact key - `server`, `client`,\n" +
+        "`public/...`, `chunk/client/...`, `layer`.",
+    };
+  }
+  return { status: "ok", dir };
+}

package/src/publish/client.ts

@@ -0,0 +1,177 @@
+import fs from "fs";
+import { getJson, postJson } from "./contentHost";
+import {
+  ArtifactsResponse,
+  DeclareBody,
+  DeclareResponse,
+  PromoteResponse,
+  StatusResponse,
+  UploadSlot,
+  VerifyResponse,
+  parseArtifacts,
+  parseDeclare,
+  parsePromote,
+  parseStatus,
+  parseVerify,
+} from "./protocol";
+
+export type PublishClient = {
+  declare(body: DeclareBody): Promise<DeclareResponse>;
+  confirmArtifacts(publishId: string): Promise<ArtifactsResponse>;
+  verify(publishId: string): Promise<VerifyResponse>;
+  promote(publishId: string): Promise<PromoteResponse>;
+  status(publishId: string): Promise<StatusResponse>;
+  upload(slot: UploadSlot, file: string): Promise<void>;
+};
+
+export function createPublishClient(options: {
+  host: string;
+  token: string;
+  fetchImpl?: typeof fetch;
+}): PublishClient {
+  const { host, token } = options;
+  const fetchImpl = options.fetchImpl ?? fetch;
+  // Every call, and nothing else: not a cookie, not the project's api key, not
+  // a personal access token.
+  const headers = { Authorization: `Bearer ${token}` };
+  const publishUrl = (publishId: string, step?: string) =>
+    `${host}/v1/publish/${encodeURIComponent(publishId)}${step ? `/${step}` : ""}`;
+
+  return {
+    declare: async (body) =>
+      parseDeclare(
+        await postJson({ url: `${host}/v1/publish`, headers, body, fetchImpl }),
+      ),
+    confirmArtifacts: async (publishId) =>
+      parseArtifacts(
+        await postJson({
+          url: publishUrl(publishId, "artifacts"),
+          headers,
+          fetchImpl,
+        }),
+      ),
+    verify: async (publishId) =>
+      parseVerify(
+        await postJson({
+          url: publishUrl(publishId, "verify"),
+          headers,
+          fetchImpl,
+        }),
+      ),
+    promote: async (publishId) =>
+      parsePromote(
+        await postJson({
+          url: publishUrl(publishId, "promote"),
+          headers,
+          fetchImpl,
+        }),
+      ),
+    status: async (publishId) =>
+      parseStatus(
+        await getJson({ url: publishUrl(publishId), headers, fetchImpl }),
+      ),
+    upload: (slot, file) => putArtifact(slot, file, fetchImpl),
+  };
+}
+
+/** Object storage refused the bytes, or could not be reached. */
+export class UploadError extends Error {
+  readonly statusCode: number;
+  readonly key: string;
+  constructor(statusCode: number, key: string, message: string) {
+    super(message);
+    this.name = "UploadError";
+    this.statusCode = statusCode;
+    this.key = key;
+  }
+}
+
+/**
+ * A slot's permission to write has run out.
+ *
+ * Not a failed publish: declaring again mints fresh slots and the publish
+ * carries on from where it was, which is why this is its own kind of failure
+ * rather than an error message.
+ */
+export function isExpiredSlot(err: UploadError): boolean {
+  return err.statusCode === 403 || err.statusCode === 401;
+}
+
+async function putArtifact(
+  slot: UploadSlot,
+  file: string,
+  fetchImpl: typeof fetch,
+): Promise<void> {
+  /*
+   * Read, rather than streamed.
+   *
+   * `ContentLength` is signed into the slot's URL, so the store rejects a body
+   * of a different size - and a streamed body goes out chunked, which is a
+   * different size as far as the signature is concerned. An artifact is capped
+   * at 128 MB by the API, which is the bound this trades against.
+   */
+  const body = await fs.promises.readFile(file);
+  let res: Response;
+  try {
+    res = await fetchImpl(slot.url, {
+      method: slot.method,
+      headers: slot.headers,
+      body,
+    });
+  } catch (err) {
+    throw new UploadError(
+      0,
+      slot.key,
+      `${slot.key}: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+  if (!res.ok) {
+    // The store answers XML, and a whole error document in a CI log buries the
+    // line that matters. The status is the actionable half.
+    throw new UploadError(
+      res.status,
+      slot.key,
+      `${slot.key}: object storage answered ${res.status} ${res.statusText}`,
+    );
+  }
+}
+
+/**
+ * Run `worker` over `items`, `limit` at a time.
+ *
+ * A project can have hundreds of small artifacts, so one at a time is minutes
+ * of latency and all at once is hundreds of open sockets on a CI runner. The
+ * first failure stops the pool: there is no point uploading the rest of a
+ * publish that is not going to be promoted.
+ */
+export async function pool<T>(
+  items: T[],
+  limit: number,
+  worker: (item: T) => Promise<void>,
+): Promise<void> {
+  let next = 0;
+  let failure: unknown = undefined;
+  const run = async (): Promise<void> => {
+    for (;;) {
+      if (failure !== undefined) {
+        return;
+      }
+      const index = next++;
+      if (index >= items.length) {
+        return;
+      }
+      try {
+        await worker(items[index]);
+      } catch (err) {
+        failure = err;
+        return;
+      }
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, () => run()),
+  );
+  if (failure !== undefined) {
+    throw failure;
+  }
+}

package/src/publish/contentApi.ts

@@ -0,0 +1,210 @@
+/**
+ * The publish API's types, COPIED from the service that serves them.
+ *
+ * Source: `content/src/handlers/Api.ts` in valbuild/home, branch
+ * `claude/new-project-studio-saves-0g9ksb`, commit `ecf3b8d`. The routes below
+ * are that file's `/publish*` entries and the two types they use, verbatim,
+ * comments included, so that the two can be diffed by eye.
+ *
+ * **Copied rather than imported, because it cannot be imported.** That file
+ * lives in a private repository which publishes nothing to npm, and its own
+ * header says this is how it is kept in step: "We have also used it (by just
+ * copying it in and setting the types there) in the @valbuild/server package to
+ * check that we are more or less in sync."
+ *
+ * **When home's `Api.ts` changes, change this with it.** Everything in
+ * `protocol.ts` is checked against these types, so a copy brought up to date
+ * fails to compile wherever this CLI has not caught up. That is the whole
+ * value: without it a wire change is a runtime 500 in somebody's CI, which is
+ * how `home` and `@valbuild/server` have already diverged three times - see
+ * `homeWireContract.test.ts` in `@valbuild/server` for the last one, and
+ * `publishWireContract.test.ts` here for the fixtures that go with these types.
+ *
+ * A copy is not a guarantee, only a tripwire: nothing checks it against the
+ * service, and the parsers in `protocol.ts` are what actually holds at runtime.
+ */
+
+export type ContentPublishApi = {
+  /**
+   * Publishing a build, as a resource with a lifecycle.
+   *
+   * ```
+   * POST /publish                  declare  -> an upload slot per MISSING artifact
+   * PUT  <presigned url>           upload   -> each artifact, straight to storage
+   * POST /publish/{id}/artifacts   confirm  -> the uploads are checked
+   * POST /publish/{id}/verify      render   -> a canary build, server side
+   * POST /publish/{id}/promote     go live  -> the pointer moves
+   * GET  /publish/{id}             status
+   * ```
+   *
+   * Four steps rather than one `POST /publish`, because they fail differently
+   * and one call cannot say "the bytes are fine but it did not render" -- which
+   * is the sentence a publisher most needs. `promote` is separate from `verify`
+   * so that a dry run is the absence of a call rather than a flag.
+   *
+   * Not keyed by a project, like `/publish-target` above and for the same
+   * reason: a project token names one. That is what lets a generated repository
+   * hold one secret and no variables at all.
+   *
+   * ## What is deliberately not here
+   *
+   * No loader URL, no `vendorRev`, no `x-platform-project`. This service holds
+   * the operator relationship with the build platform and calls it; a publisher
+   * talks to this API and nothing else. The one exception is the presigned
+   * upload URL, which points at object storage -- bytes do not travel through
+   * here.
+   *
+   * It is also what makes this publishable by a project token at all. The
+   * platform's own verify step publishes a canary to a throwaway project, and a
+   * throwaway has no secrets, so the loader has nothing to check a caller
+   * against and refuses with a 503 naming a project nobody has heard of. Behind
+   * this API the caller is never involved in that exchange.
+   */
+  "/publish": {
+    POST: {
+      body: {
+        /** The build's own hash. Repeating it returns the same publish. */
+        buildHash: string;
+        /** Null only for a seed publish. See `publishPlan.ts`. */
+        commit: string | null;
+        /** The branch content saves commit to. Required when `commit` is set. */
+        branch: string | null;
+        /** Which dependency layer this was built against, sent or not. */
+        layerRev: string | null;
+        /**
+         * Whether the app links its own CSS.
+         *
+         * Build metadata the loader needs and no artifact carries, so it has to
+         * be declared. Null is a third answer -- "this build did not say" --
+         * and is not false.
+         */
+        linksOwnCss: boolean | null;
+        artifacts: {
+          /** See `publishPlan.ts` for the namespace. */
+          key: string;
+          sha256: string;
+          bytes: number;
+        }[];
+      };
+      res: {
+        publishId: string;
+        state: PublishState;
+        project: {
+          publicProjectId: string;
+          /** Null when the project has no site yet -- reported, not refused. */
+          siteUrl: string | null;
+        };
+        /**
+         * One per artifact this project does not already hold, and nothing else.
+         *
+         * So this doubles as the answer to "what is missing": there is no
+         * separate field to keep in step with it. Slots expire; a presigned PUT
+         * that answers 403 means declare again, not that the publish failed.
+         */
+        uploads: {
+          key: string;
+          url: string;
+          method: "PUT";
+          headers: Record<string, string>;
+          /** ISO 8601. */
+          expiresAt: string;
+        }[];
+        /** Keys already held, so a caller can see what it did not have to send. */
+        have: string[];
+      };
+    };
+  };
+  "/publish/:publishId": {
+    GET: {
+      res: {
+        publishId: string;
+        state: PublishState;
+        buildHash: string;
+        /** Artifact keys still not uploaded. */
+        missing: string[];
+        problems: PublishProblem[];
+      };
+    };
+  };
+  /** Everything asked for has been uploaded. The uploads are checked here. */
+  "/publish/:publishId/artifacts": {
+    POST: {
+      res: {
+        state: PublishState;
+        problems: PublishProblem[];
+      };
+    };
+  };
+  /** A canary build and render, on the build platform, with our credential. */
+  "/publish/:publishId/verify": {
+    POST: {
+      res: {
+        state: PublishState;
+        ok: boolean;
+        /** Where the canary can be looked at, when it rendered. */
+        previewUrl: string | null;
+        problems: PublishProblem[];
+      };
+    };
+  };
+  /**
+   * Move the project's pointer to this build.
+   *
+   * Refused when the commit is no longer the branch head -- which this service
+   * is the authority on (see `getGitHead`), so the rule is enforced where the
+   * data already is rather than a round trip away.
+   */
+  "/publish/:publishId/promote": {
+    POST: {
+      res: {
+        state: PublishState;
+        url: string | null;
+        commit: string | null;
+      };
+    };
+  };
+  /** Exchange a personal access token for a short-lived publish token. */
+  "/publish-token": {
+    POST: {
+      res: {
+        /** Shown once, here. Nothing can produce it again. */
+        token: string;
+        /** ISO 8601, because a `Date` does not survive JSON as one. */
+        expiresAt: string | null;
+        publicProjectId: string;
+        productionUrl: string | null;
+      };
+    };
+  };
+};
+
+/**
+ * Where a publish is. Mirrors `PublishState` in `utils/publishPlan.ts`, which
+ * owns the transition table; this is the wire spelling of it.
+ */
+export type PublishState =
+  | "awaiting-artifacts"
+  | "ready"
+  | "verified"
+  | "live"
+  | "failed"
+  | "expired";
+
+/**
+ * Why a publish is not going anywhere.
+ *
+ * `code` is the stable half -- a pipeline gates on it -- and carries both this
+ * API's own codes (`ARTIFACT_MISMATCH`, `POINTER_STALE`, the declaration codes
+ * in `publishPlan.ts`) and the build platform's `PLATFORM*` codes passed
+ * through from a verify, because rewording those would lose the only sentence
+ * that says what to change.
+ *
+ * `hint` is not decoration. A gate that merely fails is useless in somebody
+ * else's pipeline: they get a red build and no idea why.
+ */
+export type PublishProblem = {
+  code: string;
+  message: string;
+  hint?: string;
+  keys?: string[];
+};