@valbuild/server 0.137.0 → 0.138.0

package/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @valbuild/server
 
+## 0.138.0
+
+### Minor Changes
+
+- [#754](https://github.com/valbuild/val/pull/754) [`6216172`](https://github.com/valbuild/val/commit/62161729aa40f737590b0a8093c0394385fffd8d) Thanks [@freekh](https://github.com/freekh)! - Connected projects hosted on the Val platform publish as a queued job too.
+
+  For a project with a GitHub repository whose site runs on the Val platform, pressing **Publish** is now a request, just as it is for a managed project. The Studio sends the job's changes; the content service pushes them as one commit on top of whatever is on the branch at that moment, so a developer's code push in between is kept. CI then builds it. A developer's change to the site's content files that this deployment has not seen stops the publish before anything is pushed, and says so.
+
+  - A build that CI reports as failed shows "Published, not on the site yet", with **View run**. The next publish builds again.
+  - New CLI command, `val ci-report --status failed|succeeded`, for a workflow's last step. It reads the commit, the branch and the run's address from GitHub Actions, uses the same `VAL_PROJECT_TOKEN` as `val publish`, and never fails the job.
+
+  Connected projects on a host of their own (for example Vercel) publish exactly as before.
+
+- [#754](https://github.com/valbuild/val/pull/754) [`40e933d`](https://github.com/valbuild/val/commit/40e933da7fe01cfe1c3f4a43c156f0b2f8a11f62) Thanks [@freekh](https://github.com/freekh)! - Publishing a managed project is now a queued job, and a publish is either live or it did not happen.
+
+  Pressing **Publish** in a managed project no longer commits first and builds after. It asks the content service for a publish job and returns at once; the Studio tab that pressed builds the job and uploads it, and the content service checks the build renders and makes it live — recording the commit only then. So there is no longer a "Saved, not yet live" state to get stuck in, and no **Finish publishing** button to get out of it.
+
+  - A press made while another publish is going out is queued behind it, and built when its turn comes by whichever open Studio tab is free.
+  - A tab that closes after uploading costs nothing: the rest of the publish is the content service's.
+  - A publish that fails before it goes live says so, with **Try again** and **Discard changes**. Your changes are kept either way.
+  - Publishing from the site (a page that cannot build) still opens a Studio window to build it; the window now runs the publish job for the page.
+
+  Connected projects (with a repository) that are hosted on the Val platform publish as a queued job too — see the note on connected projects. Those hosted elsewhere, for example on Vercel, publish exactly as before.
+
+  For the Val server: the publish proxy now forwards the publish-job routes, `/api/val/publish-job-prepare` renders a job's sources and hands content its archive, and `/api/val/built-source` is removed.
+
+### Patch Changes
+
+- Updated dependencies [[`57af895`](https://github.com/valbuild/val/commit/57af8959cc46519a405b1ed7441623f8bf2e6d27), [`c523506`](https://github.com/valbuild/val/commit/c523506c8b5e148864f401c005b913330e436499), [`6216172`](https://github.com/valbuild/val/commit/62161729aa40f737590b0a8093c0394385fffd8d), [`40e933d`](https://github.com/valbuild/val/commit/40e933da7fe01cfe1c3f4a43c156f0b2f8a11f62), [`5876fe0`](https://github.com/valbuild/val/commit/5876fe075fdee1f76f1cdaa2488ac52c58c894a8), [`c523506`](https://github.com/valbuild/val/commit/c523506c8b5e148864f401c005b913330e436499)]:
+  - @valbuild/core@0.138.0
+  - @valbuild/ui@0.138.0
+  - @valbuild/shared@0.138.0
+
 ## 0.137.0
 
 ### Minor Changes

package/dist/declarations/src/ValOps.d.ts

@@ -442,6 +442,12 @@ export declare abstract class ValOps {
      * an older service.
      */
     sourceMode(): "managed" | "connected" | null;
+    /**
+     * Does a press of Publish run as a publish job (valbuild/home,
+     * docs/app-mode.md, "Publishing is a queued job")? Never for a project
+     * with no content service.
+     */
+    publishesAsJobs(): boolean;
     /**
      * The branch the content service keeps this project's commits on, or `null`
      * where there is no such service, or it has not said yet.

package/dist/declarations/src/ValOpsHttp.d.ts

@@ -167,6 +167,12 @@ export declare class ValOpsHttp extends ValOps {
     sourceMode(): "managed" | "connected" | null;
     /** Remembered with {@link sourceMode}, from the same response. */
     projectBranch(): string | null;
+    /**
+     * Does a press of Publish run as a publish job? Every managed project, and
+     * a connected one content says is hosted on the platform (its CI publishes
+     * through content). `false` before anything has been heard.
+     */
+    publishesAsJobs(): boolean;
     /**
      * The short-lived publish token, and when it stops being usable.
      *
@@ -230,6 +236,26 @@ export declare class ValOpsHttp extends ValOps {
         body: string;
         contentType: string;
     }>;
+    /**
+     * A publish job's prepare, sent to content: the job's sources, archived
+     * there before the job is built (valbuild/home, docs/app-mode.md, "The
+     * archive, before the seal"). With this server's publish token, as every
+     * publish call is, and never through `publishApi` -- see the allowlist.
+     */
+    prepareJob(jobId: string, body: {
+        tab: string;
+        filesDirectory: string;
+        patchedSourceFiles: Record<string, string | null>;
+        patchedBinaryFilesDescriptors: PreparedCommit["patchedBinaryFilesDescriptors"];
+        modules: PreparedCommit["moduleVersions"];
+        /** Connected: the git commit this deployment was built from. */
+        gitCommit?: string;
+    }): Promise<{
+        status: number;
+        body: string;
+        contentType: string;
+    }>;
+    private callPublishApi;
     onInit(): Promise<void>;
     getPresignedAuthNonce(profileId: string, corsOrigin: string): Promise<{
         status: "success";

package/dist/valbuild-server.cjs.dev.js

@@ -5170,6 +5170,15 @@ class ValOps {
     return null;
   }
 
+  /**
+   * Does a press of Publish run as a publish job (valbuild/home,
+   * docs/app-mode.md, "Publishing is a queued job")? Never for a project
+   * with no content service.
+   */
+  publishesAsJobs() {
+    return false;
+  }
+
   /**
    * The branch the content service keeps this project's commits on, or `null`
    * where there is no such service, or it has not said yet.
@@ -8145,6 +8154,59 @@ async function readCommittedBinaryFiles(ops, descriptors) {
   };
 }
 
+/**
+ * A publish job's prepare, split into what the tab builds and what content
+ * archives.
+ *
+ * A job's content is its base plus every change it took (valbuild/home,
+ * docs/app-mode.md, "Publishing is a queued job"). The server has two
+ * `prepare`s to hand: `chainOnly` -- this build's embedded source with every
+ * commit since applied, which is the branch's head -- and `withJob`, the same
+ * with the job's pending changes on top. Anything applied since the job
+ * started came out of the job's own change set, so `withJob` IS the job's
+ * content; were that ever not so, the seal's containment check would restart
+ * the job rather than publish it.
+ *
+ * - The BUILD needs all of `withJob`'s source files: it is a whole site.
+ * - The ARCHIVE is what the commit publishes, so content is sent only what the
+ *   job changes: the files whose text differs from the head's, and the modules
+ *   and binary files its changes touch. Everything else is the head's, and an
+ *   archive that listed it would say this commit changed it.
+ *
+ * Files are compared rather than derived from module paths because a change to
+ * a module can write a file under another path -- a `.val.json` entry of a
+ * record -- which only the prepared output knows about.
+ */
+function splitJobPrepare(input) {
+  const job = new Set(input.jobPatchIds);
+  const before = input.chainOnly.patchedSourceFiles;
+  const after = input.withJob.patchedSourceFiles;
+  const patchedSourceFiles = {};
+  for (const [path, text] of Object.entries(after)) {
+    if (!(path in before) || before[path] !== text) {
+      patchedSourceFiles[path] = text;
+    }
+  }
+  const patchedBinaryFilesDescriptors = {};
+  for (const [path, descriptor] of Object.entries(input.withJob.patchedBinaryFilesDescriptors)) {
+    if (job.has(descriptor.patchId)) {
+      patchedBinaryFilesDescriptors[path] = descriptor;
+    }
+  }
+
+  // The modules any of the job's changes were applied to.
+  const touched = new Set(Object.entries(input.withJob.appliedPatches).filter(([, applied]) => applied.some(id => job.has(id))).map(([path]) => path));
+  const modules = Object.fromEntries(Object.entries(input.withJob.moduleVersions).filter(([path]) => touched.has(path)));
+  return {
+    buildSourceFiles: after,
+    archive: {
+      patchedSourceFiles,
+      patchedBinaryFilesDescriptors,
+      modules
+    }
+  };
+}
+
 /**
  * Which unpublished changes a failed save has to throw away to make progress.
  *
@@ -8329,7 +8391,13 @@ const GetApplicablePatches = z.z.object({
    */
   project: z.z.object({
     sourceMode: z.z.union([z.z.literal("managed"), z.z.literal("connected")]),
-    branch: z.z.string()
+    branch: z.z.string(),
+    /**
+     * Whether a press of Publish is a publish JOB. Absent from a content
+     * service that predates connected jobs: then managed projects publish
+     * as jobs, and connected ones by commit, as they did.
+     */
+    publishJobs: z.z.boolean().optional()
   }).optional()
 });
 const FilesResponse = z.z.object({
@@ -8686,6 +8754,17 @@ class ValOpsHttp extends ValOps {
     return ((_this$projectExpectat3 = this.projectExpectation) === null || _this$projectExpectat3 === void 0 ? void 0 : _this$projectExpectat3.branch) ?? null;
   }
 
+  /**
+   * Does a press of Publish run as a publish job? Every managed project, and
+   * a connected one content says is hosted on the platform (its CI publishes
+   * through content). `false` before anything has been heard.
+   */
+  publishesAsJobs() {
+    const expected = this.projectExpectation;
+    if (expected === null) return false;
+    return expected.publishJobs ?? expected.sourceMode === "managed";
+  }
+
   /**
    * The short-lived publish token, and when it stops being usable.
    *
@@ -8800,22 +8879,66 @@ class ValOpsHttp extends ValOps {
    * segment gets through.
    */
   static publishApiPathAllowed(path) {
-    if (path === "/build-target" || path === "/project-source" || path === "/update-target" || path === "/publish") {
+    if (path === "/build-target" || path === "/project-source" || path === "/update-target" || path === "/publish" ||
+    // Where "View run" goes, for a connected build CI reported failed.
+    // Read only: reporting a run is CI's, with its own token.
+    path === "/ci-runs/newest") {
+      return true;
+    }
+    /*
+     * Publishing as a queued job: a press, where it is, Try again, and the
+     * tab's reports about the job it runs. NOT `/publish-jobs/{id}/prepare`:
+     * its body is the job's source files and modules, which this server
+     * computes (`prepareJob`) -- a browser allowed to send it could publish
+     * any text at all as a commit's archive.
+     */
+    if (/^\/publish-requests(\/[A-Za-z0-9_-]{1,100})?$/.test(path)) {
+      return true;
+    }
+    if (/^\/publish-jobs\/(next|[A-Za-z0-9_-]{1,100}\/(steps|renew|cancel|discard))$/.test(path)) {
       return true;
     }
     return /^\/publish\/[A-Za-z0-9_-]+(\/(artifacts|verify|promote))?$/.test(path);
   }
   async publishApi(path, init) {
-    const json = "application/json";
     if (!ValOpsHttp.publishApiPathAllowed(path)) {
       return {
         status: 403,
-        contentType: json,
+        contentType: "application/json",
         body: JSON.stringify({
           message: `'${path}' is not part of the publish API.`
         })
       };
     }
+    return this.callPublishApi(path, init);
+  }
+
+  /**
+   * A publish job's prepare, sent to content: the job's sources, archived
+   * there before the job is built (valbuild/home, docs/app-mode.md, "The
+   * archive, before the seal"). With this server's publish token, as every
+   * publish call is, and never through `publishApi` -- see the allowlist.
+   */
+  async prepareJob(jobId, body) {
+    if (!/^[A-Za-z0-9_-]{1,100}$/.test(jobId)) {
+      return {
+        status: 400,
+        contentType: "application/json",
+        body: JSON.stringify({
+          message: `'${jobId}' is not a job id.`
+        })
+      };
+    }
+    return this.callPublishApi(`/publish-jobs/${jobId}/prepare`, {
+      method: "POST",
+      body: JSON.stringify({
+        ...body,
+        root: this.root
+      })
+    });
+  }
+  async callPublishApi(path, init) {
+    const json = "application/json";
     const send = async token => fetch(`${this.contentUrl}/v1${path}`, {
       method: init.method,
       headers: {
@@ -12967,6 +13090,9 @@ const ValServer = (valModules, options, callbacks) => {
             ...(sourceMode ? {
               sourceMode
             } : {}),
+            ...(sourceMode ? {
+              publishJobs: serverOps.publishesAsJobs()
+            } : {}),
             // Not `options.config` verbatim: in proxy mode the branch the
             // server resolved is filled in where the file did not name one.
             // See `clientConfig`.
@@ -14132,8 +14258,9 @@ const ValServer = (valModules, options, callbacks) => {
         return res;
       }
     },
-    "/built-source": {
-      GET: async req => {
+    "/publish-job-prepare": {
+      POST: async req => {
+        var _options$config$files2;
         const auth = getAuth(req.cookies);
         if (auth.error) {
           return {
@@ -14143,14 +14270,93 @@ const ValServer = (valModules, options, callbacks) => {
             }
           };
         }
-        if (!(serverOps instanceof ValOpsHttp) || !serverOps.embedsSource()) {
+        if (!(serverOps instanceof ValOpsHttp) || !serverOps.publishesAsJobs() || serverOps.sourceMode() === "managed" && !serverOps.embedsSource()) {
+          return {
+            status: 409,
+            json: {
+              message: "This deployment cannot prepare a publish job for this project."
+            }
+          };
+        }
+        const refusal = serverOps.publishRefusal();
+        if (refusal) {
           return {
             status: 409,
             json: {
-              message: "This server has no source of its own to render commits into."
+              message: refusal.message
             }
           };
         }
+        const {
+          jobId,
+          tab,
+          patchIds
+        } = req.body;
+        /*
+         * CONNECTED: the job's sources are what `/save` would commit -- its
+         * own changes, applied as `/save` applies them -- and content pushes
+         * them at the seal. Nothing is built here: CI builds after the push,
+         * so the answer carries the job and no sources.
+         */
+        if (serverOps.sourceMode() === "connected") {
+          var _options$config$files;
+          const jobPatches = await serverOps.fetchPatches({
+            patchIds,
+            excludePatchOps: false
+          });
+          if (jobPatches.error) {
+            return {
+              status: 500,
+              json: {
+                message: jobPatches.error.message
+              }
+            };
+          }
+          const found = new Set(jobPatches.patches.map(patch => patch.patchId));
+          const gone = patchIds.filter(id => !found.has(id));
+          if (gone.length > 0) {
+            return {
+              status: 409,
+              json: {
+                message: `This publish's changes are no longer all there (${gone.length} discarded).`
+              }
+            };
+          }
+          const prepared = await serverOps.prepare({
+            ...serverOps.analyzePatches(jobPatches.patches, jobPatches.commits, commit),
+            ...jobPatches
+          });
+          if (prepared.hasErrors) {
+            return {
+              status: 500,
+              json: {
+                message: "This publish's changes could not be rendered into source: " + JSON.stringify(prepared.sourceFilePatchErrors).slice(0, 500)
+              }
+            };
+          }
+          const split = splitJobPrepare({
+            // Every file the job's changes wrote is what the push writes.
+            chainOnly: {
+              patchedSourceFiles: {}
+            },
+            withJob: prepared,
+            jobPatchIds: patchIds
+          });
+          const sent = await serverOps.prepareJob(jobId, {
+            tab,
+            filesDirectory: ((_options$config$files = options.config.files) === null || _options$config$files === void 0 ? void 0 : _options$config$files.directory) || "/public/val",
+            ...split.archive,
+            ...(commit !== undefined ? {
+              gitCommit: commit
+            } : {})
+          });
+          return jobPrepareAnswer(sent, {
+            sourceFiles: {},
+            binaryFiles: {},
+            binaryFilesUnread: [],
+            branch: serverOps.projectBranch()
+          });
+        }
         const chain = await serverOps.fetchPatches({
           patchIds: undefined,
           excludePatchOps: false
@@ -14164,37 +14370,63 @@ const ValServer = (valModules, options, callbacks) => {
           };
         }
         /*
-         * The patches committed SINCE this build, as though pending.
+         * The job's content, in two steps from this build's embedded source.
          *
-         * `applicable/patches` answers with the ones applied at a commit after
-         * the one this build was made from, which is exactly what the text
-         * embedded in it is missing. `analyzePatches` skips applied patches --
-         * for a save they are already in the file it reads -- so they are
-         * handed over unapplied: here the file predates them.
+         * The commits since this build, as though pending, are the branch's
+         * head. The job's pending changes on top of them are its content: whatever was applied since the job started came
+         * out of its own changes (docs/app-mode.md, "Publishing is a queued
+         * job"), and were that ever not so, the seal would restart the job
+         * rather than publish it. Chain order, both times.
          */
+        const job = new Set(patchIds);
         const committed = chain.patches.filter(patch => patch.appliedAt !== null).map(patch => ({
           ...patch,
           appliedAt: null
         }));
-        const prepared = await serverOps.prepare({
-          ...serverOps.analyzePatches(committed, chain.commits, commit),
+        const pending = chain.patches.filter(patch => patch.appliedAt === null && job.has(patch.patchId));
+        const known = new Set(chain.patches.map(patch => patch.patchId));
+        const gone = patchIds.filter(id => !known.has(id));
+        if (gone.length > 0) {
+          return {
+            status: 409,
+            json: {
+              message: `This publish's changes are no longer all there (${gone.length} discarded).`
+            }
+          };
+        }
+        const prepareOf = patches => serverOps.prepare({
+          ...serverOps.analyzePatches(patches, chain.commits, commit),
           ...chain,
-          patches: committed
+          patches
         });
-        if (prepared.hasErrors) {
+        const chainOnly = await prepareOf(committed);
+        const withJob = await prepareOf([...committed, ...pending]);
+        if (chainOnly.hasErrors || withJob.hasErrors) {
           return {
             status: 500,
             json: {
-              message: "The commits since this build could not be rendered into source: " + JSON.stringify(prepared.sourceFilePatchErrors).slice(0, 500)
+              message: "This publish's changes could not be rendered into source: " + JSON.stringify((withJob.hasErrors ? withJob : chainOnly).sourceFilePatchErrors).slice(0, 500)
             }
           };
         }
-        return {
-          status: 200,
-          json: {
-            files: prepared.patchedSourceFiles
-          }
-        };
+        const split = splitJobPrepare({
+          chainOnly,
+          withJob,
+          jobPatchIds: patchIds
+        });
+        // Read now, while the job's changes still hold their files.
+        const binaries = await readCommittedBinaryFiles(serverOps, split.archive.patchedBinaryFilesDescriptors);
+        const sent = await serverOps.prepareJob(jobId, {
+          tab,
+          filesDirectory: ((_options$config$files2 = options.config.files) === null || _options$config$files2 === void 0 ? void 0 : _options$config$files2.directory) || "/public/val",
+          ...split.archive
+        });
+        return jobPrepareAnswer(sent, {
+          sourceFiles: split.buildSourceFiles,
+          binaryFiles: binaries.files,
+          binaryFilesUnread: binaries.unread,
+          branch: serverOps.projectBranch()
+        });
       }
     },
     "/profiles": {
@@ -14649,8 +14881,8 @@ const ValServer = (valModules, options, callbacks) => {
             const managedBranch = serverOps.sourceMode() === "managed" ? serverOps.projectBranch() : null;
             const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors) : null;
             const commitToGit = () => {
-              var _options$config$files;
-              return serverOps.commit(preparedCommit, message, auth.id, ((_options$config$files = options.config.files) === null || _options$config$files === void 0 ? void 0 : _options$config$files.directory) || "/public/val", undefined,
+              var _options$config$files3;
+              return serverOps.commit(preparedCommit, message, auth.id, ((_options$config$files3 = options.config.files) === null || _options$config$files3 === void 0 ? void 0 : _options$config$files3.directory) || "/public/val", undefined,
               /*
                * Forwarded verbatim, and only the client can decide it: the
                * content API closes the group it is named without checking
@@ -16304,6 +16536,49 @@ function getIsRemoteRequired(schemas) {
   return false;
 }
 
+/**
+ * `/publish-job-prepare`'s answer from content's: the job as content has it
+ * now, and what the tab builds from. Content's refusal is passed on -- 409
+ * as 409, anything else as 502, which the tab reads as "content answered,
+ * and has counted the attempt" (see `runStudioJob`).
+ */
+function jobPrepareAnswer(sent, build) {
+  if (sent.status !== 200) {
+    let message = `The content service did not prepare the publish (${sent.status}).`;
+    try {
+      const parsed = JSON.parse(sent.body);
+      if (typeof parsed === "object" && parsed !== null && "message" in parsed && typeof parsed.message === "string") {
+        message = parsed.message;
+      }
+    } catch {
+      // not JSON: the status says enough
+    }
+    return {
+      status: sent.status === 409 ? 409 : 502,
+      json: {
+        message
+      }
+    };
+  }
+  try {
+    const prepared = internal.parseJob(JSON.parse(sent.body), "POST /v1/publish-jobs/{id}/prepare");
+    return {
+      status: 200,
+      json: {
+        job: prepared.job,
+        ...build
+      }
+    };
+  } catch (e) {
+    return {
+      status: 502,
+      json: {
+        message: e instanceof Error ? e.message : String(e)
+      }
+    };
+  }
+}
+
 async function createValServer(valModules, route, opts, config, callbacks, formatter,
 /**
  * Called after a save has applied its patches. EXPERIMENTAL — see