@sanoma-ai/workflows 0.7.1 → 0.8.0

package/src/source.ts

@@ -28,8 +28,9 @@ export interface Source {
   /** Pushes `ref` to the repository at `url`, signed in with `token` (a draft by its `branch`). */
   push?(target: { url: string; token: string; ref: string }): void | Promise<void>;
   /**
-   * The draft `id` (letters, digits, `_` and `-`), opened: made from the checked-out revision the
-   * first time, then the same one, edits and all, until its branch is deleted.
+   * The draft `id` (letters, digits, `_` and `-`), opened on its branch (`draftBranch`): made from
+   * the checked-out revision the first time, then the same one, edits and all, until its branch is
+   * deleted.
    */
   draft?(id: string): Promise<Draft>;
   /** The open drafts' ids, sorted. */
@@ -47,13 +48,27 @@ export interface Source {
   /** The URL of the repository this one was cloned from (`origin`); undefined when it has none. */
   remoteUrl?(): Promise<string | undefined>;
   /**
-   * Fast-forwards the checked-out branch to `ref` in the repository at `url`, signed in with
-   * `token`, as `push` is: only when `ref` is checked out, nothing is uncommitted and the remote's
-   * `ref` contains it; otherwise `skipped`, saying why. The checked-out revision after.
+   * Fast-forwards the branch `ref` to `ref` in the repository at `url`, signed in with `token`, as
+   * `push` is, where it is checked out: the root's checked-out branch, or an open draft's (by its
+   * `branch`), whose new commits on the host it takes so. Only when nothing is uncommitted there
+   * and the remote's `ref` contains it; otherwise `skipped`, saying why. The revision after.
    */
   follow?(target: { url: string; token: string; ref: string }): Promise<{ revision: string } | { skipped: string }>;
+  /**
+   * How many commits the source repository of the app `folder` (the URL its `apps/<folder>/app.ts`
+   * names) has on its default branch that the checked-out one lacks, of those changing what an
+   * update merges (`apps/<folder>/` and `package.json`): fetched now. An error when the app names
+   * no source.
+   */
+  behind?(folder: string): Promise<number>;
 }
 
+/** The branch of the draft `id`, as a source names it: `draft/<id>`. */
+export const draftBranch = (id: string) => `draft/${id}`;
+
+/** The draft whose branch `branch` is (`draftBranch`), or undefined for a branch that is no draft's. */
+export const draftOf = (branch: string) => (branch.startsWith("draft/") ? branch.slice("draft/".length) : undefined);
+
 /**
  * A draft: a checkout of its own, at `root`, on `branch`, which `commit` records in as
  * `Source.commit` does in its root. Its `node_modules` may be the root's own install, linked in:
@@ -63,6 +78,36 @@ export interface Draft {
   branch: string;
   root: string;
   commit: Source["commit"];
+  /**
+   * Takes an app into the draft: fetches `ref` (the default branch by default) of the repository
+   * at `url`, a source (a public company repo with one `apps/<folder>/`), and merges it in with its
+   * history (`--allow-unrelated-histories`), as one merge commit by `author`. What it merges is
+   * `apps/<folder>/` and `package.json`; every other file stays the draft's. `folder` may be left
+   * out when the source has one app. Refused when the draft has `apps/<folder>/` already.
+   */
+  take?(app: { url: string; ref?: string; folder?: string; author: string }): Promise<AppMerge>;
+  /**
+   * Merges the new commits of the app `folder`'s source (the URL its `app.ts` names, its default
+   * branch) into the draft, by `author`, as `take` merges: `apps/<folder>/` and `package.json`.
+   */
+  update?(app: { folder: string; author: string }): Promise<AppMerge>;
+}
+
+/**
+ * What taking or updating an app merged into a draft. A line both sides changed is committed with
+ * git's markers in its file (ours, the merge base and theirs), so the draft's checks fail until the
+ * agent resolves it; a file one side deleted stays as the draft had it. The merge commit keeps the
+ * source's history either way.
+ */
+export interface AppMerge {
+  /** The app's folder: `apps/<folder>/`. */
+  folder: string;
+  /** The draft's commit after the merge. */
+  revision: string;
+  /** The source's commits the merge brought in that change the app's folder or `package.json`; 0 when the draft had them all, and nothing was merged. */
+  merged: number;
+  /** The files git could not merge, relative to the root, committed with conflict markers; empty when it merged clean. */
+  conflicts: string[];
 }
 
 /** A draft's change: the files it touches, relative to the root, and git's `--stat` of it. */

package/src/version.ts

@@ -63,24 +63,33 @@ export function computeVersion({
 }): string {
   const named = process.env.DBOS__APPVERSION?.trim();
   if (named) return `${appName}@${named}`;
-  const workflows = definitions
-    .map(
-      (wf) =>
-        [
-          wf.name,
-          wf.run.toString(),
-          opIdsOf(wf),
-          // The input schema is parsed outside steps on replay, so a change to it must change the version.
-          inputSchemaOf(wf),
-        ] as const,
-    )
-    .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
-  const hash = createHash("sha256")
-    .update(JSON.stringify([appName, RUNTIME_VERSION, STEP_LAYOUT, installedDbosVersion(), ...workflows]))
-    .digest("hex");
-  return `${appName}@${hash}`;
+  const workflows = definitions.map(codeOf).toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  return `${appName}@${sha256([appName, RUNTIME_VERSION, STEP_LAYOUT, installedDbosVersion(), ...workflows])}`;
 }
 
+/**
+ * A hash of the built-in `change` workflow's own code, which each change run carries (its
+ * `changeCode` attribute) so a worker on another version of the app may take it over
+ * (`carryChangeRuns` in worker.ts): what `computeVersion` hashes of that one workflow, with
+ * `RUNTIME_VERSION`, `STEP_LAYOUT` and the installed DBOS version. Never the company's workflows
+ * nor `DBOS__APPVERSION`, so a merge that changes only the company's code leaves it as it is.
+ */
+export function changeCodeOf(change: WorkflowDefinition<any, any>): string {
+  return sha256([RUNTIME_VERSION, STEP_LAYOUT, installedDbosVersion(), codeOf(change)]);
+}
+
+const sha256 = (parts: unknown[]) => createHash("sha256").update(JSON.stringify(parts)).digest("hex");
+
+/** What the hashes read of a workflow: its name, the source of its `run`, its operations and its input schema. */
+const codeOf = (wf: WorkflowDefinition<any, any>) =>
+  [
+    wf.name,
+    wf.run.toString(),
+    opIdsOf(wf),
+    // The input schema is parsed outside steps on replay, so a change to it must change the version.
+    inputSchemaOf(wf),
+  ] as const;
+
 /** The workflow's input schema as the hash sees it: its JSON Schema, or its text when zod can't convert it. */
 function inputSchemaOf(wf: WorkflowDefinition<any, any>): unknown {
   try {

package/src/worker.ts

@@ -4,19 +4,19 @@ import { buildCtx, decideStart, isInfrastructureError, startHoldOf } from "./cal
 import { dbosStatusesOf } from "./client.ts";
 import { declaredWakeup, type ResolvedConfig, resolveConfig, type SanomaConfig } from "./config.ts";
 import { storedSecrets, watchCredentials } from "./credentials.ts";
-import { DBOS_DATABASE_OPTIONS, lazyPool, prepareDatabase } from "./db.ts";
+import { type Db, DBOS_DATABASE_OPTIONS, lazyPool, prepareDatabase, stampVersion } from "./db.ts";
 import { deadlineRegistry, type Deadlines, idle, onOwnQueue, type SelfStop, stopWhenIdle } from "./deadlines.ts";
 import type { WorkflowDefinition } from "./define.ts";
 import { credentialsOf, unconfigured, unconfiguredSummary } from "./env.ts";
 import { errorInfo, parseOrThrow, PolicyDeniedError } from "./errors.ts";
 import { entry, ownedLedger, skipped, write, writeFailure } from "./ledger.ts";
-import { dbosLogger, quietIfSandbox, setLogLevel, warn } from "./log.ts";
+import { dbosLogger, info, quietIfSandbox, setLogLevel, warn } from "./log.ts";
 import { lineStartsOf, readSource } from "./ast.ts";
 import { callsOf, outlineWithSource } from "./outline.ts";
 import { currentRun, type Run, type RunArgs, type WorkerState, type WorkflowOutline } from "./run.ts";
 import { applyScheduled, nextScheduledFire, type ScheduledStarter, scheduledStarter } from "./schedules.ts";
 import type { Source } from "./source.ts";
-import { approverLabel, errorMessage, WORKFLOW_ACTION } from "./shared.ts";
+import { approverLabel, CHANGE_WORKFLOW, errorMessage, WORKFLOW_ACTION } from "./shared.ts";
 
 export interface Worker {
   /**
@@ -144,6 +144,12 @@ export async function startWorker(config: SanomaConfig, options: WorkerOptions =
     secrets: storedSecrets(() => secrets.db(), "sanoma"),
     workflows: resolved.workflows,
     outlines: new Map([...resolved.workflows].map(([name, wf]) => [name, outlineOf(wf)])),
+    // `registered` is filled below, before any run starts: every name of this config's is registered then.
+    startRun: async (name, runId, args) => {
+      const fn = registered.get(name) as unknown as ((args: RunArgs) => Promise<unknown>) | undefined;
+      if (!fn) throw new Error(`The worker has no workflow "${name}" to start`);
+      return DBOS.startWorkflow(fn, { workflowID: runId })(args);
+    },
     stopped: false,
     deadlines: deadlineRegistry(),
   };
@@ -216,6 +222,8 @@ export async function startWorker(config: SanomaConfig, options: WorkerOptions =
         );
       }
     }
+    // Change runs move to the latest version only: a rollback's worker, unless promoted, leaves them be.
+    if (isLatest) await carryChangeRuns(resolved, () => secrets.db());
     await DBOS.registerQueue(resolved.queueName);
     // The schedules are the latest version's: an older worker leaves them as the newer set them.
     if (isLatest && options.schedules !== false) {
@@ -282,6 +290,41 @@ function outlineOf(wf: WorkflowDefinition<any, any>): WorkflowOutline {
   return { file: outline.file, lineStarts: lineStartsOf(source!), calls: callsOf(outline.nodes) };
 }
 
+/**
+ * Takes over the unfinished change runs of other versions whose change code is this worker's,
+ * keeping each run's id, history and unread messages: a change's own deploy restarts the worker on
+ * the merged code, a new version, which DBOS would never give the run to. Each is stamped with this
+ * version (`stampVersion`), and a PENDING one queued again on the app's queue (`resumeWorkflows`);
+ * one stopped between the two is PENDING on this version, which DBOS's recovery takes. Not
+ * `rewindWorkflow`, which takes only a cancelled run and deletes its unread messages. It assumes one
+ * live worker per database, as a deploy does: an older one still running such a run runs it on
+ * until it stops. A failure is warned about, and the worker starts.
+ */
+async function carryChangeRuns({ appName, version, changeCode, queueName }: ResolvedConfig, db: () => Promise<Db>) {
+  try {
+    const runs = await DBOS.listWorkflows({
+      workflowName: CHANGE_WORKFLOW,
+      status: ["PENDING", "ENQUEUED"],
+      applicationName: appName,
+      attributes: { changeCode },
+      loadInput: false,
+      loadOutput: false,
+    });
+    const theirs = runs
+      .filter((r) => r.applicationVersion && r.applicationVersion !== version)
+      .map((r) => r.workflowID);
+    if (!theirs.length) return;
+    const carried = await stampVersion(await db(), theirs, version);
+    const pending = carried.filter((r) => r.pending).map((r) => r.id);
+    if (pending.length) await DBOS.resumeWorkflows(pending, { queueName });
+    if (carried.length) {
+      info(`carried ${carried.length} change run(s) onto version ${version}: ${carried.map((r) => r.id).join(", ")}`);
+    }
+  } catch (err) {
+    warn(`carrying the change runs of other versions onto ${version} failed, so they stay there: ${errorMessage(err)}`);
+  }
+}
+
 /**
  * Warns about unfinished runs this worker will never pick up: started on another version of
  * the app, or queued on another queue (such as the single "sanoma" queue before queues were