@sanoma-ai/workflows 0.7.1 → 0.8.1

package/src/sites.ts

@@ -1,11 +1,11 @@
 import { existsSync, readdirSync } from "node:fs";
-import { join, relative, sep } from "node:path";
+import { basename, join, relative, sep } from "node:path";
 import { childrenOf, type LintProblem, type Node, readSource, type SourceFile } from "./ast.ts";
 import { type CedarPolicySource, visitorPermits } from "./cedar.ts";
 import { keyName } from "./datafile.ts";
 import type { ConfigDescription } from "./describe.ts";
 import { stringValue, unwrap } from "./outline.ts";
-import { filesUnder, isDirectory } from "./paths.ts";
+import { APPS_DIR, appFolders, filesUnder, isDirectory } from "./paths.ts";
 import { START } from "./schema.ts";
 import {
   ANONYMOUS,
@@ -14,6 +14,7 @@ import {
   DATA_VENDOR,
   LOCALE,
   SITE_NAME,
+  siteKey,
   VISITORS,
   WORKFLOW_ACTION,
 } from "./shared.ts";
@@ -64,12 +65,16 @@ export interface SitePage {
   content?: string;
 }
 
-/** A site, `sites/<name>/`, and what it calls. */
+/** A site, `sites/<name>/` or an app's `apps/<app>/sites/<name>/`, and what it calls. */
 export interface SiteEntry {
-  /** Its folder's name, where it is mounted: `/sites/<name>`. */
+  /** Its folder's name, `<app>/<name>` for an app's (`siteKey`): where it is mounted, `/sites/<name>`. */
   name: string;
+  /** The app whose folder holds it (`apps/<app>/sites/`); absent for one at the root. */
+  app?: string;
   /** `defineSite({ title })` in its `site.ts`, when that is a string literal. */
   title?: string;
+  /** `defineSite({ icon })` in its `site.ts`: a Lucide icon's name, as written. */
+  icon?: string;
   /** `defineSite({ public: true })`: served to visitors, not people. */
   public?: true;
   /** `defineSite({ login })`, the connector whose login half signs visitors in, as written: one that names none is among `problems`. */
@@ -109,12 +114,14 @@ const PARAMS: ReadonlySet<string> = new Set(["params"]);
 const ROUTE_MAKERS: ReadonlySet<string> = new Set(["createFileRoute", "createLazyFileRoute"]);
 
 /**
- * The sites under `root`: each folder of `sites/` with a `routes/` whose name is a URL segment,
- * by name (as `@sanoma-ai/app/build`'s `sitesOf` mounts them). Reads `routes/**` and
+ * The sites under `root` (`sitesUnder`): each folder of `sites/`, then of each app's
+ * `apps/<app>/sites/`, with a `routes/` whose name is a URL segment, by key (as
+ * `@sanoma-ai/app/build`'s `sitesOf` mounts them). Reads `routes/**` and
  * `components/**` (`.ts` and `.tsx`, less tests and declarations) of each with oxc, never running
  * them: each page (`SitePage`, from a route's id), each `<RunForm workflow="…" />` and each use of `api`, both imported from `sanoma/site`,
  * called or passed on (`queryFn: api.data.leads.list`):
- * `api.<name>` and `api.<name>.<method>` are a workflow, `api.data.<collection>.<op>` a collection,
+ * `api.<name>` and `api.<name>.<method>` are a workflow, `api.data.<collection>.<op>` (an app's
+ * `api.data.<app>.<collection>.<op>`) a collection,
  * and `api.<vendor>.<resource>.<op>` an operation. A name that is not written out (a variable, a
  * template with holes, a computed member) is skipped, as is a use through a namespace import or a
  * destructured `api`. A call to what `description` does not have, or to an operation a page may
@@ -133,9 +140,11 @@ export function readSites(
   policies?: readonly CedarPolicySource[],
 ): SiteEntry[] {
   if (root === undefined) return [];
-  const dir = join(root, "sites");
   const workflows = new Set(description.workflows.map((wf) => wf.name));
   const collections = new Set(description.collections.map((c) => c.name));
+  // `api.data.<app>.<collection>` passed on as a value reads as a call of `<app>`'s method: it is none.
+  const isAppCollection = ({ kind, name, method }: Written) =>
+    kind === "collection" && collections.has(`${name}.${method}`);
   const ops = new Map(description.ops.map((op) => [op.id, op]));
   // A workflow's api name to its name: two of one api name are a problem on the workflows.
   const byApiName = new Map(description.workflows.map((wf) => [apiNameOf(wf.name), wf.name]));
@@ -193,11 +202,11 @@ export function readSites(
     return { call: { kind, target, via, ...at }, ...(problem && { problem }) };
   };
 
-  return siteNames(dir).map((name) => {
-    const read = readSiteFiles(root, name);
-    const resolved = read.flatMap(({ uses }) => uses.map(resolve));
+  return sitesUnder(root).map(({ name, app, dir }) => {
+    const read = readSiteFiles(root, dir);
+    const resolved = read.flatMap(({ uses }) => uses.filter((use) => !isAppCollection(use)).map(resolve));
     const pages = pagesOf(read);
-    const siteFile = `sites/${name}/site.ts`;
+    const siteFile = `${dir}/site.ts`;
     const { problems: optionProblems, ...options } = siteOptionsOf(join(root, siteFile), connectors);
     const calls = resolved.map((r) => r.call);
     const problems = [
@@ -212,7 +221,7 @@ export function readSites(
     const warnings = options.public
       ? read.flatMap(({ route, warning }) => (warning && !pages.get(route!.path)?.content ? [warning] : []))
       : [];
-    return { name, ...options, pages: sorted, calls, problems, warnings };
+    return { name, ...(app && { app }), ...options, pages: sorted, calls, problems, warnings };
   });
 }
 
@@ -225,9 +234,9 @@ function pagesOf(read: readonly { route?: SitePage }[]): Map<string, SitePage> {
   return pages;
 }
 
-/** Each of site `name`'s files under `root`, as `readSiteFile` reads it, by path. */
-const readSiteFiles = (root: string, name: string) =>
-  SITE_DIRS.flatMap((sub) => filesUnder(join(root, "sites", name, sub), isSource))
+/** Each file of the site in `dir` (relative to `root`), as `readSiteFile` reads it, by path. */
+const readSiteFiles = (root: string, dir: string) =>
+  SITE_DIRS.flatMap((sub) => filesUnder(join(root, dir, sub), isSource))
     .toSorted()
     .flatMap((path) => {
       const file = readSource(path);
@@ -241,8 +250,8 @@ const readSiteFiles = (root: string, name: string) =>
  */
 export function siteContentPages(root: string): Record<string, string> {
   return Object.fromEntries(
-    siteNames(join(root, "sites")).flatMap((name) =>
-      [...pagesOf(readSiteFiles(root, name)).values()].flatMap(({ path, content }) =>
+    sitesUnder(root).flatMap(({ dir }) =>
+      [...pagesOf(readSiteFiles(root, dir)).values()].flatMap(({ path, content }) =>
         content ? [[path, content]] : [],
       ),
     ),
@@ -257,13 +266,44 @@ export function siteContentPages(root: string): Record<string, string> {
 export function siteLocales(root: string): Record<string, string[]> {
   const none = new Set<string>();
   return Object.fromEntries(
-    siteNames(join(root, "sites")).flatMap((name) => {
-      const { locales } = siteOptionsOf(join(root, "sites", name, "site.ts"), { vendors: none, logins: none });
+    sitesUnder(root).flatMap(({ name, dir }) => {
+      const { locales } = siteOptionsOf(join(root, dir, "site.ts"), { vendors: none, logins: none });
       return locales ? [[name, locales]] : [];
     }),
   );
 }
 
+/**
+ * Each site's icon (`defineSite({ icon })`, a Lucide icon's name as written) under `root`, by name,
+ * for the sites whose `site.ts` gives one: the icons the app's build bundles (`virtual:sanoma/icons`).
+ * Read as `readSites` reads it, never run.
+ */
+export function siteIcons(root: string): Record<string, string> {
+  const none = new Set<string>();
+  return Object.fromEntries(
+    sitesUnder(root).flatMap(({ name, dir }) => {
+      const { icon } = siteOptionsOf(join(root, dir, "site.ts"), { vendors: none, logins: none });
+      return icon === undefined ? [] : [[name, icon]];
+    }),
+  );
+}
+
+/**
+ * The sites under `root`, by key (`siteKey`): each folder of `sites/` with a `routes/` whose name
+ * is a URL segment, then each app's `apps/<app>/sites/`, with its folder relative to `root`.
+ */
+export const sitesUnder = (root: string): { name: string; app?: string; dir: string }[] => [
+  ...siteNames(join(root, "sites")).map((name) => ({ name, dir: `sites/${name}` })),
+  ...appFolders(root).flatMap((folder) => {
+    const app = basename(folder);
+    return siteNames(join(folder, "sites")).map((name) => ({
+      name: siteKey(app, name),
+      app,
+      dir: `${APPS_DIR}/${app}/sites/${name}`,
+    }));
+  }),
+];
+
 /** The sites in `dir`: each folder with a `routes/` whose name is a URL segment, sorted. */
 const siteNames = (dir: string): string[] =>
   isDirectory(dir)
@@ -379,9 +419,12 @@ function contentKind(callee: Node, content: ReadonlySet<string>): string | undef
 function apiPath(member: Node, api: ReadonlySet<string>): Pick<Written, "kind" | "name" | "method"> | undefined {
   const path = memberPath(member, api);
   if (!path?.length) return undefined;
-  const [first, second, third] = path as [string, ...string[]];
+  const [first, second] = path as [string, ...string[]];
+  // `api.data.<collection>.<op>`, or an app's, `api.data.<app>.<collection>.<op>`.
   if (first === DATA_VENDOR)
-    return path.length === 3 ? { kind: "collection", name: second!, method: third } : undefined;
+    return path.length === 3 || path.length === 4
+      ? { kind: "collection", name: path.slice(1, -1).join("."), method: path.at(-1) }
+      : undefined;
   if (path.length === 3) return { kind: "op", name: path.join(".") };
   if (path.length > 2) return undefined;
   return { kind: "workflow", name: first, ...(second !== undefined && { method: second }) };
@@ -414,14 +457,14 @@ const unread = (what: string, as: string) => `${what} is read without running si
 /**
  * What `site.ts` says of its site, read without running it, and what is wrong there: its default
  * export must be `defineSite({ … })` written out, so it can be read at all; then `title` when it is
- * a string, `public`, `login` and `locales`, which must be literals. `login` is kept as written; one on a
+ * a string, `icon`, `public`, `login` and `locales`, which must be literals. `login` is kept as written; one on a
  * site that is not public, or naming no connector of the config with a login half, is a problem.
  * `locales` is kept only when it has no problem.
  */
 function siteOptionsOf(
   path: string,
   connectors: SiteConnectors,
-): Pick<SiteEntry, "title" | "public" | "login" | "locales"> & { problems: LintProblem[] } {
+): Pick<SiteEntry, "title" | "icon" | "public" | "login" | "locales"> & { problems: LintProblem[] } {
   const file = readSource(path);
   if (typeof file === "string") return { problems: [] };
   const problems: LintProblem[] = [...file.problems];
@@ -453,6 +496,8 @@ function siteOptionsOf(
     options?.properties.findLast((p: Node) => isWritten(p) && keyName(p.key) === name);
 
   const title = stringValue(property("title")?.value);
+  // Cosmetic: one that is not a string (tsc refuses it) is shown as none, never a problem that stops a site.
+  const icon = stringValue(property("icon")?.value);
   const publicAt = property("public");
   const flag = unwrap(publicAt?.value);
   const isPublic = flag?.type === "Literal" && flag.value === true;
@@ -495,6 +540,7 @@ function siteOptionsOf(
   if (localesMessage) problem(localesAt, localesMessage);
   return {
     ...(title !== undefined && { title }),
+    ...(icon !== undefined && { icon }),
     ...(isPublic && { public: true as const }),
     ...(login !== undefined && { login }),
     ...(localesAt && !localesMessage && isPublic && !loginAt && { locales }),

package/src/source.ts

@@ -28,8 +28,9 @@ export interface Source {
   /** Pushes `ref` to the repository at `url`, signed in with `token` (a draft by its `branch`). */
   push?(target: { url: string; token: string; ref: string }): void | Promise<void>;
   /**
-   * The draft `id` (letters, digits, `_` and `-`), opened: made from the checked-out revision the
-   * first time, then the same one, edits and all, until its branch is deleted.
+   * The draft `id` (letters, digits, `_` and `-`), opened on its branch (`draftBranch`): made from
+   * the checked-out revision the first time, then the same one, edits and all, until its branch is
+   * deleted.
    */
   draft?(id: string): Promise<Draft>;
   /** The open drafts' ids, sorted. */
@@ -47,13 +48,27 @@ export interface Source {
   /** The URL of the repository this one was cloned from (`origin`); undefined when it has none. */
   remoteUrl?(): Promise<string | undefined>;
   /**
-   * Fast-forwards the checked-out branch to `ref` in the repository at `url`, signed in with
-   * `token`, as `push` is: only when `ref` is checked out, nothing is uncommitted and the remote's
-   * `ref` contains it; otherwise `skipped`, saying why. The checked-out revision after.
+   * Fast-forwards the branch `ref` to `ref` in the repository at `url`, signed in with `token`, as
+   * `push` is, where it is checked out: the root's checked-out branch, or an open draft's (by its
+   * `branch`), whose new commits on the host it takes so. Only when nothing is uncommitted there
+   * and the remote's `ref` contains it; otherwise `skipped`, saying why. The revision after.
    */
   follow?(target: { url: string; token: string; ref: string }): Promise<{ revision: string } | { skipped: string }>;
+  /**
+   * How many commits the source repository of the app `folder` (the URL its `apps/<folder>/app.ts`
+   * names) has on its default branch that the checked-out one lacks, of those changing what an
+   * update merges (`apps/<folder>/` and `package.json`): fetched now. An error when the app names
+   * no source.
+   */
+  behind?(folder: string): Promise<number>;
 }
 
+/** The branch of the draft `id`, as a source names it: `draft/<id>`. */
+export const draftBranch = (id: string) => `draft/${id}`;
+
+/** The draft whose branch `branch` is (`draftBranch`), or undefined for a branch that is no draft's. */
+export const draftOf = (branch: string) => (branch.startsWith("draft/") ? branch.slice("draft/".length) : undefined);
+
 /**
  * A draft: a checkout of its own, at `root`, on `branch`, which `commit` records in as
  * `Source.commit` does in its root. Its `node_modules` may be the root's own install, linked in:
@@ -63,6 +78,36 @@ export interface Draft {
   branch: string;
   root: string;
   commit: Source["commit"];
+  /**
+   * Takes an app into the draft: fetches `ref` (the default branch by default) of the repository
+   * at `url`, a source (a public company repo with one `apps/<folder>/`), and merges it in with its
+   * history (`--allow-unrelated-histories`), as one merge commit by `author`. What it merges is
+   * `apps/<folder>/` and `package.json`; every other file stays the draft's. `folder` may be left
+   * out when the source has one app. Refused when the draft has `apps/<folder>/` already.
+   */
+  take?(app: { url: string; ref?: string; folder?: string; author: string }): Promise<AppMerge>;
+  /**
+   * Merges the new commits of the app `folder`'s source (the URL its `app.ts` names, its default
+   * branch) into the draft, by `author`, as `take` merges: `apps/<folder>/` and `package.json`.
+   */
+  update?(app: { folder: string; author: string }): Promise<AppMerge>;
+}
+
+/**
+ * What taking or updating an app merged into a draft. A line both sides changed is committed with
+ * git's markers in its file (ours, the merge base and theirs), so the draft's checks fail until the
+ * agent resolves it; a file one side deleted stays as the draft had it. The merge commit keeps the
+ * source's history either way.
+ */
+export interface AppMerge {
+  /** The app's folder: `apps/<folder>/`. */
+  folder: string;
+  /** The draft's commit after the merge. */
+  revision: string;
+  /** The source's commits the merge brought in that change the app's folder or `package.json`; 0 when the draft had them all, and nothing was merged. */
+  merged: number;
+  /** The files git could not merge, relative to the root, committed with conflict markers; empty when it merged clean. */
+  conflicts: string[];
 }
 
 /** A draft's change: the files it touches, relative to the root, and git's `--stat` of it. */

package/src/version.ts

@@ -63,24 +63,33 @@ export function computeVersion({
 }): string {
   const named = process.env.DBOS__APPVERSION?.trim();
   if (named) return `${appName}@${named}`;
-  const workflows = definitions
-    .map(
-      (wf) =>
-        [
-          wf.name,
-          wf.run.toString(),
-          opIdsOf(wf),
-          // The input schema is parsed outside steps on replay, so a change to it must change the version.
-          inputSchemaOf(wf),
-        ] as const,
-    )
-    .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
-  const hash = createHash("sha256")
-    .update(JSON.stringify([appName, RUNTIME_VERSION, STEP_LAYOUT, installedDbosVersion(), ...workflows]))
-    .digest("hex");
-  return `${appName}@${hash}`;
+  const workflows = definitions.map(codeOf).toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  return `${appName}@${sha256([appName, RUNTIME_VERSION, STEP_LAYOUT, installedDbosVersion(), ...workflows])}`;
 }
 
+/**
+ * A hash of the built-in `change` workflow's own code, which each change run carries (its
+ * `changeCode` attribute) so a worker on another version of the app may take it over
+ * (`carryChangeRuns` in worker.ts): what `computeVersion` hashes of that one workflow, with
+ * `RUNTIME_VERSION`, `STEP_LAYOUT` and the installed DBOS version. Never the company's workflows
+ * nor `DBOS__APPVERSION`, so a merge that changes only the company's code leaves it as it is.
+ */
+export function changeCodeOf(change: WorkflowDefinition<any, any>): string {
+  return sha256([RUNTIME_VERSION, STEP_LAYOUT, installedDbosVersion(), codeOf(change)]);
+}
+
+const sha256 = (parts: unknown[]) => createHash("sha256").update(JSON.stringify(parts)).digest("hex");
+
+/** What the hashes read of a workflow: its name, the source of its `run`, its operations and its input schema. */
+const codeOf = (wf: WorkflowDefinition<any, any>) =>
+  [
+    wf.name,
+    wf.run.toString(),
+    opIdsOf(wf),
+    // The input schema is parsed outside steps on replay, so a change to it must change the version.
+    inputSchemaOf(wf),
+  ] as const;
+
 /** The workflow's input schema as the hash sees it: its JSON Schema, or its text when zod can't convert it. */
 function inputSchemaOf(wf: WorkflowDefinition<any, any>): unknown {
   try {

package/src/worker.ts

@@ -4,19 +4,19 @@ import { buildCtx, decideStart, isInfrastructureError, startHoldOf } from "./cal
 import { dbosStatusesOf } from "./client.ts";
 import { declaredWakeup, type ResolvedConfig, resolveConfig, type SanomaConfig } from "./config.ts";
 import { storedSecrets, watchCredentials } from "./credentials.ts";
-import { DBOS_DATABASE_OPTIONS, lazyPool, prepareDatabase } from "./db.ts";
+import { type Db, DBOS_DATABASE_OPTIONS, lazyPool, prepareDatabase, stampVersion } from "./db.ts";
 import { deadlineRegistry, type Deadlines, idle, onOwnQueue, type SelfStop, stopWhenIdle } from "./deadlines.ts";
 import type { WorkflowDefinition } from "./define.ts";
 import { credentialsOf, unconfigured, unconfiguredSummary } from "./env.ts";
 import { errorInfo, parseOrThrow, PolicyDeniedError } from "./errors.ts";
 import { entry, ownedLedger, skipped, write, writeFailure } from "./ledger.ts";
-import { dbosLogger, quietIfSandbox, setLogLevel, warn } from "./log.ts";
+import { dbosLogger, info, quietIfSandbox, setLogLevel, warn } from "./log.ts";
 import { lineStartsOf, readSource } from "./ast.ts";
 import { callsOf, outlineWithSource } from "./outline.ts";
 import { currentRun, type Run, type RunArgs, type WorkerState, type WorkflowOutline } from "./run.ts";
 import { applyScheduled, nextScheduledFire, type ScheduledStarter, scheduledStarter } from "./schedules.ts";
 import type { Source } from "./source.ts";
-import { approverLabel, errorMessage, WORKFLOW_ACTION } from "./shared.ts";
+import { approverLabel, CHANGE_WORKFLOW, errorMessage, WORKFLOW_ACTION } from "./shared.ts";
 
 export interface Worker {
   /**
@@ -144,6 +144,12 @@ export async function startWorker(config: SanomaConfig, options: WorkerOptions =
     secrets: storedSecrets(() => secrets.db(), "sanoma"),
     workflows: resolved.workflows,
     outlines: new Map([...resolved.workflows].map(([name, wf]) => [name, outlineOf(wf)])),
+    // `registered` is filled below, before any run starts: every name of this config's is registered then.
+    startRun: async (name, runId, args) => {
+      const fn = registered.get(name) as unknown as ((args: RunArgs) => Promise<unknown>) | undefined;
+      if (!fn) throw new Error(`The worker has no workflow "${name}" to start`);
+      return DBOS.startWorkflow(fn, { workflowID: runId })(args);
+    },
     stopped: false,
     deadlines: deadlineRegistry(),
   };
@@ -216,6 +222,8 @@ export async function startWorker(config: SanomaConfig, options: WorkerOptions =
         );
       }
     }
+    // Change runs move to the latest version only: a rollback's worker, unless promoted, leaves them be.
+    if (isLatest) await carryChangeRuns(resolved, () => secrets.db());
     await DBOS.registerQueue(resolved.queueName);
     // The schedules are the latest version's: an older worker leaves them as the newer set them.
     if (isLatest && options.schedules !== false) {
@@ -282,6 +290,41 @@ function outlineOf(wf: WorkflowDefinition<any, any>): WorkflowOutline {
   return { file: outline.file, lineStarts: lineStartsOf(source!), calls: callsOf(outline.nodes) };
 }
 
+/**
+ * Takes over the unfinished change runs of other versions whose change code is this worker's,
+ * keeping each run's id, history and unread messages: a change's own deploy restarts the worker on
+ * the merged code, a new version, which DBOS would never give the run to. Each is stamped with this
+ * version (`stampVersion`), and a PENDING one queued again on the app's queue (`resumeWorkflows`);
+ * one stopped between the two is PENDING on this version, which DBOS's recovery takes. Not
+ * `rewindWorkflow`, which takes only a cancelled run and deletes its unread messages. It assumes one
+ * live worker per database, as a deploy does: an older one still running such a run runs it on
+ * until it stops. A failure is warned about, and the worker starts.
+ */
+async function carryChangeRuns({ appName, version, changeCode, queueName }: ResolvedConfig, db: () => Promise<Db>) {
+  try {
+    const runs = await DBOS.listWorkflows({
+      workflowName: CHANGE_WORKFLOW,
+      status: ["PENDING", "ENQUEUED"],
+      applicationName: appName,
+      attributes: { changeCode },
+      loadInput: false,
+      loadOutput: false,
+    });
+    const theirs = runs
+      .filter((r) => r.applicationVersion && r.applicationVersion !== version)
+      .map((r) => r.workflowID);
+    if (!theirs.length) return;
+    const carried = await stampVersion(await db(), theirs, version);
+    const pending = carried.filter((r) => r.pending).map((r) => r.id);
+    if (pending.length) await DBOS.resumeWorkflows(pending, { queueName });
+    if (carried.length) {
+      info(`carried ${carried.length} change run(s) onto version ${version}: ${carried.map((r) => r.id).join(", ")}`);
+    }
+  } catch (err) {
+    warn(`carrying the change runs of other versions onto ${version} failed, so they stay there: ${errorMessage(err)}`);
+  }
+}
+
 /**
  * Warns about unfinished runs this worker will never pick up: started on another version of
  * the app, or queued on another queue (such as the single "sanoma" queue before queues were