@sanoma-ai/workflows 0.7.0 → 0.8.0

package/src/client.ts

@@ -21,6 +21,18 @@ import { dbosClient, lazyPool, type LazyPool } from "./db.ts";
 import { type CredentialStatus, credentialsOf } from "./env.ts";
 import { type ApprovalState, Principal, type WorkflowDefinition } from "./define.ts";
 import { APPLY_WORKFLOW, type ApplyInput, type ApplyReport } from "./apply.ts";
+import {
+  ADMIN,
+  BASE,
+  type ChangeEvent,
+  ChangeEventSchema,
+  ChangeInput,
+  type ChangeReport,
+  type ChangeState,
+  followBase,
+  initialChangeState,
+  repositoryOf,
+} from "./change.ts";
 import { DRIFT_WORKFLOW, type DriftReport } from "./drift.ts";
 import { isDirect } from "./direct.ts";
 import {
@@ -32,8 +44,10 @@ import {
   parseOrThrow,
   PolicyDeniedError,
   SanomaError,
+  vendorCodeOf,
 } from "./errors.ts";
 import { insertRecord, type LedgerRecord, type OwnedLedger, ownedLedger } from "./ledger.ts";
+import { warn } from "./log.ts";
 import type { CallContext, Driver, Op } from "./op.ts";
 import { allowAll, type RecordedDecision } from "./policy.ts";
 import { admit, readPeople } from "./resources.ts";
@@ -42,8 +56,15 @@ import type { Secrets } from "./secret.ts";
 import {
   BUILT_IN_WORKFLOWS,
   CALL_WORKFLOW,
+  CHANGE_EVENT,
+  CHANGE_MERGE,
+  CHANGE_TOPIC,
+  CHANGE_WORKFLOW,
+  changeIdOf,
   DATA_VENDOR,
+  ENDED_STATUSES,
   errorMessage,
+  HOST_CHECK_AGENT,
   isEnded,
   isVisitor,
   refusalOf,
@@ -51,6 +72,14 @@ import {
   SCHEDULE_AGENT,
   WORKFLOW_ACTION,
 } from "./shared.ts";
+import { draftOf, type RemoteEvent, type RepoRef, type Source } from "./source.ts";
+
+/** The company's repository on its host, for `remoteEvent`. */
+type Repository = Awaited<ReturnType<typeof repositoryOf>>;
+
+/** Whether two references name one repository: a host's names are compared without case. */
+const sameRepo = (a: RepoRef, b: RepoRef) =>
+  a.owner.toLowerCase() === b.owner.toLowerCase() && a.name.toLowerCase() === b.name.toLowerCase();
 
 /** A principal, or its id alone. */
 const PrincipalOrId = z.union([
@@ -154,6 +183,27 @@ export interface StartOptions {
   sandbox?: string;
 }
 
+/** A change as the app shows it: its run, where the change is (`ChangeState`), and how it ended. */
+export interface ChangeView {
+  runId: string;
+  status: RunStatus;
+  startedBy?: Principal;
+  /** When the run started, and when its row last changed (epoch ms), as `RunSummary` has them. */
+  createdAt: number;
+  updatedAt?: number;
+  approvals: ApprovalState[];
+  /** What the run last published (`CHANGE_EVENT`); `opening`, with no branch yet, before it has. */
+  state: ChangeState;
+  /** The run's output, once it has finished. */
+  report?: ChangeReport;
+}
+
+/** Which changes `SanomaClient.changes` lists: `open` (not ended) or not, and at most `limit` (20), newest first. */
+export interface ChangesFilter {
+  open?: boolean;
+  limit?: number;
+}
+
 /** What `SanomaClient.call` takes besides the operation's id and its input. */
 export interface CallOptions {
   /**
@@ -182,6 +232,11 @@ export interface ClientOptions {
    * `dataStore`.
    */
   resolved?: ResolvedConfig;
+  /**
+   * The company's repository, as the worker is given it: what `remoteEvent` acts for (its
+   * `origin`) and fetches a draft's new commits into. Without one, `remoteEvent` does nothing.
+   */
+  source?: Source;
 }
 
 /** Talks to the runtime from another process (the app, a script), through the shared Postgres. */
@@ -263,9 +318,19 @@ export class SanomaClient {
       options.sandbox,
       "`sandbox` must be the name of a scenario in the config's `scenarios/`",
     );
+    if (workflow.name === CHANGE_WORKFLOW && !isChangeRunOf((parsed as ChangeInput).id, runId)) {
+      throw new SanomaError("invalid_input", "A change is started with openChange, which names its run", { runId });
+    }
     const hold = await this.startHold(workflow.name, parsed, startedBy, runId);
     const args: RunArgs = { input, startedBy, ...(sandbox === undefined ? {} : { sandbox }), ...(hold && { hold }) };
-    const { onBehalfOf } = startedBy;
+    // Kept on the run's status row too, so a listing tells sandbox runs apart without its input,
+    // and says whom an agent's run is for (DBOS keeps the principal's id and groups itself); a
+    // change run's code, so a worker on another version with the same takes it over.
+    const attributes = {
+      sandbox,
+      onBehalfOf: startedBy.onBehalfOf,
+      changeCode: workflow.name === CHANGE_WORKFLOW ? this.config.changeCode : undefined,
+    };
     this.options.onWrite?.();
     // An id DBOS has already returns that run, unless it is another workflow's, which DBOS refuses.
     const handle = await this.dbos
@@ -277,9 +342,7 @@ export class SanomaClient {
           applicationName: this.config.appName,
           authenticatedUser: startedBy.id,
           authenticatedRoles: startedBy.groups ?? [],
-          // Kept on the run's status row too, so a listing tells sandbox runs apart without its input,
-          // and says whom an agent's run is for (DBOS keeps the principal's id and groups itself).
-          ...((sandbox !== undefined || onBehalfOf !== undefined) && { attributes: { sandbox, onBehalfOf } }),
+          ...(Object.values(attributes).some((v) => v !== undefined) && { attributes }),
         },
         args,
       )
@@ -524,6 +587,21 @@ export class SanomaClient {
     return row ? { runId: row.workflowID, output: row.output, at: row.updatedAt ?? row.createdAt } : null;
   }
 
+  /**
+   * Cancels every run of the app that has not ended (queued, running or waiting), as a demo's
+   * `reseed` starts afresh: its moments' runs from the last seed would otherwise wait beside the
+   * new ones.
+   */
+  async cancelUnfinished(): Promise<void> {
+    const rows = await this.dbos.listWorkflows({
+      status: dbosStatusesOf("queued", "running"),
+      applicationName: this.config.appName,
+      loadInput: false,
+      loadOutput: false,
+    });
+    if (rows.length) await this.dbos.cancelWorkflows(rows.map((r) => r.workflowID));
+  }
+
   /** The run's audit record, in order, from the ledger store. */
   async ledger(runId: string): Promise<LedgerRecord[]> {
     await this.mustExist(runId);
@@ -760,6 +838,230 @@ export class SanomaClient {
     return row.status === "SUCCESS" ? row.output : await this.awaitResult(runId, 30_000);
   }
 
+  /**
+   * Starts the built-in change workflow on the draft `input.id` (`ChangeInput`): its run id is
+   * `change:<id>`, or `change:<id>~<n>` once that run has ended (the second `~2`). While a run of
+   * the change has not ended, returns its id and starts nothing.
+   */
+  async openChange(input: ChangeInput, options: StartOptions): Promise<string> {
+    const { id } = parseOrThrow(ChangeInput, input, "Not a change to open");
+    const runs = await this.changeRuns(id);
+    const open = runs.find((r) => !isEnded(runStatus(r.status, [])));
+    if (open) return open.workflowID;
+    const runId = runs.length ? `${changeRunId(id)}~${runs.length + 1}` : changeRunId(id);
+    return this.start(this.config.workflows.get(CHANGE_WORKFLOW)!, input, { ...options, runId });
+  }
+
+  /**
+   * The changes, newest first, each draft's by its latest run (`change`): `open` ones (not ended)
+   * or the others, else all; at most `limit` (20).
+   */
+  async changes({ open, limit = 20 }: ChangesFilter = {}): Promise<ChangeView[]> {
+    const runs = await this.dbos.listWorkflows({
+      workflowName: CHANGE_WORKFLOW,
+      applicationName: this.config.appName,
+      sortDesc: true,
+      loadInput: false,
+      loadOutput: false,
+    });
+    // A draft's later run (`~2`) is its change now: an earlier one is neither open nor done.
+    const ended = new Set<string>(dbosStatusesOf(...ENDED_STATUSES));
+    const ids = [...Map.groupBy(runs, (run) => changeIdOf(run.workflowID)).values()]
+      .map(([run]) => run!)
+      .filter((run) => open === undefined || ended.has(run.status) !== open)
+      .slice(0, limit)
+      .map((run) => run.workflowID);
+    if (ids.length === 0) return [];
+    const rows = await this.dbos.listWorkflows({ workflowIDs: ids, sortDesc: true, loadInput: true, loadOutput: true });
+    return Promise.all(rows.map((row) => this.changeView(row)));
+  }
+
+  /** The change of the draft `id`: its latest run's (`openChange`), or undefined when it has none. */
+  async change(id: string): Promise<ChangeView | undefined> {
+    const [row] = await this.changeRuns(id, { load: true });
+    return row && this.changeView(row);
+  }
+
+  /**
+   * Tells the change of the draft `id` what happened (`ChangeEvent`), as `options.by`: new commits
+   * on its branch, a close asked for, or its pull request closed or merged on the host. The event is
+   * queued for its latest run, which reads it when it next waits; a pending approval of the merge
+   * (Publish, or a hold on it) is withdrawn as `by` (C4's `withdraw`), so a run waiting on it reads
+   * the event now. A `close` is refused (`not_approver`) unless `by` may withdraw that approval; the
+   * other kinds are facts, so a withdrawal `by` may not make is skipped and the run's merge refuses a
+   * head that moved. `run_not_found` when the draft has no change, `run_ended` when its run has ended.
+   * With `key`, an event told again under the same key is queued once while the run has not read it.
+   */
+  async changeEvent(id: string, event: ChangeEvent, options: { by: Principal | string; key?: string }): Promise<void> {
+    const by = await this.admitted(
+      parseOrThrow(PrincipalOrId, options?.by, '`by` must be a principal, such as "alice" or { id: "alice" }'),
+    );
+    const given = parseOrThrow(ChangeEventSchema, event, "Not a change event");
+    // A close is asked for by whoever asks it, whatever the event names.
+    const told: ChangeEvent = given.kind === "close" ? { kind: "close", by: by.id } : given;
+    const [row] = await this.changeRuns(id);
+    if (!row) throw new SanomaError("run_not_found", `The draft ${id} has no change`, { id });
+    const runId = row.workflowID;
+    const approvals = await this.approvals(runId);
+    const status = runStatus(row.status, approvals);
+    if (isEnded(status)) {
+      throw new SanomaError("run_ended", `Run ${runId} has ${status}; it takes no more events`, { runId, status });
+    }
+    const held = approvals.find((a) => a.status === "pending" && a.covers.includes(CHANGE_MERGE));
+    // A close is whoever may withdraw Publish's, or with none pending, the starter's or admin's.
+    const refusal = withdrawalRefusal(held ?? { approver: { group: ADMIN } }, by, row.authenticatedUser);
+    if (refusal !== undefined && told.kind === "close") {
+      if (held) throw notApprover(runId, held, refusal);
+      throw new SanomaError("not_approver", refusal, { runId, reason: refusal });
+    }
+    this.options.onWrite?.();
+    await this.dbos.send(runId, told, CHANGE_TOPIC, options.key);
+    if (held && refusal === undefined) {
+      await this.withdraw(runId, held.id, WITHDRAWAL[told.kind], { by, timeoutSeconds: 5 }).catch((err: unknown) => {
+        // Decided or ended meanwhile: the run reads the event when it next waits, if it does.
+        if (!["already_decided", "run_ended", "no_pending_approval"].includes(errorCode(err) ?? "")) throw err;
+      });
+    }
+  }
+
+  /**
+   * Does what a host's webhook says happened (`RemoteEvent`, from a remote half's `webhook`) to the
+   * company's repository, the `origin` of `ClientOptions.source` (`repositoryOf`); an event of
+   * another repository, or with no source, is ignored. A delivery told again does nothing more.
+   *
+   * - `pushed` to `main`: the checkout follows `main`, then apply on the declared hosts
+   *   (`ResolvedConfig.hosts`) as `host-check`, as the run `deploy:<head>`, once per head; nothing
+   *   with no host, nor while a change is merging or deploying, which does both itself.
+   * - `pushed` to a draft's branch (`draftOf`) with its change open: the host's commits fetched into
+   *   the draft (`Source.follow` on its branch), then the change told `commits`, as `by`; nothing
+   *   when the change or the draft is at that head already (the run's own push), or when the draft
+   *   cannot fast-forward to it (warned).
+   * - `closed` for the open change's pull request: the change told `merged` (with its merge commit)
+   *   or `closed`, as `by`. The run checks either on the host before it believes it.
+   */
+  async remoteEvent(event: RemoteEvent, options: { by: Principal | string }): Promise<void> {
+    const main = event.kind === "pushed" && event.branch === BASE;
+    const id = draftOf(event.branch);
+    if (!main && id === undefined) return;
+    const repository = await this.repository();
+    if (!repository || !sameRepo(repository.repo, event.repo)) return;
+    if (main) return this.deployPush(repository, event.head);
+    const change = await this.change(id!);
+    if (!change || isEnded(change.status)) return;
+    let told: ChangeEvent;
+    let key: string;
+    if (event.kind === "closed") {
+      // A pull request of an earlier change of the draft is not this one's.
+      if (event.number !== change.state.pullRequest?.number) return;
+      told = event.merged && event.sha !== undefined ? { kind: "merged", sha: event.sha } : { kind: "closed" };
+      key = `${event.number}:${event.sha ?? "closed"}`;
+    } else {
+      if (change.state.head === event.head || !(await this.fetched(repository, event.branch, event.head))) return;
+      told = { kind: "commits" };
+      key = event.head;
+    }
+    await this.changeEvent(id!, told, { by: options.by, key: `${event.kind}:${event.branch}:${key}` }).catch(
+      (err: unknown) => {
+        // Ended, or gone, since it was read: nothing to tell.
+        if (!["run_ended", "run_not_found"].includes(errorCode(err) ?? "")) throw err;
+      },
+    );
+  }
+
+  /** After a push of `head` to `main` (`remoteEvent`): the checkout follows it, then apply on the declared hosts. */
+  private async deployPush({ source }: Repository, head: string): Promise<void> {
+    const runId = `deploy:${head}`;
+    const [deployed, phases] = await Promise.all([this.run(runId), this.openChangePhases()]);
+    if (deployed || phases.some((phase) => phase === "merging" || phase === "deploying")) return;
+    // First, as a change's deploy does: the hosts, and what apply reads, are then main's.
+    const moved = await followBase(this.config.remotes, this.storedSecrets, source);
+    if (!("revision" in moved)) {
+      // Not on stale files, and nothing recorded as `deploy:<head>`: a redelivery may deploy it.
+      warn(
+        `${BASE} was not deployed: the checkout did not follow it: ${"skipped" in moved ? moved.skipped : moved.error}`,
+      );
+      return;
+    }
+    const hosts = await this.config.hosts();
+    if (hosts.length) await this.apply({ only: hosts }, { startedBy: HOST_CHECK_AGENT, runId });
+  }
+
+  /** Where each open change is (`ChangeState.phase`), read from what its run last published. */
+  private async openChangePhases(): Promise<(ChangeState["phase"] | undefined)[]> {
+    const rows = await this.dbos.listWorkflows({
+      workflowName: CHANGE_WORKFLOW,
+      applicationName: this.config.appName,
+      status: dbosStatusesOf("queued", "running"),
+      loadInput: false,
+      loadOutput: false,
+    });
+    return Promise.all(
+      rows.map(async (r) => (await this.dbos.getEvent<ChangeState>(r.workflowID, CHANGE_EVENT, 0))?.phase),
+    );
+  }
+
+  /**
+   * Brings `head`, pushed to the draft's `branch` on the host, into the draft: false when it holds
+   * it already, or cannot take it (no `follow`, a commit of its own, or one uncommitted: warned).
+   */
+  private async fetched({ source, url, host }: Repository, branch: string, head: string): Promise<boolean> {
+    const local = await Promise.resolve(source.head(branch)).catch(() => undefined);
+    if (local === head) return false;
+    const moved = source.follow
+      ? await source.follow({ url, token: await host.token(), ref: branch })
+      : { skipped: "this repository cannot follow its remote" };
+    if ("skipped" in moved) {
+      warn(`${branch} did not take the host's commits: ${moved.skipped}`);
+      return false;
+    }
+    return true;
+  }
+
+  /** The repository of `ClientOptions.source` on its host (`repositoryOf`); undefined with no source, remote or half for it. */
+  private async repository(): Promise<Repository | undefined> {
+    const { source } = this.options;
+    if (!source) return undefined;
+    return repositoryOf(this.config.remotes, this.storedSecrets, source).catch((err: unknown) => {
+      if (["no_remote", "no_remote_half"].includes(vendorCodeOf(err) ?? "")) return undefined;
+      throw err;
+    });
+  }
+
+  /** The runs of the draft `id`'s change, newest first: `change:<id>` and `change:<id>~<n>`. */
+  private async changeRuns(id: string, { load = false } = {}) {
+    const base = changeRunId(id);
+    const rows = await this.dbos.listWorkflows({
+      workflow_id_prefix: base,
+      workflowName: CHANGE_WORKFLOW,
+      applicationName: this.config.appName,
+      sortDesc: true,
+      loadInput: load,
+      loadOutput: load,
+    });
+    return rows.filter((r) => isChangeRunOf(id, r.workflowID));
+  }
+
+  /** A change run's row as the app shows it: its summary, its last `ChangeState` and, once finished, its report. */
+  private async changeView(row: Parameters<SanomaClient["summarize"]>[0] & { input?: unknown; output?: unknown }) {
+    const [summary, published] = await Promise.all([
+      this.summarize(row),
+      this.dbos.getEvent<ChangeState>(row.workflowID, CHANGE_EVENT, 0),
+    ]);
+    const input = (row.input as [RunArgs])[0].input as ChangeInput;
+    const state = published ?? initialChangeState(input);
+    const { runId, status, startedBy, createdAt, updatedAt, approvals } = summary;
+    return {
+      runId,
+      status,
+      ...(startedBy && { startedBy }),
+      createdAt,
+      ...(updatedAt !== undefined && { updatedAt }),
+      approvals,
+      state,
+      ...(row.status === "SUCCESS" && { report: row.output as ChangeReport }),
+    } satisfies ChangeView;
+  }
+
   /**
    * Stores a value for a variable a driver's `env` declares, as `by`, and tells the workers: each
    * loads it into its environment, unless its environment sets the variable itself. Refuses
@@ -1034,5 +1336,19 @@ const alreadyDecided = (runId: string, a: ApprovalState) =>
     },
   );
 
+/** A change's first run's id: `change:<draft id>`. */
+const changeRunId = (id: string) => `change:${id}`;
+
+/** True when `runId` is a run of the draft `id`'s change: `change:<id>` or `change:<id>~<n>`. */
+const isChangeRunOf = (id: string, runId: string | undefined) => runId !== undefined && changeIdOf(runId) === id;
+
+/** Why a change event withdraws the merge's pending approval. */
+const WITHDRAWAL: Record<ChangeEvent["kind"], string> = {
+  commits: "The change has new commits",
+  close: "The change is being closed",
+  closed: "The pull request was closed on the host",
+  merged: "The pull request was merged on the host",
+};
+
 const notApprover = (runId: string, a: ApprovalState, reason: string) =>
   new SanomaError("not_approver", `${reason} (${a.id})`, { runId, approvalId: a.id, approver: a.approver, reason });

package/src/collection.ts

@@ -12,7 +12,17 @@ import {
   type Op,
   type OpSpec,
 } from "./op.ts";
-import { checkCollection, DATA_VENDOR, type FIELD_TYPES, LIST_LIMIT, type ListQuery, type Saved } from "./shared.ts";
+import { callerFile } from "./define.ts";
+import { appOfFile } from "./paths.ts";
+import {
+  checkCollection,
+  DATA_VENDOR,
+  type FIELD_TYPES,
+  LIST_LIMIT,
+  type ListQuery,
+  qualified,
+  type Saved,
+} from "./shared.ts";
 
 // A company's own records: a collection is declared with plain values (a name, a key and typed
 // fields), and its `get`, `list`, `save` and `delete` are operations of the vendor `data`, under
@@ -62,7 +72,10 @@ export interface Collection<
 > {
   readonly [COLLECTION]: true;
   readonly kind: "collection";
+  /** Its name: as written at the root; in an app's folder, `<app>.<name>` (`app`), though its type says the written one. */
   readonly name: N;
+  /** The app whose folder declared it (`apps/<app>/resources/data.ts`); absent for one at the root. */
+  readonly app?: string;
   readonly key: K;
   readonly fields: F;
   readonly title?: string;
@@ -77,7 +90,9 @@ export interface Collection<
  * `collection({ name: "outbound", key: "company", fields: { company: "text", stage: { enum: ["new", "won"] } } })`.
  * Its operations are `data.<name>.get`, `.list`, `.save` and `.delete`, which a workflow names in
  * `uses` and calls on `ctx.data.<name>`. The name is a lowercase letter then letters and digits;
- * the key is a required text field, each record's own and unique.
+ * the key is a required text field, each record's own and unique. In an app's folder
+ * (`apps/<app>/resources/data.ts`) the runtime names it `<app>.<name>`: `data.<app>.<name>.get`,
+ * called on `ctx.data.<app>.<name>`.
  */
 export function collection<
   const N extends string,
@@ -85,7 +100,10 @@ export function collection<
   const K extends RequiredTextKeys<F>,
 >(declaration: { name: N; key: K; fields: F; title?: string }): Collection<N, F, K> {
   checkCollection(declaration);
-  const { name, key, fields, title } = declaration;
+  const { key, fields, title } = declaration;
+  // Frame 2 of `callerFile` is this function's caller: the data file.
+  const app = appOfFile(callerFile());
+  const name = qualified(app, declaration.name);
   const record = z.strictObject(
     Object.fromEntries(
       Object.entries(fields).map(([field, kind]) => {
@@ -99,6 +117,7 @@ export function collection<
     [COLLECTION]: true,
     kind: "collection",
     name,
+    ...(app && { app }),
     key,
     fields,
     ...(title !== undefined && { title }),