@sanoma-ai/workflows 0.0.0-stage → 0.1.0

package/src/db.ts

@@ -0,0 +1,197 @@
+import { DBOS, DBOSClient } from "@dbos-inc/dbos-sdk";
+import { Client, Pool } from "pg";
+import { warn } from "./log.ts";
+import { errorMessage } from "./shared.ts";
+
+// The runtime's own tables live in the config's database, beside DBOS's: the credentials and the
+// ledger. What they share is here: the one preparation of a database, a pool that opens once it
+// is prepared and survives a dropped connection, and a table creation two processes may race.
+
+/** A connection or a pool: what the queries take. */
+export type Db = Pick<Pool | Client, "query">;
+
+/** Where the credentials set from the app live. */
+export const CREDENTIALS_TABLE = "sanoma_credentials";
+/** Where the Postgres ledger keeps its records. */
+export const LEDGER_TABLE = "sanoma_ledger";
+
+/** The runtime's tables, each with the word the messages name it by. */
+const TABLES: readonly [what: string, ddl: string][] = [
+  [
+    "credentials",
+    `CREATE TABLE IF NOT EXISTS ${CREDENTIALS_TABLE} (name text PRIMARY KEY, value text NOT NULL, ` +
+      "set_by text NOT NULL, set_at timestamptz NOT NULL DEFAULT now())",
+  ],
+  [
+    "ledger",
+    // `n` is the write order, `id` the record's, which the first write of takes, and `seq` the
+    // record's place in its run: a run reads in `seq` order, then write order.
+    `CREATE TABLE IF NOT EXISTS ${LEDGER_TABLE} (n bigserial PRIMARY KEY, id text NOT NULL UNIQUE, ` +
+      `run_id text NOT NULL, seq integer NOT NULL, record jsonb NOT NULL); ` +
+      `CREATE INDEX IF NOT EXISTS ${LEDGER_TABLE}_run_seq ON ${LEDGER_TABLE} (run_id, seq, n)`,
+  ],
+];
+
+/** The Postgres error code of `err`, or an empty string when it has none. */
+export const pgCode = (err: unknown): string => (err as { code?: unknown } | null)?.code?.toString() ?? "";
+
+/** Postgres's codes for a row type or a table another process made first. */
+const MADE_ALREADY = new Set(["23505", "42P07"]);
+
+/** True for the failure of a create that another process won: what was to be made is there. */
+export const madeAlready = (err: unknown): boolean => MADE_ALREADY.has(pgCode(err));
+
+/**
+ * A pool on the config's database for `what` (a word for the messages). An idle connection that
+ * drops (Postgres restarted) is replaced on the next query; unheard, it would end the process.
+ */
+export function openPool(databaseUrl: string, what: string): Pool {
+  const pool = new Pool({ connectionString: databaseUrl });
+  pool.on("error", (err) => warn(`a ${what} connection failed: ${errorMessage(err)}`));
+  return pool;
+}
+
+/**
+ * What every DBOS handle on the config's database is given (`DBOS.setConfig`, `dbosClient`): DBOS
+ * is told not to set Postgres's `idle_in_transaction_session_timeout` on each connection of its
+ * pool. It sends that `SET` from the pool's `connect` event without awaiting it (dbos-sdk 5.2.11,
+ * dist/src/system_database.js:691), so the pool hands the connection out while it runs: two
+ * queries at once on one client, which pg deprecates. `prepareDatabase` sets the same 60 s guard
+ * on the database instead. Remove the switch when DBOS awaits its `SET`.
+ */
+export const DBOS_DATABASE_OPTIONS = { systemDatabaseIdleTransactionTimeoutMs: 0 } as const;
+
+/**
+ * A DBOS client of the app `applicationName` on the database at `databaseUrl`, as every client of
+ * the runtime, its tests and `@sanoma-ai/testing` make one (`DBOS_DATABASE_OPTIONS`).
+ */
+export const dbosClient = (databaseUrl: string, applicationName?: string): Promise<DBOSClient> =>
+  DBOSClient.create({
+    systemDatabaseUrl: databaseUrl,
+    ...(applicationName !== undefined && { applicationName }),
+    ...DBOS_DATABASE_OPTIONS,
+  });
+
+/**
+ * The `idle_in_transaction_session_timeout` `prepareDatabase` sets: DBOS's own default
+ * (`DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS`, 60_000, in dbos-sdk 5.2.11 dist/src/system_database.js),
+ * how long a frozen or unreachable client may hold the system database's locks.
+ */
+const IDLE_IN_TRANSACTION_TIMEOUT = "60s";
+
+/** Postgres's code for a statement the role lacks the privilege for. */
+const INSUFFICIENT_PRIVILEGE = "42501";
+
+/** A pool opened on first use, once its database is prepared, and ended by `close`. */
+export interface LazyPool {
+  db(): Promise<Pool>;
+  close(): Promise<void>;
+}
+
+/**
+ * A pool for `what` on the database at `databaseUrl`, opened on the first `db()` once the
+ * database is prepared (`prepareDatabase`). A preparation that fails lets the pool go, so the
+ * next `db()` tries again. `close` ends the pool, for good: a `db()` after it is refused.
+ */
+export function lazyPool(databaseUrl: string, what: string): LazyPool {
+  let pool: Pool | undefined;
+  let closed = false;
+  return {
+    async db() {
+      if (closed) throw new Error(`the ${what} store is closed`);
+      const current = (pool ??= openPool(databaseUrl, what));
+      try {
+        await prepareDatabase(databaseUrl);
+      } catch (err) {
+        if (pool === current) pool = undefined;
+        await current.end().catch(() => undefined);
+        throw err;
+      }
+      return current;
+    },
+    async close() {
+      closed = true;
+      const current = pool;
+      pool = undefined;
+      await current?.end();
+    },
+  };
+}
+
+/** Each database prepared, or being prepared, by its URL. A failed preparation is forgotten. */
+const prepared = new Map<string, Promise<void>>();
+
+/**
+ * Makes the database at `databaseUrl` ready for the runtime, once per process and URL: the
+ * database itself and DBOS's tables when they are not there yet (as a worker's launch makes
+ * them; credentials may be set, and a ledger read, before any worker has run), then the
+ * credentials and ledger tables, and its idle-transaction timeout (`guardIdleTransactions`).
+ * Two processes preparing one database both succeed. A
+ * preparation that fails is forgotten, so the next call tries again.
+ */
+export function prepareDatabase(databaseUrl: string): Promise<void> {
+  let preparing = prepared.get(databaseUrl);
+  if (!preparing) {
+    preparing = prepare(databaseUrl).catch((err: unknown) => {
+      if (prepared.get(databaseUrl) === preparing) prepared.delete(databaseUrl);
+      throw err;
+    });
+    prepared.set(databaseUrl, preparing);
+  }
+  return preparing;
+}
+
+async function prepare(databaseUrl: string): Promise<void> {
+  try {
+    await DBOS.migrate(databaseUrl);
+  } catch (err) {
+    // The database, or a DBOS table, made by another process meanwhile: it is there.
+    if (!madeAlready(err)) throw new Error(`could not prepare the database: ${errorMessage(err)}`, { cause: err });
+  }
+  const conn = new Client({ connectionString: databaseUrl });
+  await conn.connect();
+  try {
+    for (const [what, ddl] of TABLES) await createTable(conn, ddl, what);
+    await guardIdleTransactions(conn);
+  } finally {
+    await conn.end();
+  }
+}
+
+/**
+ * Sets `idle_in_transaction_session_timeout` on the database, for each session that starts after,
+ * as DBOS would per connection (`DBOS_DATABASE_OPTIONS`), unless the server, the role or the
+ * database sets it already. One process at a time (an advisory lock), so two preparing at once
+ * both succeed. A role that may not alter the database leaves it, quietly: its administrator decides.
+ */
+async function guardIdleTransactions(db: Db): Promise<void> {
+  try {
+    await db.query(`DO $$ BEGIN
+      PERFORM pg_advisory_xact_lock(hashtext('sanoma.idle_in_transaction_session_timeout'));
+      IF current_setting('idle_in_transaction_session_timeout') = '0' AND NOT EXISTS (
+        SELECT FROM pg_db_role_setting s JOIN pg_database d ON d.oid = s.setdatabase, unnest(s.setconfig) c
+        WHERE d.datname = current_database() AND s.setrole = 0 AND c LIKE 'idle_in_transaction_session_timeout=%'
+      ) THEN
+        EXECUTE format('ALTER DATABASE %I SET idle_in_transaction_session_timeout = %L', current_database(),
+          '${IDLE_IN_TRANSACTION_TIMEOUT}');
+      END IF;
+    END $$`);
+  } catch (err) {
+    if (pgCode(err) !== INSUFFICIENT_PRIVILEGE) {
+      throw new Error(`could not prepare the database: ${errorMessage(err)}`, { cause: err });
+    }
+  }
+}
+
+/**
+ * Runs `ddl` (`CREATE TABLE IF NOT EXISTS`, and the like) on `db`. Two processes racing to create
+ * the same table both succeed; any other failure names the `what` table, with its cause.
+ */
+export async function createTable(db: Db, ddl: string, what: string): Promise<void> {
+  try {
+    await db.query(ddl);
+  } catch (err) {
+    if (madeAlready(err)) return;
+    throw new Error(`could not prepare the ${what} table: ${errorMessage(err)}`, { cause: err });
+  }
+}

package/src/define.ts

@@ -0,0 +1,285 @@
+import { fileURLToPath } from "node:url";
+import { getCallSites } from "node:util";
+import { z } from "zod";
+import { isOp, type Op } from "./op.ts";
+import { approverLabel } from "./shared.ts";
+
+/**
+ * Who is acting: the person or service that starts a run or decides an approval. `id` is
+ * whatever the deployment identifies people by; `groups` are the roles it vouches for.
+ */
+export interface Principal {
+  id: string;
+  groups?: string[];
+}
+
+export const Principal: z.ZodType<Principal> = z.object({
+  id: z.string().regex(/\S/, "id must not be blank"),
+  groups: z.array(z.string()).optional(),
+});
+
+export type Builtin = "approval" | "sleep" | "all";
+export type Use = Op | Builtin;
+
+/** Who may decide an approval: one person, by `Principal.id`, or anyone in a group, by `Principal.groups`. */
+export type Approver = string | { group: string };
+
+export const Approver: z.ZodType<Approver> = z.union(
+  [z.string().regex(/\S/), z.object({ group: z.string().regex(/\S/) })],
+  { error: 'needs an approver: a name, or { group: "name" }' },
+);
+
+/** Why `by` may not decide the approval, naming who may. */
+export const notApprover = ({ approver }: Pick<ApprovalState, "approver">, by: Principal): string =>
+  typeof approver === "string"
+    ? `${by.id} is not the approver; ${approverLabel(approver)} is`
+    : `${by.id} is not in ${approverLabel(approver)}`;
+
+/** True when the two name the same approver: the same id, or the same group. */
+export const sameApprover = (a: Approver, b: Approver): boolean =>
+  typeof a === "string" || typeof b === "string" ? a === b : a.group === b.group;
+
+const notOps = "covers must be a list of operations";
+
+/** `covers` as given (operations), parsed to their ids. */
+export const Covers = z
+  .array(
+    z.object({ id: z.string() }, { error: notOps }).transform((op) => op.id),
+    { error: notOps },
+  )
+  .optional();
+
+export interface ApprovalRequest {
+  /** Who must approve: a person's id, or `{ group }` for anyone in it. Phase 4 replaces this with a Cedar decision. */
+  approver: Approver;
+  /**
+   * The operations the approval stands for, so a policy can let their calls through once it
+   * is approved (`a.covers.includes(op.id)`). None unless given.
+   */
+  covers?: Op[];
+  links?: string[];
+  details?: string;
+}
+
+/**
+ * Checks a `ctx.approval` request as a workflow may give it, so the UI and the ledger get what
+ * the types say, and turns `covers` into op ids.
+ */
+export const ApprovalRequestSchema = z.object(
+  {
+    approver: Approver,
+    covers: Covers,
+    links: z.array(z.string(), { error: "links must be a list of strings" }).optional(),
+    details: z.string({ error: "details must be a string" }).optional(),
+  },
+  { error: "needs a request: { approver, covers?, links?, details? }" },
+);
+
+/** One approval in a run, as the run sees it. Published as the run's "approvals" event. */
+export interface ApprovalState extends Omit<ApprovalRequest, "covers"> {
+  id: string;
+  title: string;
+  /** Who asked: the workflow (`ctx.approval`), or the policy holding an operation call. */
+  requestedBy: "workflow" | "policy";
+  /**
+   * The operations the approval stands for, by id. A policy hold covers the call it held and
+   * any `covers` the policy added; a workflow's approval covers the `covers` it named, or none.
+   */
+  covers: string[];
+  /** For a policy request: the operation call held, by op id. */
+  op?: string;
+  /** For a policy request: the ledger `seq` of the call held, which its `op.called` record has. */
+  opSeq?: number;
+  /** For a policy request: the held call's input. */
+  input?: unknown;
+  status: "pending" | "approved" | "rejected";
+  requestedAt: number;
+  decidedBy?: string;
+  decidedAt?: number;
+  note?: string;
+  /** The `id` of the decision message the run decided with, when it had one. */
+  decidedWith?: string;
+  /**
+   * Messages that were ignored: decisions sent by someone who may not decide the approval,
+   * and messages that were not a decision. `by` is the sender, when the message named one, and
+   * `id` the decision message's own, when it had one.
+   */
+  refused: { by?: string; id?: string; at: number; reason: string }[];
+}
+
+export interface ApprovalResult {
+  approvedBy: string;
+  at: number;
+  note?: string;
+}
+
+/**
+ * How long `ctx.sleep` waits: until a time (an ISO 8601 date-time with an offset, such as
+ * `2026-10-07T09:00:00Z`, or epoch milliseconds), or for a duration whose parts add up. A time
+ * already past waits not at all.
+ */
+export type SleepRequest =
+  | { until: string | number }
+  | { ms?: number; seconds?: number; minutes?: number; hours?: number; days?: number };
+
+const time = "must be an ISO 8601 date-time with an offset, such as 2026-10-07T09:00:00Z, or epoch milliseconds";
+const duration = z.number({ error: "must be a number" }).nonnegative("must not be negative").optional();
+
+/** Checks a `ctx.sleep` request that names a time. */
+export const SleepUntil = z.strictObject({
+  until: z.union([z.iso.datetime({ offset: true, error: time }), z.number({ error: time })], { error: time }),
+}) satisfies z.ZodType<SleepRequest>;
+
+/** Checks a `ctx.sleep` request that names a duration. */
+export const SleepFor = z.strictObject({
+  ms: duration,
+  seconds: duration,
+  minutes: duration,
+  hours: duration,
+  days: duration,
+}) satisfies z.ZodType<SleepRequest>;
+
+/** Checks a `ctx.all` argument: a list of functions, each one member. */
+export const AllMembers = z.array(
+  z.custom<() => unknown>((m) => typeof m === "function", "each member must be a function, such as () => ctx.…"),
+  { error: "needs a list of functions: ctx.all([() => …, () => …])" },
+);
+
+/**
+ * What a step is called, beside its operation: `ctx.ghost.post.create(input, { name: "Draft" })`,
+ * `ctx.sleep({ days: 1 }, { name: "Wait a day" })`. A string literal, or a top-level `const` of
+ * the workflow's file holding one, so the outline reads it. Without one, a call is named by its
+ * operation's id, a sleep `Sleep`; an approval by its title. Names are unique in a workflow, so a
+ * workflow that calls one operation twice names the calls.
+ */
+export interface StepOptions {
+  name?: string;
+}
+
+type OpsOf<U extends readonly Use[]> = Extract<U[number], Op>;
+
+type CtxOps<O extends Op> = {
+  readonly [V in O["vendor"]]: {
+    readonly [R in Extract<O, { vendor: V }>["resource"]]: {
+      readonly [N in Extract<O, { vendor: V; resource: R }>["name"]]: Extract<
+        O,
+        { vendor: V; resource: R; name: N }
+      > extends Op<any, any, any, infer I, infer Out>
+        ? (input: I, options?: StepOptions) => Promise<Out>
+        : never;
+    };
+  };
+};
+
+interface ApprovalCtx {
+  /** Waits, durably, until the approver (or someone in the approver group) approves. Throws `RejectedError` on a rejection. */
+  approval(title: string, request: ApprovalRequest): Promise<ApprovalResult>;
+}
+
+interface SleepCtx {
+  /** Waits, durably, until a time or for a duration. Survives worker restarts. */
+  sleep(request: SleepRequest, options?: StepOptions): Promise<void>;
+}
+
+interface AllCtx {
+  /**
+   * Runs the members one after another, in order, and returns their outputs in that order. It
+   * declares a fan-out for the ledger and the graph; it adds no concurrency. The first member
+   * that throws stops the group with its error, and later members never run. Not nestable.
+   */
+  all<const T extends readonly (() => Promise<unknown>)[]>(
+    members: T,
+  ): Promise<{ -readonly [K in keyof T]: Awaited<ReturnType<T[K]>> }>;
+}
+
+interface BaseCtx {
+  /** The current time, recorded so replays see the same value. Use instead of `Date.now()`. */
+  now(): Promise<number>;
+  readonly runId: string;
+}
+
+export type Ctx<U extends readonly Use[]> = BaseCtx &
+  CtxOps<OpsOf<U>> &
+  ("approval" extends U[number] ? ApprovalCtx : unknown) &
+  ("sleep" extends U[number] ? SleepCtx : unknown) &
+  ("all" extends U[number] ? AllCtx : unknown);
+
+/**
+ * A schema as JSON Schema (draft 2020-12). By default it describes what a caller sends
+ * (`io: "input"`: defaults stay optional); `io: "output"` describes what parsing returns.
+ * What JSON Schema can't express is left open. Throws when zod can't convert the schema at all.
+ */
+export function jsonSchemaOf(
+  schema: z.ZodType,
+  io: "input" | "output" = "input",
+  refs?: ReadonlyMap<unknown, string>,
+): Record<string, unknown> {
+  return z.toJSONSchema(schema, {
+    io,
+    target: "draft-2020-12",
+    unrepresentable: "any",
+    // A schema `refs` names, inside another, becomes a `$ref` to it (a resource type's state).
+    ...(refs?.size && {
+      override: ({ zodSchema, jsonSchema, path }) => {
+        const ref = path.length > 0 ? refs.get(zodSchema) : undefined;
+        if (ref === undefined) return;
+        for (const key of Object.keys(jsonSchema)) delete (jsonSchema as Record<string, unknown>)[key];
+        Object.assign(jsonSchema, { $ref: ref });
+      },
+    }),
+  });
+}
+
+export interface WorkflowDefinition<U extends readonly Use[] = readonly Use[], S extends z.ZodType = z.ZodType> {
+  readonly kind: "workflow";
+  readonly name: string;
+  readonly title?: string;
+  readonly trigger: "manual";
+  readonly input: S;
+  /** Every operation and built-in the workflow may call. `ctx` is built from this list and nothing else. */
+  readonly uses: U;
+  readonly run: (ctx: Ctx<U>, input: z.output<S>) => Promise<unknown>;
+  /**
+   * The file that defined it, absolute. Set by defineWorkflow from its call site: the outline is
+   * read from it, the worker holds each run to that outline, and the app shows it as the code view.
+   */
+  readonly file: string;
+}
+
+/** The ids of the operations a workflow `uses`, in the order declared. */
+export const opIdsOf = (wf: Pick<WorkflowDefinition, "uses">): string[] => wf.uses.filter(isOp).map((op) => op.id);
+
+export function defineWorkflow<const U extends readonly Use[], S extends z.ZodType>(
+  def: Omit<WorkflowDefinition<U, S>, "kind" | "file">,
+): WorkflowDefinition<U, S> {
+  if (!/^[a-z][a-z0-9-]*$/.test(def.name)) {
+    throw new Error(`Workflow name "${def.name}" must be lowercase letters, digits and dashes`);
+  }
+  const file = callerFile();
+  if (file === undefined) {
+    throw new Error(
+      `Workflow "${def.name}" was defined from code with no file (an eval, or a bundle without one): ` +
+        "its outline is read from the file, and a run is held to the outline",
+    );
+  }
+  return Object.freeze({ kind: "workflow", ...def, file });
+}
+
+/**
+ * The file of the code that called the function calling this one (`defineWorkflow`'s caller),
+ * from V8's call sites, as a path; undefined where there is none.
+ */
+export function callerFile(): string | undefined {
+  // Frame 0 is this function, frame 1 its caller, frame 2 theirs: by index, since a bundle may hold all three.
+  return scriptPath(getCallSites(3)[2]?.scriptName);
+}
+
+/** A V8 script name (a `file:` URL for a module, a path for a script) as a path; undefined for none. */
+export function scriptPath(script: string | undefined): string | undefined {
+  if (!script?.startsWith("file:")) return script || undefined;
+  try {
+    return fileURLToPath(script);
+  } catch {
+    return undefined;
+  }
+}