@sanoma-ai/workflows 0.6.0 → 0.7.1

package/src/wakeup.ts

@@ -0,0 +1,139 @@
+import type { z } from "zod";
+import { hasPath } from "./host.ts";
+import type { Op } from "./op.ts";
+import type { References, Resource } from "./resource.ts";
+import type { DeclaredResource } from "./resources.ts";
+
+// The `Wakeup` contract: what a resource type that wakes a stopped instance at a time declares
+// about itself, beside `Host`. Declaring one in a data file is the whole switch: the worker then
+// registers its next deadline with it before it stops, and the wakeup calls the host's
+// `GET {url}/wake` then: a wake is "resume if stopped", so one that arrives late, twice or for a
+// replaced registration starts a worker that finds nothing to do and stops again. The contract is
+// here; the worker's side is `deadlines.ts`.
+
+/** What a wakeup's state says of it. */
+export interface WakeupOutputs {
+  healthy: boolean;
+}
+
+/** What the runtime calls a wakeup's `register` operation with. */
+export interface WakeupRegistration {
+  /** When to wake the instance, as an ISO 8601 date-time. */
+  at: string;
+}
+
+/**
+ * A resource type's `wakeup`: the field it names the host in, the operation the runtime
+ * registers each deadline through, and how its state reads.
+ */
+export interface WakeupSpec {
+  inputs: {
+    /**
+     * The dotted path of the declared field that names the host's declaration: a
+     * `fields.references` entry, resolved `by` a field of the host's state holding its URL.
+     */
+    host: string;
+  };
+  /**
+   * The id of the type's own register operation, `<vendor>.<type>.<op>`: an operation in the
+   * type's own group of its connector, beside `read` and `import` (the way Supabase adds
+   * `project.health` beside `project`). The runtime calls it with `WakeupRegistration`,
+   * at least once per registration; each replaces whatever the wakeup held.
+   */
+  register: string;
+  outputs: (state: Record<string, unknown>) => WakeupOutputs;
+}
+
+/** The operations `defineResource` gives a type itself: never its register, or a database's `storeUrl`. */
+export const RESOURCE_OPS = new Set(["read", "import", "plan", "create", "update", "delete"]);
+
+/**
+ * Throws what is wrong with a resource type's `wakeup`: a type apply cannot change (not
+ * `writable`), a host (stopped when its wake is due), a `host` that names no field or no
+ * reference resolved `by` a field, a `register` that is not an operation of the type's own group
+ * or is one of its resource operations, `outputs` that is not a function. Whether the connector
+ * has that operation, and the reference names a host, is `checkWakeupOps`'s, which
+ * `resolveConfig` runs.
+ */
+export function checkWakeup(
+  what: string,
+  wakeup: WakeupSpec,
+  type: {
+    vendor: string;
+    type: string;
+    schema: z.ZodObject;
+    fields: { references?: References };
+    writable?: boolean;
+    host?: unknown;
+  },
+): void {
+  const problems: string[] = [];
+  if (!type.writable) problems.push("it must be writable, so apply can create and update it");
+  if (type.host) problems.push("a host cannot be its own wakeup: it is stopped when the wake is due");
+  const host = wakeup.inputs?.host;
+  const reference = typeof host === "string" ? type.fields.references?.[host] : undefined;
+  if (typeof host !== "string" || !hasPath(type.schema, host)) problems.push(`inputs.host names no field (${host})`);
+  else if (!reference) problems.push(`inputs.host must be a reference to the host (fields.references), not ${host}`);
+  else if (!reference.by?.length)
+    problems.push("inputs.host must be resolved `by` the field of the host's state that holds its URL");
+  const group = `${type.vendor}.${type.type}.`;
+  const register: unknown = wakeup.register;
+  const name = typeof register === "string" && register.startsWith(group) ? register.slice(group.length) : "";
+  if (!/^\w+$/.test(name))
+    problems.push(`register must name an operation of its own group, ${group}<op> (${String(register)})`);
+  else if (RESOURCE_OPS.has(name)) problems.push(`register must be an operation of its own, not the type's ${name}`);
+  if (typeof wakeup.outputs !== "function") problems.push("outputs must be a function of the state");
+  if (problems.length) throw new Error(`${what}: its wakeup: ${problems.join("; ")}`);
+}
+
+/** A registration every register operation must take: the input `checkWakeupOps` tries. */
+const SAMPLE: WakeupRegistration = { at: new Date(0).toISOString() };
+
+/**
+ * True when `op`'s input takes `sample` and keeps each of its fields: one that strips a field
+ * takes none of it. A wakeup's `register` is checked with it.
+ */
+function keepsInput(op: Op, sample: Record<string, unknown>): boolean {
+  try {
+    const parsed = op.input.safeParse(sample);
+    const data = parsed.data as Record<string, unknown> | undefined;
+    return parsed.success && Object.keys(sample).every((key) => data?.[key] !== undefined);
+  } catch {
+    // An async refinement, which a sync parse cannot run: no input the runtime can check.
+    return false;
+  }
+}
+
+/** The host types a wakeup type may name: its `inputs.host` reference's. */
+export const wakeupHostTypes = (type: Pick<Resource, "fields" | "wakeup">): string[] =>
+  type.wakeup ? [type.fields.references?.[type.wakeup.inputs.host]?.type ?? []].flat() : [];
+
+/**
+ * Throws when a wakeup type's host reference names a type of the config that is no host, or its
+ * `register` is not among the config's operations, or does not take
+ * `WakeupRegistration`. Here rather than in `defineConnector`, which sees one connector before
+ * its package merges in the operations it adds to a resource's group (Supabase's pattern).
+ */
+export function checkWakeupOps(types: ReadonlyMap<string, Resource>, ops: ReadonlyMap<string, Op>): void {
+  for (const [id, type] of types) {
+    if (!type.wakeup) continue;
+    // The types it may name that the config has, each a host; one it lacks no data file can declare.
+    const notHosts = wakeupHostTypes(type).filter((t) => types.has(t) && !types.get(t)!.host);
+    if (notHosts.length) throw new Error(`${id}: its wakeup's host may name ${notHosts.join(", ")}, which is no host`);
+    const op = ops.get(type.wakeup.register);
+    if (!op) throw new Error(`${id}: its wakeup's register, ${type.wakeup.register}, is no operation of its connector`);
+    if (!keepsInput(op, { ...SAMPLE })) throw new Error(`${id}: its wakeup's register, ${op.id}, must take { at }`);
+  }
+}
+
+/** The one wakeup the data files declare: its declaration, its type, and its binding when it is bound. */
+export interface DeclaredWakeup {
+  /** `<file>#<export>`, as `DeclaredResource.id`. */
+  id: string;
+  type: Resource;
+  declared: DeclaredResource;
+  /** The vendor's id of the object the lockfile binds it to; absent when unbound. */
+  binding?: string;
+  /** The declared host its `inputs.host` names: its declaration's id and its type, a host's (`checkWakeupOps`). */
+  host?: { id: string; type: Resource };
+}

package/src/worker.ts

@@ -1,10 +1,11 @@
-import { DBOS } from "@dbos-inc/dbos-sdk";
+import { DBOS, type ScheduledWorkflowFn } from "@dbos-inc/dbos-sdk";
 import { awaitApproval } from "./approvals.ts";
 import { buildCtx, decideStart, isInfrastructureError, startHoldOf } from "./call.ts";
 import { dbosStatusesOf } from "./client.ts";
-import { type ResolvedConfig, resolveConfig, type SanomaConfig } from "./config.ts";
+import { declaredWakeup, type ResolvedConfig, resolveConfig, type SanomaConfig } from "./config.ts";
 import { storedSecrets, watchCredentials } from "./credentials.ts";
 import { DBOS_DATABASE_OPTIONS, lazyPool, prepareDatabase } from "./db.ts";
+import { deadlineRegistry, type Deadlines, idle, onOwnQueue, type SelfStop, stopWhenIdle } from "./deadlines.ts";
 import type { WorkflowDefinition } from "./define.ts";
 import { credentialsOf, unconfigured, unconfiguredSummary } from "./env.ts";
 import { errorInfo, parseOrThrow, PolicyDeniedError } from "./errors.ts";
@@ -13,8 +14,9 @@ import { dbosLogger, quietIfSandbox, setLogLevel, warn } from "./log.ts";
 import { lineStartsOf, readSource } from "./ast.ts";
 import { callsOf, outlineWithSource } from "./outline.ts";
 import { currentRun, type Run, type RunArgs, type WorkerState, type WorkflowOutline } from "./run.ts";
+import { applyScheduled, nextScheduledFire, type ScheduledStarter, scheduledStarter } from "./schedules.ts";
 import type { Source } from "./source.ts";
-import { errorMessage, WORKFLOW_ACTION } from "./shared.ts";
+import { approverLabel, errorMessage, WORKFLOW_ACTION } from "./shared.ts";
 
 export interface Worker {
   /**
@@ -23,14 +25,36 @@ export interface Worker {
    * The worker names them in one line at start, unless started `quiet`.
    */
   readonly unconfigured: readonly string[];
+  /**
+   * Its runs' deadlines: each sleeping run's, and the sources added here (a schedule's next
+   * fire), which a stop registers the earliest of with the wakeup.
+   */
+  readonly deadlines: Deadlines;
+  /**
+   * True when nothing this worker would run can run soon (`idle` in deadlines.ts): no run executing,
+   * none queued, every unfinished one blocked in a sleep or an approval, the next deadline past
+   * `idleHorizonMs`. False once stopped.
+   */
+  idle(): Promise<boolean>;
+  /** Stops the worker; a second call waits for the first. */
   stop(): Promise<void>;
 }
 
+/**
+ * How many times DBOS may start a run again before it gives up on it
+ * (`MAX_RECOVERY_ATTEMPTS_EXCEEDED`). Every start of a worker counts one for each unfinished run
+ * of its version, and a worker that stops when idle starts again at each wake and each request:
+ * at one every five minutes, DBOS's default of 100 would fail a run waiting on an approval within
+ * nine hours. 100,000 is a year of that; a run that crashes its process at every recovery is still
+ * given up on, after as many restarts.
+ */
+const MAX_RECOVERY_ATTEMPTS = 100_000;
+
 // DBOS registrations are process-wide and survive shutdown, so a name is registered once, and
 // its function reads the state of the worker running when a run starts, definition and all; the
-// run keeps that state.
+// run keeps that state. Each registered function, by name, is what a schedule fires.
 let current: WorkerState | undefined;
-const registered = new Set<string>();
+const registered = new Map<string, ScheduledWorkflowFn>();
 
 export interface WorkerOptions {
   /** The least severe level of the runtime's and DBOS's messages to show, for the whole process: `warn` by default. */
@@ -50,6 +74,31 @@ export interface WorkerOptions {
   promote?: boolean;
   /** The company's repository, which apply commits the lockfile to; the launcher passes its checkout. */
   source?: Source;
+  /**
+   * How soon a deadline keeps the worker up (`idle`): 60 s by default, so apply's short sleeps
+   * and other short waits never stop it.
+   */
+  idleHorizonMs?: number;
+  /** How often a worker whose data files declare a wakeup checks whether it may stop: every 15 s by default. */
+  stopCheckMs?: number;
+  /**
+   * Given, and with a wakeup declared, the worker stops itself when idle (`stopCheckMs`); without
+   * it, nobody would start another, so it never does. Called once the worker has stopped itself, with the deadline it registered with the
+   * wakeup (null for none): a supervisor starts a fresh worker on the next write or wake, since a
+   * stopped one cannot launch again.
+   */
+  onStopped?: (stopped: SelfStop) => void;
+  /**
+   * Apply the workflows' schedules (`trigger: { schedule }`): on by default. `runScenarios` turns
+   * it off, on a database of its own, so no schedule is applied there to start a live run.
+   */
+  schedules?: boolean;
+  /**
+   * The config as the caller resolved it (`resolveConfig(config, options)`), for a worker that
+   * shares its drivers with the caller: the instance fakes of `fakes: "instance"`, which a demo's
+   * client and reseed call too. The caller closes its `dataStore`, as it made it.
+   */
+  resolved?: ResolvedConfig;
 }
 
 /**
@@ -61,16 +110,29 @@ export interface WorkerOptions {
  */
 export async function startWorker(config: SanomaConfig, options: WorkerOptions = {}): Promise<Worker> {
   // Check everything before touching the state a running worker reads.
-  const resolved = resolveConfig(config);
+  const resolved = options.resolved ?? resolveConfig(config);
   setLogLevel(options.logLevel);
+  // Read once, when it matters: a worker stops itself, registering its next deadline there, only
+  // when its caller hears it stop (`onStopped`).
+  const wakeup = options.onStopped || config.wakeup === true ? await declaredWakeup(config) : undefined;
+  // The wakeup service wakes the instances that stop: one that runs it must not stop with its worker.
+  if (config.wakeup === true && wakeup) {
+    throw new Error(
+      "The config runs the wakeup service (`wakeup: true`) and declares a wakeup of its own: an instance that runs the service must not stop, so leave out one of them",
+    );
+  }
   // The config's own, or a Postgres store made for this worker, which `stop` then closes.
   const ledger = ownedLedger(resolved);
   const secrets = lazyPool(resolved.databaseUrl, "secrets");
+  let stopRule: { close(): void } | undefined;
+  let stopping: Promise<void> | undefined;
+  const horizonMs = options.idleHorizonMs ?? 60_000;
   const state: WorkerState = {
     app: resolved.appName,
     ops: resolved.ops,
     drivers: resolved.drivers,
     fakes: resolved.fakes,
+    ...(resolved.instanceFakes && { instanceFakes: resolved.instanceFakes }),
     ...(resolved.scenarios && { scenarios: resolved.scenarios }),
     ...(options.source && { source: options.source }),
     types: resolved.types,
@@ -83,26 +145,38 @@ export async function startWorker(config: SanomaConfig, options: WorkerOptions =
     workflows: resolved.workflows,
     outlines: new Map([...resolved.workflows].map(([name, wf]) => [name, outlineOf(wf)])),
     stopped: false,
+    deadlines: deadlineRegistry(),
   };
   const credentials = await watchCredentials(resolved.databaseUrl, resolved.driversByVendor);
   const worker: Worker = {
     unconfigured: [...credentialsOf(resolved.driversByVendor, (name) => process.env[name])]
       .filter(([vendor, statuses]) => unconfigured(vendor, statuses))
       .map(([vendor]) => vendor),
-    async stop() {
-      state.stopped = true;
-      // Each is closed whatever the one before did; the first failure is what stop throws.
-      let failed: { error: unknown } | undefined;
-      for (const close of [
-        () => DBOS.shutdown(),
-        () => credentials.close(),
-        () => secrets.close(),
-        () => ledger.close(),
-        async () => resolved.dataStore?.close(),
-      ]) {
-        await close().catch((error: unknown) => (failed ??= { error }));
-      }
-      if (failed) throw failed.error;
+    deadlines: state.deadlines,
+    async idle() {
+      return !state.stopped && idle(state.deadlines, resolved, horizonMs);
+    },
+    stop() {
+      // A stop that failed may be tried again.
+      return (stopping ??= (async () => {
+        state.stopped = true;
+        stopRule?.close();
+        // Each is closed whatever the one before did; the first failure is what stop throws.
+        let failed: { error: unknown } | undefined;
+        for (const close of [
+          () => DBOS.shutdown(),
+          () => credentials.close(),
+          () => secrets.close(),
+          () => ledger.close(),
+          async () => (options.resolved ? undefined : resolved.dataStore?.close()),
+        ]) {
+          await close().catch((error: unknown) => (failed ??= { error }));
+        }
+        if (failed) {
+          stopping = undefined;
+          throw failed.error;
+        }
+      })());
     },
   };
   // Runs recovered during launch start before it returns, so they must already find this
@@ -112,10 +186,7 @@ export async function startWorker(config: SanomaConfig, options: WorkerOptions =
   let launched = false;
   try {
     for (const name of resolved.workflows.keys()) {
-      if (!registered.has(name)) {
-        register(name);
-        registered.add(name);
-      }
+      if (!registered.has(name)) registered.set(name, register(name));
     }
     current = state;
     // Before DBOS's pool connects, so its connections start with the database's idle-transaction
@@ -135,6 +206,7 @@ export async function startWorker(config: SanomaConfig, options: WorkerOptions =
     // DBOS gives runs queued without a version only to the app's latest version, which is the
     // newest one registered. A worker started on code seen before (a rollback) is not it.
     const latest = (await DBOS.getLatestApplicationVersion()).versionName;
+    const isLatest = latest === resolved.version || options.promote === true;
     if (latest !== resolved.version) {
       if (options.promote) await DBOS.setLatestApplicationVersion(resolved.version);
       else {
@@ -145,12 +217,34 @@ export async function startWorker(config: SanomaConfig, options: WorkerOptions =
       }
     }
     await DBOS.registerQueue(resolved.queueName);
+    // The schedules are the latest version's: an older worker leaves them as the newer set them.
+    if (isLatest && options.schedules !== false) {
+      await applyScheduled(resolved, registered, () => secrets.db());
+      // A schedule's next tick is a deadline: a worker that stops when quiet registers it with its wakeup.
+      state.deadlines.addSource(() => nextScheduledFire(resolved, new Date()));
+    }
     await warnAboutStrandedRuns(resolved);
     const summary = unconfiguredSummary(worker.unconfigured, resolved.driversByVendor.size);
     if (summary && !options.quiet) warn(summary);
     if (resolved.policyProblems.length && !options.quiet) {
       warn(`the policies do not hold, so every call and start is refused:\n  ${resolved.policyProblems.join("\n  ")}`);
     }
+    if (wakeup && options.onStopped) {
+      const registerId = wakeup.type.wakeup!.register;
+      stopRule = stopWhenIdle({
+        // The row is one more of the runtime's tables: the secrets' pool, on the same database.
+        db: () => secrets.db(),
+        wakeup: wakeup.binding ?? wakeup.id,
+        // `resolveConfig` checked the operation is the config's (`checkWakeupOps`).
+        register: { op: resolved.ops.get(registerId)!, fn: resolved.drivers.get(registerId) },
+        secrets: state.secrets,
+        deadlines: state.deadlines,
+        idle: () => worker.idle(),
+        everyMs: options.stopCheckMs ?? 15_000,
+        stop: () => worker.stop(),
+        onStopped: options.onStopped,
+      });
+    }
   } catch (err) {
     // Once DBOS is launched, the caller gets no worker to stop: stop it here, so no run goes on
     // in a worker nobody holds, and the next startWorker launches afresh.
@@ -202,9 +296,7 @@ async function warnAboutStrandedRuns({ appName, version, queueName }: ResolvedCo
   });
   const otherVersion = runs.filter((r) => r.applicationVersion && r.applicationVersion !== version);
   // Recovery moves a worker's own runs to DBOS's internal queues, which are fine.
-  const otherQueue = runs.filter(
-    (r) => !otherVersion.includes(r) && r.queueName && r.queueName !== queueName && !r.queueName.startsWith("_dbos_"),
-  );
+  const otherQueue = runs.filter((r) => !otherVersion.includes(r) && !onOwnQueue(r.queueName, queueName));
   if (!otherVersion.length && !otherQueue.length) return;
   const ids = [...otherVersion, ...otherQueue].map((r) => r.workflowID);
   warn(
@@ -216,22 +308,46 @@ async function warnAboutStrandedRuns({ appName, version, queueName }: ResolvedCo
   );
 }
 
-/** Registers the name with DBOS: its runs take the definition of that name from the running worker. */
-function register(name: string) {
-  DBOS.registerWorkflow((args: RunArgs) => quietIfSandbox(args.sandbox !== undefined, () => runWorkflow(name, args)), {
-    name,
-  });
+/**
+ * Registers the name with DBOS: its runs take the definition of that name from the running worker.
+ * A start passes its `RunArgs`; a schedule's tick passes the time it is for (and DBOS's context).
+ */
+function register(name: string): ScheduledWorkflowFn {
+  const fn = DBOS.registerWorkflow(
+    (start: RunArgs | Date) =>
+      start instanceof Date
+        ? runWorkflow(name, start)
+        : quietIfSandbox(start.sandbox !== undefined, () => runWorkflow(name, start)),
+    { name, maxRecoveryAttempts: MAX_RECOVERY_ATTEMPTS },
+  );
+  return fn as unknown as ScheduledWorkflowFn;
 }
 
-/** One run of the workflow registered as `name`, on the state of the worker running when it starts. */
-async function runWorkflow(name: string, { input, startedBy, sandbox, hold }: RunArgs): Promise<unknown> {
+/**
+ * One run of the workflow registered as `name`, on the state of the worker running when it
+ * starts, counted busy in its deadlines but while it waits: started with its `RunArgs`, or by its
+ * schedule for the time `start`, as the `schedule` agent with the input `{}`, the policy deciding
+ * that start at the top of the run.
+ */
+async function runWorkflow(name: string, start: RunArgs | Date): Promise<unknown> {
   const state = current;
   // A stopped worker's state stays current while DBOS shuts down.
   if (!state || state.stopped) throw new Error(`Run of "${name}" started with no worker running`);
+  return state.deadlines.running(DBOS.workflowID!, () => runOn(state, name, start));
+}
+
+async function runOn(state: WorkerState, name: string, start: RunArgs | Date): Promise<unknown> {
   const wf = state.workflows.get(name);
   const outline = state.outlines.get(name);
   // Registered by an earlier worker in this process, whose config had it; this one's has not.
   if (!wf || !outline) throw new Error(`Run of "${name}" refused: this worker's config has no workflow of that name`);
+  let scheduled: ({ at: Date } & ScheduledStarter) | undefined;
+  let args: RunArgs;
+  if (start instanceof Date) {
+    scheduled = { at: start, ...(await scheduledStarter(state)) };
+    args = { input: {}, startedBy: scheduled.startedBy };
+  } else args = start;
+  const { input, startedBy, sandbox, hold } = args;
   const run: Run = {
     id: DBOS.workflowID!,
     workflow: wf.name,
@@ -245,25 +361,42 @@ async function runWorkflow(name: string, { input, startedBy, sandbox, hold }: Ru
     outline,
     state,
   };
-  await write(run, entry(run, { type: "run.started", input }));
+  await write(
+    run,
+    entry(run, { type: "run.started", input, ...(scheduled && { scheduledFor: scheduled.at.toISOString() }) }),
+  );
   let output: unknown;
   try {
     try {
       // The one parse of the input: the client checked it, but sent it as given.
       const parsed = parseOrThrow(wf.input, input, `The input does not match ${wf.name}'s schema`);
-      // The policy held the start: the body waits for each hold's approval in turn, the policy
-      // asked again once one is approved, and a rejection or a deny fails the run.
       const action = `${WORKFLOW_ACTION}${wf.name}`;
-      for (let next = hold; next;) {
-        const held = { op: action, input, reasons: next.reasons };
-        await awaitApproval(run, next.title, { approver: next.approver, covers: [action] }, undefined, held);
+      /** The policy's answer to this start, in a step; a deny fails the run. */
+      const decide = async () => {
         const decision = await DBOS.runStep(
           async () => decideStart(state.policy, wf.name, parsed, startedBy, run.approvals),
-          {
-            name: `policy:${action}`,
-          },
+          { name: `policy:${action}` },
         );
         if (decision.kind === "deny") throw new PolicyDeniedError(action, decision.reason);
+        return decision;
+      };
+      if (scheduled) {
+        if (scheduled.refused) throw new Error(`The schedule agent was not admitted: ${scheduled.refused}`);
+        // A schedule has no one to wait for, and its next tick would start another run meanwhile.
+        const decision = await decide();
+        if (decision.kind === "approve") {
+          throw new PolicyDeniedError(
+            action,
+            `it needs ${approverLabel(decision.approver)} to approve it, and a scheduled start waits for no approval`,
+          );
+        }
+      }
+      // The policy held the start: the body waits for each hold's approval in turn, the policy
+      // asked again once one is approved, and a rejection or a deny fails the run.
+      for (let next = hold; next;) {
+        const held = { op: action, input, reasons: next.reasons };
+        await awaitApproval(run, next.title, { approver: next.approver, covers: [action] }, undefined, held);
+        const decision = await decide();
         next = decision.kind === "approve" ? startHoldOf(wf.name, decision) : undefined;
       }
       if (sandbox !== undefined) {