@sanoma-ai/workflows 0.0.0-stage → 0.2.0

package/src/worker.ts

@@ -0,0 +1,264 @@
+import { DBOS } from "@dbos-inc/dbos-sdk";
+import { buildCtx, isInfrastructureError } from "./call.ts";
+import { dbosStatusesOf } from "./client.ts";
+import { type ResolvedConfig, resolveConfig, type SanomaConfig } from "./config.ts";
+import { storedSecrets, watchCredentials } from "./credentials.ts";
+import { DBOS_DATABASE_OPTIONS, lazyPool, prepareDatabase } from "./db.ts";
+import type { WorkflowDefinition } from "./define.ts";
+import { credentialsOf, unconfigured, unconfiguredSummary } from "./env.ts";
+import { errorInfo, parseOrThrow } from "./errors.ts";
+import { entry, ownedLedger, skipped, write, writeFailure } from "./ledger.ts";
+import { dbosLogger, quietIfSandbox, setLogLevel, warn } from "./log.ts";
+import { lineStartsOf, readSource } from "./ast.ts";
+import { callsOf, outlineWithSource } from "./outline.ts";
+import { currentRun, type Run, type RunArgs, type WorkerState, type WorkflowOutline } from "./run.ts";
+import type { Source } from "./source.ts";
+import { errorMessage } from "./shared.ts";
+
+export interface Worker {
+  /**
+   * The vendors whose drivers could not run when the worker started, in the config's order: a
+   * live call to one fails, saying why (`unconfigured` in env.ts), until its variables are set.
+   * The worker names them in one line at start, unless started `quiet`.
+   */
+  readonly unconfigured: readonly string[];
+  stop(): Promise<void>;
+}
+
+// DBOS registrations are process-wide and survive shutdown, so a name is registered once, and
+// its function reads the state of the worker running when a run starts, definition and all; the
+// run keeps that state.
+let current: WorkerState | undefined;
+const registered = new Set<string>();
+
+export interface WorkerOptions {
+  /** The least severe level of the runtime's and DBOS's messages to show, for the whole process: `warn` by default. */
+  logLevel?: string;
+  /**
+   * Start without naming the vendors not configured yet (`unconfigured` still does): for a worker
+   * whose runs use the fakes alone, as `runScenarios`'s.
+   */
+  quiet?: boolean;
+  /**
+   * Make this worker's version the app's latest, so runs queued without a version come here.
+   * A brand-new version becomes the latest on its own; set this only when deliberately starting
+   * a worker on a version seen before (a rollback). Off by default, because an old instance
+   * restarting during a rolling deploy would otherwise take the new workers' runs.
+   */
+  promote?: boolean;
+  /** The company's repository, which apply commits the lockfile to; the launcher passes its checkout. */
+  source?: Source;
+}
+
+/**
+ * Registers the workflows, connects to Postgres and recovers any runs that were interrupted.
+ * When a driver declares variables the environment leaves unset, it first loads their stored
+ * values into `process.env` and keeps them current while it runs. It starts whether or not they
+ * are all there: it prints one line naming the vendors that lack one (`unconfigured`), and a live
+ * call to such a vendor fails, saying why, as the call's own final failure, until they are set.
+ */
+export async function startWorker(config: SanomaConfig, options: WorkerOptions = {}): Promise<Worker> {
+  // Check everything before touching the state a running worker reads.
+  const resolved = resolveConfig(config);
+  setLogLevel(options.logLevel);
+  // The config's own, or a Postgres store made for this worker, which `stop` then closes.
+  const ledger = ownedLedger(resolved);
+  const secrets = lazyPool(resolved.databaseUrl, "secrets");
+  const state: WorkerState = {
+    app: resolved.appName,
+    ops: resolved.ops,
+    drivers: resolved.drivers,
+    fakes: resolved.fakes,
+    ...(resolved.scenarios && { scenarios: resolved.scenarios }),
+    ...(options.source && { source: options.source }),
+    types: resolved.types,
+    steps: resolved.steps,
+    policy: resolved.policy,
+    ledger: ledger.store,
+    secrets: storedSecrets(() => secrets.db(), "sanoma"),
+    workflows: resolved.workflows,
+    outlines: new Map([...resolved.workflows].map(([name, wf]) => [name, outlineOf(wf)])),
+    stopped: false,
+  };
+  const credentials = await watchCredentials(resolved.databaseUrl, resolved.driversByVendor);
+  const worker: Worker = {
+    unconfigured: [...credentialsOf(resolved.driversByVendor, (name) => process.env[name])]
+      .filter(([vendor, statuses]) => unconfigured(vendor, statuses))
+      .map(([vendor]) => vendor),
+    async stop() {
+      state.stopped = true;
+      // Each is closed whatever the one before did; the first failure is what stop throws.
+      let failed: { error: unknown } | undefined;
+      for (const close of [
+        () => DBOS.shutdown(),
+        () => credentials.close(),
+        () => secrets.close(),
+        () => ledger.close(),
+      ]) {
+        await close().catch((error: unknown) => (failed ??= { error }));
+      }
+      if (failed) throw failed.error;
+    },
+  };
+  // Runs recovered during launch start before it returns, so they must already find this
+  // state. If starting fails, a worker still running in the process gets its own back, once
+  // DBOS has stopped: until then a run DBOS dispatches finds this state stopped, and is refused.
+  const previous = current;
+  let launched = false;
+  try {
+    for (const name of resolved.workflows.keys()) {
+      if (!registered.has(name)) {
+        register(name);
+        registered.add(name);
+      }
+    }
+    current = state;
+    // Before DBOS's pool connects, so its connections start with the database's idle-transaction
+    // timeout (`DBOS_DATABASE_OPTIONS`); once per process and database.
+    await prepareDatabase(resolved.databaseUrl);
+    DBOS.setConfig({
+      name: resolved.appName,
+      systemDatabaseUrl: resolved.databaseUrl,
+      ...DBOS_DATABASE_OPTIONS,
+      // A logger of our own replaces DBOS's OTLP log export, so only while DBOS exports none (it
+      // does on DBOS Cloud); its level is then the threshold `setLogLevel` set, DBOS's own `logLevel` unread.
+      ...(process.env.DBOS__CLOUD === "true" ? { logLevel: options.logLevel ?? "warn" } : { logger: dbosLogger }),
+      applicationVersion: resolved.version,
+    });
+    await DBOS.launch();
+    launched = true;
+    // DBOS gives runs queued without a version only to the app's latest version, which is the
+    // newest one registered. A worker started on code seen before (a rollback) is not it.
+    const latest = (await DBOS.getLatestApplicationVersion()).versionName;
+    if (latest !== resolved.version) {
+      if (options.promote) await DBOS.setLatestApplicationVersion(resolved.version);
+      else {
+        warn(
+          `this worker runs version ${resolved.version} but the app's latest is ${latest}; ` +
+            "runs queued without a version go to the latest. Start with { promote: true } to take them here",
+        );
+      }
+    }
+    await DBOS.registerQueue(resolved.queueName);
+    await warnAboutStrandedRuns(resolved);
+    const summary = unconfiguredSummary(worker.unconfigured, resolved.driversByVendor.size);
+    if (summary && !options.quiet) warn(summary);
+  } catch (err) {
+    // Once DBOS is launched, the caller gets no worker to stop: stop it here, so no run goes on
+    // in a worker nobody holds, and the next startWorker launches afresh.
+    let stopFailed: { error: unknown } | undefined;
+    if (launched) await worker.stop().catch((error: unknown) => (stopFailed = { error }));
+    else {
+      await credentials
+        .close()
+        .catch((e: unknown) => warn(`closing the credentials listener failed: ${errorMessage(e)}`));
+    }
+    current = previous;
+    if (stopFailed) {
+      throw new AggregateError(
+        [err, stopFailed.error],
+        `${errorMessage(err)} (and stopping DBOS failed too: ${errorMessage(stopFailed.error)})`,
+        { cause: err },
+      );
+    }
+    throw err;
+  }
+  return worker;
+}
+
+/**
+ * The workflow's outline, read from its file, which every run is held to. A workflow whose
+ * outline cannot be read does not start: nothing could say where its calls are.
+ */
+function outlineOf(wf: WorkflowDefinition<any, any>): WorkflowOutline {
+  const { outline, source } = outlineWithSource(wf, readSource(wf.file));
+  if ("error" in outline) {
+    throw new Error(
+      `Workflow "${wf.name}" cannot be outlined, so its runs could not be held to its code: ${outline.error}`,
+    );
+  }
+  return { file: outline.file, lineStarts: lineStartsOf(source!), calls: callsOf(outline.nodes) };
+}
+
+/**
+ * Warns about unfinished runs this worker will never pick up: started on another version of
+ * the app, or queued on another queue (such as the single "sanoma" queue before queues were
+ * named per app, or the sandbox queue before sandbox runs went on the app's).
+ */
+async function warnAboutStrandedRuns({ appName, version, queueName }: ResolvedConfig) {
+  const runs = await DBOS.listWorkflows({
+    status: dbosStatusesOf("queued", "running"),
+    applicationName: appName,
+    loadInput: false,
+    loadOutput: false,
+  });
+  const otherVersion = runs.filter((r) => r.applicationVersion && r.applicationVersion !== version);
+  // Recovery moves a worker's own runs to DBOS's internal queues, which are fine.
+  const otherQueue = runs.filter(
+    (r) => !otherVersion.includes(r) && r.queueName && r.queueName !== queueName && !r.queueName.startsWith("_dbos_"),
+  );
+  if (!otherVersion.length && !otherQueue.length) return;
+  const ids = [...otherVersion, ...otherQueue].map((r) => r.workflowID);
+  warn(
+    `${otherVersion.length} unfinished run(s) of "${appName}" belong to another version and ${otherQueue.length} ` +
+      `wait on another queue, so this worker (version ${version}, queue ${queueName}) will not run them: ` +
+      `${ids.slice(0, 5).join(", ")}${ids.length > 5 ? ", …" : ""}. Run the version that started them, or move each ` +
+      `with DBOSClient.forkWorkflow(id, step, { applicationVersion: "${version}", queueName: "${queueName}" }) ` +
+      `and cancel the original; resumeWorkflow(id, { queueName: "${queueName}" }) moves a run on this version to this queue.`,
+  );
+}
+
+/** Registers the name with DBOS: its runs take the definition of that name from the running worker. */
+function register(name: string) {
+  DBOS.registerWorkflow((args: RunArgs) => quietIfSandbox(args.sandbox !== undefined, () => runWorkflow(name, args)), {
+    name,
+  });
+}
+
+/** One run of the workflow registered as `name`, on the state of the worker running when it starts. */
+async function runWorkflow(name: string, { input, startedBy, sandbox }: RunArgs): Promise<unknown> {
+  const state = current;
+  // A stopped worker's state stays current while DBOS shuts down.
+  if (!state || state.stopped) throw new Error(`Run of "${name}" started with no worker running`);
+  const wf = state.workflows.get(name);
+  const outline = state.outlines.get(name);
+  // Registered by an earlier worker in this process, whose config had it; this one's has not.
+  if (!wf || !outline) throw new Error(`Run of "${name}" refused: this worker's config has no workflow of that name`);
+  const run: Run = {
+    id: DBOS.workflowID!,
+    workflow: wf.name,
+    actor: startedBy,
+    ...(sandbox === undefined ? {} : { sandbox }),
+    approvals: [],
+    seq: 0,
+    tail: Promise.resolve(),
+    inAll: false,
+    ended: false,
+    outline,
+    state,
+  };
+  await write(run, entry(run, { type: "run.started", input }));
+  let output: unknown;
+  try {
+    try {
+      // The one parse of the input: the client checked it, but sent it as given.
+      const parsed = parseOrThrow(wf.input, input, `The input does not match ${wf.name}'s schema`);
+      if (sandbox !== undefined) {
+        // Imported here, so a live worker never loads the Gherkin parser or faker.
+        const { seedSandbox } = await import("./sandbox.ts");
+        await seedSandbox(run, sandbox);
+      }
+      output = await currentRun.run(run, () => wf.run(buildCtx(wf, run), parsed));
+    } finally {
+      // Before the outcome is written: a call still queued must find the run ended.
+      run.ended = true;
+    }
+  } catch (err) {
+    const record = entry(run, { type: "run.failed", error: errorInfo(err) });
+    if (isInfrastructureError(err, state)) skipped(record, `the run was interrupted (${errorMessage(err)})`);
+    else await writeFailure(run, record, err);
+    throw err;
+  }
+  await write(run, entry(run, { type: "run.finished", output }));
+  return output;
+}

package/src/writer.ts

@@ -0,0 +1,225 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { FormatConfig } from "oxfmt";
+import { type Node, parse } from "./ast.ts";
+import {
+  type DataFileExport,
+  type DataFileImport,
+  type DataValue,
+  isReference,
+  keyName,
+  readDataFile,
+  SECRET_ENTRIES,
+} from "./datafile.ts";
+import { isSecret } from "./secret.ts";
+
+// The data-file writer: what the app's Adopt, and setting up a company, write into the files
+// `datafile.ts` reads. A file is changed in place, one declaration at a time, and everything
+// outside that declaration is kept byte for byte; what is written is formatted with oxfmt, under
+// the repository's own options, so it reads as if a person had written it and formatted it.
+
+/** A whole data file to render: its imports and its resources, in order. */
+export interface DataFileContent {
+  imports: readonly Pick<DataFileImport, "source" | "imported" | "local">[];
+  exports: readonly Pick<DataFileExport, "name" | "vendor" | "type" | "desired">[];
+}
+
+/**
+ * oxfmt's options from `<root>/.oxfmtrc.json`, what `oxfmt` itself reads there, less what only
+ * its CLI uses (`ignorePatterns`, `overrides`); none, oxfmt's defaults, without the file.
+ */
+export function formatOptions(root: string): FormatConfig {
+  let text: string;
+  try {
+    text = readFileSync(join(root, ".oxfmtrc.json"), "utf8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return {};
+    throw err;
+  }
+  const { $schema: _, ignorePatterns: __, overrides: ___, ...options } = JSON.parse(text) as Record<string, unknown>;
+  return options as FormatConfig;
+}
+
+/**
+ * `source`, a data file's text, with fields of its resource `exportName` set to new values:
+ * `fields` by dotted path, a list item's with its index, as a drift report names them
+ * (`required_pull_request_reviews.0.dismiss_stale_reviews`). A field the declaration does not
+ * have is added at the end of the object that should hold it. Only that `export const` changes:
+ * it is formatted with oxfmt under `options` (`formatOptions`), and every other byte of the file
+ * stays as it was.
+ *
+ * A value naming a secret adds `import { secret } from "sanoma/resources"` after the file's
+ * imports when it has none.
+ *
+ * Throws when the file does not parse, declares no such resource, or a path cannot be set: it
+ * runs through a value that is not an object or a list, or ends at a reference to another
+ * resource (a person changes a reference; a vendor's value never replaces one).
+ */
+export async function setFields(
+  source: string,
+  exportName: string,
+  fields: Readonly<Record<string, DataValue>>,
+  options: FormatConfig = {},
+): Promise<string> {
+  let text = source;
+  // One field at a time, each against the file as the last left it: no offset goes stale.
+  for (const [path, value] of Object.entries(fields)) {
+    const { start, end, replacement } = editAt(declarationOf(text, exportName).object, path, value);
+    text = text.slice(0, start) + replacement + text.slice(end);
+  }
+  if (Object.values(fields).some(holdsSecret)) text = withSecretImport(text);
+  // The declaration on its own: a top-level statement formats the same in its file as alone.
+  const { start, end } = declarationOf(text, exportName).statement;
+  const formatted = await format(text.slice(start, end), options);
+  return text.slice(0, start) + formatted.trimEnd() + text.slice(end);
+}
+
+/** What a data file imports `secret` from when it names one (`readDataFile` lists no such import). */
+const SECRET_IMPORT = { source: "sanoma/resources", imported: "secret", local: "secret" };
+
+/**
+ * A whole data file: the imports, one per source, then each resource as `export const`, then
+ * `export default` naming them all, formatted with oxfmt under `options`. `readDataFile` reads it
+ * back as it was given: what `readDataFile` returns renders the file again. A file whose values
+ * name a secret also imports `secret` from "sanoma/resources".
+ */
+export async function renderDataFile({ imports, exports }: DataFileContent, options: FormatConfig = {}) {
+  const names = new Map<string, string[]>();
+  const hasSecret = imports.some((i) => i.local === SECRET_IMPORT.local);
+  const all = !hasSecret && exports.some((e) => holdsSecret(e.desired)) ? [...imports, SECRET_IMPORT] : imports;
+  for (const { source, imported, local } of all) {
+    const list = names.get(source) ?? [];
+    list.push(imported === local ? local : `${imported} as ${local}`);
+    names.set(source, list);
+  }
+  const parts = [
+    [...names].map(([source, list]) => `import { ${list.join(", ")} } from ${JSON.stringify(source)};`).join("\n"),
+    // The top-level object on lines of its own, so oxfmt keeps it expanded as people write it.
+    ...exports.map(({ name, vendor, type, desired }) => {
+      const body = Object.entries(desired).map(([key, value]) => `${keyOf(key)}: ${literal(value)},\n`);
+      return `export const ${name} = ${vendor}.${type}({${body.length ? `\n${body.join("")}` : ""}});`;
+    }),
+    `export default [${exports.map((e) => e.name).join(", ")}];`,
+  ];
+  return format(`${parts.filter(Boolean).join("\n\n")}\n`, options);
+}
+
+/** `text` with `secret` imported after its imports, unless it imports it already. */
+function withSecretImport(text: string): string {
+  const imports = parse("/data-file.ts", text).program.body.filter((s: Node) => s.type === "ImportDeclaration");
+  const has = imports.some(
+    (s: Node) =>
+      SECRET_ENTRIES.has(s.source.value) &&
+      s.specifiers.some((sp: Node) => (sp.imported?.name ?? sp.imported?.value) === "secret"),
+  );
+  if (has) return text;
+  const at = imports.at(-1)?.end ?? 0;
+  const line = `import { ${SECRET_IMPORT.local} } from ${JSON.stringify(SECRET_IMPORT.source)};`;
+  return at ? `${text.slice(0, at)}\n${line}${text.slice(at)}` : `${line}\n${text}`;
+}
+
+/** The `export const` of the resource `name` in `text`, and the object literal its constructor takes. */
+function declarationOf(text: string, name: string): { statement: Node; object: Node } {
+  // The name is never resolved, and a relative import is checked against the file's own directory.
+  const filename = "/data-file.ts";
+  const { program, errors } = parse(filename, text);
+  if (errors.length) throw new Error(`The data file does not parse: ${errors[0]!.message}`);
+  const found = readDataFile(program, filename).exports.find((e) => e.name === name);
+  if (!found) throw new Error(`The data file declares no resource ${name}`);
+  const statement: Node = program.body.find((s) => s.start === found.start);
+  return { statement, object: statement.declaration.declarations[0].init.arguments[0] };
+}
+
+/**
+ * What setting `path` to `value` in `object` replaces, and with what. A key may hold a dot
+ * (`metadata: { "app.owner": … }`), and a drift report's path does not say so: at each object,
+ * the longest run of the path's segments that names a field is that field.
+ */
+function editAt(object: Node, path: string, value: DataValue): { start: number; end: number; replacement: string } {
+  const keys = path.split(".");
+  let node = object;
+  for (let i = 0; i < keys.length;) {
+    if (node.type === "ObjectExpression") {
+      const names = new Map<string, Node>(node.properties.map((p: Node) => [fieldName(p), p]));
+      let j = keys.length;
+      while (j > i && !names.has(keys.slice(i, j).join("."))) j--;
+      if (j === i) return added(node, keys.slice(i), path, value);
+      node = names.get(keys.slice(i, j).join("."))!.value;
+      i = j;
+    } else if (node.type === "ArrayExpression" && /^\d+$/.test(keys[i]!) && node.elements[Number(keys[i])]) {
+      node = node.elements[Number(keys[i])];
+      i++;
+    } else {
+      const at = keys.slice(0, i).join(".");
+      throw new Error(`Cannot set ${path}: ${at} is not an object or a list with item ${keys[i]}`);
+    }
+  }
+  const named = referenceIn(node);
+  if (named) {
+    throw new Error(`Cannot set ${path}: it names the resource ${named}; change a reference in the file itself`);
+  }
+  return { start: node.start, end: node.end, replacement: literal(value) };
+}
+
+/** The first resource a value names, itself or anywhere inside it: a vendor's value never replaces one. */
+function referenceIn(node: Node): string | undefined {
+  if (node.type === "Identifier") return node.name;
+  const items =
+    node.type === "ArrayExpression" ? node.elements : node.type === "ObjectExpression" ? node.properties : [];
+  for (const item of items) {
+    const named = item && referenceIn(item.type === "Property" ? item.value : item);
+    if (named) return named;
+  }
+  return undefined;
+}
+
+/**
+ * The edit that adds `keys` (the first missing, then any under it) with `value` at the end of
+ * `object`. Only objects are made under it: a list item's index there is refused.
+ */
+function added(object: Node, [key, ...under]: string[], path: string, value: DataValue) {
+  const index = under.find((k) => /^\d+$/.test(k));
+  if (index !== undefined) throw new Error(`Cannot set ${path}: the file has no list to hold item ${index}`);
+  const nested = under.reduceRight<DataValue>((v, k) => Object.fromEntries([[k, v]]), value);
+  const field = `${keyOf(key!)}: ${literal(nested)}`;
+  const last = object.properties.at(-1);
+  return last
+    ? { start: last.end, end: last.end, replacement: `, ${field}` }
+    : { start: object.start, end: object.end, replacement: `{ ${field} }` };
+}
+
+/** A property's name, as `readDataFile` reads it. */
+const fieldName = (p: Node): string | undefined => (p.type === "Property" && !p.computed ? keyName(p.key) : undefined);
+
+/** A field's name in source: bare when it can be, else a string. */
+const keyOf = (key: string) => (/^[A-Za-z_$][\w$]*$/.test(key) ? key : JSON.stringify(key));
+
+/** True when a value names a secret, itself or anywhere inside it. */
+const holdsSecret = (value: DataValue): boolean =>
+  isSecret(value) ||
+  (typeof value === "object" && value !== null && !isReference(value) && Object.values(value).some(holdsSecret));
+
+/** A value in the data-file subset, as source, on one line: oxfmt lays it out. */
+function literal(value: DataValue): string {
+  if (isReference(value)) return value.name;
+  if (isSecret(value)) return `secret(${JSON.stringify(value.$secret)}${value.generate ? ", { generate: true }" : ""})`;
+  if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value);
+  if (typeof value === "number") {
+    if (!Number.isFinite(value)) throw new Error(`${value} cannot be written in a data file: write a finite number`);
+    return Object.is(value, -0) ? "0" : String(value);
+  }
+  if (Array.isArray(value)) return `[${value.map(literal).join(", ")}]`;
+  if (typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) {
+    const fields = Object.entries(value).map(([key, v]) => `${keyOf(key)}: ${literal(v)}`);
+    return fields.length ? `{ ${fields.join(", ")} }` : "{}";
+  }
+  throw new Error(`${String(value)} cannot be written in a data file: ${typeof value} is not a data value`);
+}
+
+/** `text` formatted by oxfmt, loaded only when a file is written. */
+async function format(text: string, options: FormatConfig): Promise<string> {
+  const oxfmt = await import("oxfmt");
+  const { code, errors } = await oxfmt.format("data-file.ts", text, options);
+  if (errors.length) throw new Error(`oxfmt could not format the data file: ${errors[0]!.message}`);
+  return code;
+}