@sanoma-ai/workflows 0.7.1 → 0.8.1

package/src/apps.ts

@@ -0,0 +1,162 @@
+import { existsSync, readdirSync } from "node:fs";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { childrenOf, type LintProblem, type Node, readSource } from "./ast.ts";
+import { keyName } from "./datafile.ts";
+import { stringValue, unwrap } from "./outline.ts";
+import { APPS_DIR, filesUnder, inside, isDirectory, RELATIVE } from "./paths.ts";
+import { APP_NAME } from "./shared.ts";
+
+// A company's apps: each folder of `apps/` holds the kinds the root does (workflows, resources,
+// policies, scenarios, sites), merged from a source repository whose URL its `app.ts` names, or
+// made here. What a folder declares is named with the folder (`qualified`): never written, read
+// from where the declaration's file is (`appOfFile`).
+
+/** An app: a folder of `apps/`, by its name, and the source repository its `app.ts` names. */
+export interface AppEntry {
+  /** The folder's name, which prefixes what it declares (`crm.contacts`) and is the app's title. */
+  name: string;
+  /** The folder, relative to the root, with `/`: `apps/crm`. */
+  dir: string;
+  /** `defineApp({ source })` in its `app.ts`: the repository it was taken from; absent for the company's own app. */
+  source?: string;
+}
+
+/** What `app.ts` declares: the source repository's URL, when the app was taken from one. */
+export interface AppDefinition {
+  source?: string;
+}
+
+/**
+ * Declares an app in its folder's `app.ts`, read without running it: `defineApp({ source:
+ * "https://github.com/acme/crm" })`, the repository it was taken from; `defineApp({})` for an app
+ * the company made itself.
+ */
+export function defineApp(definition: AppDefinition): AppDefinition {
+  return Object.freeze({ ...definition });
+}
+
+/** What is wrong with a company's apps, at a file relative to the root. */
+export type AppProblem = LintProblem & { file: string };
+
+/** The apps under `root` (`appsAt`), without what is wrong with them. */
+export const readApps = (root: string | undefined): AppEntry[] => appsAt(root).apps;
+
+/**
+ * The apps under `root`: each folder of `apps/` whose name is an `APP_NAME`, sorted, with what is
+ * wrong in its `app.ts` and each folder refused for its name as `problems`. `app.ts` is read
+ * with oxc, never run: its default export must be `defineApp({ … })`, from `sanoma`, written out,
+ * and `source` a string.
+ */
+export function appsAt(root: string | undefined): { apps: AppEntry[]; problems: AppProblem[] } {
+  const dir = root === undefined ? undefined : join(root, APPS_DIR);
+  if (dir === undefined || !isDirectory(dir)) return { apps: [], problems: [] };
+  const apps: AppEntry[] = [];
+  const problems: AppProblem[] = [];
+  const folders = readdirSync(dir, { withFileTypes: true })
+    .filter((d) => d.isDirectory())
+    .map((d) => d.name)
+    .toSorted();
+  for (const name of folders) {
+    const at = `${APPS_DIR}/${name}`;
+    if (!APP_NAME.test(name)) {
+      problems.push({
+        file: at,
+        line: 1,
+        column: 1,
+        message: `${at} is not an app: an app's folder name is a lowercase letter, then lowercase letters and digits`,
+      });
+      continue;
+    }
+    const read = appOptionsOf(join(dir, name, "app.ts"));
+    problems.push(...read.problems.map((p) => ({ file: `${at}/app.ts`, ...p })));
+    apps.push({ name, dir: at, ...(read.source !== undefined && { source: read.source }) });
+  }
+  return { apps, problems };
+}
+
+/** Where `defineApp` comes from in `app.ts`. */
+const APP_SOURCES: ReadonlySet<string> = new Set(["sanoma", "@sanoma-ai/workflows"]);
+
+/** `app.ts`'s `source`, read without running it, and what is wrong there; nothing for no file. */
+function appOptionsOf(path: string): { source?: string; problems: LintProblem[] } {
+  if (!existsSync(path)) return { problems: [] };
+  const file = readSource(path);
+  if (typeof file === "string") return { problems: [{ line: 1, column: 1, message: file }] };
+  const problems: LintProblem[] = [...file.problems];
+  const problem = (at: Node, message: string) => problems.push({ ...file.at(at.start), message });
+  const unread =
+    'app.ts is read without running it: its default export is `defineApp({ source })` from "sanoma", written out';
+  const exported = file.program.body.find((s: Node) => s.type === "ExportDefaultDeclaration");
+  const call = unwrap(exported?.declaration);
+  const callee = unwrap(call?.callee);
+  const options = unwrap(call?.arguments?.[0]);
+  const defineApps = new Set<string>();
+  for (const statement of file.program.body) {
+    if (statement.type !== "ImportDeclaration" || !APP_SOURCES.has(statement.source.value)) continue;
+    for (const s of statement.specifiers) {
+      if (s.type === "ImportSpecifier" && keyName(s.imported) === "defineApp") defineApps.add(s.local.name);
+    }
+  }
+  const written =
+    call?.type === "CallExpression" &&
+    callee?.type === "Identifier" &&
+    defineApps.has(callee.name) &&
+    (options === undefined || options.type === "ObjectExpression");
+  if (!written) {
+    problem(exported ?? file.program, unread);
+    return { problems };
+  }
+  const property = options?.properties.findLast(
+    (p: Node) => p.type === "Property" && !p.computed && keyName(p.key) === "source",
+  );
+  const source = property && stringValue(property.value);
+  if (property && source === undefined)
+    problem(property, "`source` is read without running app.ts: write it as a string");
+  return { ...(source !== undefined && { source }), problems };
+}
+
+/** A source file under an app: `.ts` or `.tsx`, but a declaration. */
+const isSource = (name: string) => /\.tsx?$/.test(name) && !name.endsWith(".d.ts");
+
+/**
+ * Each import, from a file under one app's folder, of a file under another's, at its place: a
+ * warning, since a company may take one app without the other. Static imports, re-exports and
+ * `import("…")` with a literal path; relative specifiers only.
+ */
+export function crossAppImports(root: string | undefined, apps: readonly AppEntry[]): AppProblem[] {
+  if (root === undefined) return [];
+  const warnings: AppProblem[] = [];
+  for (const app of apps) {
+    const own = join(root, app.dir);
+    for (const path of filesUnder(own, isSource)) {
+      const file = readSource(path);
+      if (typeof file === "string") continue;
+      const rel = relative(root, path).split(sep).join("/");
+      const visit = (node: Node) => {
+        if (!node) return;
+        const specifier =
+          node.type === "ImportDeclaration" ||
+          node.type === "ExportAllDeclaration" ||
+          node.type === "ExportNamedDeclaration"
+            ? node.source?.value
+            : node.type === "ImportExpression"
+              ? stringValue(node.source)
+              : undefined;
+        if (typeof specifier === "string" && RELATIVE.test(specifier)) {
+          const target = resolve(dirname(path), specifier);
+          const other = apps.find((a) => a !== app && inside(join(root, a.dir), target));
+          if (other) {
+            warnings.push({
+              file: rel,
+              ...file.at(node.start),
+              message: `imports "${specifier}" from the app ${other.name}: a company may take ${app.name} without it, so reach it through ctx or api instead`,
+            });
+          }
+        }
+        for (const child of childrenOf(node)) visit(child);
+      };
+      visit(file.program);
+    }
+  }
+  return warnings;
+}

package/src/call-workflow.ts

@@ -11,6 +11,9 @@ export { CALL_WORKFLOW } from "./shared.ts";
 // approval, so the approval reaches the Inbox like any run's. It is a workflow like any other,
 // held to the same lint: the call goes through `ctx`, under the policy, in the ledger.
 
+/** An operation on `ctx`. */
+type OpCall = (input: unknown) => Promise<unknown>;
+
 /** The call workflow's input: the operation's id, and its input. */
 export const CallInput = z.object({ op: z.string(), input: z.unknown() });
 
@@ -37,8 +40,13 @@ export function callWorkflow(
     uses,
     // `ctx` refuses an operation that is not in `uses`: one that may not be called directly, or has no driver.
     run: async (ctx, input) => {
-      const [vendor = "", resource = "", name = ""] = input.op.split(".");
-      return ctx[vendor]![resource]![name]!(input.input);
+      // Four segments for an app's collection: `data.<app>.<collection>.<op>`.
+      const [vendor = "", a = "", b = "", c] = input.op.split(".");
+      return c === undefined
+        ? ctx[vendor]![a]![b]!(input.input)
+        : (ctx as unknown as Record<string, Record<string, Record<string, Record<string, OpCall>>>>)[vendor]![a]![b]![
+            c
+          ]!(input.input);
     },
   });
 }

package/src/call.ts

@@ -156,9 +156,12 @@ export function buildCtx(wf: WorkflowDefinition<any, any>, run: Run): any {
     };
   const members: Record<string, any> = {};
   for (const op of uses.filter(isOp)) {
-    const vendor = (members[op.vendor] ??= {});
-    const resource = (vendor[op.resource] ??= {});
-    resource[op.name] = placed(op.id, (node, input) =>
+    // By the id's segments: an app's collection, `data.crm.contacts.get`, is `ctx.data.crm.contacts.get`.
+    const group = op.id
+      .split(".")
+      .slice(0, -1)
+      .reduce<Record<string, any>>((at, key) => (at[key] ??= {}), members);
+    group[op.name] = placed(op.id, (node, input) =>
       serial(run, () => callOp(run, op.id, input, { node: node.path, name: named(node) ?? op.id })),
     );
   }

package/src/cedar.ts

@@ -1,11 +1,11 @@
 import { createHash } from "node:crypto";
 import { readFileSync } from "node:fs";
-import { join, relative, sep } from "node:path";
+import { relative, sep } from "node:path";
 import type { CedarValueJson, DetailedError, EntityJson, Expr, PolicyJson } from "@cedar-policy/cedar-wasm/nodejs";
 import { locator } from "./ast.ts";
 import type { ResolvedConfig } from "./config.ts";
 import type { ApprovalState, Approver, Principal } from "./define.ts";
-import { filesUnder } from "./paths.ts";
+import { filesUnder, kindDirs } from "./paths.ts";
 import {
   allow,
   allowAll,
@@ -89,12 +89,14 @@ const ON_BEHALF_OF = "on-behalf-of";
 
 type Place = { line: number; column: number };
 
-/** `<root>/policies/**\/*.cedar`, sorted by path, each relative to the root with `/` between its parts. */
+/** `<root>/policies/**\/*.cedar`, then each app's `apps/<app>/policies/**\/*.cedar`, sorted by path in each, each relative to the root with `/` between its parts. */
 export function readPolicyFiles(root: string): { file: string; text: string }[] {
-  return filesUnder(join(root, POLICIES_DIR), (name) => name.endsWith(".cedar")).map((path) => ({
-    file: relative(root, path).split(sep).join("/"),
-    text: readFileSync(path, "utf8"),
-  }));
+  return kindDirs(root, POLICIES_DIR)
+    .flatMap((dir) => filesUnder(dir, (name) => name.endsWith(".cedar")))
+    .map((path) => ({
+      file: relative(root, path).split(sep).join("/"),
+      text: readFileSync(path, "utf8"),
+    }));
 }
 
 /**

package/src/change-steps.ts

@@ -0,0 +1,274 @@
+import { readFile } from "node:fs/promises";
+import { join, relative, sep } from "node:path";
+import { DBOS } from "@dbos-inc/dbos-sdk";
+import { withdrawApprovals } from "./approvals.ts";
+import { decide } from "./call.ts";
+import {
+  ADMIN,
+  type ChangeDecision,
+  type ChangeDeploy,
+  type ChangeEvent,
+  ChangeEventSchema,
+  type ChangeFacts,
+  type ChangeState,
+  type ChangeVerified,
+  followBase,
+  repositoryOf,
+  isRules,
+  sourceOfRun,
+  touchesOf,
+} from "./change.ts";
+import type { ChangeSteps } from "./change-workflow.ts";
+import type { DriftReport } from "./drift.ts";
+import { isRunControlError } from "./errors.ts";
+import { warn } from "./log.ts";
+import { inside } from "./paths.ts";
+import { DECLARES_PEOPLE } from "./resources.ts";
+import { currentRun, type Run } from "./run.ts";
+import {
+  AGENTS,
+  APPLY_WORKFLOW,
+  CHANGE_EVENT,
+  CHANGE_MERGE,
+  CHANGE_TOPIC,
+  DRIFT_WORKFLOW,
+  errorMessage,
+} from "./shared.ts";
+import type { CheckResult, RemoteHalf } from "./source.ts";
+
+// The change workflow's steps that are not one of its operations: checking and planning the
+// draft, asking the policy ahead, waiting for what happens to the change, retiring approvals,
+// deploying and verifying after the merge, and publishing where the change is. Here, not in
+// change-workflow.ts, which is held to the workflow lint and imports no DBOS; each reads its run
+// from `currentRun`.
+
+const runNow = (): Run => {
+  const run = currentRun.getStore();
+  if (!run) throw new Error("A change step ran outside its run");
+  return run;
+};
+
+/** True when the file at `path` is a people file (`DECLARES_PEOPLE`), one `readPeople` would read; false when there is none. */
+const fileDeclaresPeople = async (path: string) => DECLARES_PEOPLE.test(await readFile(path, "utf8").catch(() => ""));
+
+/** Which events win when several wait: one that ends the change over one that plans it again. */
+const WEIGHT: Record<ChangeEvent["kind"], number> = { commits: 0, close: 1, closed: 2, merged: 3 };
+
+/** How long one `recv` waits before it asks again; a change waits as long as it takes. */
+const RECV_SECONDS = 24 * 60 * 60;
+
+/**
+ * The next event on the change's topic within `timeoutSeconds`, or undefined for none; a message
+ * that is no change event is skipped, with a warning.
+ */
+async function received(run: Run, timeoutSeconds: number): Promise<ChangeEvent | undefined> {
+  for (;;) {
+    const raw = await run.state.deadlines.waiting(run.id, { kind: "message" }, () =>
+      DBOS.recv<unknown>(CHANGE_TOPIC, { timeoutSeconds }),
+    );
+    if (raw === null || raw === undefined) return undefined;
+    const event = ChangeEventSchema.safeParse(raw);
+    if (event.success) return event.data;
+    warn(`run ${run.id}: a message on ${CHANGE_TOPIC} is no change event, so it is ignored`);
+  }
+}
+
+/**
+ * Of `first` and the events waiting on the change's topic, read without waiting, the one that
+ * weighs most; undefined for none. A `merged` or `closed` someone told is checked on the host
+ * (`settle`) as it is read, so a claim the pull request does not bear out weighs as `commits`.
+ */
+async function drain(
+  run: Run,
+  settle: () => Promise<ChangeEvent>,
+  first?: ChangeEvent,
+): Promise<ChangeEvent | undefined> {
+  const checked = (event: ChangeEvent) => (event.kind === "merged" || event.kind === "closed" ? settle() : event);
+  let strongest = first && (await checked(first));
+  for (let event = await received(run, 0); event; event = await received(run, 0)) {
+    const told = await checked(event);
+    if (strongest === undefined || WEIGHT[told.kind] > WEIGHT[strongest.kind]) strongest = told;
+  }
+  return strongest;
+}
+
+/**
+ * Starts a run of the built-in `name` as a child of this one, with the id `<run>:<suffix>` (so a
+ * replay finds it), as the run's own actor, and waits for its output, counted as a wait for
+ * another run: undefined and the error's message when it failed.
+ */
+async function child(
+  run: Run,
+  name: string,
+  suffix: string,
+  input: unknown,
+): Promise<{ runId: string; output?: unknown; error?: string }> {
+  const runId = `${run.id}:${suffix}`;
+  const handle = await run.state.startRun(name, runId, {
+    input,
+    startedBy: run.actor,
+    ...(run.sandbox !== undefined && { sandbox: run.sandbox }),
+  });
+  try {
+    return { runId, output: await run.state.deadlines.waiting(run.id, { kind: "run" }, () => handle.getResult()) };
+  } catch (err) {
+    if (isRunControlError(err)) throw err;
+    return { runId, error: errorMessage(err) };
+  }
+}
+
+/** A file's path relative to `root` with `/`, as git names it; undefined when it is outside. */
+const pathIn = (root: string, file: string) =>
+  inside(root, file) ? relative(root, file).split(sep).join("/") : undefined;
+
+/** What `changeSteps` is given: the connectors' remote halves, and the declared hosts' ids, read from the data files. */
+export interface ChangeStepsOptions {
+  remotes: ReadonlyMap<string, RemoteHalf>;
+  /** The resources the data files declare whose type is a host (`<file>#<export>`): none when no connector declares one. */
+  hosts: () => Promise<string[]>;
+}
+
+/** The change workflow's steps for a config. */
+export function changeSteps({ remotes, hosts }: ChangeStepsOptions): ChangeSteps {
+  /** What the host says became of the pull request: merged or closed, else (still open) `commits`, to plan again. */
+  const settled = (number: number) =>
+    DBOS.runStep(
+      async (): Promise<ChangeEvent> => {
+        const { host } = await repositoryOf(remotes, runNow().state.secrets);
+        const pr = await host.pullRequest(number);
+        if (pr.state === "merged") return { kind: "merged", sha: pr.mergeSha ?? pr.head };
+        return pr.state === "closed" ? { kind: "closed" } : { kind: "commits" };
+      },
+      { name: "change:settled" },
+    );
+
+  return {
+    check: (id) =>
+      DBOS.runStep(
+        async (): Promise<CheckResult> => {
+          const source = sourceOfRun();
+          if (!source.check) throw new Error("This repository cannot check a draft (`Source.check`)");
+          return source.check(id);
+        },
+        { name: "change:check" },
+      ),
+
+    plan: (id, check) =>
+      DBOS.runStep(
+        async (): Promise<ChangeFacts> => {
+          const run = runNow();
+          const source = sourceOfRun();
+          if (!source.diff || !source.draft) throw new Error("This repository keeps no drafts (`Source.diff`)");
+          const [{ files }, draft] = await Promise.all([source.diff(id), source.draft(id)]);
+          const touches = touchesOf(files);
+          const touched = new Set(touches);
+          // The waiting runs of a workflow whose file the change edits, which a publish cannot move.
+          const edited = [...run.state.workflows]
+            .filter(([, wf]) => {
+              const path = wf.file === undefined ? undefined : pathIn(source.root, wf.file);
+              return path !== undefined && touched.has(path);
+            })
+            .map(([name]) => name);
+          const pending = edited.length
+            ? await DBOS.listWorkflows({
+                status: "PENDING",
+                workflowName: edited,
+                applicationName: run.state.app,
+                loadInput: false,
+                loadOutput: false,
+              })
+            : [];
+          const stranded = pending
+            .map((r) => r.workflowID)
+            .filter((runId) => runId !== run.id)
+            .toSorted();
+          // A people file as main has it or as the draft makes it: one added, edited or emptied.
+          const people = await Promise.all(
+            files
+              .filter((file) => file.startsWith("resources/"))
+              .flatMap((file) => [join(source.root, file), join(draft.root, file)].map(fileDeclaresPeople)),
+          );
+          return {
+            touches,
+            strandedRuns: stranded.length,
+            stranded,
+            checksOk: check.ok,
+            rules: people.includes(true) || touches.some(isRules),
+          };
+        },
+        { name: "change:plan" },
+      ),
+
+    ahead: (merge) =>
+      DBOS.runStep(
+        async (): Promise<ChangeDecision> => {
+          const run = runNow();
+          const op = run.state.ops.get(CHANGE_MERGE)!;
+          const decision = await decide(run, op, op.input.parse(merge));
+          if (decision.kind === "deny") return { answer: "deny", reason: decision.reason };
+          if (decision.kind === "approve") {
+            return {
+              answer: "hold",
+              approver: decision.approver,
+              ...(decision.title !== undefined && { title: decision.title }),
+              ...(decision.reasons !== undefined && { reasons: decision.reasons }),
+            };
+          }
+          // Allowed: Publish is still someone's to press: whoever drafted it, else admin for an agent acting for no one.
+          const { actor } = run;
+          const drafter = actor.onBehalfOf ?? (actor.groups?.includes(AGENTS) ? undefined : actor.id);
+          return { answer: "allow", approver: drafter ?? { group: ADMIN } };
+        },
+        { name: "change:policy" },
+      ),
+
+    async next(number) {
+      const run = runNow();
+      for (;;) {
+        // An event comes from a request (`SanomaClient.changeEvent`), which wakes a stopped worker: no deadline.
+        const event = await received(run, RECV_SECONDS);
+        if (event) return (await drain(run, () => settled(number), event))!;
+      }
+    },
+
+    queued: (number) => drain(runNow(), () => settled(number)),
+
+    retire: (reason) => withdrawApprovals(runNow(), CHANGE_MERGE, reason),
+
+    async deploy() {
+      const run = runNow();
+      // The checkout moves to the merge first, so the hosts and what apply reads are the merged ones.
+      // Merged already: a deploy that fails is the change's to report, not a failed run.
+      const follow: ChangeDeploy["follow"] = await DBOS.runStep(() => followBase(remotes, run.state.secrets), {
+        name: "change:follow",
+      });
+      const declared = run.state.workflows.has(APPLY_WORKFLOW)
+        ? await DBOS.runStep(hosts, { name: "change:hosts" })
+        : [];
+      if (!declared.length) return { follow };
+      const { runId, error } = await child(run, APPLY_WORKFLOW, "deploy", { only: declared });
+      return { follow, apply: { runId, hosts: declared, ...(error !== undefined && { error }) } };
+    },
+
+    async verify({ apply }) {
+      const run = runNow();
+      if (apply === undefined || !run.state.workflows.has(DRIFT_WORKFLOW)) return undefined;
+      const { runId, output, error } = await child(run, DRIFT_WORKFLOW, "verify", {});
+      const applied = new Set(apply.hosts);
+      const read = ((output as DriftReport | undefined)?.resources ?? []).filter((r) => applied.has(r.id));
+      // A host drift could not read is no drift: it is said in `error`.
+      const drifted = read.filter((r) => r.status !== "clean" && r.status !== "error").map((r) => r.id);
+      const unread = read.filter((r) => r.status === "error").map((r) => r.id);
+      const errors = [error, ...(unread.length ? [`could not read ${unread.join(", ")}`] : [])].filter(
+        (e) => e !== undefined,
+      );
+      return { runId, drifted, ...(errors.length && { error: errors.join("; ") }) } satisfies ChangeVerified;
+    },
+
+    closeDraft: (id) => DBOS.runStep(async () => sourceOfRun().closeDraft?.(id), { name: "change:close-draft" }),
+
+    publish: async (state: ChangeState) => {
+      await DBOS.setEvent(CHANGE_EVENT, state);
+    },
+  };
+}