@sanoma-ai/workflows 0.1.0 → 0.4.0

package/src/resources.ts

@@ -1,12 +1,31 @@
-import { readdirSync, readFileSync, statSync } from "node:fs";
+import { readFileSync, statSync } from "node:fs";
 import { dirname, join, relative, resolve, sep } from "node:path";
 import { parsed } from "./ast.ts";
-import { rootOf, type SanomaConfig } from "./config.ts";
-import { type DataFile, type DataFileExport, type DataValue, readDataFile, Reference } from "./datafile.ts";
+import {
+  type DataFile,
+  type DataFileExport,
+  type DataValue,
+  readDataFile,
+  Reference,
+  SECRET_ENTRIES,
+} from "./datafile.ts";
+import { type Binding, type Bindings, LOCKFILE, readLockfile } from "./lockfile.ts";
 import { type Connector, VENDOR } from "./op.ts";
+import { filesUnder } from "./paths.ts";
+import { peopleConnector } from "./person.ts";
 import type { Span } from "./outline.ts";
 import { type Declared, type Resource, resourceTypesOf } from "./resource.ts";
-import { problemLine } from "./shared.ts";
+import { Principal } from "./define.ts";
+import { NotAdmittedError } from "./errors.ts";
+import {
+  AGENTS,
+  declaresPeople,
+  type PeopleResource,
+  personOf,
+  principalOf,
+  problemLine,
+  RUNTIME_AGENTS,
+} from "./shared.ts";
 
 /** A resource a data file declares, as read from the file without running it. */
 export interface DeclaredResource {
@@ -16,8 +35,12 @@ export interface DeclaredResource {
   file: string;
   vendor: string;
   type: string;
-  /** Its import id: what the type's `find` makes of its fields. */
+  /** What people call it: its identity, else its export's name. */
   name: string;
+  /** The import id that finds it unbound: what its type's `find` makes of its fields; absent when the type has none. */
+  identity?: string;
+  /** The vendor's id of the object the lockfile binds it to (`LOCKFILE`); absent when unbound. */
+  binding?: string;
   /** Its `export const` statement, as UTF-16 offsets into the file's text. */
   span: Span;
   /** Its fields, as its type checked them: each resource it names as that resource's `name`. */
@@ -29,6 +52,22 @@ export interface DeclaredResource {
   refs: Record<string, string>;
 }
 
+/**
+ * A collection `resources/data.ts` declares, as read without running it: where it is, for the
+ * Resources page. Its fields are the collection's own (`describeConfig`'s `collections`), from
+ * the same file, which the runtime imports.
+ */
+export interface DeclaredCollection {
+  /** `<file>#<export>`, as a resource's: `resources/data.ts#outbound`. */
+  id: string;
+  /** `resources/data.ts`. */
+  file: string;
+  /** The collection's name, its operations' middle segment (`data.<name>.get`). */
+  name: string;
+  /** Its `export const` statement, as UTF-16 offsets into the file's text. */
+  span: Span;
+}
+
 /** Something wrong in the data files, or in finding them. */
 export interface ResourceProblem {
   /**
@@ -43,26 +82,118 @@ export interface ResourceProblem {
   message: string;
 }
 
-/** What the data files declare: each resource read without problems, and the problems. */
+/** What the data files declare: each resource read without problems, the lockfile's bindings, and the problems. */
 export interface DataFiles {
   /** By file and then in file order, less any with a problem, and any that names one. */
   resources: DeclaredResource[];
+  /** The lockfile's bindings, by key (`<file>#<export>`), as read: none when it is missing or has a problem. */
+  bindings: Bindings;
+  /** Each binding whose key no data file declares, by key. */
+  orphans: (Binding & { key: string })[];
+  /** The collections `resources/data.ts` declares, in file order. */
+  collections: DeclaredCollection[];
   /** By file, line and column. */
   problems: ResourceProblem[];
 }
 
+/** What the data files declare of the people: the resources `principalOf` takes, and what is wrong in their files. */
+export interface DeclaredPeople {
+  people: DeclaredResource[];
+  /**
+   * The problems in each file that declares (or tries to declare) a person, agent or group, and
+   * an id two of them share or one of the runtime's own agents (`RUNTIME_AGENTS`) takes. While
+   * there are any, nobody is admitted: a broken file never admits more than it says.
+   */
+  problems: ResourceProblem[];
+}
+
+/** A file that imports the runtime's constructors, `sanoma` (by any local name): the files `readPeople` reads. */
+const DECLARES_PEOPLE = /\{[^}]*\bsanoma\b[^}]*\}\s*from\s*["'](?:sanoma|@sanoma-ai\/workflows)\/resources["']/;
+/** The last read of the people under each root, by what its data files were then (`filesStamp`). */
+const peopleRead = new Map<string, { stamp: string; read: DeclaredPeople }>();
+
+/** Each data file under `root` with its size and modification time: unchanged while the files are. */
+const filesStamp = (root: string) =>
+  dataFilesUnder(join(root, "resources"))
+    .map((path) => {
+      // Gone since it was listed (an editor's save by rename): stamped as such, and read next time.
+      const stat = statSync(path, { bigint: true, throwIfNoEntry: false });
+      return `${path} ${stat?.size} ${stat?.mtimeNs}`;
+    })
+    .join("\n");
+
 /**
- * The resources the config's data files declare, read without importing or running them; throws
- * one error listing every problem, each at `file:line:column`, for a test or CI to fail on.
- * `describeConfig` lists the same, with the problems beside them instead.
+ * The people, agents and groups the data files under `root` declare (`sanoma.person`, `.agent`,
+ * `.group`), read without running them, as `principalOf` takes them, with what is wrong in their
+ * files; none without a root. Read again only when a data file changed (by its size and time),
+ * so a new colleague is admitted once their file is saved, and a request costs a `stat` per file.
  */
-export function readResources(config: SanomaConfig): DeclaredResource[] {
-  const { resources, problems } = readDataFiles(rootOf(config), config.connectors);
+export function readPeople(root: string | undefined): DeclaredPeople {
+  if (root === undefined) return { people: [], problems: [] };
+  const stamp = filesStamp(root);
+  const cached = peopleRead.get(root);
+  if (cached?.stamp === stamp) return cached.read;
+  const read = readDataFiles(root, [peopleConnector], undefined, (source) => DECLARES_PEOPLE.test(source));
+  const people = read.resources;
+  // The people are never bound, so the lockfile's problems are not theirs.
+  const own = read.problems.filter((p) => p.file !== LOCKFILE);
+  const ids = new Map<string, DeclaredResource>();
+  for (const r of people) {
+    if (r.type === "group") {
+      if (r.name === AGENTS) {
+        own.push({
+          file: r.file,
+          message: `${r.id} is named "${AGENTS}", the group every agent is in: give it another name`,
+        });
+      }
+      continue;
+    }
+    const id = r.name;
+    const other = ids.get(id);
+    if (other || RUNTIME_AGENTS.includes(id)) {
+      own.push({
+        file: r.file,
+        message: other
+          ? `${r.id} has the id "${id}", which ${other.id} has: give each person and agent their own`
+          : `${r.id} has the id "${id}", one of the runtime's own agents': give it another`,
+      });
+    }
+    ids.set(id, r);
+  }
+  const declared = { people, problems: own };
+  peopleRead.set(root, { stamp, read: declared });
+  return declared;
+}
+
+/**
+ * Who `asking` is admitted as, by the people the data files declare (`readPeople`): the declared
+ * principal (`principalOf`) of the id it names, a person's or an agent's; for a sign-in
+ * (`signIn`), of the person it matches (`personOf`: email, GitHub login, then id), never an agent.
+ * While no person is declared, whoever names an id is taken as given. Throws `NotAdmittedError`
+ * for anyone else, and for everyone while the people's files have problems.
+ */
+export function admit(
+  asking: { id?: string; email?: string; github?: string; groups?: string[]; onBehalfOf?: string },
+  { people, problems }: { people: readonly PeopleResource[]; problems: readonly ResourceProblem[] },
+  signIn = false,
+): Principal {
+  const who = asking.github ?? asking.id ?? asking.email ?? "This sign-in";
   if (problems.length) {
-    const lines = problems.map((p) => `  ${problemLine(p)}`);
-    throw new Error(`The resources' data files have problems:\n${lines.join("\n")}`);
+    // The runtime's own agents are declared by no file, so a broken one does not stop them (`check`'s scenarios).
+    if (!signIn && asking.id !== undefined && RUNTIME_AGENTS.includes(asking.id)) {
+      return principalOf(asking.id, people)!;
+    }
+    throw new NotAdmittedError(who, problems.map(problemLine));
   }
-  return resources;
+  if (!declaresPeople(people)) {
+    const given = Principal.safeParse(asking).data;
+    if (given) return given;
+  } else {
+    const id = signIn ? personOf(asking, people) : asking.id;
+    const admitted = id === undefined ? undefined : principalOf(id, people);
+    if (admitted) return admitted;
+  }
+  throw new NotAdmittedError(who);
 }
 
 interface FileEntry {
@@ -82,12 +213,23 @@ interface FileEntry {
  *
  * Problems: anything outside the subset, a constructor or data file the config does not know, a
  * type the connector does not declare, a value the type refuses (a reference where its type
- * takes none among them), a reference cycle, and two resources of one type with one name.
+ * takes none among them), a reference cycle, and two resources of one type with one identity.
+ *
+ * The collections `resources/data.ts` declares are listed as they are read (`collections`),
+ * never matched to a connector, a resource type or the lockfile: the runtime imports that file.
+ *
+ * Each declaration is joined to its binding in the lockfile (`readLockfile`) by its key,
+ * `<file>#<export>`; a binding no export declares is an orphan. Problems: the lockfile's, a
+ * binding of another type than its declaration's, and two declarations bound to one object.
+ *
+ * `partial` reads part of the files: only those whose source it passes, and in them, an import of
+ * a connector not in `connectors`, or of a file left out, is left unread rather than reported.
  */
 export function readDataFiles(
   root: string | undefined,
   connectors: readonly Connector<any, any>[],
   types: ReadonlyMap<string, Resource> = resourceTypesOf(connectors),
+  partial?: (source: string) => boolean,
 ): DataFiles {
   const problems: ResourceProblem[] = [];
   if (root === undefined) {
@@ -95,13 +237,20 @@ export function readDataFiles(
       message:
         "The config has no root, so its data files cannot be found: make it with defineConfig, which records its file, or set its `root`",
     });
-    return { resources: [], problems };
+    return { resources: [], bindings: {}, orphans: [], collections: [], problems };
   }
+  const lock = readLockfile(root);
+  problems.push(...lock.problems);
 
   const files = new Map<string, FileEntry>();
+  const left = new Set<string>();
   for (const path of dataFilesUnder(join(root, "resources"))) {
     const rel = relative(root, path).split(sep).join("/");
     const source = readFileSync(path, "utf8");
+    if (partial && !partial(source)) {
+      left.add(path);
+      continue;
+    }
     const { program, problems: syntax, at } = parsed(source, path);
     const data = readDataFile(program, path, join(root, "resources"));
     const problem = (offset: number, message: string) => problems.push({ file: rel, ...at(offset), message });
@@ -116,6 +265,8 @@ export function readDataFiles(
     const { id, info } = connector[VENDOR];
     if (info?.package) entries.set(`${info.package}/resources`, id);
   }
+  // The runtime's own (`peopleConnector`, package `sanoma`) is imported from either name of its entry.
+  if (entries.get("sanoma/resources") === "sanoma") for (const entry of SECRET_ENTRIES) entries.set(entry, "sanoma");
 
   // What each file's names stand for, and every resource read by its id.
   const vendors = new Map<FileEntry, Map<string, string>>();
@@ -127,6 +278,7 @@ export function readDataFiles(
     for (const imp of file.data.imports) {
       if (imp.kind === "constructors") {
         const vendor = entries.get(imp.source);
+        if (vendor === undefined && partial) continue;
         if (vendor === undefined) {
           file.problem(
             imp.start,
@@ -142,7 +294,9 @@ export function readDataFiles(
         }
         continue;
       }
-      const other = files.get(resolve(dirname(path), imp.source));
+      const target = resolve(dirname(path), imp.source);
+      const other = files.get(target);
+      if (!other && left.has(target)) continue;
       if (!other) {
         file.problem(imp.start, `"${imp.source}" is not a data file under resources/`);
       } else if (!other.data.declared.includes(imp.imported)) {
@@ -175,7 +329,7 @@ export function readDataFiles(
     visiting.push(id);
     const result = build(file, exp);
     visiting.pop();
-    if (result) {
+    if (result?.declared.name !== undefined) {
       const { vendor, type, name } = result.declared;
       const key = `${vendor}.${type} ${JSON.stringify(name)}`;
       const first = byName.get(key);
@@ -231,26 +385,68 @@ export function readDataFiles(
     }
   }
 
+  // Each binding against its declaration: of its type, and the only one bound to its object.
+  const boundBy = new Map<string, string>();
+  for (const [id, { file, exp }] of declarations) {
+    const binding = lock.bindings[id];
+    // An unknown constructors import was reported with the import.
+    const vendor = vendors.get(file)!.get(exp.vendor);
+    if (binding === undefined || vendor === undefined) continue;
+    const type = `${vendor}.${exp.type}`;
+    const object = `${binding.type} ${binding.id}`;
+    const first = boundBy.get(object);
+    const why =
+      binding.type !== type
+        ? `${id} is bound to a ${binding.type} in ${LOCKFILE}, but declares a ${type}`
+        : first !== undefined
+          ? `${id} is bound to the same ${object} as ${first} in ${LOCKFILE}: bind each object once`
+          : undefined;
+    if (why === undefined) boundBy.set(object, id);
+    else {
+      // Left out, as any declaration with a problem, and so is any that names it.
+      file.problem(exp.start, why);
+      done.set(id, undefined);
+    }
+  }
+  const keys = new Set([...files.values()].flatMap((f) => f.data.declared.map((name) => `${f.rel}#${name}`)));
+  const orphans = Object.entries(lock.bindings)
+    .filter(([key]) => !keys.has(key))
+    .map(([key, binding]) => ({ key, ...binding }));
+
   const resources: DeclaredResource[] = [];
   for (const [id, { file, exp }] of declarations) {
     const result = declare(id);
     if (!result) continue;
-    const { vendor, type, name, desired } = result.declared;
-    resources.push({ id, file: file.rel, vendor, type, name, span: [exp.start, exp.end], desired, refs: result.refs });
+    const { vendor, type, name: identity, desired } = result.declared;
+    const binding = lock.bindings[id]?.id;
+    resources.push({
+      id,
+      file: file.rel,
+      vendor,
+      type,
+      name: identity ?? exp.name,
+      ...(identity !== undefined && { identity }),
+      ...(binding !== undefined && { binding }),
+      span: [exp.start, exp.end],
+      desired,
+      refs: result.refs,
+    });
   }
+  const collections = [...files.values()].flatMap(({ rel, data }) =>
+    data.collections.map((c) => ({
+      id: `${rel}#${c.export}`,
+      file: rel,
+      name: c.name,
+      span: [c.start, c.end] as Span,
+    })),
+  );
   problems.sort(
     (a, b) =>
       (a.file ?? "").localeCompare(b.file ?? "") || (a.line ?? 0) - (b.line ?? 0) || (a.column ?? 0) - (b.column ?? 0),
   );
-  return { resources, problems };
+  return { resources, bindings: lock.bindings, orphans, collections, problems };
 }
 
-/** The data files under `dir`, sorted: regular `.ts` files, less tests, declarations and `node_modules`; none when `dir` is not a directory. */
-function dataFilesUnder(dir: string): string[] {
-  if (!statSync(dir, { throwIfNoEntry: false })?.isDirectory()) return [];
-  return readdirSync(dir, { recursive: true, withFileTypes: true })
-    .filter((d) => d.isFile() && d.name.endsWith(".ts") && !d.name.endsWith(".test.ts") && !d.name.endsWith(".d.ts"))
-    .map((d) => join(d.parentPath, d.name))
-    .filter((path) => !relative(dir, path).split(sep).includes("node_modules"))
-    .toSorted();
-}
+/** The data files under `dir`, sorted: regular `.ts` files, less tests, declarations and `node_modules`. */
+const dataFilesUnder = (dir: string): string[] =>
+  filesUnder(dir, (name) => name.endsWith(".ts") && !name.endsWith(".test.ts") && !name.endsWith(".d.ts"));

package/src/run.ts

@@ -1,10 +1,14 @@
-import type { ApprovalState, Principal, WorkflowDefinition } from "./define.ts";
+import { AsyncLocalStorage } from "node:async_hooks";
+import type { Collection } from "./collection.ts";
+import type { ApprovalState, Approver, Principal, WorkflowDefinition } from "./define.ts";
 import type { Fake } from "./fake.ts";
 import type { LedgerStore } from "./ledger.ts";
 import type { DriverFn, Op } from "./op.ts";
 import type { CallNode } from "./outline.ts";
 import type { Policy } from "./policy.ts";
 import type { Resource } from "./resource.ts";
+import type { Secrets } from "./secret.ts";
+import type { Source } from "./source.ts";
 import type { Steps } from "./steps.ts";
 
 /** What the DBOS workflow receives: the workflow's input, who started the run, and for a sandbox run its scenario. */
@@ -13,6 +17,18 @@ export interface RunArgs {
   startedBy: Principal;
   /** The scenario a sandbox run is seeded from; the run calls the fakes instead of the drivers. */
   sandbox?: string;
+  /**
+   * The policy's hold on the start (`workflow:<name>`), as `SanomaClient.start` was answered: the
+   * run waits for this approval before its body runs.
+   */
+  hold?: StartHold;
+}
+
+/** A start the policy held: whom it asks, the approval's title, and the rules that held it. */
+export interface StartHold {
+  approver: Approver;
+  title: string;
+  reasons?: string[];
 }
 
 /**
@@ -31,11 +47,19 @@ export interface WorkerState {
   fakes: Map<string, () => Fake<any, any>>;
   /** Where sandbox runs' scenarios are, the resource types their steps may name, and the steps the connectors ship. */
   scenarios?: URL;
+  /** The worker's `source`, when it was given one. */
+  source?: Source;
   types: ReadonlyMap<string, Resource>;
+  /** The collections, by name, which scenarios' steps may name (`data.<name>`). */
+  collections: ReadonlyMap<string, Collection>;
   steps: readonly Steps[];
   /** Its `version`, when it has one, is recorded with each of its decisions. */
   policy: Policy;
+  /** The config's directory, whose data files declare the people an approval's decider is checked against (`readPeople`). */
+  root?: string;
   ledger: LedgerStore;
+  /** The stored secrets, for a live run's drivers (`CallContext.secrets`). */
+  secrets: Secrets;
   /**
    * The config's workflows by name, the built-in `drift` among them. A run takes its definition
    * from here, by the name it was registered under: a process registers each name with DBOS once,
@@ -62,6 +86,8 @@ export interface Run {
   sandbox?: string;
   /** A sandbox run's own fakes' functions, by operation id, from `seedSandbox`; unset for a live run. */
   fakeDrivers?: Map<string, DriverFn>;
+  /** A sandbox run's own secrets, in memory, from `seedSandbox`; unset for a live run. */
+  fakeSecrets?: Secrets;
   approvals: ApprovalState[];
   /** The next ledger `seq`. Advanced only outside steps, so a replay counts the same way. */
   seq: number;
@@ -89,3 +115,9 @@ export interface WorkflowOutline {
   lineStarts: readonly number[];
   calls: readonly CallNode[];
 }
+
+/**
+ * The run whose workflow body is running: what a built-in's own steps read of it (who started
+ * it, whether it is a sandbox run, the worker's source and policy), as no `ctx` member gives them.
+ */
+export const currentRun = new AsyncLocalStorage<Run>();

package/src/sandbox.ts

@@ -6,6 +6,7 @@ import type { Fake, ResourceFake } from "./fake.ts";
 import { entry, type LedgerRecord, write } from "./ledger.ts";
 import type { DriverFn } from "./op.ts";
 import type { Run } from "./run.ts";
+import { memorySecrets } from "./secret.ts";
 import { type Given, loadScenarios, type Scenario } from "./scenario.ts";
 import { errorMessage } from "./shared.ts";
 
@@ -90,6 +91,8 @@ export function seedResource(
 export async function seedSandbox(run: Run, name: string): Promise<void> {
   const { state } = run;
   let copies: Map<string, DriverFn> | undefined;
+  // The run's secrets, in memory as its fakes are: seeding and its calls share them.
+  const secrets = memorySecrets();
   const { seeds } = await DBOS.runStep(
     async () => {
       // Read in the step, so a recovered run never reads a feature file edited or renamed since.
@@ -126,7 +129,7 @@ export async function seedSandbox(run: Run, name: string): Promise<void> {
       for (const [i, given] of scenario.given.entries()) {
         if ("fault" in given) continue;
         if ("op" in given) {
-          const call = { idempotencyKey: `${run.id}:seed:${i}`, runId: run.id, opId: given.op, attempt: 1 };
+          const call = { idempotencyKey: `${run.id}:seed:${i}`, runId: run.id, opId: given.op, attempt: 1, secrets };
           const output = await drivers.get(given.op)!(given.input, call);
           seeded.push({ op: given.op, input: given.input, output });
           continue;
@@ -153,5 +156,6 @@ export async function seedSandbox(run: Run, name: string): Promise<void> {
     );
   }
   run.fakeDrivers = copies;
+  run.fakeSecrets = secrets;
   await write(run, entry(run, { type: "scenario.seeded", scenario: name, seeds }));
 }