@sanoma-ai/workflows 0.11.17 → 0.11.18

package/src/sandbox-runner.ts

@@ -0,0 +1,187 @@
+import { createServer } from "node:http";
+import type { AddressInfo } from "node:net";
+import { homedir } from "node:os";
+import { setTimeout as delay } from "node:timers/promises";
+import { type WebSocket, WebSocketServer } from "ws";
+import { localBox } from "./local-sandbox.ts";
+import {
+  FILE,
+  frame,
+  RUNNER_HEALTH,
+  RUNNER_PORT,
+  RUNNER_PROTOCOL,
+  type RunnerAnswer,
+  type RunnerHealth,
+  type RunnerRequest,
+  STDERR,
+  STDIN,
+  STDOUT,
+  unframe,
+} from "./runner-protocol.ts";
+import type { Process } from "./sandbox-contract.ts";
+import { errorMessage } from "./shared.ts";
+
+// The runner: the server of ours a backend starts inside a remote box, which `runnerBox` talks to
+// over a WebSocket (the protocol is in `runner-protocol.ts`). It serves this machine as the local
+// backend's box does (`localBox`, whose code it bundles): commands are `bash -c` in process groups
+// of their own, a relative path or `cwd` is under the home. Built into one file,
+// `dist/sandbox-runner.cjs` (`ws` bundled in), run as `node sandbox-runner.cjs`.
+//
+// It listens on `SANOMA_RUNNER_HOST:SANOMA_RUNNER_PORT` (default `0.0.0.0:7700`) and asks no
+// credential: the vendor's edge authenticates every request to the port, the upgrade too
+// (Agent37: the API key, on a port never made public). Anything inside the box already runs as
+// the box's user, so a secret of the runner's own would guard nothing.
+
+const port = Number(process.env.SANOMA_RUNNER_PORT ?? RUNNER_PORT);
+const host = process.env.SANOMA_RUNNER_HOST ?? "0.0.0.0";
+
+/**
+ * How long an exited command's output may go quiet before its exit is sent, in ms: `localBox` has
+ * waited for its pipes already, unless what it left running holds them. Output held back for a
+ * slow connection is not quiet.
+ */
+const LAST_OUTPUT = 50;
+/** How much a connection may hold unsent before a command's output waits for it, in bytes. */
+const UNSENT = 1 << 20;
+/** How often a connection is pinged: Agent37's edge closes one silent for 15 minutes. */
+const BEAT = 30_000;
+
+// Its commands' environment is its own, but for where it listens.
+const { SANOMA_RUNNER_PORT: _, SANOMA_RUNNER_HOST: __, ...own } = process.env;
+const box = localBox("runner", homedir(), own as Record<string, string>);
+
+const server = createServer((req, res) => {
+  if (req.url !== RUNNER_HEALTH) return void res.writeHead(404).end();
+  res
+    .writeHead(200, { "content-type": "application/json" })
+    .end(JSON.stringify({ protocol: RUNNER_PROTOCOL, pid: process.pid } satisfies RunnerHealth));
+});
+new WebSocketServer({ server })
+  .on("connection", serve)
+  .on("error", (error) => console.error(`the runner's WebSocket server: ${error.message}`));
+server.listen(port, host, () => {
+  const { address, port: bound } = server.address() as AddressInfo;
+  console.log(`sanoma runner listening on ${address}:${bound}`);
+});
+// Asked to end (the box shuts down): what it started ends with it.
+process.on("SIGTERM", () => void box.stop().finally(() => process.exit(0)));
+
+/** One client's connection: its commands, by the ids it gave them. */
+function serve(ws: WebSocket) {
+  // Promises: a command's kill and its stdin may come before it has started, and wait for it.
+  const processes = new Map<number, Promise<Process>>();
+  const inputs = new Map<number, Promise<WritableStreamDefaultWriter<Uint8Array>>>();
+  /** The writes whose bytes are yet to come. */
+  const writes = new Map<number, { path: string; mode?: number }>();
+  const send = (data: Uint8Array | string) => ws.readyState === ws.OPEN && ws.send(data);
+  const say = (answer: RunnerAnswer) => send(JSON.stringify(answer));
+  const beat = setInterval(() => ws.ping(), BEAT);
+  // A frame ws refuses (one too large, one malformed) closes this connection, not the runner.
+  ws.on("error", (error) => console.error(`a connection: ${error.message}`));
+  ws.on("close", () => {
+    clearInterval(beat);
+    // A dropped connection kills nothing: its commands go on, their stdin ended, their output dropped.
+    for (const id of inputs.keys()) endInput(id);
+  });
+  ws.on("message", (data: Buffer, binary) => {
+    // A frame that is not the protocol's is answered, never a crash of the runner.
+    if (binary) {
+      if (data.length < 5) return say({ op: "error", message: "a binary frame is an id, a stream and bytes" });
+      return received(unframe(data));
+    }
+    let request: RunnerRequest;
+    try {
+      request = JSON.parse(String(data));
+    } catch {
+      return say({ op: "error", message: "a text frame is JSON" });
+    }
+    if (typeof request !== "object" || request === null) return say({ op: "error", message: "a request is an object" });
+    handle(request).catch((error: unknown) => say({ op: "error", id: request.id, message: errorMessage(error) }));
+  });
+
+  async function handle(request: RunnerRequest): Promise<unknown> {
+    switch (request.op) {
+      case "run": {
+        const { id, command, cwd, env, stdin } = request;
+        const starting = box.run(command, { cwd, env, stdin });
+        processes.set(id, starting);
+        if (stdin) {
+          const input = starting.then((started) => started.stdin.getWriter());
+          // A run that fails to start is answered below; its stdin's rejection is not another.
+          input.catch(() => {});
+          inputs.set(id, input);
+        }
+        try {
+          const started = await starting;
+          const moved = { at: Date.now(), done: false };
+          const pumped = Promise.all([
+            pump(id, STDOUT, started.stdout, moved),
+            pump(id, STDERR, started.stderr, moved),
+          ]).finally(() => (moved.done = true));
+          const { exitCode } = await started.wait();
+          // What it printed last goes out first, unless what it left running holds its pipes.
+          moved.at = Date.now();
+          while (!moved.done && Date.now() - moved.at < LAST_OUTPUT) await Promise.race([pumped, delay(10)]);
+          return say({ op: "exit", id, code: exitCode });
+        } finally {
+          processes.delete(id);
+          inputs.delete(id);
+        }
+      }
+      case "stdin-end":
+        return endInput(request.id);
+      case "kill":
+        return processes.get(request.id)?.then((started) => started.kill(request.signal));
+      case "read": {
+        const file = await box.read(request.path);
+        return file ? send(frame(request.id, FILE, file)) : say({ op: "absent", id: request.id });
+      }
+      case "write":
+        return writes.set(request.id, request);
+      case "stop":
+        await box.stop();
+        return say({ op: "ok", id: request.id });
+      default:
+        throw new Error(`no such request: ${(request as { op?: unknown }).op}`);
+    }
+  }
+
+  function received({ id, stream, bytes }: ReturnType<typeof unframe>) {
+    if (stream === STDIN)
+      return void inputs
+        .get(id)
+        ?.then((writer) => writer.write(bytes))
+        .catch(() => {});
+    const write = writes.get(id);
+    if (stream !== FILE || !write) return;
+    writes.delete(id);
+    box.write(write.path, bytes, { mode: write.mode }).then(
+      () => say({ op: "ok", id }),
+      (error: unknown) => say({ op: "error", id, message: errorMessage(error) }),
+    );
+  }
+
+  /** Ends the stdin of the command `id`, once it has started. */
+  function endInput(id: number) {
+    const input = inputs.get(id);
+    inputs.delete(id);
+    return input?.then((writer) => writer.close()).catch(() => {});
+  }
+
+  /** Sends what `from` prints, until it ends or fails, keeping in `moved` when it last sent or waited to. */
+  async function pump(id: number, stream: number, from: ReadableStream<Uint8Array>, moved: { at: number }) {
+    try {
+      for await (const chunk of from) {
+        // Its output waits for a slow connection, and so the command does, its pipe full.
+        while (ws.readyState === ws.OPEN && ws.bufferedAmount > UNSENT) {
+          moved.at = Date.now();
+          await delay(10);
+        }
+        send(frame(id, stream, chunk));
+        moved.at = Date.now();
+      }
+    } catch {
+      // A pipe that failed has nothing more to say.
+    }
+  }
+}

package/src/scenario.ts

@@ -12,6 +12,7 @@ import {
   type Step as GherkinStep,
 } from "@cucumber/messages";
 import type { z } from "zod";
+import { targetOf } from "./call.ts";
 import type { RunSummary, SanomaClient } from "./client.ts";
 import type { Approver, Principal } from "./define.ts";
 import type { ResolvedConfig } from "./config.ts";
@@ -44,10 +45,12 @@ import {
   DATA_VENDOR,
   declaresPeople,
   DRIFT_WORKFLOW,
+  entryOpsOf,
   errorMessage,
   fitsOp,
   flatten,
   groupNamesOf,
+  isBatch,
   isEnded,
   principalIdsOf,
   problemLine,
@@ -446,6 +449,12 @@ function rulesOf(scope: Scope): { rules: Rule[]; parameterTypes: ParameterTypeSp
     { name: "field", regexp: /\S+/, transformer: (s) => s },
   ];
   const opOf = (id: unknown) => scope.ops.get(id as string)!;
+  /** An operation a `Then` step reads the ledger for: never a batch, which the ledger records as its entries. */
+  const recordedOp = (id: unknown): string => {
+    const op = opOf(id);
+    if (isBatch(op)) throw new Error(`${op.id} is recorded as its entries: name ${entryOpsOf(op).join(" or ")}`);
+    return op.id;
+  };
   const workflowOf = (name: unknown) => scope.workflows.get(name as string)!;
   // The people as the root's data files declare them when a step names one (`readPeople`, read
   // again only once a file changed): while they declare no one, any id and any group go.
@@ -596,33 +605,33 @@ function rulesOf(scope: Scope): { rules: Rule[]; parameterTypes: ParameterTypeSp
       source: "{op} was called with",
       takes: TAKES_OBJECT,
       read: ([op], pickle) =>
-        expecting({ op: op as string, input: fieldsFrom(pickle, opOf(op).input, inputOf(op), true), called: true }),
+        expecting({ op: recordedOp(op), input: fieldsFrom(pickle, opOf(op).input, inputOf(op), true), called: true }),
     },
     ...[true, false].map((called): Rule => ({
       kind: "then",
       source: `{op} was ${called ? "" : "not "}called`,
-      read: ([op]) => expecting({ op: op as string, called }),
+      read: ([op]) => expecting({ op: recordedOp(op), called }),
     })),
     {
       kind: "then",
       source: "{op} was denied",
-      read: ([op]) => expecting({ op: op as string, decision: "denied" }),
+      read: ([op]) => expecting({ op: recordedOp(op), decision: "denied" }),
     },
     {
       kind: "then",
       source: "{op} was denied with {string}",
-      read: ([op, reason]) => expecting({ op: op as string, decision: "denied", reason: reason as string }),
+      read: ([op, reason]) => expecting({ op: recordedOp(op), decision: "denied", reason: reason as string }),
     },
     {
       kind: "then",
       source: "{op} was held for {who}",
-      read: ([op, who]) => expecting({ op: op as string, decision: "held", approver: who as string }),
+      read: ([op, who]) => expecting({ op: recordedOp(op), decision: "held", approver: who as string }),
     },
     {
       kind: "then",
       source: "{op} was held for group {string}",
       read: ([op, group]) =>
-        expecting({ op: op as string, decision: "held", approver: { group: groupNamed(group as string) } }),
+        expecting({ op: recordedOp(op), decision: "held", approver: { group: groupNamed(group as string) } }),
     },
     {
       kind: "then",
@@ -1207,6 +1216,7 @@ function testWorld(built: Built, { driver, fake, live }: FeatureTarget) {
             record({
               type: "op.called",
               op: op.id,
+              ...targetOf(op, input),
               effect: op.effect,
               input,
               decision: { kind: "allow" },

package/src/schema.ts

@@ -13,7 +13,15 @@ import { recordSchemaOf } from "./collection.ts";
 import type { ResolvedConfig } from "./config.ts";
 import { jsonSchemaOf } from "./define.ts";
 import { isObject } from "./resource.ts";
-import { CHANGE_MERGE, DATA_VENDOR, EFFECTS, groupNamesOf, type PeopleResource, WORKFLOW_ACTION } from "./shared.ts";
+import {
+  CHANGE_MERGE,
+  DATA_VENDOR,
+  EFFECTS,
+  groupNamesOf,
+  isBatch,
+  type PeopleResource,
+  WORKFLOW_ACTION,
+} from "./shared.ts";
 
 // The Cedar schema a company's policies are validated and decided against: made from the
 // resolved config and the people, in memory for the evaluator, and written as text to
@@ -90,7 +98,8 @@ const WRITES = new Set(["save", "delete"]);
  * - `entity Group enum [...]`, the declared groups, `agents` and `visitors`, so a policy naming another fails
  *   validation; `entity Principal in [Group] { onBehalfOf?: Principal }`; `Collection`;
  *   `Target in [Collection]`; `Workflow`;
- * - an action per operation id, in its effect's group (`effect:money`), on a `Target`, with
+ * - an action per operation id (a collection's `saveMany` excepted: the policy decides each of
+ *   its entries as a `save` or `delete`), in its effect's group (`effect:money`), on a `Target`, with
  *   `context` `{ input, before? }`: `input` from the operation's input schema, `before` the record
  *   as it was for a collection's `save` and `delete`; `sanoma.change.merge` on a `Change`, whose
  *   attributes are a change's facts (`ChangeFacts`), so a rule reads `resource.touches`;
@@ -112,7 +121,9 @@ function buildSchema(
   });
   const actions: CedarSchemaJson[string]["actions"] = { [START]: {} };
   for (const effect of EFFECTS) actions[`effect:${effect}`] = {};
-  for (const op of [...resolved.ops.values()].toSorted((a, b) => a.id.localeCompare(b.id))) {
+  // A collection's batch is no action: the policy decides each of its entries as a save or a delete.
+  const ops = [...resolved.ops.values()].filter((op) => !isBatch(op));
+  for (const op of ops.toSorted((a, b) => a.id.localeCompare(b.id))) {
     const collection =
       op.vendor === DATA_VENDOR && WRITES.has(op.name) ? resolved.collections.get(op.resource) : undefined;
     const before = collection && typeOf(recordSchemaOf(collection));

package/src/secret.ts

@@ -101,14 +101,18 @@ export type Resolved<T> = T extends Secret
 
 /**
  * `value` with each secret marker in it replaced by the stored value: generated and stored first
- * for one with `generate` and none stored. A marker naming nothing stored fails the call, for good.
+ * for one with `generate` and none stored. A marker naming nothing stored fails the call, for good,
+ * before the driver changes anything at the vendor (`secret_not_set`, which the ledger tells).
  */
 export async function resolveSecrets<T>(value: T, secrets: Secrets): Promise<Resolved<T>> {
   const resolve = async (v: unknown): Promise<unknown> => {
     if (isSecret(v)) {
       const stored = v.generate ? await secrets.ensure(v.$secret, generateSecret) : await secrets.get(v.$secret);
       if (stored === undefined) {
-        throw new DriverError(`The secret ${v.$secret} is not set: store it before applying`, { retryable: false });
+        throw new DriverError(
+          `The secret ${v.$secret} has no value: store it (or, where sanoma.config.ts binds it, set what the binding reads), then try again`,
+          { retryable: false, vendorCode: "secret_not_set" },
+        );
       }
       return stored;
     }

package/src/shared.ts

@@ -55,6 +55,14 @@ export const APPLY_WORKFLOW = "apply";
  */
 export const DATA_VENDOR = "data";
 
+/** The one operation of a collection that stands for others: `saveMany`, each entry a `save` or `delete`. */
+export const BATCH = "saveMany";
+export const isBatch = (op: Pick<OpEntry, "vendor" | "name">): boolean =>
+  op.vendor === DATA_VENDOR && op.name === BATCH;
+/** The ids of the operations a batch's entries are: its collection's `save` and `delete`. */
+export const entryOpsOf = (op: Pick<OpEntry, "resource">): string[] =>
+  ["save", "delete"].map((name) => `${DATA_VENDOR}.${op.resource}.${name}`);
+
 /** Every effect an operation may have, each an action group (`effect:<effect>`) of the Cedar schema. */
 export const EFFECTS = ["read", "write", "publish", "send", "money", "access"] as const satisfies readonly Effect[];
 
@@ -68,11 +76,16 @@ export const WORKFLOW_ACTION = "workflow:";
  */
 export function previewRequests(
   workflow: Pick<WorkflowEntry, "name" | "ops">,
-  ops: ReadonlyMap<string, Pick<OpEntry, "effect">>,
+  ops: ReadonlyMap<string, Pick<OpEntry, "effect" | "vendor" | "resource" | "name">>,
 ): Omit<PreviewRequest, "principal">[] {
+  // A batch is asked as its entries, the save and the delete the policy decides: no rule names it.
+  const asked = workflow.ops.flatMap((id) => {
+    const op = ops.get(id);
+    return op && isBatch(op) ? entryOpsOf(op) : [id];
+  });
   return [
     { action: { id: `${WORKFLOW_ACTION}${workflow.name}` } },
-    ...workflow.ops.map((id) => {
+    ...[...new Set(asked)].map((id) => {
       const effect = ops.get(id)?.effect;
       return { action: effect === undefined ? { id } : { id, effect }, inRun: true };
     }),
@@ -203,6 +216,12 @@ type Range<V> = { gt?: V; gte?: V; lt?: V; lte?: V };
 /** How many records a collection's `list` answers: `default` when the query names no `limit`, `max` at most. */
 export const LIST_LIMIT = { default: 100, max: 1000 } as const;
 
+/** How many entries (records and deletes together) a collection's `saveMany` takes at most. */
+export const BATCH_LIMIT = 1000;
+
+/** What a collection's `delete` answers, and `saveMany` for each of its deletes: whether there was a record. */
+export type Deleted = { key: string; deleted: boolean };
+
 /** The kinds of value a field holds: text, a number, true or false, a date (ISO 8601) or a person's id. */
 export const FIELD_TYPES = ["text", "number", "boolean", "date", "person"] as const;
 
@@ -673,10 +692,27 @@ export interface WorkflowFn<I, O> {
 /** An operation a site may call directly (a read, or a collection's write), as the person on the page. */
 export type OpFn<I, O> = (input: I) => Promise<O>;
 
-/** A collection as a site's `api.data.<name>` has it: its four operations (`Saved`, `ListQuery`). */
+/** A collection as a site's `api.data.<name>` has it: its six operations (`Saved`, `ListQuery`, `Deleted`). */
 export interface CollectionApi<R> {
   get: OpFn<{ key: string }, Saved<R> | null>;
   list(query?: ListQuery<R>): Promise<Saved<R>[]>;
+  /** How many records there are, or match `where` (as `list` takes it). */
+  count(query?: Pick<ListQuery<R>, "where">): Promise<{ count: number }>;
   save: OpFn<{ record: R; ifAbsent?: boolean; ifRecord?: Record<string, unknown> }, Saved<R>>;
-  delete: OpFn<{ key: string }, { deleted: boolean }>;
+  /**
+   * Saves and deletes together, all or none, at most `BATCH_LIMIT` entries, each key once: the
+   * records' receipts then the deletes' answers, in the order sent. The policy decides each entry
+   * as its own `save` or `delete`; a deny refuses the batch (`policy_denied`, with the `denied`
+   * keys), a hold holds it whole, and a record an entry's `ifRecord` or `ifAbsent` refuses refuses
+   * it (status 409, `record_changed`) with every such entry in `conflicts: { key, current }[]`.
+   */
+  saveMany: OpFn<
+    {
+      records?: { record: R; ifAbsent?: boolean; ifRecord?: Record<string, unknown> }[];
+      deletes?: { key: string; ifRecord?: Record<string, unknown> }[];
+    },
+    (Saved<R> | Deleted)[]
+  >;
+  /** Deletes the record, or with `ifRecord` only while it still matches (status 409, `record_changed`, otherwise). */
+  delete: OpFn<{ key: string; ifRecord?: Record<string, unknown> }, Deleted>;
 }

package/src/sites.ts

@@ -16,6 +16,8 @@ import {
   BUILT_IN_WORKFLOWS,
   DATA_VENDOR,
   DOMAIN_NAME,
+  entryOpsOf,
+  isBatch,
   LOCALE,
   SITE_NAME,
   siteKey,
@@ -80,7 +82,7 @@ export function readSites(
    */
   const refused = (call: SiteCall, login: string | undefined): string | undefined => {
     const why = `a visitor's call to ${KIND_NAMES[call.kind]} "${call.target}" is refused`;
-    let action: string;
+    let actions: string[];
     let group: string;
     if (call.kind === "workflow") {
       if (BUILT_IN_WORKFLOWS.has(call.target)) return `${why}: a visitor may not start the runtime's own workflows`;
@@ -88,22 +90,24 @@ export function readSites(
       if (call.method !== undefined && call.method !== "start") {
         return `${why}: a visitor reads only the result of a run it started`;
       }
-      [action, group] = [`${WORKFLOW_ACTION}${call.target}`, START];
+      [actions, group] = [[`${WORKFLOW_ACTION}${call.target}`], START];
     } else {
-      action = call.kind === "op" ? call.target : `${DATA_VENDOR}.${call.target}.${call.method}`;
+      const action = call.kind === "op" ? call.target : `${DATA_VENDOR}.${call.target}.${call.method}`;
       const op = ops.get(action);
       if (!op) return undefined;
-      group = `effect:${op.effect}`;
+      // A batch is its entries to the policy: the save and the delete each need a permit.
+      [actions, group] = [isBatch(op) ? entryOpsOf(op) : [action], `effect:${op.effect}`];
     }
     if (!permits) return undefined;
     const [scopes, principal] =
       login === undefined
         ? [[ANONYMOUS], `== Sanoma::Principal::"${ANONYMOUS}"`]
         : [[ANONYMOUS, VISITORS], `in Sanoma::Group::"${VISITORS}"`];
-    if (permits(new Set(scopes), action, group)) return undefined;
+    const missing = actions.filter((action) => !permits(new Set(scopes), action, group));
+    if (!missing.length) return undefined;
     return (
-      `${why}: no permit in policies/ names ${scopes.join(" or ")} for ${action}. ` +
-      `Add: permit (principal ${principal}, action == Sanoma::Action::"${action}", resource);`
+      `${why}: no permit in policies/ names ${scopes.join(" or ")} for ${missing.join(" or ")}. ` +
+      `Add: ${missing.map((a) => `permit (principal ${principal}, action == Sanoma::Action::"${a}", resource);`).join(" ")}`
     );
   };
 

package/src/source.ts

@@ -40,6 +40,20 @@ export interface Source {
   diff?(id: string): Promise<DraftDiff>;
   /** Closes the draft `id`: its checkout removed and its branch deleted, whatever it held; nothing when it is gone. */
   closeDraft?(id: string): Promise<void>;
+  /**
+   * The draft `id`'s branch as a git bundle (`git bundle create`), for a clone elsewhere (an agent
+   * box's) to fetch: its commits since `since`, a commit that clone has, or the whole branch when
+   * `since` is absent or a commit this repository lacks; no bytes when the clone has them all.
+   * The branch's head beside it. A bundle holds commits and the branch's name alone: no remote,
+   * no config, no credential.
+   */
+  bundle?(id: string, opts?: { since?: string }): Promise<{ bytes: Uint8Array; head: string }>;
+  /**
+   * Takes a bundle of the draft `id`'s branch made elsewhere (an agent box's new commits on it):
+   * its commits fetched, then the draft's checkout fast-forwarded to them. Refused unless that is
+   * a fast-forward that leaves what the checkout holds uncommitted in place. The branch's head after.
+   */
+  unbundle?(id: string, bytes: Uint8Array): Promise<{ head: string }>;
   /**
    * `sanoma check` on the draft `id`, as a child in its checkout, one at a time in the process. A
    * check that gives no result (a config that does not load, no install, a timeout) is a result