@sanoma-ai/workflows 0.9.0 → 0.10.0

package/src/credentials.ts

@@ -1,296 +1,109 @@
-import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
-import { setTimeout as sleep } from "node:timers/promises";
-import { Client } from "pg";
-import { credentialsOf } from "./env.ts";
-import { CREDENTIALS_TABLE as TABLE, type Db, prepareDatabase } from "./db.ts";
-import { info, warn } from "./log.ts";
-import type { Driver } from "./op.ts";
-import type { Secrets } from "./secret.ts";
-import { errorMessage } from "./shared.ts";
+import { createCipheriv, createDecipheriv, randomBytes, randomUUID } from "node:crypto";
+import { CREDENTIALS_TABLE as TABLE, type Db } from "./db.ts";
+import {
+  SecretAccess,
+  SecretStoreError,
+  type SecretAddress,
+  type SecretConfiguration,
+  type SecretStore,
+  type SecretValue,
+} from "./secret-store.ts";
+import { deploymentKey } from "./deployment-key.ts";
 
-// Where the credentials set from the app live: one table in the config's database, beside DBOS's
-// own (made by `prepareDatabase` in db.ts). A value is read here only, by the worker to load it
-// into its environment or hand it to a driver (`storedSecrets`) and by the client to check it,
-// and leaves none of them. The write (`storeCredential`, `ensureCredential`) and the read
-// (`storedCredentials`) are the only places that touch `value`, which holds it encrypted
-// (`encrypt`) under `SANOMA_SECRETS_KEY`.
-
-/** Where a change is announced, with the variable's name as the payload. */
-const CHANNEL = "sanoma_credentials";
-
-/** The variable holding the key the stored values are encrypted with: 32 bytes, base64. */
 export const SECRETS_KEY = "SANOMA_SECRETS_KEY";
-
-/** What starts an encrypted value; a value without it is a plaintext one stored before encryption, read as it is. */
 const V1 = "v1:";
-const CIPHER = "aes-256-gcm";
-const TAG_BYTES = 16;
-
-/** The key from the environment, or an error naming the variable. */
-function secretsKey(): Buffer {
-  const text = process.env[SECRETS_KEY];
-  if (!text) {
-    throw new Error(
-      `${SECRETS_KEY} is not set: it encrypts the stored credentials; set it to 32 random bytes, base64 (\`openssl rand -base64 32\`)`,
-    );
-  }
-  const key = Buffer.from(text, "base64");
-  if (key.length !== 32) throw new Error(`${SECRETS_KEY} must be 32 bytes, base64; it is ${key.length} bytes`);
-  return key;
-}
 
-/**
- * `name`'s value encrypted for the table: `v1:<iv, base64>:<ciphertext and tag, base64>`,
- * AES-256-GCM, with the name as associated data, so a value moved to another row does not decrypt.
- */
-export function encrypt(name: string, value: string): string {
-  const iv = randomBytes(12);
-  const cipher = createCipheriv(CIPHER, secretsKey(), iv).setAAD(Buffer.from(name));
+/** AES-256-GCM authenticated to the complete namespace/path identity. No plaintext format. */
+export function encrypt(identity: string, value: string): string {
+  const nonce = randomBytes(12);
+  const cipher = createCipheriv("aes-256-gcm", deploymentKey(SECRETS_KEY), nonce).setAAD(Buffer.from(identity));
   const sealed = Buffer.concat([cipher.update(value, "utf8"), cipher.final(), cipher.getAuthTag()]);
-  return `${V1}${iv.toString("base64")}:${sealed.toString("base64")}`;
+  return `${V1}${nonce.toString("base64")}:${sealed.toString("base64")}`;
 }
 
-/** The value `name`'s row holds: decrypted, or a plaintext one as it is (it is encrypted on its next write). */
-export function decrypt(name: string, stored: string): string {
-  if (!stored.startsWith(V1)) return stored;
-  const key = secretsKey();
-  const [iv = "", body = ""] = stored.slice(V1.length).split(":");
-  const sealed = Buffer.from(body, "base64");
+export function decrypt(identity: string, stored: string): string {
+  if (!stored.startsWith(V1)) throw new SecretStoreError("invalid_value");
+  const key = deploymentKey(SECRETS_KEY);
   try {
-    const decipher = createDecipheriv(CIPHER, key, Buffer.from(iv, "base64")).setAAD(Buffer.from(name));
-    decipher.setAuthTag(sealed.subarray(-TAG_BYTES));
-    return Buffer.concat([decipher.update(sealed.subarray(0, -TAG_BYTES)), decipher.final()]).toString("utf8");
-  } catch (err) {
-    throw new Error(
-      `the stored ${name} cannot be decrypted: ${SECRETS_KEY} is not the key it was stored with, or the row was not stored as ${name}`,
-      { cause: err },
-    );
+    const parts = stored.slice(V1.length).split(":");
+    if (parts.length !== 2) throw new Error("format");
+    const nonce = Buffer.from(parts[0]!, "base64");
+    const sealed = Buffer.from(parts[1]!, "base64");
+    if (nonce.length !== 12 || sealed.length < 16) throw new Error("format");
+    const decipher = createDecipheriv("aes-256-gcm", key, nonce).setAAD(Buffer.from(identity));
+    decipher.setAuthTag(sealed.subarray(-16));
+    return Buffer.concat([decipher.update(sealed.subarray(0, -16)), decipher.final()]).toString("utf8");
+  } catch {
+    throw new SecretStoreError("invalid_value");
   }
 }
 
-/** The stored rows of these names, values included: for loading into an environment or checking, never for showing. */
-export async function storedCredentials(
-  db: Db,
-  names: readonly string[],
-): Promise<Map<string, { value: string; setBy: string; setAt: string }>> {
-  const { rows } = await db.query<{ name: string; value: string; set_by: string; set_at: Date }>(
-    `SELECT name, value, set_by, set_at FROM ${TABLE} WHERE name = ANY($1)`,
-    [names],
-  );
-  return new Map(
-    rows.map((r) => [r.name, { value: decrypt(r.name, r.value), setBy: r.set_by, setAt: r.set_at.toISOString() }]),
-  );
-}
-
-/** Upserts the value and announces it, in one statement, so a worker never hears of a change it cannot read. */
-export async function storeCredential(
-  db: Db,
-  { name, value, setBy }: { name: string; value: string; setBy: string },
-): Promise<void> {
-  await db.query(
-    `WITH stored AS (INSERT INTO ${TABLE} (name, value, set_by) VALUES ($1, $2, $3) ` +
-      "ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, set_by = EXCLUDED.set_by, set_at = now() " +
-      `RETURNING name) SELECT pg_notify('${CHANNEL}', name) FROM stored`,
-    [name, encrypt(name, value), setBy],
-  );
-}
-
-/**
- * The stored value of `name`; when there is none, `make()`'s, stored first and announced. Of two
- * at once, the first stored wins, and both return it.
- */
-export async function ensureCredential(
-  db: Db,
-  { name, make, setBy }: { name: string; make: () => string; setBy: string },
-): Promise<string> {
-  const stored = (await storedCredentials(db, [name])).get(name);
-  if (stored) return stored.value;
-  await db.query(
-    `WITH stored AS (INSERT INTO ${TABLE} (name, value, set_by) VALUES ($1, $2, $3) ON CONFLICT (name) DO NOTHING ` +
-      `RETURNING name) SELECT pg_notify('${CHANNEL}', name) FROM stored`,
-    [name, encrypt(name, make()), setBy],
-  );
-  return (await storedCredentials(db, [name])).get(name)!.value;
-}
-
-/**
- * The secrets store a live driver is given (`CallContext.secrets`): a variable the environment
- * sets wins, as it does for a driver's `env`; else the stored value. `ensure` stores what it
- * makes as `setBy`.
- */
-export function storedSecrets(db: () => Promise<Db>, setBy: string): Secrets {
+type Row = { value: string; version: string; set_by: string; set_at: Date };
+const identity = ({ namespace, path }: SecretAddress) => JSON.stringify([namespace, path]);
+const decoded = (address: SecretAddress, row: Row): SecretValue => ({
+  value: decrypt(identity(address), row.value),
+  version: row.version,
+  updatedBy: row.set_by,
+  updatedAt: row.set_at.toISOString(),
+});
+
+/** One company's database connection supplies authority; namespace separates deployments within it. */
+export function postgresSecretStore(db: () => Promise<Db>): SecretStore {
   return {
-    get: async (name) => process.env[name] || (await storedCredentials(await db(), [name])).get(name)?.value,
-    ensure: async (name, make) => process.env[name] || ensureCredential(await db(), { name, make, setBy }),
-  };
-}
-
-/** Deletes the row, announcing it when there was one. */
-export async function deleteCredential(db: Db, name: string): Promise<void> {
-  await db.query(
-    `WITH gone AS (DELETE FROM ${TABLE} WHERE name = $1 RETURNING name) SELECT pg_notify('${CHANNEL}', name) FROM gone`,
-    [name],
-  );
-}
-
-/** Stops a worker's credentials listener. */
-export interface CredentialsWatch {
-  close(): Promise<void>;
-}
-
-/** The longest wait between two attempts to reconnect the listener, and between two warnings that one failed. */
-const RETRY_MAX_MS = 30_000;
-
-/**
- * Loads the stored values of the declared variables the environment does not set into
- * `process.env`, then keeps them current: on each change announced, and on reconnecting, it
- * loads them again, so a value set replaces the one before and a value cleared is deleted. A
- * name the environment set when this started is never touched, and the names it did set are
- * deleted on `close`. Logs, per vendor, how many variables each load changed, never a value, and
- * warns of a reloaded value its schema refuses. A listener that drops reconnects, waiting longer
- * after each failure, up to 30 s, until it is closed. With no such variable, it connects to nothing.
- */
-export async function watchCredentials(
-  databaseUrl: string,
-  drivers: ReadonlyMap<string, Driver>,
-): Promise<CredentialsWatch> {
-  const vendors = new Map<string, string>();
-  for (const [vendor, list] of credentialsOf(drivers, () => undefined)) {
-    for (const { name } of list) if (!process.env[name] && !vendors.has(name)) vendors.set(name, vendor);
-  }
-  const names = [...vendors.keys()];
-  if (!names.length) return { close: async () => undefined };
-  /** The listener's connection, from when it connects until it drops or closes. */
-  let db: Client | undefined;
-  let closed = false;
-  let reconnecting = false;
-  // One load at a time, so close() can wait for it; the changes announced during one make one more.
-  let loading: Promise<void> | undefined;
-  let stale = false;
-
-  async function load(first: boolean) {
-    const conn = db;
-    if (closed || !conn) return;
-    const rows = await storedCredentials(conn, names);
-    // Nothing touches the environment once closed: a worker or an app started after may own it.
-    if (closed) return;
-    const changed = names.filter((name) => rows.get(name)?.value !== process.env[name]);
-    const counts = new Map<string, number>();
-    for (const name of changed) {
-      const value = rows.get(name)?.value;
-      if (value === undefined) delete process.env[name];
-      else process.env[name] = value;
-      const vendor = vendors.get(name)!;
-      counts.set(vendor, (counts.get(vendor) ?? 0) + 1);
-    }
-    const verb = first ? "loaded" : "reloaded";
-    for (const [vendor, n] of counts) info(`credentials ${verb} for ${vendor} (${n} variable${n === 1 ? "" : "s"})`);
-    // At start, startWorker warns of an invalid one, naming it.
-    if (first) return;
-    for (const list of credentialsOf(drivers, (name) =>
-      changed.includes(name) ? process.env[name] : undefined,
-    ).values()) {
-      for (const c of list) {
-        if (c.status === "invalid")
-          warn(`the stored ${c.name} is invalid (${c.problem}): calls that read it will fail`);
-      }
-    }
-  }
-
-  function reload(first = false): Promise<void> {
-    if (loading) {
-      stale = true;
-      return loading;
-    }
-    loading = (async () => {
-      try {
-        do {
-          stale = false;
-          await load(first);
-          first = false;
-        } while (stale);
-      } finally {
-        loading = undefined;
-      }
-    })();
-    return loading;
-  }
-
-  async function open(first: boolean) {
-    const conn = new Client({ connectionString: databaseUrl, keepAlive: true, connectionTimeoutMillis: 10_000 });
-    // Before anything can fail: an error nobody listens for ends the process.
-    conn.on("error", (err) => lost(conn, err));
-    // Before LISTEN, so no change announced from then on is missed.
-    conn.on("notification", () => {
-      if (!closed && conn === db) {
-        reload().catch((err: unknown) => warn(`could not reload the credentials: ${errorMessage(err)}`));
-      }
-    });
-    try {
-      if (first) await prepareDatabase(databaseUrl);
-      await conn.connect();
-      if (closed) {
-        await conn.end();
-        return;
-      }
-      db = conn;
-      await conn.query(`LISTEN ${CHANNEL}`);
-      await reload(first);
-    } catch (err) {
-      if (db === conn) db = undefined;
-      await conn.end().catch(() => undefined);
-      throw err;
-    }
-  }
-
-  /** A connection that dropped: the one listening is replaced; one still connecting fails its own open. */
-  function lost(conn: Client, err: unknown) {
-    if (closed || conn !== db) return;
-    db = undefined;
-    void conn.end().catch(() => undefined);
-    warn(`the credentials listener failed (${errorMessage(err)}); reconnecting`);
-    void reconnect();
-  }
-
-  async function reconnect() {
-    if (reconnecting) return;
-    reconnecting = true;
-    let wait = 1_000;
-    let warnedAt = 0;
-    try {
-      for (;;) {
-        await sleep(wait, undefined, { ref: false });
-        if (closed) return;
-        try {
-          await open(false);
-          return;
-        } catch (err) {
-          if (Date.now() - warnedAt >= RETRY_MAX_MS) {
-            warnedAt = Date.now();
-            warn(`could not reconnect the credentials listener (${errorMessage(err)}); trying again`);
-          }
-          wait = Math.min(wait * 2, RETRY_MAX_MS);
-        }
-      }
-    } finally {
-      reconnecting = false;
-    }
-  }
-
-  try {
-    await open(true);
-  } catch (err) {
-    closed = true;
-    throw err;
-  }
-  return {
-    async close() {
-      closed = true;
-      await loading?.catch(() => undefined);
-      for (const name of names) delete process.env[name];
-      const conn = db;
-      db = undefined;
-      await conn?.end();
+    id: "postgres",
+    capabilities: Object.freeze({
+      writable: true,
+      maxBytes: 65_536,
+      conditionalWrites: "atomic",
+      deletion: "destroy",
+      changeDetection: "read",
+    }),
+    async validate(namespace) {
+      if (!namespace) throw new SecretStoreError("invalid_configuration");
+      deploymentKey(SECRETS_KEY);
+      await (await db()).query(`SELECT namespace FROM ${TABLE} LIMIT 0`);
+    },
+    async read(address) {
+      const { rows } = await (
+        await db()
+      ).query<Row>(`SELECT value, version, set_by, set_at FROM ${TABLE} WHERE namespace = $1 AND name = $2`, [
+        address.namespace,
+        address.path,
+      ]);
+      return rows[0] && decoded(address, rows[0]);
+    },
+    async write(address, value, { expectedVersion, by }) {
+      if (Buffer.byteLength(value) > 65_536) throw new SecretStoreError("too_large");
+      const version = randomUUID();
+      const sealed = encrypt(identity(address), value);
+      const query =
+        expectedVersion === null
+          ? `INSERT INTO ${TABLE} (namespace,name,value,version,set_by) VALUES ($1,$2,$3,$4,$5) ON CONFLICT (namespace,name) DO NOTHING RETURNING value,version,set_by,set_at`
+          : `UPDATE ${TABLE} SET value=$3,version=$4,set_by=$5,set_at=now() WHERE namespace=$1 AND name=$2 AND version=$6 RETURNING value,version,set_by,set_at`;
+      const params = [
+        address.namespace,
+        address.path,
+        sealed,
+        version,
+        by,
+        ...(expectedVersion === null ? [] : [expectedVersion]),
+      ];
+      const { rows } = await (await db()).query<Row>(query, params);
+      if (!rows[0]) throw new SecretStoreError("conflict");
+      return decoded(address, rows[0]);
+    },
+    async delete(address, { expectedVersion }) {
+      const { rowCount } = await (
+        await db()
+      ).query(`DELETE FROM ${TABLE} WHERE namespace=$1 AND name=$2 AND version=$3`, [
+        address.namespace,
+        address.path,
+        expectedVersion,
+      ]);
+      if (!rowCount) throw new SecretStoreError("conflict");
     },
   };
 }
+
+export const configuredSecrets = (config: SecretConfiguration, db: () => Promise<Db>): SecretAccess =>
+  new SecretAccess(config, postgresSecretStore(db));

package/src/cucumber.ts

@@ -173,7 +173,8 @@ const literal = (path: string) => path.replaceAll(/[*?[\]()+@]/g, "[$&]").replac
  * examples row at that line. `makeWorld` makes each scenario's World, given the report so far
  * (every file and pickle, read before the first scenario starts). A dry run matches the steps
  * and runs none. A step's error that is a bug in code (`isBug`) is rethrown once the run is
- * over, rather than reported as the step's failure.
+ * over, rather than reported as the step's failure. `timeoutMs` is how long a step may take
+ * (Cucumber's 5 s by default).
  */
 export async function runFeatures(
   library: ISupportCodeLibrary,
@@ -184,7 +185,15 @@ export async function runFeatures(
     line,
     dryRun = false,
     isBug,
-  }: { cwd: string; files: string[]; line?: number; dryRun?: boolean; isBug: (err: unknown) => boolean },
+    timeoutMs,
+  }: {
+    cwd: string;
+    files: string[];
+    line?: number;
+    dryRun?: boolean;
+    isBug: (err: unknown) => boolean;
+    timeoutMs?: number;
+  },
 ): Promise<Report> {
   const report: Report = { query: new Query(), texts: new Map(), parseErrors: [] };
   let bug: unknown;
@@ -200,7 +209,11 @@ export async function runFeatures(
   }
   const configuration: IRunOptions = {
     sources: { defaultDialect: "en", paths: files.map(literal), names: [], tagExpression: "", order: "defined" },
-    support: { ...library, World } as ISupportCodeLibrary,
+    support: {
+      ...library,
+      World,
+      ...(timeoutMs !== undefined && { defaultTimeout: timeoutMs }),
+    } as ISupportCodeLibrary,
     runtime: {
       dryRun,
       failFast: false,

package/src/db.ts

@@ -31,10 +31,29 @@ export const CHAT_MESSAGES_TABLE = "sanoma_chat_messages";
 
 /** The runtime's tables, each with the word the messages name it by. */
 const TABLES: readonly [what: string, ddl: string][] = [
+  [
+    "connections",
+    `CREATE TABLE IF NOT EXISTS sanoma_connections (
+    id text PRIMARY KEY, vendor text NOT NULL, method text NOT NULL, owner text, rule text,
+    account text NOT NULL, scopes jsonb NOT NULL, revision text NOT NULL, definition text NOT NULL,
+    credential text, status text NOT NULL DEFAULT 'connected', updated_by text NOT NULL,
+    updated_at timestamptz NOT NULL DEFAULT now())`,
+  ],
   [
     "credentials",
-    `CREATE TABLE IF NOT EXISTS ${CREDENTIALS_TABLE} (name text PRIMARY KEY, value text NOT NULL, ` +
-      "set_by text NOT NULL, set_at timestamptz NOT NULL DEFAULT now())",
+    // A table from before 0.10.0 (one row per name, each value sealed to its name alone) holds
+    // nothing 0.10.0 can read: it goes, and its credentials are set again.
+    // One table for the look and the drop: the one the name resolves to.
+    `DO $$ BEGIN
+      IF to_regclass('${CREDENTIALS_TABLE}') IS NOT NULL AND NOT EXISTS (
+        SELECT FROM pg_attribute
+        WHERE attrelid = to_regclass('${CREDENTIALS_TABLE}') AND attname = 'namespace' AND NOT attisdropped
+      ) THEN
+        DROP TABLE ${CREDENTIALS_TABLE};
+      END IF;
+    END $$; ` +
+      `CREATE TABLE IF NOT EXISTS ${CREDENTIALS_TABLE} (namespace text NOT NULL, name text NOT NULL, value text NOT NULL, ` +
+      "version text NOT NULL, set_by text NOT NULL, set_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY(namespace, name))",
   ],
   [
     "ledger",

package/src/define.ts

@@ -1,3 +1,5 @@
+import type { ConnectionRef } from "./connection.ts";
+import type { WorkflowEntry } from "./description.ts";
 import { fileURLToPath } from "node:url";
 import { getCallSites } from "node:util";
 import { z } from "zod";
@@ -24,7 +26,8 @@ export const Principal: z.ZodType<Principal> = z.object({
   onBehalfOf: z.string().regex(/\S/, "onBehalfOf must not be blank").optional(),
 });
 
-export type Builtin = "approval" | "sleep" | "all";
+/** A built-in a workflow may call, as its description lists them. */
+export type Builtin = WorkflowEntry["builtins"][number];
 export type Use = Op | Builtin;
 
 /** Who may decide an approval: one person, by `Principal.id`, or anyone in a group, by `Principal.groups`. */
@@ -77,6 +80,8 @@ export const ApprovalRequestSchema = z.object(
 
 /** One approval in a run, as the run sees it. Published as the run's "approvals" event. */
 export interface ApprovalState extends Omit<ApprovalRequest, "covers"> {
+  /** Connection binding revision approved by this policy hold. */
+  connection?: string;
   id: string;
   title: string;
   /** Who asked: the workflow (`ctx.approval`), or the policy holding an operation call. */
@@ -179,7 +184,7 @@ type VendorCtx<O extends Op, V> = {
       O,
       { vendor: V; resource: R; name: N }
     > extends Op<any, any, any, infer I, infer Out>
-      ? (input: I, options?: StepOptions) => Promise<Out>
+      ? (input: I, options?: StepOptions & { connection?: ConnectionRef<Extract<V, string>> }) => Promise<Out>
       : never;
   };
 };

package/src/definitions.ts

@@ -0,0 +1,31 @@
+// What an app's `app.ts` and a site's `site.ts` declare themselves with. Dependency-free: the
+// loader (`load.ts`) serves these to those files as `sanoma` and `sanoma/site`, the browser-safe
+// entry (`shared.ts`) re-exports `defineSite`, and the app's `sanoma/site` calls it.
+
+/** What `app.ts` declares: the source repository's URL, when the app was taken from one. */
+export interface AppDefinition {
+  source?: string;
+}
+
+/**
+ * Declares an app in its folder's `app.ts`, its default export: `defineApp({ source:
+ * "https://github.com/acme/crm" })`, the repository it was taken from; `defineApp({})` for an app
+ * the company made itself.
+ */
+export function defineApp(definition: AppDefinition): AppDefinition {
+  return Object.freeze({ ...definition });
+}
+
+/** What a site's `site.ts` declares, as its default export; the app's `sanoma/site` types `icon` as a Lucide icon's name. */
+export interface SiteDefinition {
+  title?: string;
+  icon?: string;
+  public?: boolean;
+  login?: string;
+  locales?: readonly [string, ...string[]];
+}
+
+/** Declares a site in its folder's `site.ts`, its default export: `defineSite({ title: "Help desk" })`. */
+export function defineSite<S extends SiteDefinition>(site: S): Readonly<S> {
+  return Object.freeze({ ...site });
+}

package/src/deployment-key.ts

@@ -0,0 +1,23 @@
+import { readFileSync } from "node:fs";
+
+/** Bootstrap only: never resolves through the secret store that the key unlocks. */
+export function deploymentKey(
+  name: "SANOMA_SECRETS_KEY" | "SANOMA_SIGNING_KEY",
+  env: NodeJS.ProcessEnv = process.env,
+): Buffer {
+  const value = env[name];
+  const file = env[`${name}_FILE`];
+  if (value && file) throw new Error(`Set only one of ${name} and ${name}_FILE`);
+  let encoded = value;
+  if (file) {
+    try {
+      encoded = readFileSync(file, "utf8").trim();
+    } catch {
+      throw new Error(`${name}_FILE could not be read`);
+    }
+  }
+  if (!encoded || !/^[A-Za-z0-9+/]{43}=$/.test(encoded) || Buffer.from(encoded, "base64").length !== 32) {
+    throw new Error(`${name} must be 32 random bytes in base64; supply ${name} or ${name}_FILE`);
+  }
+  return Buffer.from(encoded, "base64");
+}