@sanoma-ai/workflows 0.11.12 → 0.11.14

package/src/cloud-identity.ts

@@ -1,6 +1,8 @@
 import { createHash, randomBytes } from "node:crypto";
 import { Compute, OAuth2Client } from "google-auth-library";
+import { and, eq, gt, lte, sql } from "drizzle-orm";
 import type { Db } from "./db.ts";
+import { identityCredentials, identityDeployments } from "./db/identity.ts";
 import { enrollmentBroker } from "./identity-broker.ts";
 export { provisioningIdentityBroker, attachedGoogleIdTokenIssuer } from "./provisioning-identity.ts";
 export type { ProvisioningIdentity, ProvisioningIdToken } from "./provisioning-identity.ts";
@@ -64,31 +66,28 @@ REVOKE ALL ON ALL TABLES IN SCHEMA sanoma_identity FROM PUBLIC;
 export function postgresIdentityEnrollments(database: () => Promise<Db>): IdentityEnrollments {
   return {
     async active(hash) {
-      const { rows } = await (
+      const [row] = await (
         await database()
-      ).query<{
-        deployment: string;
-        company: string;
-        environment: string;
-        service_account: string;
-        generation: string;
-      }>(
-        `SELECT d.id AS deployment, d.company, d.environment, d.service_account, c.generation
-          FROM sanoma_identity.credentials c JOIN sanoma_identity.deployments d ON d.id = c.deployment
-          WHERE c.hash = $1 AND NOT c.revoked AND d.enabled
-            AND c.valid_from <= clock_timestamp() AND c.valid_until > clock_timestamp()`,
-        [hash],
-      );
-      const row = rows[0];
-      return (
-        row && {
-          deployment: row.deployment,
-          company: row.company,
-          environment: row.environment,
-          serviceAccount: row.service_account,
-          generation: Number(row.generation),
-        }
-      );
+      )
+        .select({
+          deployment: identityDeployments.id,
+          company: identityDeployments.company,
+          environment: identityDeployments.environment,
+          serviceAccount: identityDeployments.serviceAccount,
+          generation: identityCredentials.generation,
+        })
+        .from(identityCredentials)
+        .innerJoin(identityDeployments, eq(identityDeployments.id, identityCredentials.deployment))
+        .where(
+          and(
+            eq(identityCredentials.hash, hash),
+            eq(identityCredentials.revoked, false),
+            eq(identityDeployments.enabled, true),
+            lte(identityCredentials.validFrom, sql`clock_timestamp()`),
+            gt(identityCredentials.validUntil, sql`clock_timestamp()`),
+          ),
+        );
+      return row;
     },
   };
 }

package/src/collection.ts

@@ -1,6 +1,9 @@
 import { isDeepStrictEqual } from "node:util";
 import { z } from "zod";
-import { DATA_TABLE, type Db } from "./db.ts";
+import * as drizzle from "drizzle-orm";
+import { and, eq, or, sql, type SQL } from "drizzle-orm";
+import type { Db } from "./db.ts";
+import { data } from "./db/company.ts";
 import { defineFake, type Fake } from "./fake.ts";
 import { isObject } from "./resource.ts";
 import {
@@ -29,7 +32,7 @@ import {
 // A company's own records: a collection is declared with plain values (a name, a key and typed
 // fields), and its `get`, `list`, `save` and `delete` are operations of the vendor `data`, under
 // the policy and in the ledger like any connector's. The records are JSON rows of one table the
-// runtime owns (`DATA_TABLE`), in the config's database; a sandbox run keeps them in memory.
+// runtime owns (`data`), in the config's database; a sandbox run keeps them in memory.
 
 export type FieldType = (typeof FIELD_TYPES)[number];
 
@@ -266,7 +269,7 @@ export function collectionsConnector(list: readonly Collection[]): Connector<typ
   return { ...defineConnector(DATA_VENDOR, {}), ...Object.fromEntries(ops) } as Connector<typeof DATA_VENDOR, DataOps>;
 }
 
-const COLUMNS = "key, record, updated_at, updated_by";
+const COLUMNS = { key: data.key, record: data.record, updatedAt: data.updatedAt, updatedBy: data.updatedBy };
 const recordExists = () =>
   new DriverError("Collection record already exists", { retryable: false, status: 409, vendorCode: "record_exists" });
 const recordChanged = () =>
@@ -278,14 +281,10 @@ const recordChanged = () =>
 const sameJson = (a: Record<string, unknown>, b: Record<string, unknown>) =>
   isDeepStrictEqual(JSON.parse(JSON.stringify(a)), JSON.parse(JSON.stringify(b)));
 type SaveInput = { record: Record<string, unknown>; ifAbsent?: boolean; ifRecord?: Record<string, unknown> };
-const RANGE = { gt: ">", gte: ">=", lt: "<", lte: "<=" } as const;
+// By namespace: the fake names a range's bounds `gt`, `gte`, `lt` and `lte`.
+const RANGE = { gt: drizzle.gt, gte: drizzle.gte, lt: drizzle.lt, lte: drizzle.lte };
 
-interface Row {
-  key: string;
-  record: Record<string, unknown>;
-  updated_at: Date;
-  updated_by: string;
-}
+type Row = Pick<typeof data.$inferSelect, keyof typeof COLUMNS>;
 
 /** A `list` input as its schema parses it. */
 interface Listing {
@@ -295,58 +294,70 @@ interface Listing {
 }
 
 /**
- * The collections' driver: their records in `DATA_TABLE` of the database `store` is on, through
+ * The collections' driver: their records in the `data` table of the database `store` is on, through
  * the pool a run's calls share, or one connection in a transaction (a direct write, with its
  * ledger record). Every value reaches Postgres as a parameter, field names included; the field
  * names and operators are the ones the input's schema accepts.
  */
 export function collectionsDriver(list: readonly Collection[], store: { db(): Promise<Db> }): Driver {
-  const query = async (sql: string, params: unknown[]) => (await store.db()).query<Row>(sql, params);
   const impl = list.map((c) => {
     const saved = (row: Row) =>
       answer(c, {
         key: row.key,
         record: row.record,
-        updatedAt: row.updated_at.toISOString(),
-        updatedBy: row.updated_by,
+        updatedAt: row.updatedAt.toISOString(),
+        updatedBy: row.updatedBy,
       });
+    const here = (key: string) => and(eq(data.collection, c.name), eq(data.key, key));
     const ops: Record<string, (input: any, call: CallContext) => Promise<unknown>> = {
       get: async ({ key }: { key: string }) => {
-        const sql = `SELECT ${COLUMNS} FROM ${DATA_TABLE} WHERE collection = $1 AND key = $2`;
-        const [row] = (await query(sql, [c.name, key])).rows;
+        const [row] = await (await store.db()).select(COLUMNS).from(data).where(here(key));
         return row ? saved(row) : null;
       },
-      list: async (listing: Listing) => (await query(...listSql(c, listing))).rows.map(saved),
+      list: async (listing: Listing) => {
+        const { where, order } = listClauses(c, listing);
+        const rows = await (
+          await store.db()
+        )
+          .select(COLUMNS)
+          .from(data)
+          .where(where)
+          .orderBy(...order)
+          .limit(listing.limit);
+        return rows.map(saved);
+      },
       save: async ({ record, ifAbsent, ifRecord }: SaveInput, call) => {
+        const db = await store.db();
+        const key = record[c.key] as string;
+        const by = updatedBy(call);
         if (ifRecord !== undefined) {
           // PostgreSQL locks and rechecks the predicate on a concurrent update. A lost-response
           // retry may recover its own identical result, but never overwrite another actor's state.
-          const retry = "record = $3::jsonb AND updated_by = $4";
-          const sql = `UPDATE ${DATA_TABLE} SET record = $3::jsonb,
-            updated_at = CASE WHEN ${retry} THEN updated_at ELSE now() END, updated_by = $4
-            WHERE collection = $1 AND key = $2 AND (record = $5::jsonb OR (${retry}))
-            RETURNING ${COLUMNS}`;
-          const params = [c.name, record[c.key], JSON.stringify(record), updatedBy(call), JSON.stringify(ifRecord)];
-          const row = (await query(sql, params)).rows[0];
+          const retry = and(eq(data.record, record), eq(data.updatedBy, by))!;
+          const [row] = await db
+            .update(data)
+            .set({ record, updatedAt: sql`CASE WHEN ${retry} THEN ${data.updatedAt} ELSE now() END`, updatedBy: by })
+            .where(and(here(key), or(eq(data.record, ifRecord), retry)))
+            .returning(COLUMNS);
           if (!row) throw recordChanged();
           return saved(row);
         }
         // The conflict row is locked by Postgres: never implement an insert-only claim as get + save.
         // An identical retry by the same actor keeps the first receipt and never overwrites later progress.
         const conflict = ifAbsent
-          ? `record = ${DATA_TABLE}.record WHERE ${DATA_TABLE}.record = EXCLUDED.record AND ${DATA_TABLE}.updated_by = EXCLUDED.updated_by`
-          : "record = EXCLUDED.record, updated_at = now(), updated_by = EXCLUDED.updated_by";
-        const sql =
-          `INSERT INTO ${DATA_TABLE} (collection, key, record, updated_by) VALUES ($1, $2, $3, $4) ` +
-          `ON CONFLICT (collection, key) DO UPDATE SET ${conflict} RETURNING ${COLUMNS}`;
-        const params = [c.name, record[c.key], JSON.stringify(record), updatedBy(call)];
-        const row = (await query(sql, params)).rows[0];
+          ? { set: { record: sql`${data.record}` }, setWhere: and(eq(data.record, record), eq(data.updatedBy, by)) }
+          : { set: { record, updatedAt: sql`now()`, updatedBy: by } };
+        const [row] = await db
+          .insert(data)
+          .values({ collection: c.name, key, record, updatedBy: by })
+          .onConflictDoUpdate({ target: [data.collection, data.key], ...conflict })
+          .returning(COLUMNS);
         if (!row) throw recordExists();
         return saved(row);
       },
       delete: async ({ key }: { key: string }) => {
-        const sql = `DELETE FROM ${DATA_TABLE} WHERE collection = $1 AND key = $2`;
-        return { deleted: ((await query(sql, [c.name, key])).rowCount ?? 0) > 0 };
+        const gone = await (await store.db()).delete(data).where(here(key)).returning({ key: data.key });
+        return { deleted: gone.length > 0 };
       },
     };
     return [c.name, ops];
@@ -369,40 +380,37 @@ const jsonTypesOf = (c: Collection): Record<string, JsonType> =>
 const SQL_TYPE = { string: "text", number: "numeric", boolean: "boolean" } as const;
 
 /**
- * The `list` query on Postgres, and its parameters. A field's value is compared as its declared
+ * The `list` query's filter and order on Postgres. A field's value is compared as its declared
  * type, text byte by byte (`COLLATE "C"`, whatever the database's collation, as the fake
  * compares it); a missing value, or one of another type, is SQL's null: it matches no filter and
  * sorts last.
  */
-function listSql(c: Collection, { where = {}, order, limit }: Listing): [string, unknown[]] {
+function listClauses(c: Collection, { where = {}, order }: Listing): { where: SQL | undefined; order: SQL[] } {
   const types = jsonTypesOf(c);
-  const params: unknown[] = [c.name];
-  const param = (value: unknown) => `$${params.push(value)}`;
   const field = (name: string) => {
     const type = types[name]!;
-    const json = `record -> ${param(name)}::text`;
-    const typed = `CASE WHEN jsonb_typeof(${json}) = '${type}' THEN (${json} #>> '{}')::${SQL_TYPE[type]} END`;
-    return type === "string" ? `(${typed}) COLLATE "C"` : typed;
+    const sqlType = sql.raw(SQL_TYPE[type]);
+    const json = sql`${data.record} -> ${name}::text`;
+    const typed = sql`CASE WHEN jsonb_typeof(${json}) = ${type} THEN (${json} #>> '{}')::${sqlType} END`;
+    return type === "string" ? sql`(${typed}) COLLATE "C"` : typed;
   };
-  const value = (name: string, v: unknown) => `${param(v)}::${SQL_TYPE[types[name]!]}`;
-  const conditions = ["collection = $1"];
+  const value = (name: string, v: unknown) => sql`${v}::${sql.raw(SQL_TYPE[types[name]!])}`;
+  const conditions: SQL[] = [eq(data.collection, c.name)];
   for (const [name, condition] of Object.entries(where)) {
     if (condition === undefined) continue;
-    if (!isObject(condition)) conditions.push(`${field(name)} = ${value(name, condition)}`);
+    if (!isObject(condition)) conditions.push(eq(field(name), value(name, condition)));
     else {
       for (const [op, v] of Object.entries(condition)) {
-        const sign = RANGE[op as keyof typeof RANGE];
-        if (v !== undefined) conditions.push(`${field(name)} ${sign} ${value(name, v)}`);
+        if (v !== undefined) conditions.push(RANGE[op as keyof typeof RANGE](field(name), value(name, v)));
       }
     }
   }
-  const byKey = `key COLLATE "C"`;
+  const byKey = sql`${data.key} COLLATE "C"`;
   const by =
     order === undefined
-      ? byKey
-      : `${field(order.replace(/^-/, ""))} ${order.startsWith("-") ? "DESC" : "ASC"} NULLS LAST, ${byKey}`;
-  const sql = `SELECT ${COLUMNS} FROM ${DATA_TABLE} WHERE ${conditions.join(" AND ")} ORDER BY ${by} LIMIT ${param(limit)}`;
-  return [sql, params];
+      ? [byKey]
+      : [sql`${field(order.replace(/^-/, ""))} ${sql.raw(order.startsWith("-") ? "DESC" : "ASC")} NULLS LAST`, byKey];
+  return { where: and(...conditions), order: by };
 }
 
 /** Orders two values of one type as Postgres does under `COLLATE "C"`: strings by their UTF-8 bytes. */

package/src/config.ts

@@ -35,7 +35,7 @@ import {
   collectionsFake,
   isCollection,
 } from "./collection.ts";
-import { lazyPool, type LazyPool } from "./db.ts";
+import { lazyDb, type LazyDb } from "./db.ts";
 import { isDirectory, kindDirs, SCENARIOS_DIR } from "./paths.ts";
 import { callerFile, type Use, type WorkflowDefinition } from "./define.ts";
 import { DRIFT_WORKFLOW, driftWorkflow } from "./drift.ts";
@@ -77,7 +77,7 @@ import { runtimeDriver, sandboxRuntimeDriver, sanoma } from "./runtime-ops.ts";
 import { tokenStore } from "./tokens.ts";
 import type { Steps } from "./steps.ts";
 import { checkSchedule } from "./schedules.ts";
-import { changeCodeOf, computeVersion } from "./version.ts";
+import { builtInCodeOf, computeVersion } from "./version.ts";
 import { type DeclaredResource, readPeople, type ResourceProblem } from "./resources.ts";
 import { checkWakeupOps, type DeclaredWakeup, type EnvWakeup, HOST_URL } from "./wakeup.ts";
 import { instanceUrl, WAKEUP_TOKEN } from "./wakeup-service.ts";
@@ -174,7 +174,7 @@ export interface Login {
 export interface ResolvedConfig {
   secrets: SecretConfiguration;
   connections: ConnectionRegistry;
-  connectionPool: LazyPool;
+  connectionStore: LazyDb;
   identityProvisioning: ReturnType<typeof configuredIdentityProvisioning>;
   /**
    * Scopes workflows, queues and versions in the system database: the `name` of
@@ -184,8 +184,11 @@ export interface ResolvedConfig {
   databaseUrl: string;
   /** The DBOS application version: `<appName>@<DBOS__APPVERSION, or a hash of the workflows>`. */
   version: string;
-  /** The hash of the built-in change workflow's own code (`changeCodeOf`), which each change run carries. */
-  changeCode: string;
+  /**
+   * Each built-in workflow's code hash (`builtInCodeOf`), by its name, which each of its runs carries:
+   * a worker on another version of the app takes over a run whose code is its own (`carryBuiltInRuns`).
+   */
+  builtInCodes: Readonly<Record<string, string>>;
   /** The DBOS queue runs are started on: `sanoma:<appName>`. */
   queueName: string;
   /** The config's `login`, checked (`checkLogin`); the built-in `login-code` workflow sends through its `sender`, from its `from`. */
@@ -206,6 +209,8 @@ export interface ResolvedConfig {
   driversByVendor: Map<string, Driver>;
   /** The makers of the fakes the connectors attach, by vendor id in `connectors` order: each sandbox run makes its own. */
   fakes: Map<string, () => Fake<any, any>>;
+  /** The preparations the connectors attach (`ConnectorSetup.prepare`), by vendor id in `connectors` order: the worker runs each at its start. */
+  preparations: ReadonlyMap<string, NonNullable<ConnectorSetup["prepare"]>>;
   /** The makers of the login halves the connectors attach, by vendor id in `connectors` order: the app makes one per public site that names it. */
   logins: ReadonlyMap<string, NonNullable<ConnectorSetup["login"]>>;
   /** The makers of the model halves the connectors attach, by vendor id in `connectors` order: the app makes each once for its model registry. */
@@ -235,11 +240,11 @@ export interface ResolvedConfig {
   /** The config's collections, by name: their operations are in `ops` and `drivers`, their fake in `fakes`, under `DATA_VENDOR`. */
   collections: ReadonlyMap<string, Collection>;
   /**
-   * The pool the collections' driver keeps their records through, when there are any: opened on
+   * The database the collections' driver keeps their records in, when there are any: opened on
    * its first call, and closed by whoever calls the driver (the worker, and the client, which
    * calls it directly, when each stops).
    */
-  dataStore?: LazyPool;
+  dataStore?: LazyDb;
   /**
    * By name; each checked against `ops` and `drivers`. The config's, the built-ins `drift`
    * (`DRIFT_WORKFLOW`) and `apply` (`APPLY_WORKFLOW`) when its connectors declare resource types,
@@ -347,15 +352,15 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
   const hostTyped = [...types.values()].some((type) => type.host);
   // Built-in: the collections' operations, `data.<name>.<op>`, on the config's database.
   const databaseUrl = resolveDatabaseUrl(config);
-  const connectionPool = lazyPool(databaseUrl, "connections");
+  const connectionStore = lazyDb(databaseUrl, "connections");
   const identityProvisioning = configuredIdentityProvisioning(config.identityProvisioning);
   const connections = new ConnectionRegistry(
     connectionCatalog(config.connectors, config.connections, config.connectionRules),
-    () => connectionPool.db(),
-    configuredSecrets(config.secrets ?? {}, () => connectionPool.db()),
+    () => connectionStore.db(),
+    configuredSecrets(config.secrets ?? {}, () => connectionStore.db()),
   );
   const declared = [...collections.values()];
-  const dataStore = declared.length ? lazyPool(databaseUrl, "data") : undefined;
+  const dataStore = declared.length ? lazyDb(databaseUrl, "data") : undefined;
   // Built-in, in every config: a change's operations, which the template's policy names
   // (`sanoma.change.merge`), the listing of apps a page may call (`sanoma.apps.list`), and what a
   // workflow asks the runtime to do for it (`sanoma`, runtime-ops.ts).
@@ -380,9 +385,9 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
       connectionDriver(connections),
       appsListDriver,
       runtimeDriver({
-        tokens: tokenStore(() => connectionPool.db()),
+        tokens: tokenStore(() => connectionStore.db()),
         identities: identityProvisioning.store,
-        inspect: inspectRun(() => connectionPool.db(), appName),
+        inspect: inspectRun(() => connectionStore.db(), appName),
       }),
       ...(hostTyped ? [sourceDriver] : []),
       ...(dataStore ? [collectionsDriver(declared, dataStore)] : []),
@@ -452,20 +457,25 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
   return {
     secrets: config.secrets ?? {},
     connections,
-    connectionPool,
+    connectionStore,
     identityProvisioning,
     appName,
     databaseUrl,
     // The built-in's code and operations are part of the app's version, like the config's own. A
     // bundle's code is not the source's: it carries the hashes of the root it was read from.
     version: bundled?.version ?? computeVersion({ appName, workflows: [...names.values()] }),
-    changeCode: bundled?.changeCode ?? changeCodeOf(names.get(CHANGE_WORKFLOW)!),
+    builtInCodes:
+      bundled?.builtInCodes ??
+      Object.fromEntries(
+        [...names.values()].filter((wf) => BUILT_IN_WORKFLOWS.has(wf.name)).map((wf) => [wf.name, builtInCodeOf(wf)]),
+      ),
     queueName: `sanoma:${appName}`,
     ...(login && { login }),
     ops,
     drivers,
     driversByVendor,
     fakes,
+    preparations: makersOf(config.connectors, "prepare"),
     logins: makersOf(config.connectors, "login"),
     models: makersOf(config.connectors, "model"),
     remotes,
@@ -626,8 +636,8 @@ function makersOf<K extends keyof ConnectorSetup>(
 export function readRootSnapshot(config: SanomaConfig, resolved: ResolvedConfig = resolveConfig(config)): RootSnapshot {
   const root = rootOf(config);
   if (root === undefined) throw new Error("A config with no root has no files to carry");
-  const { appName, version, changeCode } = resolved;
-  return { appName, version, changeCode, policies: readPolicyFiles(root), people: readPeople(root) };
+  const { appName, version, builtInCodes } = resolved;
+  return { appName, version, builtInCodes, policies: readPolicyFiles(root), people: readPeople(root) };
 }
 
 /**
@@ -762,9 +772,9 @@ async function readDeclared(
   const keep = new Set<string>();
   const leftOut = new Set<string>();
   if (root !== undefined) {
-    const { deriveHosts, hostFacts } = await import("./host-derive.ts");
+    const { deriveHosts, hostFacts, hostRootFacts } = await import("./host-derive.ts");
     const facts = await hostFacts({
-      root,
+      rootFacts: () => hostRootFacts(root),
       resources,
       types,
       drivers: options.drivers,