@sanoma-ai/workflows 0.9.1 → 0.10.0

package/src/run.ts

@@ -1,3 +1,4 @@
+import type { ConnectionRegistry } from "./connection.ts";
 import { AsyncLocalStorage } from "node:async_hooks";
 import type { Collection } from "./collection.ts";
 import type { DeadlineRegistry } from "./deadlines.ts";
@@ -38,6 +39,7 @@ export interface StartHold {
  * from a stopped worker never reads a newer worker's drivers or policy.
  */
 export interface WorkerState {
+  connections?: ConnectionRegistry;
   /** The config's `appName`, recorded on every ledger record. */
   app: string;
   /** The operations as the config's connectors declare them, by id. */

package/src/sandbox.ts

@@ -1,5 +1,6 @@
 import { DBOS } from "@dbos-inc/dbos-sdk";
 import { indexDrivers, sandboxDrivers } from "./config.ts";
+import { CALL_WORKFLOW, CallInput } from "./call-workflow.ts";
 import { opIdsOf } from "./define.ts";
 import { SanomaError } from "./errors.ts";
 import type { Fake, ResourceFake } from "./fake.ts";
@@ -88,7 +89,7 @@ export function seedResource(
  * run whose seeding is replayed from it has none, and fails, since a sandbox run does not
  * survive a worker restart.
  */
-export async function seedSandbox(run: Run, name: string): Promise<void> {
+export async function seedSandbox(run: Run, name: string, input: unknown): Promise<void> {
   const { state } = run;
   let copies: Map<string, DriverFn> | undefined;
   // The run's secrets, in memory as its fakes are: seeding and its calls share them.
@@ -114,7 +115,10 @@ export async function seedSandbox(run: Run, name: string): Promise<void> {
         [...sandboxDrivers(state.ops), ...Array.from(fakes.values(), (f) => f.driver)],
         state.ops,
       );
-      const uses = opIdsOf(state.workflows.get(run.workflow)!);
+      // The built-in call workflow selects exactly one operation. Other direct operations
+      // (including live connection mutations) need no fake unless this scenario calls them.
+      const uses =
+        run.workflow === CALL_WORKFLOW ? [CallInput.parse(input).op] : opIdsOf(state.workflows.get(run.workflow)!);
       const seeding = scenario.given.flatMap((g) => ("op" in g ? [g.op] : []));
       const missing = [...new Set([...uses, ...seeding])].filter((id) => !drivers.has(id));
       if (missing.length) {

package/src/scenarios.ts

@@ -5,6 +5,7 @@ import { type LedgerRecord, memoryLedger } from "./ledger.ts";
 import { type Check, check, drive, loadScenarios, type Scenario, type Scope } from "./scenario.ts";
 import { errorMessage } from "./shared.ts";
 import { startWorker } from "./worker.ts";
+import { memorySecretStore } from "./secret-store.ts";
 
 /*
  * `@sanoma-ai/workflows/scenarios`: runs a config's scenarios as sandbox runs, with no test
@@ -77,7 +78,12 @@ export async function runScenarios(
   config: SanomaConfig,
   { databaseUrl, startedBy, timeoutMs, log }: RunScenariosOptions,
 ): Promise<{ passed: string[]; failed: ScenarioFailure[] }> {
-  const shared: SanomaConfig = { ...config, databaseUrl, ledger: memoryLedger() };
+  const shared: SanomaConfig = {
+    ...config,
+    databaseUrl,
+    ledger: memoryLedger(),
+    secrets: { store: memorySecretStore(), namespace: "scenarios" },
+  };
   const resolved = resolveConfig(shared);
   const { scenarios, errors } = await loadScenarios(resolved);
   const failed: ScenarioFailure[] = errors.map(({ file, message }) => ({

package/src/schema.ts

@@ -104,6 +104,16 @@ function buildSchema(
       memberOf: [{ id: `effect:${op.effect}` }],
       appliesTo: applies(op.id === CHANGE_MERGE ? CHANGE : "Target", {
         ...input(op.input),
+        connection: optional(
+          record({
+            id: STRING,
+            vendor: STRING,
+            method: STRING,
+            account: STRING,
+            revision: STRING,
+            scopes: { type: "Set", element: STRING },
+          }),
+        ),
         ...(before && { before: optional(before) }),
       }),
     };

package/src/secret-store-check.ts

@@ -0,0 +1,93 @@
+import { SecretAccess, SecretStoreError, type SecretStore } from "./secret-store.ts";
+
+const assert = (condition: unknown) => {
+  if (!condition) throw new Error("Conformance failure");
+};
+
+const rejects = async (run: () => Promise<unknown>, code: string) => {
+  try {
+    await run();
+  } catch (error) {
+    assert(error instanceof SecretStoreError && error.code === code);
+    return;
+  }
+  throw new Error("Expected refusal");
+};
+
+export interface SecretStoreCheck {
+  name: string;
+  ok: boolean;
+}
+
+/**
+ * Destructive conformance checks on a fresh, disposable namespace. Never use a production
+ * namespace. The caller supplies a store scoped to test credentials and cleans up retained
+ * versions/namespaces according to the provider's retention rules. No result contains values.
+ */
+export async function checkSecretStore(store: SecretStore, namespace: string): Promise<SecretStoreCheck[]> {
+  const access = new SecretAccess({ store, namespace }, store);
+  const other = new SecretAccess({ store, namespace: `${namespace}-other` }, store);
+  const results: SecretStoreCheck[] = [];
+  const check = async (name: string, run: () => Promise<void>) => {
+    try {
+      await run();
+      results.push({ name, ok: true });
+    } catch {
+      results.push({ name, ok: false });
+    }
+  };
+  await check("validate and missing read", async () => {
+    await access.validate();
+    assert((await access.read("ABSENT")) === undefined);
+  });
+  await check("external ownership", async () => {
+    const external = new SecretAccess(
+      { store, namespace, bindings: { EXISTING: { path: "existing", management: "external" } } },
+      store,
+    );
+    await rejects(() => external.write("EXISTING", "value", { expectedVersion: null, by: "check" }), "read_only");
+    await rejects(() => external.delete("EXISTING", { expectedVersion: "v1", by: "check" }), "read_only");
+  });
+  if (!store.capabilities.writable) {
+    await check("read-only provider", () =>
+      rejects(() => access.write("VALUE", "value", { expectedVersion: null, by: "check" }), "read_only"),
+    );
+    return results;
+  }
+  if (store.capabilities.conditionalWrites === "none") {
+    await check("unsupported conditional writes", () =>
+      rejects(() => access.ensure("VALUE", () => "value"), "unsupported"),
+    );
+    return results;
+  }
+  await check("concurrent create and namespace separation", async () => {
+    const values = await Promise.all(Array.from({ length: 8 }, (_, i) => access.ensure("RACE", () => String(i))));
+    assert(new Set(values).size === 1);
+    assert((await other.get("RACE")) === undefined);
+  });
+  await check("replace rejects stale versions", async () => {
+    const one = await access.write("REPLACE", "one", { expectedVersion: null, by: "check" });
+    const two = await access.write("REPLACE", "two", { expectedVersion: one.version, by: "check" });
+    assert(one.version !== two.version);
+    await rejects(() => access.write("REPLACE", "stale", { expectedVersion: one.version, by: "check" }), "conflict");
+    assert((await access.get("REPLACE")) === "two");
+  });
+  await check("UTF-8 size rejection", () =>
+    rejects(
+      () =>
+        access.write("OVERSIZE", "x".repeat(store.capabilities.maxBytes + 1), { expectedVersion: null, by: "check" }),
+      "too_large",
+    ),
+  );
+  await check("deletion semantics", async () => {
+    const one = await access.write("DELETE", "one", { expectedVersion: null, by: "check" });
+    if (store.capabilities.deletion === "unsupported") {
+      await rejects(() => access.delete("DELETE", { expectedVersion: one.version, by: "check" }), "unsupported");
+      return;
+    }
+    await rejects(() => access.delete("DELETE", { expectedVersion: "incorrect", by: "check" }), "conflict");
+    await access.delete("DELETE", { expectedVersion: one.version, by: "check" });
+    assert((await access.read("DELETE")) === undefined);
+  });
+  return results;
+}

package/src/secret-store.ts

@@ -0,0 +1,314 @@
+import { deploymentKey } from "./deployment-key.ts";
+/** Opaque values belong only in trusted runtime code, never workflow history or descriptions. */
+export interface SecretValue {
+  value: string;
+  version: string;
+  updatedBy?: string;
+  updatedAt?: string;
+}
+
+/** A store instance already authenticates as one deployment. A namespace is not tenant authorization. */
+export interface SecretAddress {
+  namespace: string;
+  path: string;
+}
+
+export interface SecretStoreCapabilities {
+  writable: boolean;
+  /** UTF-8 bytes, including any serialized credential bundle. */
+  maxBytes: number;
+  /** `coordinated` covers cooperating Sanoma writers only; `none` cannot ensure or replace safely. */
+  conditionalWrites: "atomic" | "coordinated" | "none";
+  deletion: "destroy" | "soft-delete" | "unsupported";
+  changeDetection: "read" | "watch";
+}
+
+export type SecretStoreErrorCode =
+  | "invalid_configuration"
+  | "unavailable"
+  | "denied"
+  | "conflict"
+  | "read_only"
+  | "unsupported"
+  | "too_large"
+  | "invalid_value";
+
+/** Intentionally carries no provider message, cause, path or value. */
+export class SecretStoreError extends Error {
+  readonly code: SecretStoreErrorCode;
+  readonly retryable: boolean;
+  constructor(code: SecretStoreErrorCode) {
+    super(`Secret store: ${code}`);
+    this.name = "SecretStoreError";
+    this.code = code;
+    this.retryable = code === "unavailable";
+  }
+}
+
+export interface SecretStore {
+  /** Non-secret adapter identifier, e.g. `postgres` or `aws-secrets-manager`. */
+  readonly id: string;
+  readonly capabilities: Readonly<SecretStoreCapabilities>;
+  /** Check scoped access without listing an account or returning secret material. */
+  validate(namespace: string): Promise<void>;
+  read(address: SecretAddress): Promise<SecretValue | undefined>;
+  /** null means create only. A string means replace only that exact current version. */
+  write(
+    address: SecretAddress,
+    value: string,
+    options: { expectedVersion: string | null; by: string },
+  ): Promise<SecretValue>;
+  /** Delete only the exact version; report the retention behavior advertised in capabilities. */
+  delete(address: SecretAddress, options: { expectedVersion: string; by: string }): Promise<void>;
+}
+
+export type SecretBinding =
+  | { path: string; management: "sanoma" | "external"; env?: never; bootstrap?: never }
+  | { env: string; management?: "external"; path?: never; bootstrap?: never }
+  | { bootstrap: "SANOMA_SIGNING_KEY" | "SANOMA_SECRETS_KEY"; management?: "external"; env?: never; path?: never };
+
+export interface SecretConfiguration {
+  /** Omitted: built-in encrypted Postgres, supplied by the runtime. */
+  store?: SecretStore;
+  /** Defaults to `default`. Choose a distinct namespace and identity for each deployment. */
+  namespace?: string;
+  /** Names without a binding resolve to the same name in this namespace. */
+  bindings?: Readonly<Record<string, SecretBinding>>;
+}
+
+export interface SecretLocation {
+  store: string;
+  namespace: string;
+  path: string;
+  management: "sanoma" | "external";
+}
+
+const errorCodes = new Set<string>([
+  "invalid_configuration",
+  "unavailable",
+  "denied",
+  "conflict",
+  "read_only",
+  "unsupported",
+  "too_large",
+  "invalid_value",
+]);
+const storeError = (error: unknown): error is SecretStoreError =>
+  typeof error === "object" &&
+  error !== null &&
+  (error as SecretStoreError).name === "SecretStoreError" &&
+  errorCodes.has((error as SecretStoreError).code);
+
+/** Provider failures are sanitized even when an SDK throws a credential-bearing response. */
+export async function secretStoreCall<T>(run: () => Promise<T>): Promise<T> {
+  try {
+    return await run();
+  } catch (error) {
+    if (storeError(error)) throw new SecretStoreError(error.code);
+    throw new SecretStoreError("unavailable");
+  }
+}
+
+const namePattern = /^[A-Z][A-Z0-9_]*$/;
+const environmentPattern = /^[A-Za-z_][A-Za-z0-9_]*$/;
+const nonempty = (value: unknown): value is string =>
+  typeof value === "string" && value.trim().length > 0 && !value.includes("\0");
+
+/** Structural validation only: no secret reads, provider calls or environment access. */
+const invalid = () => {
+  throw new SecretStoreError("invalid_configuration");
+};
+export function checkSecretConfiguration(config: SecretConfiguration): void {
+  if (!config || typeof config !== "object" || Array.isArray(config)) invalid();
+  if (config.namespace !== undefined && !nonempty(config.namespace)) invalid();
+  if (config.store !== undefined) {
+    const s = config.store;
+    const c = s?.capabilities;
+    if (
+      !nonempty(s?.id) ||
+      !/^[a-z][a-z0-9-]{0,63}$/.test(s.id) ||
+      !c ||
+      typeof c.writable !== "boolean" ||
+      !Number.isSafeInteger(c.maxBytes) ||
+      c.maxBytes < 1 ||
+      !["atomic", "coordinated", "none"].includes(c.conditionalWrites) ||
+      !["destroy", "soft-delete", "unsupported"].includes(c.deletion) ||
+      !["read", "watch"].includes(c.changeDetection) ||
+      [s.validate, s.read, s.write, s.delete].some((fn) => typeof fn !== "function")
+    )
+      invalid();
+  }
+  if (
+    config.bindings !== undefined &&
+    (!config.bindings || typeof config.bindings !== "object" || Array.isArray(config.bindings))
+  )
+    invalid();
+  for (const [name, b] of Object.entries(config.bindings ?? {})) {
+    if (!namePattern.test(name) || !b || typeof b !== "object") invalid();
+    if (["SANOMA_SIGNING_KEY", "SANOMA_SECRETS_KEY"].includes(name) && !("env" in b) && !("bootstrap" in b)) invalid();
+    if ("bootstrap" in b) {
+      if (
+        !["SANOMA_SIGNING_KEY", "SANOMA_SECRETS_KEY"].includes(b.bootstrap!) ||
+        b.path !== undefined ||
+        b.env !== undefined ||
+        (b.management !== undefined && b.management !== "external")
+      )
+        invalid();
+    } else if ("env" in b) {
+      if (
+        typeof b.env !== "string" ||
+        !environmentPattern.test(b.env) ||
+        b.path !== undefined ||
+        (b.management !== undefined && b.management !== "external")
+      )
+        invalid();
+    } else if (!nonempty(b.path) || !["sanoma", "external"].includes(b.management)) invalid();
+  }
+}
+
+/** Enforces binding ownership before calling a provider; preserves the drivers' get/ensure interface. */
+export class SecretAccess {
+  readonly store: SecretStore;
+  readonly namespace: string;
+  private readonly bindings: Readonly<Record<string, SecretBinding>>;
+
+  private readonly env: (name: string) => string | undefined;
+
+  constructor(
+    config: SecretConfiguration,
+    defaultStore: SecretStore,
+    env: (name: string) => string | undefined = (name) => process.env[name],
+  ) {
+    this.env = env;
+    checkSecretConfiguration(config);
+    this.store = config.store ?? defaultStore;
+    checkSecretConfiguration({ store: this.store });
+    this.namespace = config.namespace ?? "default";
+    this.bindings = Object.fromEntries(
+      Object.entries(config.bindings ?? {}).map(([name, binding]) => [name, { ...binding }]),
+    );
+  }
+
+  location(name: string): SecretLocation {
+    if (!namePattern.test(name)) throw new SecretStoreError("invalid_configuration");
+    const binding = Object.hasOwn(this.bindings, name) ? this.bindings[name] : undefined;
+    if (binding?.bootstrap !== undefined)
+      return { store: "bootstrap", namespace: "", path: binding.bootstrap, management: "external" };
+    if (["SANOMA_SIGNING_KEY", "SANOMA_SECRETS_KEY"].includes(name) && !binding)
+      throw new SecretStoreError("invalid_configuration");
+    if (binding?.env !== undefined)
+      return { store: "environment", namespace: "", path: binding.env, management: "external" };
+    return {
+      store: this.store.id,
+      namespace: this.namespace,
+      path: binding?.path ?? name,
+      management: binding?.management ?? "sanoma",
+    };
+  }
+
+  async validate(): Promise<void> {
+    await secretStoreCall(() => this.store.validate(this.namespace));
+  }
+
+  async read(name: string): Promise<SecretValue | undefined> {
+    const location = this.location(name);
+    if (location.store === "bootstrap" && location.namespace === "") {
+      const value = deploymentKey(location.path as "SANOMA_SIGNING_KEY" | "SANOMA_SECRETS_KEY").toString("base64");
+      return { value, version: "bootstrap" };
+    }
+    if (location.store === "environment" && location.namespace === "") {
+      const value = this.env(location.path);
+      // No content-derived version (which would expose a hash of a low-entropy password).
+      return value === undefined ? undefined : { value, version: "environment" };
+    }
+    return secretStoreCall(() => this.store.read(location));
+  }
+
+  async get(name: string): Promise<string | undefined> {
+    return (await this.read(name))?.value;
+  }
+
+  async write(
+    name: string,
+    value: string,
+    options: { expectedVersion: string | null; by: string },
+  ): Promise<SecretValue> {
+    const location = this.writable(name);
+    if (Buffer.byteLength(value, "utf8") > this.store.capabilities.maxBytes) throw new SecretStoreError("too_large");
+    return secretStoreCall(() => this.store.write(location, value, options));
+  }
+
+  async delete(name: string, options: { expectedVersion: string; by: string }): Promise<void> {
+    const location = this.writable(name);
+    if (this.store.capabilities.deletion === "unsupported") throw new SecretStoreError("unsupported");
+    await secretStoreCall(() => this.store.delete(location, options));
+  }
+
+  async ensure(name: string, make: () => string): Promise<string> {
+    const stored = await this.read(name);
+    if (stored) return stored.value;
+    this.writable(name);
+    try {
+      return (await this.write(name, make(), { expectedVersion: null, by: "sanoma" })).value;
+    } catch (error) {
+      if (!storeError(error) || error.code !== "conflict") throw error;
+      const winner = await this.read(name);
+      if (winner) return winner.value;
+      throw error;
+    }
+  }
+
+  private writable(name: string): SecretLocation {
+    const location = this.location(name);
+    if (location.management === "external" || !this.store.capabilities.writable)
+      throw new SecretStoreError("read_only");
+    if (this.store.capabilities.conditionalWrites === "none") throw new SecretStoreError("unsupported");
+    return location;
+  }
+}
+
+/** Reference provider for tests. `atomic` is scoped to this shared in-process instance only. */
+const key = (a: SecretAddress) => JSON.stringify([a.namespace, a.path]);
+export function memorySecretStore(
+  options: { maxBytes?: number; writable?: boolean; deletion?: "destroy" | "soft-delete" } = {},
+): SecretStore {
+  const values = new Map<string, SecretValue>();
+  const retained = new Map<string, SecretValue>();
+  let generation = 0;
+  const capabilities: SecretStoreCapabilities = {
+    writable: options.writable ?? true,
+    maxBytes: options.maxBytes ?? 65_536,
+    conditionalWrites: "atomic",
+    deletion: options.deletion ?? "destroy",
+    changeDetection: "read",
+  };
+  const mutable = () => {
+    if (!capabilities.writable) throw new SecretStoreError("read_only");
+  };
+  return {
+    id: "memory",
+    capabilities: Object.freeze(capabilities),
+    async validate(namespace) {
+      if (!nonempty(namespace)) throw new SecretStoreError("invalid_configuration");
+    },
+    async read(address) {
+      const value = values.get(key(address));
+      return value && { ...value };
+    },
+    async write(address, value, { expectedVersion, by }) {
+      mutable();
+      if (Buffer.byteLength(value, "utf8") > capabilities.maxBytes) throw new SecretStoreError("too_large");
+      const current = values.get(key(address));
+      if ((current?.version ?? null) !== expectedVersion) throw new SecretStoreError("conflict");
+      const stored = { value, version: String(++generation), updatedBy: by, updatedAt: new Date().toISOString() };
+      values.set(key(address), stored);
+      return { ...stored };
+    },
+    async delete(address, { expectedVersion }) {
+      mutable();
+      if (values.get(key(address))?.version !== expectedVersion) throw new SecretStoreError("conflict");
+      if (capabilities.deletion === "soft-delete") retained.set(expectedVersion, values.get(key(address))!);
+      values.delete(key(address));
+    },
+  };
+}

package/src/shared.ts

@@ -5,7 +5,7 @@ import type { OpEntry, WorkflowEntry } from "./describe.ts";
 import type { LedgerRecord } from "./ledger.ts";
 import type { Effect } from "./op.ts";
 import type { PreviewRequest } from "./policy.ts";
-import type { DeclaredResource, ResourceProblem } from "./resources.ts";
+import type { DeclaredResource, ResourceProblem } from "./description.ts";
 import type { OutlineNode } from "./outline.ts";
 
 // Types only: resource types as a UI reads them. Erased from the bundle.
@@ -18,7 +18,7 @@ export type {
   ResourceFields,
   ResourceSpec,
 } from "./resource.ts";
-export type { DeclaredResource, ResourceProblem } from "./resources.ts";
+export type { At, DeclaredResource, ResourceProblem } from "./description.ts";
 export type { DriftField, DriftReport, DriftResult, DriftStatus } from "./drift.ts";
 export type { ApplyPlan, ApplyReport, ApplyResult, PlanAnswer, PlanStep } from "./apply.ts";
 export type { CredentialStatus } from "./env.ts";
@@ -31,6 +31,8 @@ export type { AvailableApp, AvailableApps } from "./apps-list.ts";
 
 // Sanoma's claims, data (`sanoma/claims` re-exports them).
 export * from "./claims.ts";
+// A site's `site.ts` declares itself with it (`sanoma/site` calls it, typing its icon).
+export { defineSite, type SiteDefinition } from "./definitions.ts";
 
 /**
  * What a `ctx.sleep` without a `{ name }` is called, in its outline node and its ledger record
@@ -490,17 +492,40 @@ export function flatten(nodes: readonly OutlineNode[]): OutlineNode[] {
   return nodes.flatMap((node) => [node, ...flatten(childLists(node).flat())]);
 }
 
-/** Where a data-file problem is, `file:line:column`; undefined for one that is the config's, with no file. */
-export const problemAt = (p: ResourceProblem): string | undefined =>
-  p.file === undefined ? undefined : `${p.file}:${p.line}:${p.column}`;
+/**
+ * Where a problem is, `file:line:column`, `file#export` or `file`; undefined for one that is the
+ * config's, with no file.
+ */
+export const problemAt = (p: ResourceProblem): string | undefined => {
+  if (p.file === undefined) return undefined;
+  if (p.line !== undefined) return `${p.file}:${p.line}:${p.column ?? 1}`;
+  return p.export === undefined ? p.file : `${p.file}#${p.export}`;
+};
+
+/**
+ * A problem as one line: `file:line:column: message`, `file#export: message` or `file: message`
+ * without a position, or the message alone without a file.
+ */
+export function problemLine(p: ResourceProblem): string {
+  const at = problemAt(p);
+  return at === undefined ? p.message : `${at}: ${p.message}`;
+}
 
 /**
- * A problem as one line: `file:line:column: message`, or `file: message` without a position, or
- * the message alone without a file.
+ * Where the export `name` (`default` for the default export) is in a file's text, as UTF-16
+ * offsets: from its `export` to the next `export`, `import`, `const` or comment at the start of a
+ * line, or the end, less trailing blank space. Read from the text alone, for highlighting it; undefined
+ * when no line starts it.
  */
-export function problemLine(p: { file?: string; line?: number; column?: number; message: string }): string {
-  if (p.file === undefined) return p.message;
-  return p.line === undefined ? `${p.file}: ${p.message}` : `${p.file}:${p.line}:${p.column ?? 1}: ${p.message}`;
+export function exportSpan(text: string, name: string): [number, number] | undefined {
+  const declared = name === "default" ? "default" : `const\\s+${name.replaceAll("$", "\\$")}`;
+  const start = new RegExp(`^export\\s+${declared}(?![\\w$])`, "m").exec(text);
+  if (!start) return undefined;
+  const after = start.index + start[0].length;
+  // A comment at the start of a line is the next declaration's.
+  const next = /^(?:(?:export|import|const)\b|\/[/*])/m.exec(text.slice(after));
+  const end = next ? after + next.index : text.length;
+  return [start.index, start.index + text.slice(start.index, end).trimEnd().length];
 }
 
 /** A warning as one line, as `problemLine` with `warning: ` before its message: what is worth fixing but not wrong. */