@sanoma-ai/workflows 0.11.3 → 0.11.5

package/src/apply.ts

@@ -10,12 +10,13 @@ import {
   importsOf,
   observeBy,
   observed,
+  refOf,
   type ResolveOp,
   type Revision,
   sourceUrl,
 } from "./drift.ts";
 import { isRunControlError } from "./errors.ts";
-import { type HostOutputs, wakeUrlOf } from "./host.ts";
+import { detaching, type HostOutputs, wakeUrlOf } from "./host.ts";
 import type { Binding, Bindings, Rebind } from "./lockfile.ts";
 import type { Op } from "./op.ts";
 import {
@@ -72,7 +73,13 @@ export type ApplyInput = z.infer<typeof ApplyInput>;
 export type PlanStep = { id: string; vendor: string; type: string; connection?: ConnectionRef } & (
   | { action: "create"; planned: Record<string, unknown>; unknown: string[] }
   | { action: "update" | "replace"; binding: string; fields: DriftField[]; replace: string[]; unknown?: string[] }
-  | { action: "bind"; binding: string; source: "found" | "given" }
+  | {
+      action: "bind";
+      binding: string;
+      source: "found" | "given";
+      /** The orphaned lockfile key that held the object found, whose binding the declaration takes over. */
+      from?: string;
+    }
   | { action: "gone"; binding: string; answer?: PlanAnswer }
   | { action: "orphan"; key: string; binding: string; answer?: PlanAnswer }
   | { action: "none"; binding: string }
@@ -113,6 +120,8 @@ export interface ApplyReport {
 export interface ApplyDeclared extends DriftDeclared {
   bindings: Bindings;
   orphans: (Binding & { key: string })[];
+  /** The declarations left out with a problem (a host that derives, which apply cannot give all it needs), which an input may still name. */
+  leftOut?: string[];
   rebinds: Rebind[];
 }
 
@@ -133,7 +142,13 @@ export interface ApplySteps {
    * another run made since this one read it stays, and commits it with `message`; a sandbox run
    * writes nothing.
    */
-  write(key: string, binding: Binding | undefined, message: string): Promise<{ written: boolean; revision?: string }>;
+  /** Writes `key`'s binding (none: removed), and removes `moved`, the key it moved from, in the same commit. */
+  write(
+    key: string,
+    binding: Binding | undefined,
+    message: string,
+    moved?: string,
+  ): Promise<{ written: boolean; revision?: string }>;
 }
 
 type State = Record<string, unknown>;
@@ -193,7 +208,7 @@ interface Known {
   /** The lockfile's bindings, as the run has changed them so far. */
   bindings: Bindings;
   /** The clone URL and commit a host wanted runs. */
-  hosts: ReadonlyMap<string, { source: string; revision: string }>;
+  hosts: ReadonlyMap<string, { source: string; revision: string; ref: string }>;
   /** Where each host's wakeup reaches it, as its state says (`wakeUrl`, else `url`). */
   urls: ReadonlyMap<string, string>;
   /**
@@ -232,11 +247,12 @@ function desiredOf(
     const named = holderOf(out, wakeup.inputs.host);
     if (url && named) named.holder[named.key] = url;
   }
-  // A host is given its source's clone URL and the commit its ref names now.
+  // A host is given its source's clone URL, its ref (the default branch when it names none) and the commit it names now.
   const run = hosts.get(r.id);
   if (host && run) {
     for (const [path, value] of [
       [host.inputs.source, run.source],
+      [host.inputs.ref, run.ref],
       [host.revision, run.revision],
     ] as const) {
       const place = holderOf(out, path);
@@ -263,7 +279,7 @@ const cannotChange = (type: Pick<Resource, "vendor" | "type" | "writable"> | und
 const unknown = (what: string, list: readonly string[], known: (id: string) => boolean): ResourceProblem[] =>
   list.filter((id) => !known(id)).map((id) => ({ message: `${what}: ${id} is no declared resource` }));
 
-/** What the input names that is neither declared nor an orphaned binding. */
+/** What the input names that is neither declared nor an orphaned binding (nor a declaration left out with a problem). */
 function inputProblems(
   input: ApplyInput,
   ids: ReadonlyMap<string, unknown>,
@@ -352,6 +368,32 @@ function fromPlan(
   return { problem: problemOf(r.id, `the vendor plans to ${p.action} it, which apply does not do here`) };
 }
 
+/**
+ * An object found by its identity that an orphaned binding holds is that binding moved (an export
+ * renamed, a deploy key apply declares now): its `bind` step takes the binding over (`from`), and
+ * the orphan is none. The orphaned keys taken over.
+ */
+function takeOver(
+  planned: PlanStep[],
+  orphans: ApplyDeclared["orphans"],
+  types: ReadonlyMap<string, Resource>,
+): Set<string> {
+  const taken = new Set<string>();
+  for (const step of planned) {
+    if (step.action !== "bind" || step.source !== "found") continue;
+    const type = `${step.vendor}.${step.type}`;
+    const held = orphans.find(
+      (o) =>
+        !taken.has(o.key) && o.id === step.binding && (o.type === type || types.get(type)?.formerly?.includes(o.type)),
+    );
+    if (held) {
+      step.from = held.key;
+      taken.add(held.key);
+    }
+  }
+  return taken;
+}
+
 /** Each orphaned binding the input wants, after the declarations, and the answers that cannot be done. */
 function orphansOf(
   orphans: ApplyDeclared["orphans"],
@@ -444,8 +486,9 @@ const exportOf = (key: string) => key.slice(key.indexOf("#") + 1);
 function messageOf(step: PlanStep, before: Binding | undefined, after: Binding | undefined): string {
   const key = keyOf(step);
   if (!after) return `Apply: unbind ${key} (${before?.type} ${before?.id})`;
-  const what = runOf(step) === "create" ? "create" : before ? "rebind" : "bind";
-  return `Apply: ${what} ${key} (${after.type} ${after.id})`;
+  const from = step.action === "bind" ? step.from : undefined;
+  const what = runOf(step) === "create" ? "create" : before || from ? "rebind" : "bind";
+  return `Apply: ${what} ${key} (${after.type} ${after.id})${from ? `, from ${from}` : ""}`;
 }
 
 /**
@@ -518,10 +561,11 @@ export function applyWorkflow(
       ...(hooks.length ? (["sleep"] as const) : []),
     ],
     run: async (ctx, input) => {
-      const { resources, problems: read, bindings: locked, orphans, rebinds } = await steps.declared();
+      const { resources, problems: read, bindings: locked, orphans, rebinds, leftOut } = await steps.declared();
       const wanted = (id: string) => input.only === undefined || input.only.includes(id);
       const declarations = new Map(resources.map((r) => [r.id, r]));
-      const problems = [...read, ...inputProblems(input, declarations, new Set(orphans.map((o) => o.key)))];
+      const nameable = new Set([...orphans.map((o) => o.key), ...(leftOut ?? [])]);
+      const problems = [...read, ...inputProblems(input, declarations, nameable)];
       // Observed as drift does, every import first, by the binding given before the lockfile's.
       const found = new Map<string, Found>();
       for (const resource of resources) {
@@ -566,19 +610,19 @@ export function applyWorkflow(
       // A host wanted runs the commit its ref names now, resolved before planning: a merge is a change
       // to it. One bound by the input needs none.
       const revisions = new Map<string, Revision>();
-      const hosts = new Map<string, { source: string; revision: string }>();
+      const hosts = new Map<string, { source: string; revision: string; ref: string }>();
       for (const { id, vendor, type, desired, refs } of resources.filter(
         (r) => wanted(r.id) && input.bind?.[r.id] === undefined,
       )) {
         const resource = types.get(`${vendor}.${type}`);
         const host = resource?.host;
         const source = host && sourceUrl(desired, refs[host.inputs.source], resource, found);
-        const ref = host && valueAt(desired, host.inputs.ref);
-        if (host && source !== undefined && typeof ref === "string" && ref) {
+        const ref = host && refOf(desired, host);
+        if (host && source !== undefined && ref !== undefined) {
           try {
-            const { revision } = await ctx.sanoma!.source!.resolve({ url: source, ref });
-            revisions.set(id, { revision });
-            hosts.set(id, { source, revision });
+            const resolved = await ctx.sanoma!.source!.resolve({ url: source, ...(ref ? { ref } : {}) });
+            revisions.set(id, { revision: resolved.revision });
+            hosts.set(id, { source, ...resolved });
           } catch (err) {
             if (isRunControlError(err)) throw err;
             revisions.set(id, { error: errorMessage(err) });
@@ -633,10 +677,19 @@ export function applyWorkflow(
         if (left.length < unfilled.length) awaited.add(resource.id);
         const unresolved = revisions.get(resource.id);
         // A host whose ref could not be resolved has no commit to run (compare says so only for one it found).
-        const next =
-          unresolved && "error" in unresolved && result.status !== "error"
+        const now = found.get(resource.id)!;
+        const host = types.get(`${resource.vendor}.${resource.type}`)?.host;
+        const attached = host && "state" in now ? detaching(host, fields, now.state) : [];
+        const next = attached.length
+          ? {
+              problem: problemOf(
+                resource.id,
+                `${attached.join(", ")} ${attached.length > 1 ? "are" : "is"} attached and no site declares a domain, so a change would detach ${attached.length > 1 ? "them" : "it"}: put \`domain\` on each site, \`defineSite({ public: true, domain })\` in sites/<site>/site.ts`,
+              ),
+            }
+          : unresolved && "error" in unresolved && result.status !== "error"
             ? { problem: problemOf(resource.id, unresolved.error) }
-            : decide(resource, result, found.get(resource.id)!, input, fields, types, usable);
+            : decide(resource, result, now, input, fields, types, usable);
         if (next.ask !== undefined) {
           try {
             const answered = await ctx[resource.vendor]![resource.type]!.plan(next.ask, {
@@ -653,7 +706,14 @@ export function applyWorkflow(
         if (next.step) add(next.step);
         if (next.problem) problems.push(next.problem);
       }
-      const orphaned = orphansOf(orphans, input, wanted, usable, types);
+      const taken = takeOver(planned, orphans, types);
+      const orphaned = orphansOf(
+        orphans.filter((o) => !taken.has(o.key)),
+        input,
+        wanted,
+        usable,
+        types,
+      );
       const plan: ApplyPlan = {
         problems: [...problems, ...orphaned.problems],
         rebinds,
@@ -734,13 +794,16 @@ export function applyWorkflow(
           } else if (todo === "delete") {
             await ctx[step.vendor]![step.type]!.delete({ id: binding }, { connection: step.connection });
             delete bindings[key];
-          } else if (todo === "bind")
+          } else if (todo === "bind") {
             bindings[key] = { type, id: binding, ...(step.connection && { connection: step.connection }) };
-          else if (todo === "forget") delete bindings[key];
+            if (step.action === "bind" && step.from) delete bindings[step.from];
+          } else if (todo === "forget") delete bindings[key];
           if (state && declared) keep(declared, state);
           const after = bindings[key];
-          if (after?.id !== before?.id) {
-            const wrote = await steps.write(key, after, messageOf(step, before, after));
+          // A new id, or the same object under its type's new name (`formerly`): written.
+          if (after?.id !== before?.id || (after && before && after.type !== before.type)) {
+            const moved = step.action === "bind" ? step.from : undefined;
+            const wrote = await steps.write(key, after, messageOf(step, before, after), moved);
             written ||= wrote.written;
             revision = wrote.revision ?? revision;
           }

package/src/config.ts

@@ -406,7 +406,14 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
   const connectors = connectorsOf(config);
   // The types the data files may declare: the vendors' and the people's, whose resources are left out.
   const declarable = resourceTypesOf(connectors);
-  const read = () => readDeclared(root, connectors, declarable, types, connections);
+  const read = (options: ReadOptions = {}) =>
+    readDeclared(root, connectors, declarable, types, connections, {
+      ...options,
+      drivers: driversByVendor,
+      builtInStore: !config.secrets?.store,
+      signingKeyBound: config.secrets?.bindings?.SANOMA_SIGNING_KEY !== undefined,
+      ...(isOp(config.login?.sender) && { sender: config.login.sender.vendor }),
+    });
   if (types.size) {
     names.set(
       DRIFT_WORKFLOW,
@@ -415,8 +422,13 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
     names.set(APPLY_WORKFLOW, applyWorkflow(types, ops, drivers, applySteps(appName, root, read)));
   }
   // Built-in, in every config: the life of a change, which deploys to the declared hosts after its merge.
-  const hosts = async () =>
-    (await read()).resources.filter((r) => types.get(`${r.vendor}.${r.type}`)?.host).map((r) => r.id);
+  // The declared hosts' ids, as the files say them: nothing derived, nothing minted.
+  const hosts = async () => {
+    const { loadDataFiles } = await import("./resources.ts");
+    return loadDataFiles(root, connectors, declarable)
+      .resources.filter((r) => types.get(`${r.vendor}.${r.type}`)?.host)
+      .map((r) => r.id);
+  };
   names.set(CHANGE_WORKFLOW, changeWorkflow(changeOps, changeSteps({ remotes, hosts })));
   const declaredObjects = () => readDeclaredObjects(root, connectors, declarable);
   // Built-in: a direct call the policy holds for an approval, made a run, when any operation may be called directly.
@@ -706,6 +718,11 @@ async function readDeclaredObjects(
   return { keys: new Map([...named, ...bound]), problems };
 }
 
+/** How a run reads the data files: `mint`, apply's, mints the keys a deriving host is given (`hostFacts`). */
+export interface ReadOptions {
+  mint?: boolean;
+}
+
 /**
  * The data files under `root` and the lockfile beside them, in one read, as a drift or apply run
  * records them: each resource with its binding, without its write-only values (a secret marker
@@ -718,12 +735,48 @@ async function readDeclared(
   declarable: ReadonlyMap<string, Resource>,
   types: ReadonlyMap<string, Resource>,
   connections: ConnectionRegistry,
+  options: ReadOptions & {
+    drivers: ReadonlyMap<string, Driver>;
+    builtInStore: boolean;
+    signingKeyBound: boolean;
+    sender?: string;
+  },
 ): Promise<Omit<ApplyDeclared, "rebinds">> {
   // Loaded when a run reads the files, not with the worker.
   const { loadDataFiles } = await import("./resources.ts");
   const { resources, bindings, orphans, problems } = loadDataFiles(root, connectors, declarable);
-  // Read inside the data-file durable step: every operation of the plan uses one pinned account.
   const run = currentRun.getStore();
+  // A host that derives is filled with what it is not told (`deriveHosts`), before it is checked
+  // or pinned: the deploy key it clones with among them. Minted only by a live apply.
+  const keep = new Set<string>();
+  const leftOut = new Set<string>();
+  if (root !== undefined) {
+    const { deriveHosts, hostFacts } = await import("./host-derive.ts");
+    const facts = await hostFacts({
+      root,
+      resources,
+      types,
+      drivers: options.drivers,
+      ...(run && { secrets: run.state.secrets }),
+      builtInStore: options.builtInStore,
+      signingKeyBound: options.signingKeyBound,
+      ...(options.sender !== undefined && { sender: options.sender }),
+      mint: !!options.mint && !!run && !run.sandbox,
+    });
+    if (facts) problems.push(...deriveHosts(resources, types, bindings, facts, { keep, leftOut }));
+  }
+  // A host apply cannot give all it needs is left out, with what names it (its wakeup), and what
+  // names that in turn: its problem says why, and nothing of it changes until it is fixed.
+  for (let grew = leftOut.size > 0; grew;) {
+    const before = leftOut.size;
+    for (const o of resources) if (Object.values(o.refs).some((id) => leftOut.has(id))) leftOut.add(o.id);
+    grew = leftOut.size > before;
+  }
+  for (const r of resources.filter((o) => leftOut.has(o.id))) resources.splice(resources.indexOf(r), 1);
+  // A host's deploy key apply declares, or cannot tell yet, is no orphan.
+  const declaredIds = new Set(resources.map((r) => r.id));
+  const kept = orphans.filter((o) => !declaredIds.has(o.key) && !keep.has(o.key));
+  // Read inside the data-file durable step: every operation of the plan uses one pinned account.
   if (run && !run.sandbox) {
     for (const resource of resources.filter((r) => types.has(`${r.vendor}.${r.type}`))) {
       try {
@@ -760,7 +813,8 @@ async function readDeclared(
         ...(connection && { connection }),
       })),
     bindings,
-    orphans,
+    orphans: kept,
+    ...(leftOut.size > 0 && { leftOut: [...leftOut] }),
     problems,
   };
 }

package/src/definitions.ts

@@ -23,6 +23,11 @@ export interface SiteDefinition {
   public?: boolean;
   login?: string;
   locales?: readonly [string, ...string[]];
+  /**
+   * A public site's domain (`www.acme.example`): the production host attaches it and serves the
+   * site at its root there; in a preview and locally the site stays at `/sites/<name>`.
+   */
+  domain?: string;
 }
 
 /** Declares a site in its folder's `site.ts`, its default export: `defineSite({ title: "Help desk" })`. */

package/src/describe.ts

@@ -41,7 +41,8 @@ import {
   problemLine,
   siteAtHostname,
 } from "./shared.ts";
-import { readSites, type SiteEntry } from "./sites.ts";
+import { readSites, type SiteEntry, siteDomains } from "./sites.ts";
+import { sourceKeyIds } from "./host.ts";
 
 // `@sanoma-ai/workflows/describe`: what a UI renders from, apart from the main entry.
 // What `describeConfig` answers, as zod schemas and the types they infer (`./description`).
@@ -85,7 +86,7 @@ export {
 // The apps under `apps/`, with what their `app.ts` declares.
 export { crossAppImports, readApps } from "./apps.ts";
 // What the sites call, read from their source: a workflow's Used by.
-export { readSites, siteContentPages, siteIcons, siteLocales, sitesUnder } from "./sites.ts";
+export { readSites, siteContentPages, siteDomains, siteIcons, siteLocales, sitesUnder } from "./sites.ts";
 // The generated SDK, `.sanoma/api.ts`: a site's `api` typed from this description.
 export { API_FILE, generateApi, writeApi, writeGenerated } from "./sdk.ts";
 // The Cedar schema, `.sanoma/schema.cedarschema`: what a company's policies are validated and decided against.
@@ -193,6 +194,7 @@ export function describeConfig(config: SanomaConfig): ConfigDescription {
     .toSorted((a, b) => a.id.localeCompare(b.id));
 
   const dataFiles = loadDataFiles(resolved.root, connectorsOf(config), resources);
+  const sourceKeys = sourceKeyIds(dataFiles.resources, resources);
   const declared = new Map(dataFiles.collections.map((d) => [d.name, d]));
   const collections: CollectionEntry[] = [...resolved.collections.values()]
     .map((c) => ({
@@ -289,12 +291,18 @@ export function describeConfig(config: SanomaConfig): ConfigDescription {
     collections,
     // An app's people are refused (`readPeople`), so the grid and the People page never show them.
     resources: dataFiles.resources.filter((r) => !inAppPeople(r)),
-    orphans: dataFiles.orphans,
+    // A host's deploy key, which apply declares, is no orphan.
+    orphans: dataFiles.orphans.filter((o) => !sourceKeys.has(o.key)),
     problems: [
       ...dataFiles.problems,
       ...appProblems,
       ...appClashes(apps, collections, sites),
-      ...domainProblems(dataFiles.resources, resources, sites),
+      ...domainProblems(
+        dataFiles.resources,
+        resources,
+        sites,
+        resolved.root === undefined ? {} : siteDomains(resolved.root),
+      ),
       ...loginProblems(resolved, dataFiles.resources),
     ],
     apps,
@@ -361,15 +369,17 @@ function appClashes(
 }
 
 /**
- * Each site a host's `domains` (its type's `host.inputs.domains`) names that is not a public
- * site, each domain it gives more than one site, and a domain that is the host's `hostname` too,
- * or another host's hostname or domain, at the host's declaration: the app serves only a public
- * site at a domain, one site at each, and itself at another, and one host holds each.
+ * Each site a host's `domains` (its type's `host.inputs.domains`; for a host that derives, the
+ * public sites' own, `declaredDomains`) names that is not a public site, each domain it gives more
+ * than one site, and a domain that is the host's `hostname` too, or another host's hostname or
+ * domain, at the host's declaration: the app serves only a public site at a domain, one site at
+ * each, and itself at another, and one host holds each.
  */
 function domainProblems(
   declared: readonly DeclaredResource[],
   types: ReadonlyMap<string, Resource>,
   sites: readonly SiteEntry[],
+  declaredDomains: Readonly<Record<string, string>>,
 ): ResourceProblem[] {
   const isPublic = new Set(sites.filter((site) => site.public).map((site) => site.name));
   const problems: ResourceProblem[] = [];
@@ -379,9 +389,12 @@ function domainProblems(
   const hold = (name: string, resource: DeclaredResource) =>
     held.set(name.toLowerCase(), (held.get(name.toLowerCase()) ?? new Set()).add(resource));
   for (const resource of declared) {
-    const inputs = types.get(`${resource.vendor}.${resource.type}`)?.host?.inputs;
+    const host = types.get(`${resource.vendor}.${resource.type}`)?.host;
+    const inputs = host?.inputs;
     const path = inputs?.domains;
-    const domains = (path === undefined ? {} : (valueAt(resource.desired, path) ?? {})) as Record<string, unknown>;
+    const domains = (
+      path === undefined ? {} : host?.derives ? declaredDomains : (valueAt(resource.desired, path) ?? {})
+    ) as Record<string, unknown>;
     const hostname = inputs?.hostname === undefined ? undefined : valueAt(resource.desired, inputs.hostname);
     const messages = Object.keys(domains)
       .filter((name) => !isPublic.has(name))

package/src/drift.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import { defineWorkflow, type WorkflowDefinition } from "./define.ts";
 import { isRunControlError } from "./errors.ts";
+import type { HostSpec } from "./host.ts";
 import type { Op } from "./op.ts";
 import { type DriftField, diffDeclared, fieldPathOf, type Resource, valueAt, withFrom } from "./resource.ts";
 import type { DeclaredResource, ResourceProblem } from "./resources.ts";
@@ -240,10 +241,11 @@ export function driftWorkflow(
         const resource = types.get(`${vendor}.${type}`);
         const host = resource?.host;
         const source = host && sourceUrl(desired, refs[host.inputs.source], resource, found);
-        const ref = host && valueAt(desired, host.inputs.ref);
-        if (host && "state" in found.get(id)! && source !== undefined && typeof ref === "string" && ref) {
+        const ref = host && refOf(desired, host);
+        if (host && "state" in found.get(id)! && source !== undefined && ref !== undefined) {
           try {
-            revisions.set(id, await ctx.sanoma!.source!.resolve({ url: source, ref }));
+            const { revision } = await ctx.sanoma!.source!.resolve({ url: source, ...(ref ? { ref } : {}) });
+            revisions.set(id, { revision });
           } catch (err) {
             if (isRunControlError(err)) throw err;
             revisions.set(id, { error: errorMessage(err) });
@@ -256,6 +258,17 @@ export function driftWorkflow(
   });
 }
 
+/**
+ * The ref a host runs: the one its data file gives, or `""` for a host that derives and names
+ * none (its repository's default branch, which `sanoma.source.resolve` tells); undefined when it
+ * has none to resolve.
+ */
+export function refOf(desired: State, host: HostSpec): string | undefined {
+  const ref = valueAt(desired, host.inputs.ref);
+  if (typeof ref === "string" && ref) return ref;
+  return host.derives && ref == null ? "" : undefined;
+}
+
 /**
  * A host's source: the clone URL its data file gives, or, when it names a declared repository
  * (`target`), the first of the reference's `by` fields that holds one in that repository's state,