@sanoma-ai/workflows 0.0.0-stage → 0.1.0

package/src/describe.ts

@@ -0,0 +1,262 @@
+import type { z } from "zod";
+import { type LintProblem, readSource, type SourceFile } from "./ast.ts";
+import { resolveConfig, type SanomaConfig } from "./config.ts";
+import { type Builtin, jsonSchemaOf, opIdsOf, type Use } from "./define.ts";
+import { DRIFT_WORKFLOW } from "./drift.ts";
+import { lintWorkflow } from "./lint.ts";
+import { type Effect, VENDOR, type VendorInfo } from "./op.ts";
+import { type Outline, outlineWithSource } from "./outline.ts";
+import { allowAll, policyOpOf } from "./policy.ts";
+import { type ResourceFields, resourceTypesOf } from "./resource.ts";
+import { type DeclaredResource, readDataFiles, type ResourceProblem } from "./resources.ts";
+
+// `@sanoma-ai/workflows/describe`: what a UI renders from, apart from the main entry.
+export { outlineWorkflow, type Outline, type OutlineNode, type Span } from "./outline.ts";
+// The reader parses data files with oxc-parser too.
+export type { LintProblem } from "./ast.ts";
+export {
+  readDataFiles,
+  readResources,
+  type DataFiles,
+  type DeclaredResource,
+  type ResourceProblem,
+} from "./resources.ts";
+
+/** What a workflow is, read from its definition: enough to draw a start form and show what it may call. */
+export interface WorkflowEntry {
+  name: string;
+  title?: string;
+  trigger: "manual";
+  /**
+   * Set for a workflow the runtime adds itself, `drift`, rather than the config: a label. It
+   * starts like any other.
+   */
+  builtin?: true;
+  /** The input, as JSON Schema (draft 2020-12), from the zod schema. */
+  input: Record<string, unknown>;
+  /** Operation ids the workflow may call, in the order declared. */
+  ops: string[];
+  /** Built-ins the workflow may call. */
+  builtins: Builtin[];
+  /** The file that defined it, absolute. */
+  file: string;
+  /** What its `run` calls, in order, read from its file (`outlineWorkflow`), which the worker holds its runs to. */
+  outline: Outline;
+  /**
+   * What is wrong in its file, by position: what `lintWorkflow` finds (syntax, an import a
+   * workflow may not make, `instanceof` against the runtime's error classes, what the outline
+   * cannot draw in each `run`), read the one way the outline is, so a check sees what the
+   * worker refuses; and, at 1:1, an outline error that is the file's (it cannot be read, or holds
+   * no workflow of this name). A file defining several workflows gives each its problems. Empty
+   * for the built-in `drift`, which the package's tests hold to the lint.
+   */
+  problems: LintProblem[];
+  /**
+   * The text the outline's spans index into: the workflow's file, with `\n` line endings.
+   * Absent only when the outline is `{ error }`. Whole files: a server may keep it from clients.
+   */
+  source?: string;
+}
+
+/** What an operation is, read from its connector: enough to badge it and show its contract. */
+export interface OpEntry {
+  id: string;
+  vendor: string;
+  resource: string;
+  name: string;
+  effect: Effect;
+  idempotent: boolean;
+  description?: string;
+  /** How a scenario's steps name it: Cucumber expressions over its input's fields. */
+  phrases?: { given?: string; expect?: string };
+  /** What a call sends, as JSON Schema (`io: "input"`: fields with defaults are optional). */
+  input: Record<string, unknown>;
+  /** What a call returns once its schema has parsed the vendor's reply (`io: "output"`). */
+  output: Record<string, unknown>;
+}
+
+/** Who a vendor is, from its connector's `VendorInfo`: enough to name it, show its logo and link to its connector. */
+export interface VendorEntry {
+  /** The connector's `title`, else the vendor's id. */
+  title: string;
+  /** The logo as `data:image/svg+xml` URLs, for an `<img>`: `dark` is for dark backgrounds. */
+  logo?: { src: string; dark?: string };
+  /** The connector's npm package name. */
+  package?: string;
+  /** The connector package's `homepage` from its package.json: where its code and README are, as an https URL. */
+  homepage?: string;
+  /** Whether the vendor's connector attaches a fake (`withSetup`), for sandbox runs. */
+  fake: boolean;
+}
+
+/** A resource type a connector declares with `defineResource`. */
+export interface ResourceTypeEntry {
+  /** `<vendor>.<type>`, such as `github.repository`. */
+  id: string;
+  vendor: string;
+  type: string;
+  title: string;
+  /** How a declared resource's import id is made, such as `name` or `repository_id:pattern`. */
+  identity: string;
+  /** Its flagged fields, by dotted path. */
+  fields: ResourceFields;
+  /**
+   * Its state, as a read returns it, as JSON Schema (`io: "output"`), with `$id`
+   * `sanoma:resource-type/<id>`: the operations' contracts `$ref` it instead of repeating it.
+   */
+  schema: Record<string, unknown>;
+  /** Its operations' ids: `<id>.import` and `<id>.read`. */
+  ops: string[];
+}
+
+/**
+ * A plain-JSON description of what a config can do: its workflows, the operations they call,
+ * and whether a policy gates them. The app server builds it from the config and sends it to the
+ * browser, so the UI shows exactly what the worker enforces. Never written by hand.
+ */
+export interface ConfigDescription {
+  appName: string;
+  /** The application version runs started now are stamped with. */
+  version: string;
+  workflows: WorkflowEntry[];
+  ops: OpEntry[];
+  /** Every vendor the operations are from, by its id (an `OpEntry`'s `vendor`). */
+  vendors: Record<string, VendorEntry>;
+  /** The resource types the connectors declare, by `id`. */
+  resourceTypes: ResourceTypeEntry[];
+  /**
+   * The resources the data files under `resources/` declare, by file and then in file order,
+   * less any with a problem and any that names one.
+   */
+  resources: DeclaredResource[];
+  /** What is wrong in the data files (`readDataFiles`), each at its file, relative to the root, line and column. A workflow's are on its entry. */
+  problems: ResourceProblem[];
+  /** `defined` is false for `allowAll`. */
+  policy: { defined: boolean; version?: string };
+}
+
+/**
+ * Describes a config. Throws what `startWorker` would refuse (see `resolveConfig`), except missing
+ * credentials (`SanomaClient.credentials` gives their statuses) and what is wrong in a workflow's
+ * file, which the worker refuses when it reads the outline: that is data, on each workflow's
+ * entry, as the data files' problems are, `problems`, beside the resources read without any.
+ * Each workflow's file is read and parsed once, however many workflows it defines.
+ */
+export function describeConfig(config: SanomaConfig): ConfigDescription {
+  const resolved = resolveConfig(config);
+  const vendors: Record<string, VendorEntry> = {};
+  // A vendor whose operations are split over several connectors is named by the first; its
+  // resource types are every connector's.
+  for (const connector of config.connectors) {
+    const { id, info } = connector[VENDOR];
+    vendors[id] ??= vendorEntry(id, info, resolved.fakes.has(id));
+  }
+  const resources = resourceTypesOf(config.connectors);
+  // Each resource type's state is described once, on its entry, and referenced from its operations.
+  const refs = new Map<z.ZodType, string>([...resources].map(([id, r]) => [r.schema, resourceTypeUri(id)]));
+  const resourceTypes: ResourceTypeEntry[] = [...resources]
+    .map(([id, { vendor, type, title, identity, fields, schema }]) => ({
+      id,
+      vendor,
+      type,
+      title,
+      identity,
+      fields,
+      schema: { $id: resourceTypeUri(id), ...toJsonSchema(schema, `${id} state`, "output") },
+      ops: [`${id}.import`, `${id}.read`],
+    }))
+    .toSorted((a, b) => a.id.localeCompare(b.id));
+
+  const ops: OpEntry[] = [...resolved.ops.values()].map((op) => ({
+    ...policyOpOf(op),
+    idempotent: op.idempotent,
+    description: op.description,
+    ...(op.phrases && { phrases: op.phrases }),
+    input: toJsonSchema(op.input, `${op.id} input`, "input", refs),
+    output: toJsonSchema(op.output, `${op.id} output`, "output", refs),
+  }));
+  ops.sort((a, b) => a.id.localeCompare(b.id));
+
+  const read = memoised(readSource);
+  const lint = memoised((file: SourceFile) => lintWorkflow(file));
+  const workflows: WorkflowEntry[] = [...resolved.workflows.values()].map((wf) => {
+    const file = read(wf.file);
+    const { outline, source, found } = outlineWithSource(wf, file);
+    // What the lint does not report: the file unread, or no workflow of this name in it.
+    const own = "error" in outline && !found && (typeof file === "string" || !file.problems.length);
+    const problems =
+      wf.name === DRIFT_WORKFLOW
+        ? []
+        : [
+            ...(typeof file === "string" ? [] : lint(file)),
+            ...(own ? [{ line: 1, column: 1, message: outline.error }] : []),
+          ];
+    return {
+      name: wf.name,
+      title: wf.title,
+      trigger: wf.trigger,
+      ...(wf.name === DRIFT_WORKFLOW && { builtin: true as const }),
+      input: toJsonSchema(wf.input, `${wf.name} input`, "input"),
+      ops: opIdsOf(wf),
+      builtins: (wf.uses as Use[]).filter((u): u is Builtin => typeof u === "string"),
+      file: wf.file,
+      outline,
+      problems,
+      ...(source === undefined ? {} : { source }),
+    };
+  });
+  workflows.sort((a, b) => a.name.localeCompare(b.name));
+  const dataFiles = readDataFiles(resolved.root, config.connectors, resources);
+
+  return {
+    appName: resolved.appName,
+    version: resolved.version,
+    workflows,
+    ops,
+    vendors,
+    resourceTypes,
+    resources: dataFiles.resources,
+    problems: dataFiles.problems,
+    policy: {
+      defined: resolved.policy !== allowAll,
+      ...(resolved.policy.version === undefined ? {} : { version: resolved.policy.version }),
+    },
+  };
+}
+
+/** `fn`, called once per argument for as long as the returned function lives. */
+function memoised<A, R>(fn: (arg: A) => R): (arg: A) => R {
+  const seen = new Map<A, R>();
+  return (arg) => {
+    if (!seen.has(arg)) seen.set(arg, fn(arg));
+    return seen.get(arg)!;
+  };
+}
+
+const svgDataUrl = (svg: string) => `data:image/svg+xml,${encodeURIComponent(svg.trim())}`;
+
+const vendorEntry = (id: string, info: VendorInfo | undefined, fake: boolean): VendorEntry => ({
+  title: info?.title ?? id,
+  ...(info?.logo && {
+    logo: { src: svgDataUrl(info.logo.svg), ...(info.logo.dark && { dark: svgDataUrl(info.logo.dark) }) },
+  }),
+  ...(info?.package && { package: info.package }),
+  ...(info?.homepage && { homepage: info.homepage }),
+  fake,
+});
+
+/** Where a resource type's state schema is, for a `$ref`. */
+const resourceTypeUri = (id: string) => `sanoma:resource-type/${id}`;
+
+function toJsonSchema(
+  schema: z.ZodType,
+  what: string,
+  io: "input" | "output",
+  refs?: ReadonlyMap<unknown, string>,
+): Record<string, unknown> {
+  try {
+    return jsonSchemaOf(schema, io, refs);
+  } catch (e) {
+    throw new Error(`Cannot describe ${what} as JSON Schema: ${(e as Error).message}`, { cause: e });
+  }
+}

package/src/drift.ts

@@ -0,0 +1,149 @@
+import { z } from "zod";
+import { defineWorkflow, type WorkflowDefinition } from "./define.ts";
+import { isRunControlError } from "./errors.ts";
+import type { Op } from "./op.ts";
+import { type DriftField, diffDeclared, type Resource } from "./resource.ts";
+import type { DeclaredResource, ResourceProblem } from "./resources.ts";
+import { type DRIFT_WORKFLOW, errorMessage } from "./shared.ts";
+
+export type { DriftField } from "./resource.ts";
+export { DRIFT_WORKFLOW } from "./shared.ts";
+
+// The built-in `drift` workflow: for each resource the data files declare, what the vendor holds
+// now against what is declared. It is a workflow like any other, held to the same lint, so every
+// vendor call goes through `ctx`: the policy sees each (effect `read`) and the ledger records it.
+
+/**
+ * How a declared resource compares: `clean` when every declared field holds its declared value,
+ * `drifted` when one does not, `gone` when the vendor has no such object, and `error` when it
+ * could not be read (the vendor's error, a policy denial, or no driver for its type).
+ */
+export type DriftStatus = "clean" | "drifted" | "gone" | "error";
+
+/** One declared resource in a drift report. */
+export interface DriftResult {
+  /** The resource's id, `<file>#<export>`: `resources/identity/github.ts#website`. */
+  id: string;
+  /** Its type within the vendor, such as `repository`. */
+  type: string;
+  vendor: string;
+  /** Its import id. */
+  name: string;
+  status: DriftStatus;
+  /** The declared fields that differ; empty unless `drifted`. */
+  fields: DriftField[];
+  /** Why it could not be read, for `error`. */
+  error?: string;
+}
+
+/** The drift workflow's output. */
+export interface DriftReport {
+  /** Epoch milliseconds, by `ctx.now()`. */
+  startedAt: number;
+  /** Epoch milliseconds, by `ctx.now()`. */
+  finishedAt: number;
+  /** What was wrong in the data files when the run read them: their resources are not in `resources`. */
+  problems: ResourceProblem[];
+  /** Each resource the data files declared when the run read them, in their order. */
+  resources: DriftResult[];
+}
+
+/**
+ * The drift workflow's input: nothing. The run reads the data files itself, so a run says what
+ * they declared when it ran, and no caller can hand it other resources to check.
+ */
+export const DriftInput = z.object({}).strict();
+
+/**
+ * What a drift run records of the data files, as its first step: each resource without its
+ * write-only values, and the problems.
+ */
+export interface DriftDeclared {
+  resources: Pick<DeclaredResource, "id" | "vendor" | "type" | "name" | "desired" | "refs">[];
+  problems: ResourceProblem[];
+}
+
+type State = Record<string, unknown>;
+
+// A resource type's import, as `ctx` gives it: by vendor and type, both only known per run.
+type ImportOp = Op<string, string, "import", { id: string }, { id: string; gone: boolean; state?: State }>;
+
+/** What one import found: the object's state, that it is gone, or why it could not be read. */
+type Found = { state: State } | { gone: true } | { error: string };
+
+/**
+ * The drift workflow for a config: it `uses` the `import` of every resource type the connectors
+ * declare (`types`) that has a driver, so its version changes with them, not with the data
+ * files. `resolveConfig` adds it to a config whose connectors declare resource types, with
+ * `declared`, the step that reads the data files.
+ *
+ * A run reads the data files (`declared`, recorded as a step, so a replay checks the same list),
+ * then imports each resource by its import id (its `name`), in the files' order: one vendor call
+ * each. An import that says gone is `gone`. Else the state and the declared fields are compared
+ * by `diffDeclared`, a reference against the state of the resource it names, read in the same run
+ * (so GitHub's node id for a repository counts as its name). A failed call is that resource's
+ * `error`, and the run goes on to the next; a failure that ends the run (`isRunControlError`)
+ * ends it.
+ */
+export function driftWorkflow(
+  types: ReadonlyMap<string, Resource>,
+  ops: ReadonlyMap<string, Op>,
+  drivers: ReadonlyMap<string, unknown>,
+  declared: () => Promise<DriftDeclared>,
+): WorkflowDefinition<readonly ImportOp[], typeof DriftInput> {
+  const uses = [...types.keys()].toSorted().flatMap((type) => {
+    const op = ops.get(`${type}.import`);
+    return op && drivers.has(op.id) ? [op as ImportOp] : [];
+  });
+  // Why a resource of each type cannot be read, if it cannot.
+  const unreadable = new Map<string, string>();
+  for (const type of types.keys()) {
+    if (!uses.some((op) => op.id === `${type}.import`)) unreadable.set(type, `no driver for ${type}.import`);
+  }
+  return defineWorkflow({
+    // The literal, held to the one name: the outline finds a workflow by a string, or a const of its own file.
+    name: "drift" satisfies typeof DRIFT_WORKFLOW,
+    title: "Check resources for drift",
+    trigger: "manual",
+    input: DriftInput,
+    uses,
+    run: async (ctx) => {
+      const startedAt = await ctx.now();
+      const { resources, problems } = await declared();
+      // Every import first, so a reference compares against what its resource holds now.
+      const found = new Map<string, Found>();
+      for (const { id, vendor, type, name } of resources) {
+        const key = `${vendor}.${type}`;
+        const why = types.has(key) ? unreadable.get(key) : `the config's connectors declare no resource type ${key}`;
+        // An if/else, not a `continue`: the outline draws the import as the way past the error.
+        if (why !== undefined) found.set(id, { error: why });
+        else {
+          try {
+            const out = await ctx[vendor]![type]!.import({ id: name });
+            found.set(id, out.gone || !out.state ? { gone: true } : { state: out.state });
+          } catch (err) {
+            if (isRunControlError(err)) throw err;
+            found.set(id, { error: errorMessage(err) });
+          }
+        }
+      }
+      const results = resources.map(({ id, vendor, type, name, desired, refs }): DriftResult => {
+        const row = { id, type, vendor, name };
+        const now = found.get(id)!;
+        if ("error" in now) return { ...row, status: "error", fields: [], error: now.error };
+        if ("gone" in now) return { ...row, status: "gone", fields: [] };
+        const resource = types.get(`${vendor}.${type}`)!;
+        // Each reference, by where it is: the values the vendor may hold there for its resource.
+        const accept: Record<string, unknown[]> = {};
+        for (const [at, target] of Object.entries(refs)) {
+          const by = resource.fields.references?.[at.replaceAll(/\.\d+(?=\.|$)/g, "")]?.by ?? [];
+          const named = found.get(target);
+          if (named && "state" in named) accept[at] = by.map((field) => named.state[field]).filter((v) => v != null);
+        }
+        const fields = diffDeclared(resource, now.state, desired, accept);
+        return { ...row, status: fields.length ? "drifted" : "clean", fields };
+      });
+      return { startedAt, finishedAt: await ctx.now(), problems, resources: results } satisfies DriftReport;
+    },
+  });
+}

package/src/env.ts

@@ -0,0 +1,122 @@
+import type { z } from "zod";
+import type { Driver, DriverEnv } from "./op.ts";
+import { credentialReady } from "./shared.ts";
+
+// A driver's `env` against an environment: what each declared variable's status is, never its
+// value. The worker, the client, the app and each live call read the declarations through here.
+
+/** One environment variable a driver declares in `env`, and whether its value lets the driver run: never the value. */
+export interface CredentialStatus {
+  /** The variable's name, such as `GHOST_ADMIN_API_KEY`. */
+  name: string;
+  /** Its one-line manual, from its schema's `.describe()`. */
+  description?: string;
+  /** It may be unset. */
+  optional: boolean;
+  /** `missing` when unset or empty; `invalid` when its schema refuses the value. */
+  status: "set" | "missing" | "invalid";
+  /** Why it is `invalid`: the schema's first issue, never the value. */
+  problem?: string;
+  /**
+   * Where the value is from, when there is one: the process environment, which wins, or the
+   * credentials stored with `SanomaClient.setCredential`. Only `SanomaClient.credentials` says.
+   */
+  source?: "environment" | "stored";
+  /** Who set the stored value (`setCredential`'s `by`). */
+  setBy?: string;
+  /** When the stored value was set, as an ISO 8601 time. */
+  setAt?: string;
+}
+
+/**
+ * Each vendor's driver's `env` (a resolved config's `driversByVendor`), by vendor in the map's
+ * order (a vendor whose driver declares none has no entry), each variable in declared order and
+ * checked against the value `lookup` gives for its name: only the declared names are looked up,
+ * an empty value counts as unset, and a value is never kept. The one walk of the declarations:
+ * the worker, the client and its checks all read them through it.
+ */
+export function credentialsOf(
+  drivers: ReadonlyMap<string, Driver>,
+  lookup: (name: string) => string | undefined,
+): Map<string, CredentialStatus[]> {
+  const map = new Map<string, CredentialStatus[]>();
+  for (const [vendor, { env }] of drivers) if (env) map.set(vendor, statusesOf(env, lookup));
+  return map;
+}
+
+/** One driver's `env`, each variable checked against `lookup`'s value, as `credentialsOf` checks it. */
+export function statusesOf(env: DriverEnv, lookup: (name: string) => string | undefined): CredentialStatus[] {
+  return Object.entries(env.shape).map(([name, schema]) => credentialStatus(name, schema, lookup(name) || undefined));
+}
+
+/**
+ * Why the vendor's drivers cannot run on `statuses`, as a call to one fails (`startWorker` names
+ * the vendor alone, `unconfiguredSummary`): `<vendor> is not configured: <problems>; <hint>`, a
+ * missing variable (not optional) as `NAME (description)`, an invalid one as `NAME is invalid:
+ * <why>`, never a value. The hint says
+ * where to set them: in the environment when an invalid value is there (the environment wins
+ * over a stored value, so the connector page cannot replace it), else on the connector page.
+ * Undefined when the drivers can run.
+ */
+export function unconfigured(vendor: string, statuses: readonly CredentialStatus[]): string | undefined {
+  const unready = statuses.filter((c) => !credentialReady(c));
+  if (!unready.length) return undefined;
+  const problems = unready.map((c) =>
+    c.status === "invalid"
+      ? `${c.name} is invalid: ${c.problem}`
+      : `${c.name}${c.description ? ` (${c.description})` : ""}`,
+  );
+  const inEnvironment = unready.some((c) => c.status === "invalid" && process.env[c.name]);
+  const hint = inEnvironment ? "set in the environment" : "set it on the connector page";
+  return `${vendor} is not configured: ${problems.join(", ")}; ${hint}`;
+}
+
+/**
+ * The one line `startWorker` prints for the vendors whose drivers cannot run yet (`vendors`, those
+ * `unconfigured` has a message for) among the config's `total`: their names alone, since a call
+ * to one fails with its own message, and its connector page says the same. Undefined for none.
+ */
+export function unconfiguredSummary(vendors: readonly string[], total: number): string | undefined {
+  if (!vendors.length) return undefined;
+  return (
+    `${vendors.length} of ${total} connectors not configured yet: ${vendors.join(", ")}; ` +
+    "set their variables on the connector pages or in the environment"
+  );
+}
+
+function credentialStatus(
+  name: string,
+  schema: DriverEnv["shape"][string],
+  value: string | undefined,
+): CredentialStatus {
+  const description = descriptionOf(schema);
+  const parsed = value === undefined ? undefined : check(schema, value);
+  const problem = parsed?.problem;
+  return {
+    name,
+    ...(description !== undefined && { description }),
+    optional: check(schema, undefined).success,
+    status: !parsed ? "missing" : parsed.success ? "set" : "invalid",
+    ...(problem !== undefined && { problem }),
+  };
+}
+
+/**
+ * `schema.safeParse(value)`, with its first issue's message as the problem. A check that throws
+ * (zod lets a `.refine()` or `.transform()` throw) fails without its error, which may carry the value.
+ */
+function check(schema: z.ZodType, value: string | undefined): { success: boolean; problem?: string } {
+  try {
+    const { success, error } = schema.safeParse(value);
+    return { success, ...(error && { problem: error.issues[0]!.message }) };
+  } catch {
+    return { success: false, problem: "its check threw" };
+  }
+}
+
+/** A schema's `.describe()`, also through `.optional()`, `.default()` and `.prefault()` in any order. */
+function descriptionOf(schema: z.ZodType): string | undefined {
+  let s: z.ZodType | undefined = schema;
+  while (s && !s.description) s = "unwrap" in s ? (s as { unwrap(): z.ZodType }).unwrap() : undefined;
+  return s?.description;
+}