@olivervea/pipelines 2.0.0-beta.3 → 2.0.0-beta.31

package/src/builder/refs.ts

@@ -1,6 +1,7 @@
-import type { RefBrand } from "../core/brands.ts";
-import type { AnyStep, InferEntry } from "../core/steps.ts";
-import type { Primitive } from "../core/types.ts";
+import type { RefBrand } from '../core/brands.ts';
+import type { AnyStep, InferEntry } from '../core/steps.ts';
+import type { External } from '../core/external.ts';
+import type { Primitive } from '../core/types.ts';
 
 /**
  * A typed reference into the run pool — what mappers see. `symbol &` makes template strings, `+`
@@ -12,7 +13,11 @@ type RefFields<T> = [NonNullable<T>] extends [Primitive]
   ? {}
   : NonNullable<T> extends readonly unknown[]
     ? {}
-    : { readonly [K in keyof NonNullable<T> & string]-?: Ref<NonNullable<T>[K] | (undefined extends T ? undefined : never)> };
+    : {
+        readonly [K in keyof NonNullable<T> & string]-?: Ref<
+          NonNullable<T>[K] | (undefined extends T ? undefined : never)
+        >;
+      };
 
 /** What a mapper may put where a T is expected: a Ref<T>, a literal, or an object of those. */
 export type Mappable<T> =
@@ -23,4 +28,9 @@ export type Mappable<T> =
         ? never
         : { [K in keyof T]: Mappable<T[K]> } & { readonly [RefBrand]?: never }); // a Ref of another model can't sneak in structurally
 
-export type MapperResult<S extends AnyStep> = { [K in keyof S["input"]]: Mappable<InferEntry<S["input"][K]>> };
+/** What a mapper provides: every input except the external ones, which come from outside the run. */
+export type MapperResult<S extends AnyStep> = {
+  [K in keyof S['input'] as S['input'][K] extends External ? never : K]: Mappable<
+    InferEntry<S['input'][K]>
+  >;
+};

package/src/core/brands.ts

@@ -1,13 +1,14 @@
 // Runtime markers. `Symbol.for` (not `Symbol()`) so that two copies of the SDK in one process — e.g.
 // a dev runner and the user's installed package — still recognise each other's objects.
-export const STEP = Symbol.for("olve.pipelines.step");
-export const IMPLEMENTATION = Symbol.for("olve.pipelines.implementation");
-export const SKIPPED = Symbol.for("olve.pipelines.skipped");
-export const PIPELINE = Symbol.for("olve.pipelines.pipeline");
-export const REF_PATH = Symbol.for("olve.pipelines.refPath");
+export const STEP = Symbol.for('olve.pipelines.step');
+export const IMPLEMENTATION = Symbol.for('olve.pipelines.implementation');
+export const SKIPPED = Symbol.for('olve.pipelines.skipped');
+export const PIPELINE = Symbol.for('olve.pipelines.pipeline');
+export const REF_PATH = Symbol.for('olve.pipelines.refPath');
 
 // Type-only brands (never exist at runtime).
 export declare const ModelBrand: unique symbol;
 export declare const RefBrand: unique symbol;
 export declare const StepBrand: unique symbol;
 export declare const SkippedBrand: unique symbol;
+export declare const ImplementationBrand: unique symbol;

package/src/core/config-refs.ts

@@ -0,0 +1,103 @@
+// Config references inside library-step definitions: collected (the step receives their entries),
+// recorded in the manifest as `{"$config": "<entry>.<path>"}`, and resolved from ctx.config at run time.
+import {
+  configRefPath,
+  configRefTarget,
+  isConfigRef,
+  type AnyConfigEntry,
+  type ConfigRef,
+} from './config.ts';
+
+/** `T` with every config reference replaced by the type of its value. */
+export type Resolved<T> =
+  T extends ConfigRef<infer U>
+    ? U
+    : T extends readonly (infer E)[]
+      ? Resolved<E>[]
+      : T extends (...args: never[]) => unknown
+        ? T
+        : T extends object
+          ? { [K in keyof T]: Resolved<T[K]> }
+          : T;
+
+/** Manifest key of a recorded config reference. */
+export const CONFIG_REF_KEY = '$config';
+/** Beside it on a reference narrowed by a `select` case: the case's condition, `<ref>.<discriminator>=<value>`. */
+export const CONFIG_CASE_KEY = '$case';
+
+function isPlain(v: unknown): v is Record<string, unknown> {
+  if (typeof v !== 'object' || v === null || Array.isArray(v)) return false;
+  const proto = Object.getPrototypeOf(v);
+  return proto === Object.prototype || proto === null;
+}
+
+/** Every config entry referenced anywhere in `v` (plain objects and arrays only), by name. */
+export function configRefsIn(
+  v: unknown,
+  into = new Map<string, AnyConfigEntry>(),
+): AnyConfigEntry[] {
+  if (isConfigRef(v)) {
+    const { entry } = configRefTarget(v);
+    if (!into.has(entry.name)) into.set(entry.name, entry);
+  } else if (Array.isArray(v)) for (const x of v) configRefsIn(x, into);
+  else if (isPlain(v)) for (const x of Object.values(v)) configRefsIn(x, into);
+  return [...into.values()];
+}
+
+/** `v` as manifest JSON: references become `{"$config": "target.beta.host"}`; literal keys starting with `$` are reserved. */
+export function recordConfigRefs(v: unknown): unknown {
+  if (isConfigRef(v)) {
+    const { path, case: kase } = configRefTarget(v);
+    for (const seg of path)
+      if (seg === '' || seg.includes('.'))
+        throw new Error(
+          `config reference '${configRefPath(v)}': field names must not be empty or contain '.'`,
+        );
+    return {
+      [CONFIG_REF_KEY]: configRefPath(v),
+      ...(kase ? { [CONFIG_CASE_KEY]: kase } : {}),
+    };
+  }
+  if (Array.isArray(v)) return v.map(recordConfigRefs);
+  if (isPlain(v)) {
+    const out: Record<string, unknown> = {};
+    for (const [k, x] of Object.entries(v)) {
+      if (k.startsWith('$'))
+        throw new Error(`key '${k}': keys starting with '$' are reserved`);
+      out[k] = recordConfigRefs(x);
+    }
+    return out;
+  }
+  return v;
+}
+
+/** `v` with every reference replaced by its value in `config` (the step's parsed config entries). */
+export function resolveConfigRefs<T>(
+  v: T,
+  config: Readonly<Record<string, unknown>>,
+): Resolved<T> {
+  if (isConfigRef(v)) {
+    const { entry, path } = configRefTarget(v);
+    if (!Object.hasOwn(config, entry.name))
+      throw new Error(`config entry '${entry.name}' was not delivered to the step`);
+    let cur: unknown = config[entry.name];
+    for (const seg of path)
+      cur =
+        cur !== null && typeof cur === 'object'
+          ? (cur as Record<string, unknown>)[seg]
+          : undefined;
+    return cur as Resolved<T>;
+  }
+  if (Array.isArray(v)) return v.map((x) => resolveConfigRefs(x, config)) as Resolved<T>;
+  if (isPlain(v)) {
+    const out: Record<string, unknown> = {};
+    for (const [k, x] of Object.entries(v)) out[k] = resolveConfigRefs(x, config);
+    return out as Resolved<T>;
+  }
+  return v as Resolved<T>;
+}
+
+/** Literal `v` (no reference inside), so define-time checks can run on it. */
+export function isLiteral(v: unknown): boolean {
+  return configRefsIn(v).length === 0;
+}

package/src/core/config.ts

@@ -0,0 +1,224 @@
+import { z } from 'zod';
+import type { Secret } from './secrets.ts';
+
+const CONFIG_REF = Symbol.for('olve.configRef');
+declare const VALUE: unique symbol;
+
+interface RefTarget {
+  readonly entry: AnyConfigEntry;
+  readonly path: readonly string[];
+  /** Set on a reference narrowed by a `select` case: `<ref>.<discriminator>=<value>`. */
+  readonly case?: string;
+}
+
+/**
+ * A typed reference to a config value (`TARGET.beta.host`), accepted in a step's settings in place of a
+ * literal. It is recorded in the manifest as `{"$config": "target.beta.host"}` and resolved from the
+ * run's config when the step runs; the step receives the entry automatically.
+ */
+export interface ConfigRef<T = unknown> {
+  readonly [CONFIG_REF]: RefTarget;
+  readonly [VALUE]?: T;
+}
+
+/** A literal or a reference to a config value of the same type. */
+export type Configurable<T> = T | ConfigRef<T>;
+
+type Fields<T> = T extends readonly unknown[] ? never : T extends object ? T : never;
+type Children<T> = [Fields<NonNullable<T>>] extends [never]
+  ? {}
+  : {
+      readonly [K in keyof Fields<NonNullable<T>> & string]-?: ConfigNode<
+        Fields<NonNullable<T>>[K] | (undefined extends T ? undefined : never)
+      >;
+    };
+/** A reference to a config value, and to each of its fields. */
+export type ConfigNode<T> = ConfigRef<T> & Children<T>;
+
+/** Fields of an entry that are its own properties; reach config fields with these names through `$`. */
+export type ReservedConfigField = 'kind' | 'name' | 'description' | 'schema' | '$';
+
+/** A named, typed value stored on the instance (one YAML document per pipeline), declared by the steps that use it. */
+export interface ConfigEntryBase<
+  N extends string = string,
+  S extends z.ZodType = z.ZodType,
+> {
+  readonly kind: 'config';
+  readonly name: N;
+  readonly description: string;
+  readonly schema: S;
+}
+export type ConfigEntry<
+  N extends string = string,
+  S extends z.ZodType = z.ZodType,
+> = ConfigEntryBase<N, S> &
+  ConfigRef<z.output<S>> &
+  Omit<Children<z.output<S>>, ReservedConfigField> & {
+    readonly $: ConfigNode<z.output<S>>;
+  };
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type AnyConfigEntry = ConfigEntryBase<string, any>;
+
+/** Config names are YAML keys: camelCase identifiers. */
+export const CONFIG_NAME = /^[a-z][a-zA-Z0-9]*$/;
+
+const NOT_REFS = new Set<PropertyKey>([
+  'then',
+  'toJSON',
+  'constructor',
+  'valueOf',
+  'toString',
+  'asymmetricMatch',
+  '$$typeof',
+  'nodeType',
+]);
+
+function node(entry: AnyConfigEntry, path: readonly string[], kase?: string): unknown {
+  const target = Object.freeze({
+    [CONFIG_REF]: Object.freeze({ entry, path, ...(kase ? { case: kase } : {}) }),
+  });
+  return new Proxy(target, {
+    get(t, prop) {
+      if (prop === CONFIG_REF) return t[CONFIG_REF];
+      if (typeof prop === 'symbol' || NOT_REFS.has(prop)) return undefined;
+      return node(entry, [...path, prop], kase);
+    },
+  });
+}
+
+/** `ref` narrowed to the union member whose `discriminator` is `value` (a `select` case). */
+export function narrowConfigRef(
+  ref: ConfigRef,
+  discriminator: string,
+  value: string,
+): unknown {
+  const { entry, path } = ref[CONFIG_REF];
+  return node(
+    entry,
+    path,
+    `${[entry.name, ...path].join('.')}.${discriminator}=${value}`,
+  );
+}
+
+export function config<const N extends string, S extends z.ZodType>(
+  name: N,
+  description: string,
+  schema: S,
+): ConfigEntry<N, S> {
+  const ref = {
+    entry: undefined as unknown as AnyConfigEntry,
+    path: [] as readonly string[],
+  };
+  const target = Object.freeze({
+    kind: 'config' as const,
+    name,
+    description,
+    schema,
+    [CONFIG_REF]: ref,
+  });
+  const entry: unknown = new Proxy(target, {
+    get(t, prop) {
+      if (prop in t) return t[prop as keyof typeof t];
+      if (typeof prop === 'symbol' || NOT_REFS.has(prop)) return undefined;
+      if (prop === '$') return node(entry as AnyConfigEntry, []);
+      return node(entry as AnyConfigEntry, [prop]);
+    },
+  });
+  ref.entry = entry as AnyConfigEntry;
+  Object.freeze(ref);
+  return entry as ConfigEntry<N, S>;
+}
+
+export function isConfigEntry(v: unknown): v is AnyConfigEntry {
+  return (
+    typeof v === 'object' &&
+    v !== null &&
+    (v as { kind?: unknown }).kind === 'config' &&
+    (v as { schema?: unknown }).schema instanceof z.ZodType
+  );
+}
+
+export function isConfigRef(v: unknown): v is ConfigRef {
+  return (
+    typeof v === 'object' &&
+    v !== null &&
+    (v as Record<symbol, unknown>)[CONFIG_REF] !== undefined
+  );
+}
+
+/** The entry and field path a reference points at, and the `select` case it was narrowed by. */
+export function configRefTarget(ref: ConfigRef): {
+  entry: AnyConfigEntry;
+  path: readonly string[];
+  case?: string;
+} {
+  return ref[CONFIG_REF];
+}
+
+/** `target.beta.host` */
+export function configRefPath(ref: ConfigRef): string {
+  const { entry, path } = configRefTarget(ref);
+  return [entry.name, ...path].join('.');
+}
+
+export const SECRET_REF_KIND = 'secretRef';
+
+/** A config field that names a secret: `{ secret: NAME }` in YAML. The value never enters the config. */
+export function secretRef<const N extends string>(s: Secret<N>) {
+  return z
+    .strictObject({ secret: z.literal(s.name) })
+    .meta({ 'x-olve-kind': SECRET_REF_KIND });
+}
+
+/** `ctx.config`: exactly the entries a step declares, after parsing (defaults applied). */
+export type ConfigValues<C extends AnyConfigEntry> = {
+  readonly [E in C as E['name']]: z.output<E['schema']>;
+};
+
+// ─── select: branching on a config union ────────────────────────────────────────────────────
+
+type IsLiteralString<T> = [T] extends [string]
+  ? string extends T
+    ? false
+    : true
+  : false;
+/** The fields of union `U` whose type is a string literal in every member: its discriminator(s). */
+export type DiscriminatorOf<U> = {
+  [K in keyof U & string]-?: IsLiteralString<U[K]> extends true ? K : never;
+}[keyof U & string];
+/** The member of `U` whose discriminator `D` is `V`. */
+export type VariantOf<U, D extends string, V> = Extract<U, { readonly [K in D]: V }>;
+
+function unwrap(schema: z.ZodType): z.ZodType {
+  let s = schema as unknown as { _zod: { def: { innerType?: z.ZodType } } };
+  for (let i = 0; i < 16 && s._zod.def.innerType; i++) s = s._zod.def.innerType as never;
+  return s as unknown as z.ZodType;
+}
+
+/**
+ * The discriminated union a reference points at: its discriminator and, per value, nothing else.
+ * Undefined when the referenced value is not a `z.discriminatedUnion`.
+ */
+export function unionAt(
+  ref: ConfigRef,
+): { discriminator: string; values: string[] } | undefined {
+  const { entry, path } = configRefTarget(ref);
+  let s = unwrap(entry.schema as z.ZodType) as unknown as {
+    _zod: { def: Record<string, unknown> };
+  };
+  for (const seg of path) {
+    const shape = s._zod.def.shape as Record<string, z.ZodType> | undefined;
+    if (!shape || !(seg in shape)) return undefined;
+    s = unwrap(shape[seg]!) as never;
+  }
+  const def = s._zod.def as {
+    discriminator?: string;
+    options?: { _zod: { propValues?: Record<string, Set<unknown>> } }[];
+  };
+  if (typeof def.discriminator !== 'string' || !Array.isArray(def.options))
+    return undefined;
+  const values = def.options
+    .flatMap((o) => [...(o._zod.propValues?.[def.discriminator!] ?? [])])
+    .filter((v): v is string => typeof v === 'string');
+  return { discriminator: def.discriminator, values };
+}

package/src/core/external.ts

@@ -0,0 +1,86 @@
+import { isModel, type AnyModel } from './models.ts';
+
+/** Who or what may supply an external input's value. */
+export type ExternalSource =
+  | { readonly kind: 'person' }
+  | { readonly kind: 'callback' }
+  | {
+      readonly kind: 'poll';
+      readonly url: string;
+      readonly valuePath: string;
+      readonly intervalSeconds: number;
+    };
+
+/** A signed-in user answers, in the UI or with `pl input`. */
+export function person(): ExternalSource {
+  return Object.freeze({ kind: 'person' as const });
+}
+
+/**
+ * Another system answers by POSTing to this run's callback URL for the input, which an earlier step hands it
+ * with `ctx.inputUrl(step, input)`. The body is the value as JSON, or `{"$decline": "reason"}`.
+ */
+export function callback(): ExternalSource {
+  return Object.freeze({ kind: 'callback' as const });
+}
+
+export interface PollOptions {
+  /** GET while the step waits; `{run.id}` and `{run.number}` are filled in. */
+  url: string;
+  /** Dot path to the value in the JSON response; `.` is the whole body. */
+  valuePath: string;
+  /** Default 30; at least 10. */
+  intervalSeconds?: number;
+}
+
+/** The service polls `url` while the step waits and answers with the value at `valuePath` once it validates. */
+export function poll(options: PollOptions): ExternalSource {
+  return Object.freeze({
+    kind: 'poll' as const,
+    url: options.url,
+    valuePath: options.valuePath,
+    intervalSeconds: options.intervalSeconds ?? 30,
+  });
+}
+
+/**
+ * A step input whose value comes from outside the run. The step waits until it is answered (the value is
+ * validated against `model`), then runs with the value in its input like any other.
+ */
+export interface External<M extends AnyModel = AnyModel> {
+  readonly kind: 'external';
+  readonly model: M;
+  readonly title: string;
+  readonly description: string;
+  readonly from: readonly ExternalSource[];
+}
+
+export interface ExternalOptions {
+  /** What is asked, shown to whoever answers. */
+  title: string;
+  description?: string;
+  /** Who may answer (default: a person). */
+  from?: readonly ExternalSource[];
+}
+
+export function external<M extends AnyModel>(
+  model: M,
+  options: ExternalOptions,
+): External<M> {
+  return Object.freeze({
+    kind: 'external' as const,
+    model,
+    title: options.title,
+    description: options.description ?? '',
+    from: Object.freeze([...(options.from ?? [person()])]),
+  });
+}
+
+export function isExternal(v: unknown): v is External {
+  return (
+    typeof v === 'object' &&
+    v !== null &&
+    (v as { kind?: unknown }).kind === 'external' &&
+    isModel((v as { model?: unknown }).model)
+  );
+}

package/src/core/handlers.ts

@@ -1,7 +1,19 @@
-import { IMPLEMENTATION, SKIPPED, type SkippedBrand } from "./brands.ts";
-import type { Ctx } from "../ctx/types.ts";
-import type { AnyStep, LibraryStep, OutputValue, SecretsOf, StepInput } from "./steps.ts";
-import type { Awaitable } from "./types.ts";
+import {
+  IMPLEMENTATION,
+  SKIPPED,
+  type ImplementationBrand,
+  type SkippedBrand,
+} from './brands.ts';
+import type { StepCtx } from '../ctx/types.ts';
+import type {
+  AnyStep,
+  CellOf,
+  ConfigOf,
+  OutputValue,
+  SecretsOf,
+  StepInput,
+} from './steps.ts';
+import type { Awaitable } from './types.ts';
 
 /** Returned by `ctx.skip(reason)`: the step produced no output on purpose. */
 export interface Skipped {
@@ -12,33 +24,68 @@ export function skipped(reason: string): Skipped {
   return Object.freeze({ [SKIPPED]: true, reason }) as unknown as Skipped;
 }
 export function isSkipped(v: unknown): v is Skipped {
-  return typeof v === "object" && v !== null && (v as Record<symbol, unknown>)[SKIPPED] === true;
+  return (
+    typeof v === 'object' &&
+    v !== null &&
+    (v as Record<symbol, unknown>)[SKIPPED] === true
+  );
 }
 
 export type HandlerResult<S extends AnyStep> = OutputValue<S> | Skipped;
-export type Handler<S extends AnyStep> = (input: StepInput<S["input"]>, ctx: Ctx<SecretsOf<S>>) => Awaitable<HandlerResult<S>>;
+/** The ctx a handler of `S` gets. */
+export type CtxOf<S extends AnyStep> = StepCtx<SecretsOf<S>, ConfigOf<S>, CellOf<S>>;
+export type Handler<S extends AnyStep> = (
+  input: StepInput<S['input']>,
+  ctx: CtxOf<S>,
+) => Awaitable<HandlerResult<S>>;
 
-export interface Implementation<S extends AnyStep = AnyStep> {
+/** A step with its handler. It is itself a step: stage it like any value. */
+export type Implementation<S extends AnyStep = AnyStep> = S & {
+  readonly [ImplementationBrand]: true;
+  /** The step as defined. */
   readonly step: S;
   readonly handler: Handler<S>;
-}
+};
 
 /** Block-bodied/async returns get no excess-property check; this restores it for handler outputs. */
 type NoExtraFields<R, T> = R extends Skipped | void
   ? unknown
-  : { [K in Exclude<keyof R, keyof T>]: `❌ '${K & string}' is not a field of the output model` };
+  : {
+      [
+        K in Exclude<keyof R, keyof T>
+      ]: `❌ '${K & string}' is not a field of the output model`;
+    };
 
 /**
- * Implement a custom step. Input, ctx and return types come from the step definition alone.
- * Default-export the result from `handlers/<step-name>.ts`.
+ * Implement a step. Input, ctx and return types come from the step definition alone. Stage the
+ * result directly; a reusable step gives itself a `kind`.
  */
 export function implement<S extends AnyStep, R extends HandlerResult<S>>(
-  step: S & (S extends LibraryStep ? "❌ library steps already carry their implementation" : unknown),
-  handler: (input: StepInput<S["input"]>, ctx: Ctx<SecretsOf<S>>) => Awaitable<R & NoExtraFields<R, OutputValue<S>>>,
+  step: S &
+    (S extends { readonly [ImplementationBrand]: true }
+      ? '❌ this step already carries its implementation'
+      : unknown),
+  handler: (
+    input: StepInput<S['input']>,
+    ctx: CtxOf<S>,
+  ) => Awaitable<R & NoExtraFields<R, OutputValue<S>>>,
 ): Implementation<S> {
-  return Object.freeze({ [IMPLEMENTATION]: true, step, handler: handler as unknown as Handler<S> }) as Implementation<S>;
+  if (isImplementation(step))
+    throw new TypeError(
+      `step '${(step as AnyStep).name}' already carries its implementation`,
+    );
+  return Object.freeze({
+    ...step,
+    [IMPLEMENTATION]: true,
+    step,
+    handler: handler as unknown as Handler<S>,
+  }) as unknown as Implementation<S>;
 }
 
 export function isImplementation(v: unknown): v is Implementation {
-  return typeof v === "object" && v !== null && (v as Record<symbol, unknown>)[IMPLEMENTATION] === true;
+  return (
+    typeof v === 'object' &&
+    v !== null &&
+    (v as Record<symbol, unknown>)[IMPLEMENTATION] === true
+  );
 }