@olivervea/pipelines 2.0.0-beta.14 → 2.0.0-beta.16

package/README.md

@@ -103,7 +103,7 @@ Rules:
 - **Trigger mappers** produce `z.input`, so fields with defaults may be left out. **Handlers, mapper refs and `when`** see `z.output`.
 - A **plain-schema input** has no model to match on, so the step needs a mapper.
 - **`optional(M)`** means "maybe skipped": `M` must still have an earlier producer.
-- **`external(M, { title, description?, from? })`** is answered from outside the run (`from`: `person()`, the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one.
+- **`external(M, { title, description?, from? })`** is answered from outside the run; `from` lists `callback()` (another system POSTs to the URL an earlier step gets from `ctx.inputUrl(step, input)`), `poll({ url, valuePath, intervalSeconds? })` and `person()` (the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one.
 
 ### Config
 
@@ -309,11 +309,12 @@ This section is everything the C# side needs in order to run the binary.
 | `OLVE_CASES` | run | Only for a step inside a `select`: a JSON array of the case conditions it sits in (`["target.prod.kind=ssh"]`); `run <step>` uses it to pick the variant of a name several cases use. |
 | `<SECRET_NAME>` | per step | Each declared secret as an env var of the **same name**: `secretKeyRef` key = name. Mount only the step's `secrets` (for poll triggers, the poll's `secrets`). Child processes inherit them. The mount is optional: an unset secret leaves the variable unset and `ctx.secrets.<NAME>` throws on access, so a handler can check `process.env.<NAME>` first and skip. |
 | `OLVE_RESULT_URL` | optional | Pre-signed **PUT** for `result.tar.gz` (see below). Unset means no upload. |
+| `OLVE_CALLBACK_URLS` | optional | JSON object mapping `<step>/<input>` → this run's callback URL of that external input (`ctx.inputUrl`). Set when the pipeline has `callback()` inputs and the instance has a public base URL. |
 | `OLVE_FILE_URLS` | optional | JSON object mapping file key → pre-signed **GET** URL, one entry per file ref in the input. The key is `<run>/<step>/<attempt>/<path>` (`path` exactly as in the ref, including a trailing `/` for directories). The URL points at the stored object (see the storage rule). |
 | `OLVE_FILES_BASE_URL` (+ `OLVE_FILES_TOKEN`) | optional | Fallback: `GET <base>/<run>/<step>/<attempt>/<object path>` with an optional `Authorization: Bearer <token>`. |
 | `OLVE_FILES_DIR` | dev | Local pool laid out as `<run>/<step>/<attempt>/files/<object path>`. |
 
-`OLVE_RESULT_URL`, `OLVE_FILE_URLS` and `OLVE_FILES_TOKEN` are removed from the environment of child processes. URLs are never logged past `?`.
+`OLVE_RESULT_URL`, `OLVE_FILE_URLS`, `OLVE_FILES_TOKEN` and `OLVE_CALLBACK_URLS` are removed from the environment of child processes. URLs are never logged past `?`.
 
 `ctx.files.fetch` looks for a file in this order:
 
@@ -384,7 +385,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
   - a nested object of mappings.
 
   An object whose only key is `ref` (with a string value) or `const` is a tag. Describe never emits a nested object with exactly that key set. Model-matched inputs are emitted already resolved (`{"ref": "build"}`).
-- **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from: [{kind: "person"}]}`. Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
+- **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}`. Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
 - **`inputTypes`** declares each input key: `{model, optional}`, or `{schema: {output, input?}}` for a plain schema.
 - **`handlerHash`** is the step's implementation identity. A step version can be computed from image + `inputTypes`/model versions + `output` model version + `secrets` + `configEntries` (and those entries' schemas) + `input` mappings + `handlerHash`. Stage placement, `when` and the step's `description` are not part of it.
   - Custom steps: `sha256(canonical {bundle, harness})`. `bundle` hashes the handler bundled on its own, with `pipeline.ts`, `models.ts` and `@olivervea/pipelines` external. It is tree-shaken and includes embedded scripts, so it changes only when that handler's code (or code it uses) changes.

package/manifest.schema.json

@@ -411,17 +411,64 @@
               "from": {
                 "description": "Who or what may answer.",
                 "items": {
-                  "additionalProperties": false,
-                  "properties": {
-                    "kind": {
-                      "const": "person",
-                      "type": "string"
+                  "oneOf": [
+                    {
+                      "additionalProperties": false,
+                      "properties": {
+                        "kind": {
+                          "const": "person",
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "kind"
+                      ],
+                      "type": "object"
+                    },
+                    {
+                      "additionalProperties": false,
+                      "properties": {
+                        "kind": {
+                          "const": "callback",
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "kind"
+                      ],
+                      "type": "object"
+                    },
+                    {
+                      "additionalProperties": false,
+                      "properties": {
+                        "intervalSeconds": {
+                          "maximum": 9007199254740991,
+                          "minimum": 10,
+                          "type": "integer"
+                        },
+                        "kind": {
+                          "const": "poll",
+                          "type": "string"
+                        },
+                        "url": {
+                          "description": "GET while waiting; {run.id} and {run.number} are filled in.",
+                          "type": "string"
+                        },
+                        "valuePath": {
+                          "description": "Dot path to the value; '.' is the whole body.",
+                          "minLength": 1,
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "kind",
+                        "url",
+                        "valuePath",
+                        "intervalSeconds"
+                      ],
+                      "type": "object"
                     }
-                  },
-                  "required": [
-                    "kind"
-                  ],
-                  "type": "object"
+                  ]
                 },
                 "minItems": 1,
                 "type": "array"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.14",
+  "version": "2.0.0-beta.16",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/core/external.ts

@@ -1,13 +1,38 @@
 import { isModel, type AnyModel } from "./models.ts";
 
 /** Who or what may supply an external input's value. */
-export type ExternalSource = { readonly kind: "person" };
+export type ExternalSource =
+  | { readonly kind: "person" }
+  | { readonly kind: "callback" }
+  | { readonly kind: "poll"; readonly url: string; readonly valuePath: string; readonly intervalSeconds: number };
 
 /** A signed-in user answers, in the UI or with `pl input`. */
 export function person(): ExternalSource {
   return Object.freeze({ kind: "person" as const });
 }
 
+/**
+ * Another system answers by POSTing to this run's callback URL for the input, which an earlier step hands it
+ * with `ctx.inputUrl(step, input)`. The body is the value as JSON, or `{"$decline": "reason"}`.
+ */
+export function callback(): ExternalSource {
+  return Object.freeze({ kind: "callback" as const });
+}
+
+export interface PollOptions {
+  /** GET while the step waits; `{run.id}` and `{run.number}` are filled in. */
+  url: string;
+  /** Dot path to the value in the JSON response; `.` is the whole body. */
+  valuePath: string;
+  /** Default 30; at least 10. */
+  intervalSeconds?: number;
+}
+
+/** The service polls `url` while the step waits and answers with the value at `valuePath` once it validates. */
+export function poll(options: PollOptions): ExternalSource {
+  return Object.freeze({ kind: "poll" as const, url: options.url, valuePath: options.valuePath, intervalSeconds: options.intervalSeconds ?? 30 });
+}
+
 /**
  * A step input whose value comes from outside the run. The step waits until it is answered (the value is
  * validated against `model`), then runs with the value in its input like any other.

package/src/ctx/index.ts

@@ -18,6 +18,8 @@ export interface CtxOptions {
   signal: AbortSignal;
   /** Where log lines go (default stderr). */
   logSink?: (line: string) => void;
+  /** `<step>/<input>` → callback URL (`ctx.inputUrl`). */
+  callbackUrls?: Readonly<Record<string, string>>;
 }
 
 export function formatLog(message: string, fields?: Record<string, unknown>): string {
@@ -118,5 +120,12 @@ export function createCtx(o: CtxOptions): Ctx {
     run: Object.freeze({ ...o.run }),
     tmpDir: o.paths.tmp,
     skip: skipped,
+    inputUrl: (step, input) => {
+      const name = typeof step === "string" ? step : step.name;
+      const url = o.callbackUrls?.[`${name}/${input}`];
+      if (!url)
+        throw new Error(`no callback URL for input '${input}' of step '${name}': it must list callback() in from, and the instance needs a public base URL`);
+      return url;
+    },
   };
 }

package/src/ctx/types.ts

@@ -113,4 +113,9 @@ export interface Ctx<Sec extends string = string, Cfg extends AnyConfigEntry = n
   readonly tmpDir: string;
   /** Runtime escape hatch: produce no output. Prefer `when` blocks. */
   skip(reason: string): Skipped;
+  /**
+   * This run's callback URL for an external input with `from: [callback()]` (of any step of the pipeline):
+   * hand it to the system that will answer. POST the value as JSON, or `{"$decline": "reason"}`.
+   */
+  inputUrl(step: string | { readonly name: string }, input: string): string;
 }

package/src/describe/describe.ts

@@ -197,11 +197,20 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
       if (failureHandler) problems.push(`${at}: failure handlers run unattended and cannot wait for input ('${key}')`);
       if (typeof e.title !== "string" || e.title.trim() === "") problems.push(`${at} external input '${key}': title must not be empty`);
       if (e.from.length === 0) problems.push(`${at} external input '${key}': from must list at least one source`);
+      if (new Set(e.from.map((f) => f.kind)).size !== e.from.length) problems.push(`${at} external input '${key}': each source kind may be listed once`);
+      if (e.from.some((f) => f.kind === "callback") && !/^[A-Za-z0-9_-]+$/.test(key))
+        problems.push(`${at} external input '${key}': a callback input's key must be letters, digits, '_' or '-' (it is part of the URL)`);
+      for (const f of e.from) {
+        if (f.kind !== "poll") continue;
+        if (!/^https?:\/\//.test(f.url)) problems.push(`${at} external input '${key}': poll url must be an absolute http(s) URL`);
+        if (!Number.isInteger(f.intervalSeconds) || f.intervalSeconds < 10) problems.push(`${at} external input '${key}': poll intervalSeconds must be an integer of at least 10`);
+        if (f.valuePath.trim() === "") problems.push(`${at} external input '${key}': poll valuePath must not be empty`);
+      }
       externalInputs[key] = {
         model: useModel(e.model, `${at} external input '${key}'`),
         title: e.title,
         description: e.description,
-        from: e.from.map((f) => ({ kind: f.kind })),
+        from: e.from.map((f) => ({ ...f })),
       };
     }
     const entries = allEntries.filter(([, e]) => !isExternal(e));

package/src/describe/manifest-schema.ts

@@ -79,7 +79,21 @@ export const StepDescriptor = z
           model: modelName.describe("The value is validated against this model's input view."),
           title: z.string().min(1).describe("What is asked, shown to whoever answers."),
           description: z.string(),
-          from: z.array(z.strictObject({ kind: z.literal("person") })).min(1).describe("Who or what may answer."),
+          from: z
+            .array(
+              z.discriminatedUnion("kind", [
+                z.strictObject({ kind: z.literal("person") }),
+                z.strictObject({ kind: z.literal("callback") }),
+                z.strictObject({
+                  kind: z.literal("poll"),
+                  url: z.string().describe("GET while waiting; {run.id} and {run.number} are filled in."),
+                  valuePath: z.string().min(1).describe("Dot path to the value; '.' is the whole body."),
+                  intervalSeconds: z.int().min(10),
+                }),
+              ]),
+            )
+            .min(1)
+            .describe("Who or what may answer."),
         }),
       )
       .optional()

package/src/harness/env.ts

@@ -13,6 +13,8 @@ export const ENV = {
   resultUrl: "OLVE_RESULT_URL",
   /** JSON object: file key (`<run>/<step>/<attempt>/<path>`) → pre-signed GET URL. Optional. */
   fileUrls: "OLVE_FILE_URLS",
+  /** JSON object: `<step>/<input>` → callback URL of that external input in this run (`ctx.inputUrl`). Optional. */
+  callbackUrls: "OLVE_CALLBACK_URLS",
   /** Base URL for file downloads: GET `<base>/<run>/<step>/<attempt>/<object path>`. Optional. */
   filesBaseUrl: "OLVE_FILES_BASE_URL",
   /** Bearer token sent with OLVE_FILES_BASE_URL requests. Optional. */
@@ -22,7 +24,19 @@ export const ENV = {
 } as const;
 
 /** Env vars never passed to child processes (they carry credentials for the harness only). */
-export const PRIVATE_ENV: readonly string[] = [ENV.resultUrl, ENV.fileUrls, ENV.filesToken];
+export const PRIVATE_ENV: readonly string[] = [ENV.resultUrl, ENV.fileUrls, ENV.filesToken, ENV.callbackUrls];
+
+/** `OLVE_CALLBACK_URLS`, parsed; empty when unset or malformed. */
+export function callbackUrlsFromEnv(): Record<string, string> {
+  const raw = process.env[ENV.callbackUrls];
+  if (!raw) return {};
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    return parsed && typeof parsed === "object" ? (parsed as Record<string, string>) : {};
+  } catch {
+    return {};
+  }
+}
 
 export const olveDir = () => process.env[ENV.dir] || "/olve";
 export const paths = () => {

package/src/harness/main.ts

@@ -12,7 +12,7 @@ import { pipelineData, type PipelineData, type StageNode } from "../builder/pipe
 import { createCtx, formatLog } from "../ctx/index.ts";
 import { DescribeError, describePipeline, manifestText } from "../describe/describe.ts";
 import type { HarnessInfo } from "../util/handler-hash.ts";
-import { ENV, paths, runInfoFromEnv } from "./env.ts";
+import { callbackUrlsFromEnv, ENV, paths, runInfoFromEnv } from "./env.ts";
 import { EXIT } from "./exit.ts";
 import { executeStep, type Envelope, type Execution, type StepFunction } from "./execute.ts";
 import { MANIFEST_BEGIN, MANIFEST_END, packResult, removeStale, sentinelBlock, uploadFile, writeEnvelope } from "./result.ts";
@@ -183,7 +183,7 @@ async function runMode(cfg: HarnessConfig, name: string | undefined, io: Harness
       exec = { envelope: { status: "failed", error: { kind: "input", message: raw.message } }, exitCode: EXIT.inputInvalid };
     } else {
       const run = runInfoFromEnv(step.name);
-      const ctx = createCtx({ run, secrets: step.secrets, paths: { root: p.root, workFiles: p.files, tmp: p.tmp }, signal: cancel.signal, logSink: io.stderr });
+      const ctx = createCtx({ run, secrets: step.secrets, paths: { root: p.root, workFiles: p.files, tmp: p.tmp }, signal: cancel.signal, logSink: io.stderr, callbackUrls: callbackUrlsFromEnv() });
       exec = await executeStep({ step, fn, rawInput: raw.value, ctx, workFiles: p.files, log: io.stderr });
     }
     return await finish(exec, io);

package/src/index.ts

@@ -6,7 +6,7 @@ export { secret, type Secret, type DeclaredSecrets } from "./core/secrets.ts";
 export { config, secretRef, isConfigRef, type ConfigEntry, type AnyConfigEntry, type ConfigValues, type ConfigRef, type ConfigNode, type Configurable, type DiscriminatorOf, type VariantOf } from "./core/config.ts";
 export { hmacSha256, githubSignature, bearer, type WebhookVerify, type WebhookEvent } from "./core/webhooks.ts";
 export { resolveConfigRefs, isLiteral, type Resolved } from "./core/config-refs.ts";
-export { external, person, type External, type ExternalOptions, type ExternalSource } from "./core/external.ts";
+export { callback, external, person, poll, type External, type ExternalOptions, type ExternalSource, type PollOptions } from "./core/external.ts";
 export { step, type Step, type AnyStep, type StepDef, type StepInput, type StepKind, type ConfigOf, type InputEntry, type InputSpec, type LiteralName, type OutputValue } from "./core/steps.ts";
 export type { Simplify } from "./core/types.ts";
 export { implement, type Implementation, type Handler, type HandlerResult, type Skipped } from "./core/handlers.ts";

package/src/library/models.ts

@@ -64,7 +64,7 @@ export const FailureContext = model(
       address: z.string(),
       /** The run's derived version, `yyyy.MM.dd.<run>`. */
       versionStamp: z.string(),
-      trigger: z.enum(["push", "poll", "manual"]),
+      trigger: z.enum(["push", "poll", "manual", "webhook"]),
       /** The run (`GET …/api/pipelines/<id>/runs/<n>`). */
       url: z.string(),
     }),

package/src/testing/index.ts

@@ -23,6 +23,8 @@ export interface TestHandlerOptions<S extends AnyStep> {
   dir?: string;
   /** Abort signal handed to ctx.signal. */
   signal?: AbortSignal;
+  /** `<step>/<input>` → URL returned by `ctx.inputUrl`. */
+  callbackUrls?: Record<string, string>;
 }
 
 export type TestResult<S extends AnyStep> = {
@@ -62,6 +64,7 @@ export async function testHandler<S extends AnyStep>(
     paths: { root: dir, workFiles: filesDir, tmp: join(dir, "tmp") },
     signal: opts.signal ?? new AbortController().signal,
     logSink: (l) => logs.push(l),
+    callbackUrls: opts.callbackUrls,
   });
   const ctx = { ...real, ...opts.ctx } as Ctx;
   // Round-trip through JSON like input.json does (drops undefined, rejects non-JSON values early).