@olivervea/pipelines 2.0.0-beta.32 → 2.0.0-beta.34

package/README.md

@@ -97,7 +97,7 @@ Rules:
 - A **plain-schema input** has no model to match on, so the step needs a mapper.
 - **`optional(M)`** means "maybe skipped": `M` must still have an earlier producer.
 - **`cells(M)`** reads every cell of a matrix step whose output is `M` (see "Matrix steps").
-- **`external(M, { title, description?, from? })`** is answered from outside the run; `from` lists `callback()` (another system POSTs to the URL an earlier step gets from `ctx.inputUrl(step, input)`), `poll({ url, valuePath, intervalSeconds? })` and `person()` (the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one.
+- **`external(M, { title, description?, from? })`** is answered from outside the run; `from` lists `callback()` (another system POSTs to the URL an earlier step gets from `ctx.inputUrl(step, input)`), `poll({ url, valuePath, intervalSeconds? })` and `person()` (the default). It needs no producer and no mapper entry; the step waits until it is answered, then the value (validated against `M`) is in its input like any other. Failure handlers can't have one. `link: { input, path?, label? }` is the link shown on the step while it waits: `input` is one of the same step's regular (non-external) inputs (typed to their names), `path` a dot path inside its value (`url` → `input.review.url`; none = the value itself is the URL), `label` defaults to `title`. A value that is not an absolute http(s) URL sets no link; the step waits all the same.
 
 ### Config
 
@@ -241,7 +241,7 @@ export const deployBeta = implement(step({ name: "deploy-beta", /* … */ }), as
 | `ctx.` | |
 |---|---|
 | `sh(cmd \| script, { env, cwd, args })`, `sh.text(…)` | Runs a command string, or an embedded script, through the image's shell. The script is copied to `$OLVE_DIR/tmp/scripts/` first. The shell is the first of `/bin/sh`, `/busybox/sh`, `/bin/bash` that exists; a shell-less image gets a clear `ShellNotFoundError`. A script's shebang is honoured when its interpreter exists. A string counts as an embedded script when it is a `/$bunfs/…` path, or an absolute path to an existing `*.sh` or `*.bash` file. Pass `{ script }` to force script mode. `args` are the script's arguments, or `$1`, `$2`… of a command string. A non-zero exit throws a `CommandError` that carries `exitCode`. |
-| `exec([argv…], { env, cwd, idleTimeoutSeconds })` | Runs a binary directly, with no shell (works on distroless). `idleTimeoutSeconds` (also on `sh`) is an inactivity watchdog: no stdout/stderr output for that long → SIGTERM, SIGKILL 10 s later, and a `CommandStalledError`. |
+| `exec([argv…], { env, cwd, idleTimeoutSeconds })` | Runs a binary directly, with no shell (works on distroless). `idleTimeoutSeconds` (also on `sh`) is an inactivity watchdog: no stdout/stderr output for that long → SIGTERM, SIGKILL 10 s later, and a `CommandStalledError`. Once a command exits, its output is read for at most 5 s more, so a leftover background child holding the pipes doesn't hang the step. |
 | `readText(path)`, `readJson(path)` | Paths are relative to the cwd. |
 | `files.fetch(ref)` | Downloads a file reference and verifies its sha256. Directory refs are extracted. Returns a local path. |
 | `output.write(name, content)`, `output.file(path, { name })`, `output.dir(path, { name })` | Adds a file to the step's output and returns its `FileRef`. A directory becomes a deterministic tar. |
@@ -343,7 +343,7 @@ This section is everything the C# side needs in order to run the binary.
 | `OLVE_FILES_BASE_URL` (+ `OLVE_FILES_TOKEN`) | optional | Fallback: `GET <base>/<run>/<step>/<attempt>/<object path>` with an optional `Authorization: Bearer <token>`. |
 | `OLVE_FILES_DIR` | dev | Local pool laid out as `<run>/<step>/<attempt>/files/<object path>`. |
 
-`OLVE_RESULT_URL`, `OLVE_FILE_URLS`, `OLVE_FILES_TOKEN` and `OLVE_CALLBACK_URLS` are removed from the environment of child processes. URLs are never logged past `?`.
+The variables in this table (the `OLVE_*` contract, credentials included) are removed from the environment of child processes `ctx.sh` and `ctx.exec` start; declared secrets are not. URLs are never logged past `?`.
 
 **Secret masking** (`run`): with `OLVE_SECRETS` set, the binary runs the step as its own child and replaces every value of those secrets with `***` in the child's stdout and stderr, a line at a time, from the pipeline's first import to its exit. That covers every way of printing: logs, `console`, direct writes, commands and the processes they start. The envelope's failure message, skip reason and links are masked too; outputs and files are data and aren't. A multi-line value is masked line by line, and values (or lines) shorter than 4 characters are left alone. Not masked: a value the step transforms before printing (escaped, encoded, cut short) and a process writing to the pod's stdout directly (`/proc/1/fd/1`).
 
@@ -366,7 +366,7 @@ It retries network errors, 5xx, 408 and 429. It always verifies `sha256`.
 { "status": "failed",    "error": { "kind": "handler" | "input" | "output", "message": "…" } }
 ```
 
-Any of the three may carry `"links": [{ "url": "https://…", "label": "…" }]`, what the handler passed to `ctx.link` (absolute http(s) URLs of at most 2048 characters, labels of at most 100, at most 10). The key is left out when there are none. The server refuses an envelope whose links break these limits.
+Any of the three may carry `"links": [{ "url": "https://…", "label": "…" }]`, what the handler passed to `ctx.link` (absolute http(s) URLs of at most 2048 characters, labels of at most 100, at most 10). The key is left out when there are none. The server refuses an envelope whose links break these limits. The attempt keeps the links its external inputs' `link` gave it while it waited first, then adds the envelope's, leaving out a URL already there.
 
 For `map-trigger`, `output` is the pipeline input after it has been parsed by the input model, so defaults are applied. Validate it again server-side.
 
@@ -422,7 +422,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
   - a nested object of mappings.
 
   An object whose only key is `ref` (with a string value) or `const` is a tag. Describe never emits a nested object with exactly that key set. Model-matched inputs are emitted already resolved (`{"ref": "build"}`).
-- **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}`. Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
+- **Step `externalInputs`** (when present) maps each external input key to `{model, title, description, from, link?}`, `from` being `{kind: "person"}`, `{kind: "callback"}` or `{kind: "poll", url, valuePath, intervalSeconds}`, and `link` `{input, path?, label?}` (omitted when not set, as are its `path` and `label`). Those keys are not in `input` or `inputTypes`; the service merges the answers into `input.json`.
 - **`inputTypes`** declares each input key: `{model, optional}`, `{model, optional: false, cells: true}` for a `cells` input (mapped with a whole-step ref to a matrix step whose output is that model), or `{schema: {output, input?}}` for a plain schema.
 - **Step `matrix`** (matrix steps only, never a failure handler) is `{axes: [{name, values}…], exclude?: [{axis: value…}…], failure: {kind: "keepRunning" | "cascadeFailure"}, maxParallel?}`. `axes` is a list because canonical JSON sorts keys; `exclude` is omitted when empty; `failure.kind` is always written. All of it is part of what the step declares.
 - **`codeHash`** is the build's identity: `sha256(canonical {bundle, harness})`. `bundle` hashes the entry point bundled with everything it imports, with `@olivervea/pipelines` external. It is tree-shaken and includes embedded scripts and dependencies, so any change to code in use changes it, and with it the manifest, which makes a new pipeline version. A step version covers what the step declares: image, `inputTypes`/model versions, `output` model version, `secrets`, `configEntries` (and those entries' schemas), `input` mappings and `config`.
@@ -445,6 +445,7 @@ Describe runs these checks; reconcile should re-run the manifest-level ones:
 - availability, ambiguity, maybe-available as required, `optional` without a producer, plain schema without a mapper;
 - mapper source analysis and recording, unknown fields in refs, mapper keys that are missing or extra;
 - `when` on an unknown or non-literal field, or with an invalid value;
+- an external input's `link` naming anything but one of the step's regular inputs, an empty path segment, a label that is empty or over 100 characters;
 - a poll header referencing an undeclared secret, duplicate triggers;
 - conflicting secret descriptions;
 - config names, one description and schema per config name, a step receiving a `secretRef` without declaring its secret, and input keys starting with `$` (reserved);

package/manifest.schema.json

@@ -584,6 +584,32 @@
                 "minItems": 1,
                 "type": "array"
               },
+              "link": {
+                "additionalProperties": false,
+                "description": "The link shown while the step waits: the absolute http(s) URL at 'path' inside the value of the step's input 'input'. Omitted when none.",
+                "properties": {
+                  "input": {
+                    "description": "One of the same step's inputs (not an external one).",
+                    "minLength": 1,
+                    "type": "string"
+                  },
+                  "label": {
+                    "description": "Default: the input's title.",
+                    "maxLength": 100,
+                    "minLength": 1,
+                    "type": "string"
+                  },
+                  "path": {
+                    "description": "Dot path inside that input's value; omitted = the value itself is the URL.",
+                    "minLength": 1,
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "input"
+                ],
+                "type": "object"
+              },
               "model": {
                 "description": "The value is validated against this model's input view.",
                 "pattern": "^[A-Za-z][A-Za-z0-9_]*$",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.32",
+  "version": "2.0.0-beta.34",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/core/external.ts

@@ -47,32 +47,49 @@ export function poll(options: PollOptions): ExternalSource {
  * A step input whose value comes from outside the run. The step waits until it is answered (the value is
  * validated against `model`), then runs with the value in its input like any other.
  */
-export interface External<M extends AnyModel = AnyModel> {
+export interface External<M extends AnyModel = AnyModel, L extends string = string> {
   readonly kind: 'external';
   readonly model: M;
   readonly title: string;
   readonly description: string;
   readonly from: readonly ExternalSource[];
+  readonly link?: ExternalLink<L>;
 }
 
-export interface ExternalOptions {
+/**
+ * The link shown on the step while it waits: the URL is the value of one of the step's other
+ * (non-external) inputs, or a field inside it.
+ */
+export interface ExternalLink<L extends string = string> {
+  /** One of the same step's non-external inputs. */
+  readonly input: L;
+  /** Dot path inside that input's value (`url`, `review.url`); omitted = the value itself is the URL. */
+  readonly path?: string;
+  /** Default: the input's title. */
+  readonly label?: string;
+}
+
+export interface ExternalOptions<L extends string = string> {
   /** What is asked, shown to whoever answers. */
   title: string;
   description?: string;
   /** Who may answer (default: a person). */
   from?: readonly ExternalSource[];
+  /** The link shown while the step waits, e.g. `{ input: "review", path: "url" }`. */
+  link?: ExternalLink<L>;
 }
 
-export function external<M extends AnyModel>(
+export function external<M extends AnyModel, const L extends string = never>(
   model: M,
-  options: ExternalOptions,
-): External<M> {
+  options: ExternalOptions<L>,
+): External<M, L> {
   return Object.freeze({
     kind: 'external' as const,
     model,
     title: options.title,
     description: options.description ?? '',
     from: Object.freeze([...(options.from ?? [person()])]),
+    ...(options.link !== undefined ? { link: Object.freeze({ ...options.link }) } : {}),
   });
 }
 

package/src/core/steps.ts

@@ -1,7 +1,7 @@
 import type { z } from 'zod';
 import { STEP, type StepBrand } from './brands.ts';
 import type { AnyModel, Infer, Optional } from './models.ts';
-import type { External } from './external.ts';
+import type { External, ExternalLink } from './external.ts';
 import type { AnyConfigEntry } from './config.ts';
 import { configRefsIn } from './config-refs.ts';
 import type { Secret } from './secrets.ts';
@@ -35,6 +35,23 @@ export type InferEntry<E> =
           : E extends z.ZodType
             ? z.output<E>
             : never;
+/** The keys of a step's inputs that are not external. */
+export type RegularInputKey<I> = {
+  [K in keyof I]: I[K] extends External ? never : K;
+}[keyof I] &
+  string;
+
+/** An external input's `link.input` must name one of the same step's non-external inputs. */
+export type ExternalLinksCheck<I> = {
+  [K in keyof I]: I[K] extends External<AnyModel, infer L>
+    ? [L] extends [never]
+      ? unknown
+      : [L] extends [RegularInputKey<I>]
+        ? unknown
+        : { readonly link?: ExternalLink<RegularInputKey<I>> }
+    : unknown;
+};
+
 /** What a handler receives: the output view of every input entry. */
 export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
 
@@ -106,7 +123,7 @@ export interface StepDef<
   description: string;
   /** Any image: the harness brings its own runtime. */
   image: string;
-  input?: I;
+  input?: I & ExternalLinksCheck<I>;
   output?: O;
   secrets?: Sec;
   /** Config entries delivered as `ctx.config` (typed to exactly these). */

package/src/ctx/sh.ts

@@ -52,6 +52,9 @@ export class CommandStalledError extends CommandError {
 /** Grace between the watchdog's SIGTERM and SIGKILL (a process blocked in a socket read may ignore TERM). */
 export const STALL_KILL_GRACE_MS = 10_000;
 
+/** How long output is still read after a command exits; a leftover child (an MSBuild node) may hold the pipes. */
+export const PIPE_GRACE_MS = 5_000;
+
 export function detectShell(candidates: readonly string[] = SHELL_CANDIDATES): string {
   for (const c of candidates) if (existsSync(c)) return c;
   throw new ShellNotFoundError();
@@ -80,6 +83,7 @@ export interface ShDeps {
   tmpDir: string;
   signal: AbortSignal;
   shellCandidates?: readonly string[];
+  pipeGraceMs?: number;
 }
 
 export function createSh(deps: ShDeps): {
@@ -208,9 +212,16 @@ export function createSh(deps: ShDeps): {
 
     try {
       const code = await proc.exited;
-      // A killed command's children may still hold the pipes open: don't wait for them after a stall.
-      if (stalled) await Promise.all(readers.map((r) => r.cancel().catch(() => {})));
-      else await Promise.all(pumps);
+      // A command's leftover children may still hold the pipes open: don't wait for them long.
+      if (!stalled) {
+        let grace: ReturnType<typeof setTimeout> | undefined;
+        await Promise.race([
+          Promise.all(pumps),
+          new Promise((r) => (grace = setTimeout(r, deps.pipeGraceMs ?? PIPE_GRACE_MS))),
+        ]);
+        clearTimeout(grace);
+      }
+      await Promise.all(readers.map((r) => r.cancel().catch(() => {})));
       if (stalled)
         throw new CommandStalledError(
           what,

package/src/describe/manifest-schema.ts

@@ -211,6 +211,30 @@ export const StepDescriptor = z
             )
             .min(1)
             .describe('Who or what may answer.'),
+          link: z
+            .strictObject({
+              input: z
+                .string()
+                .min(1)
+                .describe("One of the same step's inputs (not an external one)."),
+              path: z
+                .string()
+                .min(1)
+                .optional()
+                .describe(
+                  "Dot path inside that input's value; omitted = the value itself is the URL.",
+                ),
+              label: z
+                .string()
+                .min(1)
+                .max(100)
+                .optional()
+                .describe("Default: the input's title."),
+            })
+            .optional()
+            .describe(
+              "The link shown while the step waits: the absolute http(s) URL at 'path' inside the value of the step's input 'input'. Omitted when none.",
+            ),
         }),
       )
       .optional()

package/src/describe/step.ts

@@ -124,6 +124,7 @@ function describeExternalInputs(
 ): ExternalInputs {
   const problems = ctx.problems;
   const externalInputs: ExternalInputs = {};
+  const regular = new Set(allEntries.filter(([, e]) => !isExternal(e)).map(([k]) => k));
   for (const [key, e] of allEntries) {
     if (!isExternal(e)) continue;
     if (failureHandler)
@@ -153,16 +154,48 @@ function describeExternalInputs(
       if (f.valuePath.trim() === '')
         problems.push(`${at} external input '${key}': poll valuePath must not be empty`);
     }
+    const link = e.link;
+    if (link !== undefined) {
+      if (!regular.has(link.input))
+        problems.push(
+          `${at} external input '${key}': link input '${link.input}' must name one of the step's regular (non-external) inputs`,
+        );
+      if (link.path !== undefined && !isDotPath(link.path))
+        problems.push(
+          `${at} external input '${key}': link path must be a dot path of non-empty field names`,
+        );
+      if (
+        link.label !== undefined &&
+        (typeof link.label !== 'string' ||
+          link.label.length < 1 ||
+          link.label.length > 100)
+      )
+        problems.push(
+          `${at} external input '${key}': link label must be 1 to 100 characters`,
+        );
+    }
     externalInputs[key] = {
       model: ctx.useModel(e.model, `${at} external input '${key}'`),
       title: e.title,
       description: e.description,
       from: e.from.map((f) => ({ ...f })),
+      ...(link !== undefined
+        ? {
+            link: {
+              input: link.input,
+              ...(link.path !== undefined ? { path: link.path } : {}),
+              ...(link.label !== undefined ? { label: link.label } : {}),
+            },
+          }
+        : {}),
     };
   }
   return externalInputs;
 }
 
+const isDotPath = (path: unknown): boolean =>
+  typeof path === 'string' && path.split('.').every((segment) => segment !== '');
+
 function describeInputTypes(
   ctx: DescribeContext,
   at: string,

package/src/harness/env.ts

@@ -25,12 +25,12 @@ export const ENV = {
   filesDir: 'OLVE_FILES_DIR',
 } as const;
 
-/** Env vars never passed to child processes (they carry credentials for the harness only). */
+/** Env vars never passed to child processes: the harness's own contract, credentials included. */
 export const PRIVATE_ENV: readonly string[] = [
-  ENV.resultUrl,
-  ENV.fileUrls,
-  ENV.filesToken,
-  ENV.callbackUrls,
+  ...Object.values(ENV),
+  'OLVE_CASES',
+  'OLVE_SECRETS',
+  'OLVE_SUPERVISED',
 ];
 
 /** `OLVE_CALLBACK_URLS`, parsed; empty when unset or malformed. */

package/src/index.ts

@@ -53,6 +53,7 @@ export {
   person,
   poll,
   type External,
+  type ExternalLink,
   type ExternalOptions,
   type ExternalSource,
   type PollOptions,