@olivervea/pipelines 2.0.0-beta.3 → 2.0.0-beta.30

package/src/dev/cli.ts

@@ -1,31 +1,42 @@
 #!/usr/bin/env bun
 // olve-dev — run a pipeline from source through the same harness path as the compiled binary.
 //   olve-dev describe <dir> [--sentinels]
-//   olve-dev run <dir> <step> [--input-file f] [--olve-dir d] [--secret NAME=value]...
+//   olve-dev run <dir> <step> [--input-file f] [--config-file f] [--olve-dir d] [--secret NAME=value]...
 //   olve-dev map-trigger <dir> <trigger> --event-file f [--olve-dir d]
 // $OLVE_DIR defaults to <dir>/.olve; outputs land in <dir>/.olve/work/.
-import { copyFileSync, mkdirSync } from "node:fs";
-import { join, resolve } from "node:path";
-import { parseArgs } from "node:util";
-import { runHarness, type HarnessConfig } from "../harness/main.ts";
-import { EXIT } from "../harness/exit.ts";
-import { handlerHashes } from "../build/hashes.ts";
-import { harnessInfo, loadProject, ProjectError } from "../build/project.ts";
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+} from 'node:fs';
+import { join, resolve } from 'node:path';
+import { parseArgs } from 'node:util';
+import { runHarness, type HarnessConfig } from '../harness/main.ts';
+import { EXIT } from '../harness/exit.ts';
+import { codeHash } from '../build/hashes.ts';
+import {
+  harnessInfo,
+  loadPipeline,
+  loadProject,
+  ProjectError,
+} from '../build/project.ts';
 
 const USAGE = `usage: olve-dev describe <dir> [--sentinels]
-       olve-dev run <dir> <step> [--input-file f] [--olve-dir d] [--secret NAME=value]...
+       olve-dev run <dir> <step> [--input-file f] [--config-file f] [--olve-dir d] [--secret NAME=value]...
        olve-dev map-trigger <dir> <trigger> --event-file f [--olve-dir d]`;
 
-export async function devConfig(dir: string, withHashes: boolean): Promise<HarnessConfig> {
+export async function devConfig(
+  dir: string,
+  withHashes: boolean,
+): Promise<HarnessConfig> {
   const project = await loadProject(dir);
-  const mod = (await import(project.pipelineFile)) as { default?: unknown };
-  if (!mod.default) throw new ProjectError(`${project.pipelineFile} has no default export (export default pipeline({...})...)`);
+  const pipeline = await loadPipeline(project);
   const harness = harnessInfo();
-  const hashes = withHashes ? await handlerHashes(project, harness) : {};
   return {
-    pipeline: mod.default,
-    handlers: Object.fromEntries(project.handlers.map(([name, file]) => [name, () => import(file) as Promise<{ default: unknown }>])),
-    handlerHashes: Object.fromEntries(Object.entries(hashes).map(([k, v]) => [k, v.handlerHash])),
+    pipeline,
+    ...(withHashes ? { codeHash: await codeHash(project, harness) } : {}),
     harness,
   };
 }
@@ -37,12 +48,13 @@ export async function devMain(argv: string[]): Promise<number> {
       args: argv,
       allowPositionals: true,
       options: {
-        "input-file": { type: "string" },
-        "event-file": { type: "string" },
-        "olve-dir": { type: "string" },
-        secret: { type: "string", multiple: true },
-        sentinels: { type: "boolean", default: false },
-        help: { type: "boolean", short: "h", default: false },
+        'input-file': { type: 'string' },
+        'config-file': { type: 'string' },
+        'event-file': { type: 'string' },
+        'olve-dir': { type: 'string' },
+        secret: { type: 'string', multiple: true },
+        sentinels: { type: 'boolean', default: false },
+        help: { type: 'boolean', short: 'h', default: false },
       },
     });
   } catch (e) {
@@ -50,28 +62,43 @@ export async function devMain(argv: string[]): Promise<number> {
     return EXIT.usage;
   }
   const { values, positionals } = parsed;
-  if (values.help) return console.log(USAGE), 0;
+  if (values.help) return (console.log(USAGE), 0);
   const [mode, dir, name] = positionals;
-  if (!mode || !dir) return console.error(USAGE), EXIT.usage;
+  if (!mode || !dir) return (console.error(USAGE), EXIT.usage);
   try {
-    if (mode === "describe") return await runHarness(await devConfig(dir, true), ["describe", ...(values.sentinels ? ["--sentinels"] : [])]);
-    if (mode !== "run" && mode !== "map-trigger") return console.error(USAGE), EXIT.usage;
-    if (!name) return console.error(USAGE), EXIT.usage;
-    const olveDir = resolve(values["olve-dir"] ?? join(dir, ".olve"));
+    if (mode === 'describe')
+      return await runHarness(await devConfig(dir, true), [
+        'describe',
+        ...(values.sentinels ? ['--sentinels'] : []),
+      ]);
+    if (mode !== 'run' && mode !== 'map-trigger')
+      return (console.error(USAGE), EXIT.usage);
+    if (!name) return (console.error(USAGE), EXIT.usage);
+    const file = mode === 'run' ? values['input-file'] : values['event-file'];
+    if (mode === 'map-trigger' && !file)
+      return (console.error('--event-file is required'), EXIT.usage);
+    const olveDir = resolve(values['olve-dir'] ?? join(dir, '.olve'));
     mkdirSync(olveDir, { recursive: true });
     process.env.OLVE_DIR = olveDir;
-    process.env.TMPDIR ??= join(olveDir, "tmp");
+    process.env.TMPDIR ??= join(olveDir, 'tmp');
     for (const s of values.secret ?? []) {
-      const i = s.indexOf("=");
-      if (i <= 0) return console.error(`--secret expects NAME=value, got '${s}'`), EXIT.usage;
+      const i = s.indexOf('=');
+      if (i <= 0)
+        return (console.error(`--secret expects NAME=value, got '${s}'`), EXIT.usage);
       process.env[s.slice(0, i)] = s.slice(i + 1);
     }
-    const file = mode === "run" ? values["input-file"] : values["event-file"];
-    if (file) copyFileSync(resolve(file), join(olveDir, "input.json"));
-    else if (mode === "map-trigger") return console.error("--event-file is required"), EXIT.usage;
+    if (file) copyFileSync(resolve(file), join(olveDir, 'input.json'));
+    if (mode === 'run' && values['config-file']) {
+      const inputFile = join(olveDir, 'input.json');
+      const input = (
+        existsSync(inputFile) ? JSON.parse(readFileSync(inputFile, 'utf8')) : {}
+      ) as Record<string, unknown>;
+      input.$config = JSON.parse(readFileSync(resolve(values['config-file']), 'utf8'));
+      writeFileSync(inputFile, JSON.stringify(input));
+    }
     return await runHarness(await devConfig(dir, false), [mode, name]);
   } catch (e) {
-    if (e instanceof ProjectError) return console.error(e.message), EXIT.configInvalid;
+    if (e instanceof ProjectError) return (console.error(e.message), EXIT.configInvalid);
     console.error((e as Error)?.stack ?? String(e));
     return EXIT.configInvalid;
   }

package/src/harness/env.ts

@@ -1,58 +1,80 @@
 // The environment contract between the service and the harness. Documented in README.md.
-import { join } from "node:path";
+import { join } from 'node:path';
 
 export const ENV = {
   /** Root of the harness IO directory (default /olve). */
-  dir: "OLVE_DIR",
-  pipeline: "OLVE_PIPELINE",
-  runId: "OLVE_RUN_ID",
-  run: "OLVE_RUN",
-  attempt: "OLVE_ATTEMPT",
-  versionStamp: "OLVE_VERSION_STAMP",
+  dir: 'OLVE_DIR',
+  pipeline: 'OLVE_PIPELINE',
+  runId: 'OLVE_RUN_ID',
+  run: 'OLVE_RUN',
+  attempt: 'OLVE_ATTEMPT',
+  versionStamp: 'OLVE_VERSION_STAMP',
+  /** A matrix step's cell, as compact JSON in axis order (`{"os":"linux","arch":"x64"}`). */
+  cell: 'OLVE_CELL',
   /** Pre-signed PUT for result.tar.gz (run / map-trigger). Optional. */
-  resultUrl: "OLVE_RESULT_URL",
+  resultUrl: 'OLVE_RESULT_URL',
   /** JSON object: file key (`<run>/<step>/<attempt>/<path>`) → pre-signed GET URL. Optional. */
-  fileUrls: "OLVE_FILE_URLS",
+  fileUrls: 'OLVE_FILE_URLS',
+  /** JSON object: `<step>/<input>` → callback URL of that external input in this run (`ctx.inputUrl`). Optional. */
+  callbackUrls: 'OLVE_CALLBACK_URLS',
   /** Base URL for file downloads: GET `<base>/<run>/<step>/<attempt>/<object path>`. Optional. */
-  filesBaseUrl: "OLVE_FILES_BASE_URL",
+  filesBaseUrl: 'OLVE_FILES_BASE_URL',
   /** Bearer token sent with OLVE_FILES_BASE_URL requests. Optional. */
-  filesToken: "OLVE_FILES_TOKEN",
+  filesToken: 'OLVE_FILES_TOKEN',
   /** Local directory laid out as `<run>/<step>/<attempt>/files/<object path>` (dev). Optional. */
-  filesDir: "OLVE_FILES_DIR",
+  filesDir: 'OLVE_FILES_DIR',
 } as const;
 
 /** Env vars never passed to child processes (they carry credentials for the harness only). */
-export const PRIVATE_ENV: readonly string[] = [ENV.resultUrl, ENV.fileUrls, ENV.filesToken];
+export const PRIVATE_ENV: readonly string[] = [
+  ENV.resultUrl,
+  ENV.fileUrls,
+  ENV.filesToken,
+  ENV.callbackUrls,
+];
 
-export const olveDir = () => process.env[ENV.dir] || "/olve";
+/** `OLVE_CALLBACK_URLS`, parsed; empty when unset or malformed. */
+export function callbackUrlsFromEnv(): Record<string, string> {
+  const raw = process.env[ENV.callbackUrls];
+  if (!raw) return {};
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    return parsed && typeof parsed === 'object' ? (parsed as Record<string, string>) : {};
+  } catch {
+    return {};
+  }
+}
+
+export const olveDir = () => process.env[ENV.dir] || '/olve';
 export const paths = () => {
   const root = olveDir();
   return {
     root,
-    input: join(root, "input.json"),
-    work: join(root, "work"),
-    output: join(root, "work", "output.json"),
-    files: join(root, "work", "files"),
-    tmp: join(root, "tmp"),
-    fileUrls: join(root, "files.json"),
+    input: join(root, 'input.json'),
+    work: join(root, 'work'),
+    output: join(root, 'work', 'output.json'),
+    files: join(root, 'work', 'files'),
+    tmp: join(root, 'tmp'),
+    fileUrls: join(root, 'files.json'),
   };
 };
 
 function int(name: string, fallback: number): number {
   const v = process.env[name];
-  if (v === undefined || v === "") return fallback;
+  if (v === undefined || v === '') return fallback;
   const n = Number(v);
-  if (!Number.isInteger(n) || n < 0) throw new Error(`${name} must be a non-negative integer, got '${v}'`);
+  if (!Number.isInteger(n) || n < 0)
+    throw new Error(`${name} must be a non-negative integer, got '${v}'`);
   return n;
 }
 
 export function runInfoFromEnv(step: string) {
   return {
-    pipeline: process.env[ENV.pipeline] ?? "local",
-    id: process.env[ENV.runId] ?? "local",
+    pipeline: process.env[ENV.pipeline] ?? 'local',
+    id: process.env[ENV.runId] ?? 'local',
     number: int(ENV.run, 0),
     attempt: Math.max(1, int(ENV.attempt, 1)),
     step,
-    version: process.env[ENV.versionStamp] ?? "0.0.0-local",
+    version: process.env[ENV.versionStamp] ?? '0.0.0-local',
   };
 }

package/src/harness/execute.ts

@@ -1,20 +1,22 @@
 // The shared execution path: validate input → handler → validate output → envelope.
 // Used by the compiled harness (`run`), the dev runner and testHandler.
-import { existsSync } from "node:fs";
-import { join } from "node:path";
-import { isSkipped } from "../core/handlers.ts";
-import type { AnyStep } from "../core/steps.ts";
-import type { Ctx, RunInfo } from "../ctx/types.ts";
-import { objectPath } from "../ctx/files.ts";
-import { sha256File } from "../util/hash.ts";
-import { EXIT } from "./exit.ts";
-import { validateInput, validateOutput } from "./validate.ts";
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { isSkipped } from '../core/handlers.ts';
+import type { AnyStep } from '../core/steps.ts';
+import type { Ctx, RunInfo } from '../ctx/types.ts';
+import { objectPath } from '../ctx/files.ts';
+import { createLinks, type StepLink } from '../ctx/links.ts';
+import { sha256File } from '../util/hash.ts';
+import { EXIT } from './exit.ts';
+import { validateConfig, validateInput, validateOutput } from './validate.ts';
 
-export type FailureKind = "handler" | "input" | "output";
-export type Envelope =
-  | { status: "succeeded"; output: unknown }
-  | { status: "skipped"; reason: string }
-  | { status: "failed"; error: { kind: FailureKind; message: string } };
+export type FailureKind = 'handler' | 'input' | 'output';
+export type Envelope = (
+  | { status: 'succeeded'; output: unknown }
+  | { status: 'skipped'; reason: string }
+  | { status: 'failed'; error: { kind: FailureKind; message: string } }
+) & { links?: StepLink[] };
 
 export interface Execution {
   envelope: Envelope;
@@ -26,7 +28,7 @@ export type StepFunction = (input: Record<string, unknown>, ctx: Ctx) => unknown
 /** Signal → exit code, when the handler failed after a cancellation. */
 export function cancelExitCode(signal: AbortSignal): number | undefined {
   if (!signal.aborted) return undefined;
-  return signal.reason === "SIGINT" ? EXIT.sigint : EXIT.sigterm;
+  return signal.reason === 'SIGINT' ? EXIT.sigint : EXIT.sigterm;
 }
 
 const errorMessage = (e: unknown): string => (e instanceof Error ? e.message : String(e));
@@ -36,17 +38,29 @@ function checkOwnFileRefs(value: unknown, run: RunInfo, workFiles: string): stri
   const problems: string[] = [];
   const walk = (v: unknown, at: string) => {
     if (Array.isArray(v)) return v.forEach((x, i) => walk(x, `${at}[${i}]`));
-    if (typeof v !== "object" || v === null) return;
+    if (typeof v !== 'object' || v === null) return;
     const o = v as Record<string, unknown>;
-    if (typeof o.path === "string" && typeof o.sha256 === "string" && o.run === run.number && o.step === run.step && o.attempt === run.attempt) {
+    if (
+      typeof o.path === 'string' &&
+      typeof o.sha256 === 'string' &&
+      o.run === run.number &&
+      o.step === run.step &&
+      o.attempt === run.attempt
+    ) {
       const file = join(workFiles, objectPath(o.path));
-      if (!existsSync(file)) problems.push(`${at}: file reference '${o.path}' points to this attempt but no such output file was produced`);
-      else if (sha256File(file) !== o.sha256) problems.push(`${at}: file reference '${o.path}' has the wrong sha256 (the file changed after it was referenced)`);
+      if (!existsSync(file))
+        problems.push(
+          `${at}: file reference '${o.path}' points to this attempt but no such output file was produced`,
+        );
+      else if (sha256File(file) !== o.sha256)
+        problems.push(
+          `${at}: file reference '${o.path}' has the wrong sha256 (the file changed after it was referenced)`,
+        );
       return;
     }
     for (const [k, x] of Object.entries(o)) walk(x, `${at}.${k}`);
   };
-  walk(value, "output");
+  walk(value, 'output');
   return problems;
 }
 
@@ -58,13 +72,29 @@ export async function executeStep(args: {
   workFiles: string;
   log: (line: string) => void;
 }): Promise<Execution> {
-  const { step, ctx, log } = args;
+  const { step, log } = args;
   const input = validateInput(step, args.rawInput);
-  if (!input.ok) {
-    const message = `step '${step.name}': input does not match the step's inputs:\n  ${input.issues.join("\n  ")}`;
+  const config = validateConfig(step, args.rawInput);
+  if (!input.ok || !config.ok) {
+    const issues = [
+      ...(input.ok ? [] : input.issues),
+      ...(config.ok ? [] : config.issues),
+    ];
+    const message = `step '${step.name}': input does not match the step's inputs:\n  ${issues.join('\n  ')}`;
     log(message);
-    return { envelope: { status: "failed", error: { kind: "input", message } }, exitCode: EXIT.inputInvalid };
+    return {
+      envelope: { status: 'failed', error: { kind: 'input', message } },
+      exitCode: EXIT.inputInvalid,
+    };
   }
+  const collected = createLinks();
+  const ctx: Ctx = {
+    ...args.ctx,
+    config: Object.freeze(config.value) as Ctx['config'],
+    link: collected.link,
+  };
+  const withLinks = (e: Envelope): Envelope =>
+    collected.links.length ? { ...e, links: [...collected.links] } : e;
 
   let value: unknown;
   try {
@@ -76,19 +106,33 @@ export async function executeStep(args: {
       : `step '${step.name}' failed: ${errorMessage(e)}`;
     log(message);
     if (e instanceof Error && e.stack && !cancelled) log(e.stack);
-    return { envelope: { status: "failed", error: { kind: "handler", message } }, exitCode: cancelled ?? EXIT.handlerFailed };
+    return {
+      envelope: withLinks({ status: 'failed', error: { kind: 'handler', message } }),
+      exitCode: cancelled ?? EXIT.handlerFailed,
+    };
   }
 
   if (isSkipped(value)) {
     log(`step '${step.name}' skipped: ${value.reason}`);
-    return { envelope: { status: "skipped", reason: value.reason }, exitCode: EXIT.ok };
+    return {
+      envelope: withLinks({ status: 'skipped', reason: value.reason }),
+      exitCode: EXIT.ok,
+    };
   }
   const out = validateOutput(step, value);
-  const problems = out.ok ? checkOwnFileRefs(out.value, ctx.run, args.workFiles) : out.issues;
+  const problems = out.ok
+    ? checkOwnFileRefs(out.value, ctx.run, args.workFiles)
+    : out.issues;
   if (!out.ok || problems.length) {
-    const message = `step '${step.name}': returned value does not match ${step.output?.name ?? "its (absent) output model"}:\n  ${problems.join("\n  ")}`;
+    const message = `step '${step.name}': returned value does not match ${step.output?.name ?? 'its (absent) output model'}:\n  ${problems.join('\n  ')}`;
     log(message);
-    return { envelope: { status: "failed", error: { kind: "output", message } }, exitCode: EXIT.outputInvalid };
+    return {
+      envelope: withLinks({ status: 'failed', error: { kind: 'output', message } }),
+      exitCode: EXIT.outputInvalid,
+    };
   }
-  return { envelope: { status: "succeeded", output: out.value }, exitCode: EXIT.ok };
+  return {
+    envelope: withLinks({ status: 'succeeded', output: out.value }),
+    exitCode: EXIT.ok,
+  };
 }

package/src/harness/exit.ts

@@ -11,7 +11,7 @@ export const EXIT = {
   outputInvalid: 66,
   /** Uploading result.tar.gz to OLVE_RESULT_URL failed (after retries). The envelope was still printed. */
   uploadFailed: 74,
-  /** The pipeline is invalid: describe problems, missing/invalid handler module. */
+  /** The pipeline is invalid: describe problems, a step without an implementation. */
   configInvalid: 78,
   /** Cancelled by SIGINT / SIGTERM (128 + signal). */
   sigint: 130,

package/src/harness/index.ts

@@ -1,9 +1,20 @@
 // @olivervea/pipelines/harness — the runtime the build tool compiles into the step binary.
-export { runHarness, defaultIO, allSteps, resolveStepFunction, type HarnessConfig, type HandlerRegistry, type HarnessIO } from "./main.ts";
-export { EXIT } from "./exit.ts";
-export { ENV, paths, runInfoFromEnv } from "./env.ts";
-export { executeStep, type Envelope, type Execution, type FailureKind } from "./execute.ts";
-export { RESULT_BEGIN, RESULT_END, MANIFEST_BEGIN, MANIFEST_END } from "./result.ts";
-export { describePipeline, DescribeError, manifestText } from "../describe/describe.ts";
-export type { ManifestT as Manifest } from "../describe/manifest-schema.ts";
-export type { HarnessInfo } from "../util/handler-hash.ts";
+export {
+  runHarness,
+  defaultIO,
+  resolveStepFunction,
+  type HarnessConfig,
+  type HarnessIO,
+} from './main.ts';
+export { EXIT } from './exit.ts';
+export { ENV, paths, runInfoFromEnv } from './env.ts';
+export {
+  executeStep,
+  type Envelope,
+  type Execution,
+  type FailureKind,
+} from './execute.ts';
+export { RESULT_BEGIN, RESULT_END, MANIFEST_BEGIN, MANIFEST_END } from './result.ts';
+export { describePipeline, DescribeError, manifestText } from '../describe/describe.ts';
+export type { ManifestT as Manifest } from '../describe/manifest-schema.ts';
+export type { HarnessInfo } from '../util/code-hash.ts';