@olivervea/pipelines 2.0.0-beta.25 → 2.0.0-beta.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.25",
+  "version": "2.0.0-beta.26",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/build/build.ts

@@ -6,7 +6,7 @@ import { sha256File } from '../util/hash.ts';
 import type { HarnessInfo } from '../util/code-hash.ts';
 import { writeTarZst } from '../util/tar.ts';
 import { MANIFEST_BEGIN, MANIFEST_END, uploadFile } from '../harness/result.ts';
-import { GLIBC_INTERP, MUSL_INTERP, readInterpreter } from './elf.ts';
+import { loaderProblem } from './elf.ts';
 import { codeHash } from './hashes.ts';
 import { harnessInfo, loadProject, relPosix, type Project } from './project.ts';
 
@@ -112,14 +112,9 @@ export async function build(dir: string, opts: BuildOptions): Promise<BuildResul
     `olve-build: compiled ${target} in ${((performance.now() - t1) / 1000).toFixed(2)}s`,
   );
 
-  // 2. loader check: the harness launches the binary through /olve/lib/<loader>; a musl binary
-  //    (e.g. compiled on alpine, where bun-linux-x64 silently yields musl) would not start.
-  const expected = target === 'bun-linux-x64' ? GLIBC_INTERP : MUSL_INTERP;
-  const interp = readInterpreter(binary);
-  if (interp !== expected)
-    throw new BuildError(
-      `compiled binary has PT_INTERP ${interp ?? '(none)'}, expected ${expected} — compile on a glibc Bun image (oven/bun:<version>-slim), not alpine`,
-    );
+  // 2. loader check: on alpine, bun-linux-x64 silently yields a musl binary, which would not start.
+  const problem = loaderProblem(binary, target);
+  if (problem) throw new BuildError(problem);
 
   // 3. describe (runs the real binary: the manifest is exactly what the step pods will see)
   const d = Bun.spawnSync([binary, 'describe'], {

package/src/build/elf.ts

@@ -34,3 +34,11 @@ export function readInterpreter(path: string): string | null {
     closeSync(fd);
   }
 }
+
+/** Why `binary` won't start through `target`'s loader (the harness runs it via /olve/lib/<loader>), or undefined. */
+export function loaderProblem(binary: string, target: string): string | undefined {
+  const expected = target === 'bun-linux-x64' ? GLIBC_INTERP : MUSL_INTERP;
+  const interp = readInterpreter(binary);
+  if (interp === expected) return undefined;
+  return `compiled binary has PT_INTERP ${interp ?? '(none)'}, expected ${expected} — compile on a glibc Bun image (oven/bun:<version>-slim), not alpine`;
+}

package/src/ctx/files.ts

@@ -37,6 +37,8 @@ export interface FilesDeps {
   root: string;
   signal: AbortSignal;
   log: (message: string, fields?: Record<string, unknown>) => void;
+  /** Backoff between download retries (default: real sleep). */
+  sleep?: (ms: number) => Promise<void>;
 }
 
 function checkName(name: string, what: string): void {
@@ -122,6 +124,7 @@ async function download(
   headers: Record<string, string>,
   signal: AbortSignal,
   log: FilesDeps['log'],
+  sleep: (ms: number) => Promise<void>,
 ): Promise<void> {
   const attempts = 4;
   for (let i = 1; ; i++) {
@@ -143,13 +146,14 @@ async function download(
         attempt: i,
         error: (e as Error).message,
       });
-      await Bun.sleep(1000 * 2 ** (i - 1));
+      await sleep(1000 * 2 ** (i - 1));
     }
   }
 }
 
 export function createFetch(deps: FilesDeps) {
   const cache = new Map<string, string>();
+  const sleep = deps.sleep ?? ((ms: number) => Bun.sleep(ms));
   return async (input: FileRef): Promise<string> => {
     const parsed = fileRefSchema.safeParse(input);
     if (!parsed.success)
@@ -181,7 +185,8 @@ export function createFetch(deps: FilesDeps) {
     const baseUrl = process.env[ENV.filesBaseUrl];
     if (own && existsSync(join(deps.workFiles, obj)))
       copyFileSync(join(deps.workFiles, obj), objFile);
-    else if (urls[key]) await download(urls[key], objFile, {}, deps.signal, deps.log);
+    else if (urls[key])
+      await download(urls[key], objFile, {}, deps.signal, deps.log, sleep);
     else if (
       localDir &&
       existsSync(
@@ -201,6 +206,7 @@ export function createFetch(deps: FilesDeps) {
         token ? { authorization: `Bearer ${token}` } : {},
         deps.signal,
         deps.log,
+        sleep,
       );
     } else {
       throw new Error(

package/src/ctx/index.ts

@@ -89,9 +89,25 @@ export function createHttp(signal: AbortSignal): Http {
   };
 }
 
+/** Waits `ms`, or rejects at once when the step is cancelled. */
+function cancellableSleep(ms: number, signal: AbortSignal): Promise<void> {
+  return new Promise<void>((res, rej) => {
+    const t = setTimeout(() => {
+      signal.removeEventListener('abort', onAbort);
+      res();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(t);
+      rej(new Error('retry cancelled: the step was cancelled'));
+    };
+    signal.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
 export function createRetry(
   signal: AbortSignal,
   log: (m: string, f?: Record<string, unknown>) => void,
+  sleep: (ms: number, signal: AbortSignal) => Promise<void> = cancellableSleep,
 ) {
   return async <T>(
     opts: RetryOptions,
@@ -110,17 +126,7 @@ export function createRetry(
           of: opts.attempts,
           error: (e as Error)?.message ?? String(e),
         });
-        await new Promise<void>((res, rej) => {
-          const t = setTimeout(() => {
-            signal.removeEventListener('abort', onAbort);
-            res();
-          }, delay);
-          const onAbort = () => {
-            clearTimeout(t);
-            rej(new Error('retry cancelled: the step was cancelled'));
-          };
-          signal.addEventListener('abort', onAbort, { once: true });
-        });
+        await sleep(delay, signal);
         delay *= opts.backoff ?? 1;
       }
     }

package/src/describe/describe.ts

@@ -806,7 +806,7 @@ export function describePipeline(pipeline: unknown, opts: DescribeOptions): Mani
 export const manifestText = (m: ManifestT): string => canonicalJson(m);
 
 /**
- * The schemas at `path` inside an entry's schema: every union branch that has the field, except where
+ * The schemas at `path` inside an entry's schema (through properties and records): every union branch that has the field, except where
  * `kase` (`<entry>.<path>.<discriminator>=<value>`) narrows a union to one branch.
  */
 export function subSchemas(
@@ -848,6 +848,8 @@ export function subSchemas(
     const props = o.properties as Record<string, JsonSchema> | undefined;
     if (props && Object.hasOwn(props, rest[0]!))
       return walk(props[rest[0]!], rest.slice(1), `${here}.${rest[0]}`);
+    if (o.additionalProperties && typeof o.additionalProperties === 'object')
+      return walk(o.additionalProperties, rest.slice(1), `${here}.${rest[0]}`);
     return [];
   };
   return walk(schema, path, entry);

package/src/describe/mapper-source.ts

@@ -131,8 +131,8 @@ function tokenize(src: string): Tok[] {
       i = e + 2;
       continue;
     }
-    if (c === '`') throw new MapperError('mapper uses a template string');
-    if (c === '"' || c === "'") {
+    // Bun prints a string constant holding a newline or both quote kinds as a template literal.
+    if (c === '"' || c === "'" || c === '`') {
       let j = i + 1;
       let v = '';
       for (;;) {
@@ -140,6 +140,8 @@ function tokenize(src: string): Tok[] {
           throw new MapperError('mapper source has an unterminated string');
         const d = src[j]!;
         if (d === c) break;
+        if (c === '`' && d === '$' && src[j + 1] === '{')
+          throw new MapperError('mapper uses a template string');
         if (d === '\\') {
           v += src.slice(j, j + 2);
           j += 2;

package/src/harness/index.ts

@@ -2,7 +2,6 @@
 export {
   runHarness,
   defaultIO,
-  allSteps,
   resolveStepFunction,
   type HarnessConfig,
   type HarnessIO,

package/src/harness/main.ts

@@ -56,12 +56,6 @@ export const defaultIO: HarnessIO = {
 const USAGE =
   'usage: step describe [--sentinels] | step run <step> | step map-trigger <trigger>';
 
-export function allSteps(data: PipelineData): AnyStep[] {
-  const out: AnyStep[] = [];
-  for (const { step } of stepsWithCases(data)) if (!out.includes(step)) out.push(step);
-  return out;
-}
-
 /** Every step with the `select` case conditions (`<ref>.<discriminator>=<value>`) it sits in. */
 export function stepsWithCases(data: PipelineData): { step: AnyStep; cases: string[] }[] {
   const out: { step: AnyStep; cases: string[] }[] = [];

package/src/harness/result.ts

@@ -53,10 +53,15 @@ export async function uploadFile(
   url: string,
   file: string,
   log: (line: string) => void,
-  opts: { attempts?: number; baseDelayMs?: number } = {},
+  opts: {
+    attempts?: number;
+    baseDelayMs?: number;
+    sleep?: (ms: number) => Promise<void>;
+  } = {},
 ): Promise<void> {
   const attempts = opts.attempts ?? 5;
   const base = opts.baseDelayMs ?? 1000;
+  const sleep = opts.sleep ?? ((ms: number) => Bun.sleep(ms));
   for (let i = 1; ; i++) {
     let status = 0;
     let detail: string;
@@ -73,7 +78,7 @@ export async function uploadFile(
     if (!transient || i >= attempts)
       throw new Error(`upload to ${redactUrl(url)} failed: ${detail}`);
     log(`upload to ${redactUrl(url)} failed (${detail}); retry ${i}/${attempts - 1}`);
-    await Bun.sleep(base * 2 ** (i - 1));
+    await sleep(base * 2 ** (i - 1));
   }
 }
 

package/src/util/tar.ts

@@ -20,7 +20,6 @@ import { createGzip } from 'node:zlib';
 
 export type TarEntry =
   | { path: string; type: 'file'; mode: number; source: string }
-  | { path: string; type: 'bytes'; mode: number; data: Uint8Array }
   | { path: string; type: 'dir'; mode: number }
   | { path: string; type: 'symlink'; target: string };
 
@@ -88,7 +87,6 @@ export function writeTar(out: string, entries: readonly TarEntry[]): void {
       const link = e.type === 'symlink' ? e.target : '';
       let size = 0;
       if (e.type === 'file') size = lstatSync(e.source).size;
-      else if (e.type === 'bytes') size = e.data.length;
       const pax: string[] = [];
       if (enc.encode(name).length > 100) pax.push(paxRecord('path', name));
       if (enc.encode(link).length > 100) pax.push(paxRecord('linkpath', link));
@@ -104,10 +102,7 @@ export function writeTar(out: string, entries: readonly TarEntry[]): void {
       write(
         header(shortName, size, mode, flag, link.length > 100 ? link.slice(0, 99) : link),
       );
-      if (e.type === 'bytes') {
-        write(e.data);
-        pad(size);
-      } else if (e.type === 'file') {
+      if (e.type === 'file') {
         const src = openSync(e.source, 'r');
         try {
           const buf = new Uint8Array(1 << 20);