@davidorex/pi-workflows 0.1.0

package/src/dag.test.ts

@@ -0,0 +1,350 @@
+import { describe, it } from "node:test";
+import assert from "node:assert";
+import { extractDependencies, buildExecutionPlan, isSequential } from "./dag.ts";
+import type { StepSpec } from "./types.ts";
+import { makeSpec as makeSpecFull } from "./test-helpers.ts";
+
+/**
+ * Helper: build a minimal WorkflowSpec from a steps object.
+ */
+function makeSpec(steps: Record<string, StepSpec>) {
+  return makeSpecFull({ steps });
+}
+
+describe("extractDependencies", () => {
+  it("returns empty deps for steps with no expressions", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: { agent: "bar" },
+    });
+    const deps = extractDependencies(spec);
+    assert.strictEqual(deps.get("a")!.size, 0);
+    assert.strictEqual(deps.get("b")!.size, 0);
+  });
+
+  it("extracts deps from input expressions", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: {
+        agent: "bar",
+        input: {
+          context: "${{ steps.a.textOutput }}",
+        },
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.strictEqual(deps.get("a")!.size, 0);
+    assert.ok(deps.get("b")!.has("a"));
+  });
+
+  it("extracts deps from when condition", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: {
+        agent: "bar",
+        when: "${{ steps.a.output.shouldRun }}",
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.ok(deps.get("b")!.has("a"));
+  });
+
+  it("extracts deps from bare when condition (no ${{ }})", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: {
+        agent: "bar",
+        when: "steps.a.output.shouldRun",
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.ok(deps.get("b")!.has("a"));
+  });
+
+  it("extracts deps from gate check", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: {
+        gate: { check: "${{ steps.a.output.reproCommand }}" },
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.ok(deps.get("b")!.has("a"));
+  });
+
+  it("extracts deps from transform mapping", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: { agent: "bar" },
+      c: {
+        transform: {
+          mapping: {
+            fromA: "${{ steps.a.textOutput }}",
+            fromB: "${{ steps.b.output.data }}",
+          },
+        },
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.ok(deps.get("c")!.has("a"));
+    assert.ok(deps.get("c")!.has("b"));
+  });
+
+  it("extracts multiple deps from a single expression string", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: { agent: "bar" },
+      c: {
+        agent: "baz",
+        input: {
+          combined: "${{ steps.a.textOutput }}\n\n${{ steps.b.textOutput }}",
+        },
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.ok(deps.get("c")!.has("a"));
+    assert.ok(deps.get("c")!.has("b"));
+  });
+
+  it("extracts deps from nested input objects", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: {
+        agent: "bar",
+        input: {
+          nested: {
+            deep: {
+              value: "${{ steps.a.output.x }}",
+            },
+          },
+        },
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.ok(deps.get("b")!.has("a"));
+  });
+
+  it("extracts deps from input arrays", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: {
+        agent: "bar",
+        input: {
+          items: ["${{ steps.a.output.x }}", "static"],
+        },
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.ok(deps.get("b")!.has("a"));
+  });
+
+  it("ignores input-only references (no step deps)", () => {
+    const spec = makeSpec({
+      a: {
+        agent: "foo",
+        input: {
+          path: "${{ input.path }}",
+        },
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.strictEqual(deps.get("a")!.size, 0);
+  });
+
+  it("ignores self-references", () => {
+    const spec = makeSpec({
+      a: {
+        agent: "foo",
+        input: {
+          self: "${{ steps.a.output }}",
+        },
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.strictEqual(deps.get("a")!.size, 0);
+  });
+
+  it("ignores references to non-existent steps", () => {
+    const spec = makeSpec({
+      a: {
+        agent: "foo",
+        input: {
+          bad: "${{ steps.nonexistent.output }}",
+        },
+      },
+    });
+    const deps = extractDependencies(spec);
+    assert.strictEqual(deps.get("a")!.size, 0);
+  });
+
+  it("does not descend into loop sub-steps", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: {
+        loop: {
+          maxAttempts: 3,
+          steps: {
+            inner: {
+              agent: "bar",
+              input: { ctx: "${{ steps.a.textOutput }}" },
+            },
+          },
+        },
+      },
+    });
+    const deps = extractDependencies(spec);
+    // The loop step itself should depend on 'a' via its sub-step's expression,
+    // BUT per the design: we do NOT descend into loop sub-steps for top-level DAG.
+    // The loop step's own `input` or `when` would declare its real deps.
+    assert.strictEqual(deps.get("b")!.size, 0);
+  });
+});
+
+describe("buildExecutionPlan", () => {
+  it("fully sequential: linear chain", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: {
+        agent: "bar",
+        input: { ctx: "${{ steps.a.textOutput }}" },
+      },
+      c: {
+        agent: "baz",
+        input: { ctx: "${{ steps.b.output }}" },
+      },
+    });
+    const plan = buildExecutionPlan(spec);
+    assert.strictEqual(plan.length, 3);
+    assert.deepStrictEqual(plan[0].steps, ["a"]);
+    assert.deepStrictEqual(plan[1].steps, ["b"]);
+    assert.deepStrictEqual(plan[2].steps, ["c"]);
+    assert.ok(isSequential(plan));
+  });
+
+  it("fully parallel: no dependencies", () => {
+    const spec = makeSpec({
+      a: { agent: "foo", input: { x: "${{ input.path }}" } },
+      b: { agent: "bar", input: { x: "${{ input.path }}" } },
+      c: { agent: "baz", input: { x: "${{ input.path }}" } },
+    });
+    const plan = buildExecutionPlan(spec);
+    assert.strictEqual(plan.length, 1);
+    assert.strictEqual(plan[0].steps.length, 3);
+    assert.ok(plan[0].steps.includes("a"));
+    assert.ok(plan[0].steps.includes("b"));
+    assert.ok(plan[0].steps.includes("c"));
+    assert.ok(!isSequential(plan));
+  });
+
+  it("fan-out then merge: diamond pattern", () => {
+    const spec = makeSpec({
+      source: { agent: "foo" },
+      analyzerA: {
+        agent: "bar",
+        input: { ctx: "${{ steps.source.textOutput }}" },
+      },
+      analyzerB: {
+        agent: "baz",
+        input: { ctx: "${{ steps.source.textOutput }}" },
+      },
+      merge: {
+        transform: {
+          mapping: {
+            a: "${{ steps.analyzerA.output }}",
+            b: "${{ steps.analyzerB.output }}",
+          },
+        },
+      },
+    });
+    const plan = buildExecutionPlan(spec);
+    assert.strictEqual(plan.length, 3);
+    assert.deepStrictEqual(plan[0].steps, ["source"]);
+    assert.strictEqual(plan[1].steps.length, 2);
+    assert.ok(plan[1].steps.includes("analyzerA"));
+    assert.ok(plan[1].steps.includes("analyzerB"));
+    assert.deepStrictEqual(plan[2].steps, ["merge"]);
+  });
+
+  it("mixed: some parallel, some sequential", () => {
+    const spec = makeSpec({
+      a: { agent: "foo" },
+      b: { agent: "bar" },
+      c: {
+        agent: "baz",
+        input: { ctx: "${{ steps.a.textOutput }}" },
+      },
+      d: {
+        transform: {
+          mapping: {
+            b: "${{ steps.b.output }}",
+            c: "${{ steps.c.output }}",
+          },
+        },
+      },
+    });
+    const plan = buildExecutionPlan(spec);
+    assert.strictEqual(plan.length, 3);
+    // Layer 0: a, b (both independent)
+    assert.strictEqual(plan[0].steps.length, 2);
+    assert.ok(plan[0].steps.includes("a"));
+    assert.ok(plan[0].steps.includes("b"));
+    // Layer 1: c (depends on a)
+    assert.deepStrictEqual(plan[1].steps, ["c"]);
+    // Layer 2: d (depends on b and c)
+    assert.deepStrictEqual(plan[2].steps, ["d"]);
+  });
+
+  it("detects cycle", () => {
+    const spec = makeSpec({
+      a: {
+        agent: "foo",
+        input: { ctx: "${{ steps.b.output }}" },
+      },
+      b: {
+        agent: "bar",
+        input: { ctx: "${{ steps.a.output }}" },
+      },
+    });
+    assert.throws(
+      () => buildExecutionPlan(spec),
+      (err: Error) => err.message.includes("cycle"),
+    );
+  });
+
+  it("single step", () => {
+    const spec = makeSpec({
+      only: { agent: "foo" },
+    });
+    const plan = buildExecutionPlan(spec);
+    assert.strictEqual(plan.length, 1);
+    assert.deepStrictEqual(plan[0].steps, ["only"]);
+    assert.ok(isSequential(plan));
+  });
+
+  it("preserves YAML order within layers", () => {
+    // Steps with no deps should appear in their YAML declaration order
+    const spec = makeSpec({
+      z: { agent: "foo" },
+      a: { agent: "bar" },
+      m: { agent: "baz" },
+    });
+    const plan = buildExecutionPlan(spec);
+    assert.strictEqual(plan.length, 1);
+    // Should preserve insertion order: z, a, m
+    assert.deepStrictEqual(plan[0].steps, ["z", "a", "m"]);
+  });
+
+  it("step with when referencing another step", () => {
+    const spec = makeSpec({
+      check: { gate: { check: "echo ok" } },
+      conditional: {
+        agent: "foo",
+        when: "${{ steps.check.output.passed }}",
+      },
+    });
+    const plan = buildExecutionPlan(spec);
+    assert.strictEqual(plan.length, 2);
+    assert.deepStrictEqual(plan[0].steps, ["check"]);
+    assert.deepStrictEqual(plan[1].steps, ["conditional"]);
+  });
+});

package/src/dag.ts

@@ -0,0 +1,219 @@
+/**
+ * DAG dependency analysis and execution planning for workflow steps.
+ * Extracts `${{ steps.X }}` references, builds a dependency graph,
+ * and produces layered execution plans for parallel dispatch.
+ */
+import type { WorkflowSpec, StepSpec } from "./types.ts";
+
+/**
+ * A single layer in the execution plan.
+ * All steps in a layer are independent and can run concurrently.
+ */
+export interface ExecutionLayer {
+  steps: string[];  // step names in this layer
+}
+
+/**
+ * The full execution plan: an ordered list of layers.
+ * Steps in the same layer have no mutual dependencies.
+ * Steps in layer N depend only on steps in layers 0..N-1.
+ */
+export type ExecutionPlan = ExecutionLayer[];
+
+/**
+ * Extract all step dependencies from a workflow spec.
+ *
+ * Scans all expression-bearing fields in each step:
+ * - `input` values (recursive — expressions can be nested in objects/arrays)
+ * - `when` condition
+ * - `gate.check` (may contain ${{ }})
+ * - `transform.mapping` values (recursive)
+ * - `loop.attempts` (may be an expression)
+ *
+ * Does NOT descend into loop sub-steps — a loop step's internal steps
+ * are the loop's own concern, not part of the top-level DAG.
+ *
+ * Returns a map: stepName → Set of step names it depends on.
+ */
+export function extractDependencies(spec: WorkflowSpec): Map<string, Set<string>> {
+  const stepNames = new Set(Object.keys(spec.steps));
+  const deps = new Map<string, Set<string>>();
+
+  for (const [name, step] of Object.entries(spec.steps)) {
+    const stepDeps = new Set<string>();
+
+    // Collect all expression strings from this step
+    const expressions = collectExpressions(step);
+
+    // Parse each expression for `steps.<name>` references
+    for (const expr of expressions) {
+      const referenced = extractStepReferences(expr, stepNames);
+      for (const ref of referenced) {
+        if (ref !== name) {  // no self-dependencies
+          stepDeps.add(ref);
+        }
+      }
+    }
+
+    deps.set(name, stepDeps);
+  }
+
+  return deps;
+}
+
+/**
+ * Collect all raw expression strings from a step spec.
+ * Walks `input`, `when`, `gate.check`, `transform.mapping`, `loop.attempts`.
+ */
+function collectExpressions(step: StepSpec): string[] {
+  const exprs: string[] = [];
+
+  // input values (recursive)
+  if (step.input) {
+    collectExpressionsFromValue(step.input, exprs);
+  }
+
+  // when condition
+  if (step.when) {
+    exprs.push(step.when);
+  }
+
+  // gate check
+  if (step.gate) {
+    collectExpressionsFromValue(step.gate.check, exprs);
+  }
+
+  // transform mapping (recursive)
+  if (step.transform) {
+    collectExpressionsFromValue(step.transform.mapping, exprs);
+  }
+
+  // loop attempts expression
+  if (step.loop?.attempts) {
+    exprs.push(step.loop.attempts);
+  }
+
+  // forEach expression
+  if (step.forEach) {
+    collectExpressionsFromValue(step.forEach, exprs);
+  }
+
+  // command expression (may contain ${{ }} references)
+  if (step.command) {
+    collectExpressionsFromValue(step.command, exprs);
+  }
+
+  return exprs;
+}
+
+/**
+ * Recursively collect expression strings from a value.
+ * Walks objects, arrays, and strings looking for ${{ }} patterns.
+ */
+function collectExpressionsFromValue(value: unknown, exprs: string[]): void {
+  if (typeof value === "string") {
+    // Extract all ${{ ... }} patterns from the string
+    const regex = /\$\{\{([^}]+)\}\}/g;
+    let match: RegExpExecArray | null;
+    while ((match = regex.exec(value)) !== null) {
+      exprs.push(match[1].trim());
+    }
+    // Also handle bare expressions (no ${{ }} wrapper, e.g. in `when`)
+    if (!value.includes("${{") && value.includes("steps.")) {
+      exprs.push(value);
+    }
+  } else if (Array.isArray(value)) {
+    for (const item of value) {
+      collectExpressionsFromValue(item, exprs);
+    }
+  } else if (value !== null && typeof value === "object") {
+    for (const v of Object.values(value)) {
+      collectExpressionsFromValue(v, exprs);
+    }
+  }
+}
+
+/**
+ * Extract step name references from an expression string.
+ *
+ * Looks for patterns like:
+ * - `steps.diagnose.output`
+ * - `steps.diagnose.textOutput`
+ * - `steps.diagnose.status`
+ *
+ * Returns the set of step names referenced.
+ */
+function extractStepReferences(expr: string, validStepNames: Set<string>): Set<string> {
+  const refs = new Set<string>();
+  // Match `steps.<name>` — name is a word (alphanumeric + underscore + hyphen)
+  const regex = /steps\.([a-zA-Z_][\w-]*)/g;
+  let match: RegExpExecArray | null;
+  while ((match = regex.exec(expr)) !== null) {
+    const name = match[1];
+    if (validStepNames.has(name)) {
+      refs.add(name);
+    }
+  }
+  return refs;
+}
+
+/**
+ * Build an execution plan from a pre-computed dependency map.
+ * Performs topological sort, grouping independent steps into layers.
+ * Throws if the dependency graph contains a cycle.
+ */
+export function buildPlanFromDeps(
+  allSteps: string[],
+  deps: Map<string, Set<string>>,
+): ExecutionPlan {
+  const plan: ExecutionPlan = [];
+  const placed = new Set<string>();
+
+  while (placed.size < allSteps.length) {
+    const layer: string[] = [];
+
+    for (const step of allSteps) {
+      if (placed.has(step)) continue;
+
+      const stepDeps = deps.get(step) ?? new Set();
+      if ([...stepDeps].every((d) => placed.has(d))) {
+        layer.push(step);
+      }
+    }
+
+    if (layer.length === 0) {
+      const remaining = allSteps.filter((s) => !placed.has(s));
+      throw new Error(
+        `Dependency cycle detected among steps: ${remaining.join(", ")}`,
+      );
+    }
+
+    plan.push({ steps: layer });
+    for (const s of layer) {
+      placed.add(s);
+    }
+  }
+
+  return plan;
+}
+
+/**
+ * Build an execution plan from a workflow spec.
+ *
+ * Performs topological sort, grouping independent steps into layers.
+ * Steps within a layer can execute concurrently.
+ *
+ * Throws if the dependency graph contains a cycle.
+ */
+export function buildExecutionPlan(spec: WorkflowSpec): ExecutionPlan {
+  const deps = extractDependencies(spec);
+  return buildPlanFromDeps(Object.keys(spec.steps), deps);
+}
+
+/**
+ * Check if an execution plan is fully sequential
+ * (every layer has exactly one step).
+ */
+export function isSequential(plan: ExecutionPlan): boolean {
+  return plan.every((layer) => layer.steps.length === 1);
+}