@lchase/yak 0.5.0 → 0.6.1

package/dist/index.js

@@ -100,6 +100,16 @@ function inputNamesOf(step) {
 function dependenciesOf(step, producerOf) {
   return inputNamesOf(step).map((need) => producerOf.get(need)).filter((id) => id !== void 0);
 }
+function earlierProducerIds(step, posIndex, producerOf, order) {
+  const deps = [];
+  for (const name of inputNamesOf(step)) {
+    const producerId = producerOf.get(name);
+    if (producerId === void 0) continue;
+    const producerPos = order.get(producerId);
+    if (producerPos !== void 0 && producerPos < posIndex) deps.push(producerId);
+  }
+  return deps;
+}
 function flattenSteps(steps) {
   const result = [];
   for (const step of steps) {
@@ -191,6 +201,7 @@ var rawStepSchema = z2.lazy(
     loop: z2.object({
       until: rawExprSchema,
       budget: rawBudgetSchema,
+      concurrency: z2.number().int().positive().optional(),
       onExhausted: z2.enum(["suspend", "fail", "continue"]).optional(),
       freshContext: z2.boolean().optional(),
       body: z2.array(rawStepSchema)
@@ -281,6 +292,7 @@ function normalizeStep(raw) {
       body: raw.loop.body.map(normalizeStep),
       until: raw.loop.until,
       budget: raw.loop.budget,
+      concurrency: raw.loop.concurrency ?? 1,
       onExhausted: raw.loop.onExhausted ?? "suspend",
       freshContext: raw.loop.freshContext ?? true,
       ...raw.produces !== void 0 ? { produces: raw.produces } : {}
@@ -331,14 +343,76 @@ import { promisify } from "util";
 // src/expr/template.ts
 import jexl from "jexl";
 var PLACEHOLDER_RE = /\{\{\s*([^}]+?)\s*\}\}/g;
+var IDENTIFIER = /[A-Za-z_$][A-Za-z0-9_$]*/y;
+var NOT_A_ROOT = /* @__PURE__ */ new Set(["true", "false", "in", "null", "undefined"]);
 function extractTemplateRoots(template) {
-  const roots = /* @__PURE__ */ new Set();
+  const roots = [];
+  const seen = /* @__PURE__ */ new Set();
   for (const match of template.matchAll(PLACEHOLDER_RE)) {
-    const expr = match[1].trim();
-    const root = expr.split(/[.[]/, 1)[0];
-    if (root) roots.add(root);
+    for (const ident of identifiersInExpr(match[1])) {
+      if (seen.has(ident)) continue;
+      seen.add(ident);
+      roots.push(ident);
+    }
+  }
+  return roots;
+}
+function identifiersInExpr(expr) {
+  const found = [];
+  let prev = "other";
+  let i = 0;
+  while (i < expr.length) {
+    const c = expr[i];
+    if (c === " " || c === "	" || c === "\n" || c === "\r") {
+      i++;
+      continue;
+    }
+    if (c === '"' || c === "'") {
+      i = skipString(expr, i);
+      prev = "other";
+      continue;
+    }
+    if (c === "|" && expr[i + 1] === "|") {
+      i += 2;
+      prev = "other";
+      continue;
+    }
+    if (c === "|") {
+      i++;
+      prev = "pipe";
+      continue;
+    }
+    if (c === ".") {
+      i++;
+      prev = "dot";
+      continue;
+    }
+    IDENTIFIER.lastIndex = i;
+    const ident = IDENTIFIER.exec(expr);
+    if (ident && ident.index === i) {
+      const name = ident[0];
+      if (prev !== "dot" && prev !== "pipe" && !NOT_A_ROOT.has(name)) found.push(name);
+      i += name.length;
+      prev = "other";
+      continue;
+    }
+    i++;
+    prev = "other";
+  }
+  return found;
+}
+function skipString(expr, start) {
+  const quote = expr[start];
+  let i = start + 1;
+  while (i < expr.length) {
+    if (expr[i] === "\\") {
+      i += 2;
+      continue;
+    }
+    if (expr[i] === quote) return i + 1;
+    i++;
   }
-  return [...roots];
+  return i;
 }
 async function renderTemplate(template, inputs) {
   let result = "";
@@ -483,6 +557,7 @@ var KNOWN_AGENT_TOOLS = /* @__PURE__ */ new Set([
 async function validateWorkflow(workflow, cwd = process.cwd(), runIsolation = "none") {
   const flatSteps = flattenSteps(workflow.steps);
   checkKnownKinds(flatSteps);
+  checkLoopBodyKinds(workflow.steps);
   checkDuplicateIds(flatSteps);
   const producerOf = buildProducerMap(workflow.steps);
   checkReservedArtifactNames(flatSteps);
@@ -495,6 +570,7 @@ async function validateWorkflow(workflow, cwd = process.cwd(), runIsolation = "n
   await checkAgentPromptPlaceholders(flatSteps, cwd);
   checkSessionChainDepth(flatSteps);
   checkMapIsolation(workflow.steps, runIsolation);
+  checkLoopParallelWriters(workflow.steps);
   await checkGateSchemaKeys(flatSteps, cwd);
   await checkGateRenderPlaceholders(flatSteps, cwd);
   checkSandboxImage(flatSteps);
@@ -547,6 +623,83 @@ function checkMapIsolation(steps, runIsolation) {
     checkMapIsolation([step.step], runIsolation);
   }
 }
+var UNSUPPORTED_LOOP_BODY_KINDS = /* @__PURE__ */ new Set(["map", "loop", "gate"]);
+function checkLoopBodyKinds(steps) {
+  for (const step of steps) {
+    if (step.kind === "map") {
+      checkLoopBodyKinds([step.step]);
+      continue;
+    }
+    if (step.kind !== "loop") continue;
+    for (const bodyStep of step.body) {
+      if (UNSUPPORTED_LOOP_BODY_KINDS.has(bodyStep.kind)) {
+        throw new WorkflowValidationError(
+          `step "${bodyStep.id}": kind "${bodyStep.kind}" not supported inside a loop body in M3`
+        );
+      }
+    }
+    checkLoopBodyKinds(step.body);
+  }
+}
+function checkLoopParallelWriters(steps) {
+  for (const step of steps) {
+    if (step.kind === "loop") {
+      warnParallelWriteCapableBody(step);
+      checkLoopParallelWriters(step.body);
+    } else if (step.kind === "map") {
+      checkLoopParallelWriters([step.step]);
+    }
+  }
+}
+function writeCapableTool(step) {
+  if (step.kind !== "agent") return void 0;
+  return (step.tools ?? []).find((tool) => WRITE_CAPABLE_TOOLS.has(tool));
+}
+function waitsOn(step, targetId, stepsById, producerOf, order) {
+  const seen = /* @__PURE__ */ new Set();
+  const startPos = order.get(step.id);
+  if (startPos === void 0) return false;
+  const stack = earlierProducerIds(step, startPos, producerOf, order);
+  while (stack.length > 0) {
+    const id = stack.pop();
+    if (id === void 0 || seen.has(id)) continue;
+    if (id === targetId) return true;
+    seen.add(id);
+    const pred = stepsById.get(id);
+    const predPos = order.get(id);
+    if (!pred || predPos === void 0) continue;
+    stack.push(...earlierProducerIds(pred, predPos, producerOf, order));
+  }
+  return false;
+}
+function warnParallelWriteCapableBody(loop) {
+  const concurrency = loop.concurrency ?? 1;
+  if (concurrency <= 1 || loop.body.length < 2) return;
+  const order = new Map(loop.body.map((bodyStep, index) => [bodyStep.id, index]));
+  const stepsById = new Map(loop.body.map((bodyStep) => [bodyStep.id, bodyStep]));
+  const producerOf = buildProducerMap(loop.body);
+  const warned = /* @__PURE__ */ new Set();
+  for (let i = 0; i < loop.body.length; i++) {
+    for (let j = i + 1; j < loop.body.length; j++) {
+      const earlier = loop.body[i];
+      const later = loop.body[j];
+      if (!earlier || !later) continue;
+      if (waitsOn(later, earlier.id, stepsById, producerOf, order)) continue;
+      for (const bodyStep of [earlier, later]) {
+        if (warned.has(bodyStep.id)) continue;
+        const tool = writeCapableTool(bodyStep);
+        const command = bodyStep.kind === "command";
+        if (!tool && !command) continue;
+        const why = command ? "is a command step" : `declares write-capable tool "${tool}"`;
+        const fix = command ? "use concurrency: 1" : "use concurrency: 1 or drop the write-capable tool";
+        console.warn(
+          `step "${loop.id}": concurrency ${concurrency} runs body steps "${earlier.id}" and "${later.id}" in parallel, and "${bodyStep.id}" ${why} \u2014 concurrent body steps share one cwd and may race; ${fix}`
+        );
+        warned.add(bodyStep.id);
+      }
+    }
+  }
+}
 function checkAgentToolNames(steps) {
   for (const step of steps) {
     if (step.kind !== "agent") continue;
@@ -1038,7 +1191,7 @@ async function createWorktree(repoRoot, branchName, baseRef, worktreePath) {
 
 // src/engine/scheduler.ts
 import path14 from "path";
-import pLimit2 from "p-limit";
+import pLimit3 from "p-limit";
 import { z as z8 } from "zod";
 
 // src/adapters/claude-code.ts
@@ -1534,6 +1687,8 @@ function runCommandStep(step, cwd, extraEnv) {
 }
 
 // src/steps/loop.ts
+import assert from "assert/strict";
+import pLimit from "p-limit";
 import { z as z6 } from "zod";
 
 // src/engine/cache.ts
@@ -1709,22 +1864,18 @@ async function runLoopBody(step, ctx, resumeFrom) {
     const thisValues = /* @__PURE__ */ new Map();
     const thisHashes = /* @__PURE__ */ new Map();
     try {
-      for (const [posIndex, bodyStep] of step.body.entries()) {
-        await runOneBodyStep({
-          bodyStep,
-          posIndex,
-          iteration,
-          ctx,
-          localProducerOf,
-          bodyOrderIndex,
-          priorValues,
-          priorHashes,
-          thisValues,
-          thisHashes,
-          freshContext: step.freshContext ?? true,
-          bodySessionIds
-        });
-      }
+      await runLoopRound({
+        step,
+        iteration,
+        ctx,
+        localProducerOf,
+        bodyOrderIndex,
+        priorValues,
+        priorHashes,
+        thisValues,
+        thisHashes,
+        bodySessionIds
+      });
     } catch (err) {
       if (err instanceof CommandStepFailedError || err instanceof AgentStepFailedError) {
         await appendJournalEvent(ctx.runDir, ctx.runId, {
@@ -1808,6 +1959,80 @@ async function handleExhausted(step, ctx, iteration, tripped) {
   await writeLoopExhaustedRequest(ctx.runDir, ctx.runId, step.id, iteration, tripped);
   return "suspended";
 }
+async function runLoopRound(args) {
+  const {
+    step,
+    iteration,
+    ctx,
+    localProducerOf,
+    bodyOrderIndex,
+    priorValues,
+    priorHashes,
+    thisValues,
+    thisHashes,
+    bodySessionIds
+  } = args;
+  const cap = Math.min(step.concurrency ?? 1, ctx.globalConcurrency);
+  const limit = pLimit(cap);
+  const remaining = new Set(step.body.map((bodyStep) => bodyStep.id));
+  const completed = /* @__PURE__ */ new Set();
+  const inFlight = /* @__PURE__ */ new Map();
+  let firstError;
+  const thisRoundDeps = (bodyStep, posIndex) => earlierProducerIds(bodyStep, posIndex, localProducerOf, bodyOrderIndex);
+  const launchEligible = () => {
+    if (firstError !== void 0) return;
+    for (const [posIndex, bodyStep] of step.body.entries()) {
+      if (!remaining.has(bodyStep.id) || inFlight.has(bodyStep.id)) continue;
+      const ready = thisRoundDeps(bodyStep, posIndex).every((dep) => completed.has(dep));
+      if (!ready) {
+        if (cap === 1) break;
+        continue;
+      }
+      inFlight.set(
+        bodyStep.id,
+        limit(async () => {
+          if (firstError !== void 0) return "skipped";
+          try {
+            await runOneBodyStep({
+              bodyStep,
+              posIndex,
+              iteration,
+              ctx,
+              localProducerOf,
+              bodyOrderIndex,
+              priorValues,
+              priorHashes,
+              thisValues,
+              thisHashes,
+              freshContext: step.freshContext ?? true,
+              bodySessionIds
+            });
+            return "ok";
+          } catch (err) {
+            if (firstError === void 0) firstError = err;
+            return "error";
+          }
+        })
+      );
+    }
+  };
+  launchEligible();
+  while (inFlight.size > 0) {
+    const [doneId, status] = await Promise.race(
+      [...inFlight.entries()].map(async ([id, p]) => [id, await p])
+    );
+    inFlight.delete(doneId);
+    remaining.delete(doneId);
+    if (status === "ok") completed.add(doneId);
+    launchEligible();
+  }
+  if (firstError !== void 0) throw firstError;
+  assert.equal(
+    remaining.size,
+    0,
+    `loop "${step.id}" stalled: body step(s) [${[...remaining].join(", ")}] never became eligible`
+  );
+}
 async function runOneBodyStep(args) {
   const {
     bodyStep,
@@ -1923,7 +2148,7 @@ async function runBodyStepKind(bodyStep, inputs, ctx, iteration, freshContext, b
 
 // src/steps/map.ts
 import path13 from "path";
-import pLimit from "p-limit";
+import pLimit2 from "p-limit";
 import { z as z7 } from "zod";
 var RETRY_ATTEMPTS = 2;
 async function runMapStep(step, ctx) {
@@ -1937,7 +2162,7 @@ async function runMapStep(step, ctx) {
     await commitAllIfDirty(ctx.cwd, `yak: checkpoint before map ${step.id}`);
   }
   const items = await readArtifactRaw(ctx.runDir, step.over);
-  const limit = pLimit(Math.min(step.concurrency ?? 4, ctx.globalConcurrency));
+  const limit = pLimit2(Math.min(step.concurrency ?? 4, ctx.globalConcurrency));
   const results = new Array(items.length);
   let anyFailed = false;
   await Promise.all(
@@ -2142,7 +2367,7 @@ function trustResumedSteps(remaining, producerOf, resumeState, completed, artifa
   }
 }
 async function runEligibleSteps(workflow, ctx, resumeState) {
-  const limit = pLimit2(ctx.concurrency ?? DEFAULT_CONCURRENCY);
+  const limit = pLimit3(ctx.concurrency ?? DEFAULT_CONCURRENCY);
   const producerOf = buildProducerMap(workflow.steps);
   const remaining = new Map(workflow.steps.map((s) => [s.id, s]));
   const completed = /* @__PURE__ */ new Set();
@@ -2208,6 +2433,7 @@ async function runStep(step, ctx, artifactHashes, workflowName, agentSessionIds)
       semanticKey: semanticKey2,
       definitionKey: definitionKey2,
       outerArtifactHashes: artifactHashes,
+      globalConcurrency: ctx.concurrency ?? DEFAULT_CONCURRENCY,
       buildAdapter: (step2, iteration) => buildAdapter(ctx.adapter, ctx, workflowName, step2, iteration),
       loopContinuation: ctx.loopContinuations?.get(step.id)
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lchase/yak",
-  "version": "0.5.0",
+  "version": "0.6.1",
   "description": "An agentic workflow engine",
   "license": "MIT",
   "type": "module",