@agwab/pi-workflow 0.2.1 → 0.4.0

package/dist/engine-run-graph.js

@@ -92,6 +92,118 @@ export function reconcileDynamicGeneratedRunRecords(cwd, run, compiledFlow) {
     }
     return changed;
 }
+export function reconcileForeachGeneratedRunRecords(cwd, run, compiledFlow) {
+    let changed = false;
+    const compiledSpecIds = new Set(compiledFlow.tasks.map((task) => compiledTaskSpecId(task)));
+    const compiledTaskBySpecId = new Map(compiledFlow.tasks.map((task) => [compiledTaskSpecId(task), task]));
+    const placeholderToGeneratedSpecIds = new Map();
+    const streamingPlaceholderSpecIds = new Set();
+    for (const compiledTask of compiledFlow.tasks) {
+        const specId = compiledTaskSpecId(compiledTask);
+        if (compiledTask.foreach && foreachStreamingEnabled(compiledTask)) {
+            streamingPlaceholderSpecIds.add(specId);
+        }
+        const placeholderSpecId = foreachGeneratedPlaceholderSpecId(compiledTask, compiledFlow, specId);
+        if (!placeholderSpecId)
+            continue;
+        if (compiledTask.foreachGenerated?.placeholderSpecId !== placeholderSpecId) {
+            compiledTask.foreachGenerated = { placeholderSpecId };
+            changed = true;
+        }
+        const generated = placeholderToGeneratedSpecIds.get(placeholderSpecId) ?? [];
+        generated.push(specId);
+        placeholderToGeneratedSpecIds.set(placeholderSpecId, generated);
+    }
+    if (placeholderToGeneratedSpecIds.size === 0)
+        return changed;
+    const filteredRunTasks = [];
+    const seenGeneratedSpecIds = new Set();
+    for (const task of run.tasks) {
+        const generatedSpecIds = placeholderToGeneratedSpecIds.get(task.specId);
+        let placeholderSpecId = foreachGeneratedPlaceholderSpecId(task, compiledFlow, task.specId);
+        if (generatedSpecIds && !placeholderSpecId) {
+            const compiledTask = compiledTaskBySpecId.get(task.specId);
+            if (compiledTask?.foreach &&
+                streamingPlaceholderSpecIds.has(task.specId)) {
+                filteredRunTasks.push(task);
+                continue;
+            }
+            if (generatedSpecIds.includes(task.specId)) {
+                placeholderSpecId = task.specId;
+                task.foreachGenerated = { placeholderSpecId };
+                changed = true;
+            }
+            else {
+                changed = true;
+                continue;
+            }
+        }
+        if (placeholderSpecId && !compiledSpecIds.has(task.specId)) {
+            changed = true;
+            continue;
+        }
+        if (placeholderSpecId && seenGeneratedSpecIds.has(task.specId)) {
+            changed = true;
+            continue;
+        }
+        if (placeholderSpecId) {
+            seenGeneratedSpecIds.add(task.specId);
+            if (task.foreachGenerated?.placeholderSpecId !== placeholderSpecId) {
+                task.foreachGenerated = { placeholderSpecId };
+                changed = true;
+            }
+        }
+        filteredRunTasks.push(task);
+    }
+    const runTaskBySpecId = new Map();
+    for (const task of filteredRunTasks) {
+        if (!runTaskBySpecId.has(task.specId))
+            runTaskBySpecId.set(task.specId, task);
+    }
+    const reordered = [];
+    const usedSpecIds = new Set();
+    let nextIndex = nextTaskRecordIndex({ ...run, tasks: filteredRunTasks });
+    for (const compiledTask of compiledFlow.tasks) {
+        const specId = compiledTaskSpecId(compiledTask);
+        const existing = runTaskBySpecId.get(specId);
+        if (existing) {
+            const placeholderSpecId = compiledTask.foreachGenerated?.placeholderSpecId;
+            if (placeholderSpecId &&
+                existing.foreachGenerated?.placeholderSpecId !== placeholderSpecId) {
+                existing.foreachGenerated = { placeholderSpecId };
+                changed = true;
+            }
+            reordered.push(existing);
+            usedSpecIds.add(specId);
+            continue;
+        }
+        if (!compiledTask.foreachGenerated)
+            continue;
+        const created = createTaskRunRecord(cwd, run.runId, compiledTask, nextIndex);
+        nextIndex += 1;
+        reordered.push(created);
+        usedSpecIds.add(specId);
+        changed = true;
+    }
+    for (const task of filteredRunTasks) {
+        if (!usedSpecIds.has(task.specId))
+            reordered.push(task);
+    }
+    if (!sameTaskRecordOrder(run.tasks, reordered))
+        changed = true;
+    for (const task of reordered) {
+        if (!task.dependsOn)
+            continue;
+        const replaced = replaceForeachGeneratedDependencies(task.dependsOn, placeholderToGeneratedSpecIds, streamingPlaceholderSpecIds);
+        if (!sameStringList(task.dependsOn, replaced)) {
+            task.dependsOn = replaced;
+            changed = true;
+        }
+    }
+    if (changed)
+        run.tasks = reordered;
+    return changed;
+}
 export function assertRunTaskPositionalAlignment(run, compiledFlow) {
     const maxLength = Math.max(run.tasks.length, compiledFlow.tasks.length);
     for (let index = 0; index < maxLength; index += 1) {
@@ -141,6 +253,43 @@ export function compiledTaskSpecId(task) {
     const specId = task.specId;
     return typeof specId === "string" && specId.trim() !== "" ? specId : task.id;
 }
+function foreachGeneratedPlaceholderSpecId(task, compiledFlow, specId) {
+    const explicit = task.foreachGenerated?.placeholderSpecId;
+    if (typeof explicit === "string" && explicit.trim() !== "")
+        return explicit;
+    if (task.foreach)
+        return undefined;
+    if (task.kind !== "foreach" || !task.stageId)
+        return undefined;
+    const placeholderSpecId = foreachPlaceholderSpecId(compiledFlow, task.stageId);
+    if (!placeholderSpecId || specId === placeholderSpecId)
+        return undefined;
+    return placeholderSpecId;
+}
+function foreachPlaceholderSpecId(compiledFlow, stageId) {
+    const stage = (compiledFlow.stages ?? []).find((candidate) => candidate?.id === stageId);
+    if (stage?.type !== "foreach")
+        return undefined;
+    return `${stageId}.item`;
+}
+function replaceForeachGeneratedDependencies(dependsOn, placeholderToGeneratedSpecIds, keepPlaceholderSpecIds = new Set()) {
+    const replaced = [];
+    for (const dep of dependsOn) {
+        const generatedSpecIds = placeholderToGeneratedSpecIds.get(dep);
+        if (generatedSpecIds) {
+            if (keepPlaceholderSpecIds.has(dep))
+                replaced.push(dep);
+            replaced.push(...generatedSpecIds);
+        }
+        else
+            replaced.push(dep);
+    }
+    return [...new Set(replaced)];
+}
+function sameStringList(left, right) {
+    return (left.length === right.length &&
+        left.every((value, index) => value === right[index]));
+}
 function isLoopGeneratedCompiledTask(task, loopIds) {
     return Boolean((task.loopChild?.loopId && loopIds.has(task.loopChild.loopId)) ||
         (task.loopExhausted?.loopId && loopIds.has(task.loopExhausted.loopId)));
@@ -206,6 +355,29 @@ export function dependenciesReady(compiledTask, bySpecId, compiledFlow) {
     if (deps.length === 0)
         return true;
     const partial = stageSourcePolicy(compiledFlow, compiledTask.stageId ?? "") === "partial";
+    if (foreachStreamingEnabled(compiledTask)) {
+        let completedDependencyReady = false;
+        let runningDependencyMayHavePartialItems = false;
+        let allKnownDependenciesTerminal = true;
+        for (const dep of deps) {
+            const status = bySpecId.get(dep)?.status;
+            if (status === "completed") {
+                completedDependencyReady = true;
+                continue;
+            }
+            if (status && isTerminalTaskStatus(status)) {
+                if (!partial)
+                    return false;
+                continue;
+            }
+            if (status === "running")
+                runningDependencyMayHavePartialItems = true;
+            allKnownDependenciesTerminal = false;
+        }
+        return (completedDependencyReady ||
+            runningDependencyMayHavePartialItems ||
+            allKnownDependenciesTerminal);
+    }
     return deps.every((dep) => {
         const status = bySpecId.get(dep)?.status;
         if (status === "completed")
@@ -215,6 +387,18 @@ export function dependenciesReady(compiledTask, bySpecId, compiledFlow) {
         return false;
     });
 }
+export function foreachStreamingEnabled(compiledTask) {
+    const streaming = compiledTask.foreach?.from?.streaming;
+    return Boolean(streaming &&
+        typeof streaming === "object" &&
+        streaming.enabled === true);
+}
+export function foreachStreamingMinChunk(compiledTask) {
+    const value = compiledTask.foreach?.from?.streaming?.minChunk;
+    return typeof value === "number" && Number.isFinite(value) && value > 0
+        ? Math.floor(value)
+        : 1;
+}
 export function buildForeachGeneratedTasks(template, runtimeTask, items) {
     const seen = new Set();
     const tasks = [];
@@ -228,14 +412,15 @@ export function buildForeachGeneratedTasks(template, runtimeTask, items) {
         seen.add(taskId);
         const specId = `${template.stageId}.${taskId}`;
         const itemText = formatForeachItem(item);
-        const instructions = template.foreach.prompt.replace(/\$\{item\}/g, itemText);
+        const instructions = template.foreach.prompt.replace(/\$\{item\}/g, escapeReplacementText(itemText));
         const compiledPrompt = [
             template.foreach.injectRuntimeTask && runtimeTask
                 ? `# Task\n\n${runtimeTask}`
                 : undefined,
-            `# Workflow Stage\n\nstage=${template.stageId}\ntype=foreach\nitem=${taskId}`,
-            `# Instructions\n\n${instructions}`,
+            `# Workflow Stage\n\nstage=${template.stageId}\ntype=foreach`,
             template.foreach.roleText || undefined,
+            `# Workflow Item\n\nitem=${taskId}`,
+            `# Instructions\n\n${instructions}`,
         ]
             .filter(Boolean)
             .join("\n\n");
@@ -249,6 +434,7 @@ export function buildForeachGeneratedTasks(template, runtimeTask, items) {
             compiledPrompt,
             dependsOn: [...(template.dependsOn ?? [])],
             foreach: undefined,
+            foreachGenerated: { placeholderSpecId: template.id },
         });
     }
     return { tasks };
@@ -274,6 +460,9 @@ export function sanitizeTaskId(value) {
 function formatForeachItem(item) {
     return typeof item === "string" ? item : JSON.stringify(item);
 }
+function escapeReplacementText(value) {
+    return value.replace(/\$/g, "$$$$");
+}
 export function sourceStageIdsForFrom(from) {
     if (Array.isArray(from))
         return from.filter((item) => typeof item === "string");
@@ -321,7 +510,8 @@ export function markDagDependentsSkipped(run, compiledFlow) {
                 const status = bySpecId.get(dep)?.status;
                 return (status === "failed" ||
                     status === "interrupted" ||
-                    status === "skipped");
+                    status === "skipped" ||
+                    status === "blocked");
             });
             if (!failedDep)
                 continue;

package/dist/engine.d.ts

@@ -25,6 +25,11 @@ export interface ResumeRunSummary {
     run: WorkflowRunRecord;
     resetTaskIds: string[];
 }
+export interface StopRunSummary {
+    run: WorkflowRunRecord;
+    interruptedTaskIds: string[];
+}
+export declare function stopRun(cwd: string, runIdOrPrefix: string): Promise<StopRunSummary>;
 export declare function resumeRun(cwd: string, runIdOrPrefix: string, options?: WorkflowScheduleOptions): Promise<ResumeRunSummary>;
 export declare function resumeSupervisors(cwd: string, options?: WorkflowScheduleOptions): Promise<void>;
 export declare function watchRun(cwd: string, runId: string, options?: WorkflowScheduleOptions): void;