@agwab/pi-workflow 0.3.0 → 0.4.0

package/README.md

@@ -12,7 +12,7 @@
 
 `pi-workflow` lets Pi run named, repeatable multi-step workflows: research, code review, spec conformance checks, impact review, and project-specific team routines.
 
-Built on [`@agwab/pi-subagent`](https://github.com/AgwaB/pi-subagent), it coordinates Pi subagent workers across workflow steps, passes results between them, and records the run so it can be inspected or resumed.
+Built on [`@agwab/pi-subagent`](https://github.com/AgwaB/pi-subagent), it coordinates Pi subagent workers across workflow steps, passes results between them, and records the run so it can be inspected, stopped, or resumed.
 
 You choose a workflow and describe the task in natural language.
 
@@ -74,6 +74,8 @@ For a one-off adaptive workflow that should plan, fan out, and synthesize withou
 
 `/workflow dynamic` uses pi-workflow's built-in trusted dynamic controller and records a normal workflow run under `.pi/workflows/`. Use it when you explicitly want adaptive orchestration rather than a named reusable workflow.
 
+To interrupt a non-terminal run and stop its local supervisor watch, use `/workflow stop <run-id>`. Resume later with `/workflow resume <run-id>` if unfinished tasks should be retried.
+
 ## Usage: choose an execution mode
 
 Use the bundled `execution-router` skill when you are not sure whether a task should be handled directly, by a targeted verifier/subagent, by an existing workflow, or by a new workflow:

package/dist/artifact-graph-runtime.d.ts

@@ -1,5 +1,5 @@
 import { type WorkflowSourceManifestSource } from "./workflow-artifact-tool.js";
-import { type CompiledTask, type CompiledWorkflow, type WorkflowRunRecord, type WorkflowTaskRunRecord } from "./types.js";
+import type { CompiledTask, CompiledWorkflow, WorkflowRunRecord, WorkflowTaskRunRecord } from "./types.js";
 export declare function executeSupportTask(cwd: string, run: WorkflowRunRecord, task: WorkflowTaskRunRecord, compiledTask: CompiledWorkflow["tasks"][number]): Promise<boolean>;
 export declare function readSupportSources(cwd: string, run: WorkflowRunRecord, dependsOn: string[]): Promise<Record<string, unknown>>;
 export declare function readArtifactGraphSupportSources(cwd: string, run: WorkflowRunRecord, dependsOn: string[]): Promise<Record<string, unknown>>;

package/dist/artifact-graph-runtime.js

@@ -1,6 +1,8 @@
 import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
 import { dirname, extname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { stringifyPromptJson } from "./prompt-json.js";
+import { compactStrings } from "./strings.js";
 import { loadWorkflowHelper } from "./workflow-helpers.js";
 import { WORKFLOW_ARTIFACT_TOOL_NAME, writeWorkflowArtifactExtensionWrapper, } from "./workflow-artifact-extension.js";
 import { WORKFLOW_SOURCE_MANIFEST_SCHEMA, } from "./workflow-artifact-tool.js";
@@ -337,11 +339,14 @@ export async function prepareDagTask(cwd, run, compiledFlow, index) {
             compiledTask.compiledPrompt,
             "# Source Stage Context",
             "Use this deterministic source context packet. Prefer structuredOutput over outputPreview. Do not assume dependencies beyond this explicit packet.",
-            JSON.stringify({ ...context, missingDependencies: missing }, null, 2),
+            stringifyPromptJson({ ...context, missingDependencies: missing }),
         ].join("\n\n"),
     };
 }
 async function prepareArtifactGraphTask(cwd, run, compiledTask, task, contextDependsOn) {
+    if (compiledTask.artifactGraph?.artifactAccess === "none") {
+        return { ...compiledTask, cwd: task.cwd };
+    }
     const taskDir = dirname(fromProjectPath(cwd, task.files.result));
     const manifestPath = join(taskDir, "source-manifest.json");
     const ledgerPath = join(taskDir, "read-ledger.jsonl");
@@ -665,7 +670,7 @@ export function formatArtifactGraphSourceContext(sources, requiredReads) {
     return [
         "# Workflow Artifact Inputs",
         "Use workflow_artifact to list/read upstream workflow artifacts. Inline controlProjection fields are authoritative for the projected data they contain; use artifact reads for declared requiredReads, missing fields, or debug detail.",
-        "Projected reads must include a JSON path when using maxItems or maxChars, for example {\"action\":\"read\",\"source\":\"plan\",\"artifact\":\"control\",\"path\":\"$.factSlots\",\"maxItems\":8,\"maxChars\":2000}. For a whole artifact read, omit maxItems/maxChars.",
+        'Projected reads must include a JSON path when using maxItems or maxChars, for example {"action":"read","source":"plan","artifact":"control","path":"$.factSlots","maxItems":8,"maxChars":2000}. For a whole artifact read, omit maxItems/maxChars.',
         requiredReads.length > 0
             ? [
                 "Required reads before final output:",
@@ -673,7 +678,7 @@ export function formatArtifactGraphSourceContext(sources, requiredReads) {
             ].join("\n")
             : "No hard requiredReads are declared for this stage.",
         "Available sources:",
-        JSON.stringify(sources.map((source) => ({
+        stringifyPromptJson(sources.map((source) => ({
             source: source.source,
             taskId: source.taskId,
             specId: source.specId,
@@ -687,11 +692,11 @@ export function formatArtifactGraphSourceContext(sources, requiredReads) {
             projectionMissingPaths: source.projectionMissingPaths,
             projectionTruncated: source.projectionTruncated,
             availableArtifacts: Object.keys(source.artifacts),
-        })), null, 2),
+        }))),
     ].join("\n\n");
 }
 function uniqueStrings(values) {
-    return [...new Set(values.filter((value) => value.trim().length > 0))];
+    return compactStrings(values, { trim: false, dropWhitespaceOnly: true });
 }
 export async function readArtifactGraphControl(cwd, task) {
     const taskDir = dirname(fromProjectPath(cwd, task.files.result));

package/dist/artifact-graph-schema.js

@@ -1,5 +1,6 @@
 import { isAbsolute } from "node:path";
 import { DYNAMIC_OUTPUT_PROFILES } from "./dynamic-profiles.js";
+import { compactStrings } from "./strings.js";
 import { APPROVAL_MODES, FAST_MODES, THINKING_LEVELS, TOOL_CLASSIFICATIONS, WORKTREE_POLICIES, WorkflowValidationError, } from "./types.js";
 const TOP_LEVEL_KEYS = new Set([
     "schemaVersion",
@@ -58,10 +59,16 @@ const OUTPUT_KEYS = new Set([
     "analysis",
     "refs",
     "maxDigestChars",
+    "partial",
 ]);
+const OUTPUT_PARTIAL_KEYS = new Set(["paths"]);
 const REQUIRED_FLAG_KEYS = new Set(["required"]);
 const REFS_OUTPUT_KEYS = new Set(["required", "minItems"]);
-const INPUT_POLICY_KEYS = new Set(["requiredReads", "enforcement"]);
+const INPUT_POLICY_KEYS = new Set([
+    "requiredReads",
+    "enforcement",
+    "artifactAccess",
+]);
 const SOURCE_PROJECTION_KEYS = new Set(["include", "maxChars"]);
 const SUPPORT_KEYS = new Set(["uses", "options"]);
 const DYNAMIC_STAGE_FORBIDDEN_KEYS = new Set([
@@ -172,7 +179,8 @@ const UNTIL_KEYS = new Set([
     "all",
     "any",
 ]);
-const FOREACH_FROM_KEYS = new Set(["source", "path"]);
+const FOREACH_FROM_KEYS = new Set(["source", "path", "streaming"]);
+const FOREACH_STREAMING_KEYS = new Set(["enabled", "minChunk"]);
 const NORMAL_ARTIFACT_KINDS = new Set([
     "control",
     "analysis",
@@ -262,8 +270,48 @@ function validateStageArray(value, path, issues) {
     for (const [index, item] of value.entries()) {
         validateStage(item, `${path}[${index}]`, ids, sourceIds, issues);
     }
+    validateStreamingProducerDeclarations(value, path, issues);
     validateStageDependencyGraph(value, path, issues);
 }
+function validateStreamingProducerDeclarations(stages, path, issues) {
+    const byId = new Map();
+    for (const stage of stages) {
+        if (isRecord(stage) && typeof stage.id === "string") {
+            byId.set(stage.id, stage);
+        }
+    }
+    for (const [index, stage] of stages.entries()) {
+        if (!isRecord(stage) || !isRecord(stage.from))
+            continue;
+        const from = stage.from;
+        if (!isRecord(from.streaming) || from.streaming.enabled !== true)
+            continue;
+        const source = typeof from.source === "string" ? from.source : undefined;
+        const controlPath = typeof from.path === "string" ? from.path : undefined;
+        if (!source || !controlPath)
+            continue;
+        const sourceStage = byId.get(source);
+        const partialPaths = outputPartialPaths(sourceStage);
+        if (!partialPaths.includes(controlPath)) {
+            issues.push({
+                path: `${path}[${index}].from.streaming`,
+                message: `source stage "${source}" must declare output.partial.paths including "${controlPath}" to use streaming`,
+            });
+        }
+    }
+}
+function outputPartialPaths(stage) {
+    const output = isRecord(stage?.output) ? stage.output : undefined;
+    const partial = isRecord(output?.partial) ? output.partial : undefined;
+    return Array.isArray(partial?.paths)
+        ? compactStrings(partial.paths, {
+            trim: false,
+            unique: false,
+            dropEmpty: false,
+            dropWhitespaceOnly: false,
+        })
+        : [];
+}
 function validateStageDependencyGraph(stages, path, issues) {
     const ids = new Set(stages
         .filter(isRecord)
@@ -334,7 +382,12 @@ function extractDependencyRefs(value) {
     if (typeof value === "string")
         return [value];
     if (Array.isArray(value))
-        return value.filter((item) => typeof item === "string");
+        return compactStrings(value, {
+            trim: false,
+            unique: false,
+            dropEmpty: false,
+            dropWhitespaceOnly: false,
+        });
     if (isRecord(value) && typeof value.source === "string")
         return [value.source];
     return [];
@@ -445,7 +498,7 @@ function validateStage(value, path, siblingIds, sourceIds, issues) {
         allowControlPath: false,
     });
     validateSourceProjection(stage.sourceProjection, `${path}.sourceProjection`, issues);
-    validateInputPolicy(stage.inputPolicy, `${path}.inputPolicy`, sourceIds, issues);
+    validateInputPolicy(stage.inputPolicy, `${path}.inputPolicy`, sourceIds, issues, stage.sourceProjection);
     validateOutput(stage.output, `${path}.output`, issues);
     validateSupportStage(stage, type, path, issues);
     validateDynamicStage(stage, type, path, issues);
@@ -518,6 +571,19 @@ function validateControlPathRef(value, path, siblingIds, issues) {
             message: "must be a control JSONPath starting with $.",
         });
     }
+    if (value.streaming !== undefined) {
+        validateForeachStreaming(value.streaming, `${path}.streaming`, issues);
+    }
+}
+function validateForeachStreaming(value, path, issues) {
+    const streaming = recordAt(value, path, issues);
+    if (!streaming)
+        return;
+    rejectUnknownKeys(streaming, FOREACH_STREAMING_KEYS, path, issues);
+    if (streaming.enabled !== true) {
+        issues.push({ path: `${path}.enabled`, message: "must be true" });
+    }
+    optionalPositiveInteger(streaming.minChunk, `${path}.minChunk`, issues);
 }
 function validateKnownStageRef(stageId, path, siblingIds, issues) {
     if (stageId.trim() === "") {
@@ -539,6 +605,39 @@ function validateOutput(value, path, issues) {
     optionalPositiveInteger(output.maxDigestChars, `${path}.maxDigestChars`, issues);
     validateRequiredFlagObject(output.analysis, `${path}.analysis`, issues);
     validateRefsOutputObject(output.refs, `${path}.refs`, issues);
+    validatePartialOutput(output.partial, `${path}.partial`, issues);
+}
+function validatePartialOutput(value, path, issues) {
+    if (value === undefined)
+        return;
+    const partial = recordAt(value, path, issues);
+    if (!partial)
+        return;
+    rejectUnknownKeys(partial, OUTPUT_PARTIAL_KEYS, path, issues);
+    if (!Array.isArray(partial.paths)) {
+        issues.push({ path: `${path}.paths`, message: "must be an array" });
+        return;
+    }
+    if (partial.paths.length === 0) {
+        issues.push({ path: `${path}.paths`, message: "must not be empty" });
+    }
+    const seen = new Set();
+    for (const [index, item] of partial.paths.entries()) {
+        const itemPath = `${path}.paths[${index}]`;
+        if (typeof item !== "string" || item.trim() === "") {
+            issues.push({ path: itemPath, message: "must be a non-empty string" });
+            continue;
+        }
+        if (!item.startsWith("$.")) {
+            issues.push({
+                path: itemPath,
+                message: "must be a control JSONPath starting with $.",
+            });
+        }
+        if (seen.has(item))
+            issues.push({ path: itemPath, message: `duplicate value "${item}"` });
+        seen.add(item);
+    }
 }
 function validateControlSchemaRef(value, path, issues) {
     if (value === undefined)
@@ -568,7 +667,7 @@ function validateRefsOutputObject(value, path, issues) {
     optionalBoolean(object.required, `${path}.required`, issues);
     optionalPositiveInteger(object.minItems, `${path}.minItems`, issues);
 }
-function validateInputPolicy(value, path, siblingIds, issues) {
+function validateInputPolicy(value, path, siblingIds, issues, sourceProjection) {
     if (value === undefined)
         return;
     const policy = recordAt(value, path, issues);
@@ -579,6 +678,29 @@ function validateInputPolicy(value, path, siblingIds, issues) {
     if (policy.enforcement !== undefined && policy.enforcement !== "fail") {
         issues.push({ path: `${path}.enforcement`, message: 'must be "fail"' });
     }
+    if (policy.artifactAccess !== undefined &&
+        policy.artifactAccess !== "enabled" &&
+        policy.artifactAccess !== "none") {
+        issues.push({
+            path: `${path}.artifactAccess`,
+            message: 'must be "enabled" or "none"',
+        });
+    }
+    if (policy.artifactAccess === "none") {
+        if (Array.isArray(policy.requiredReads) &&
+            policy.requiredReads.length > 0) {
+            issues.push({
+                path: `${path}.requiredReads`,
+                message: 'must be empty when artifactAccess is "none"',
+            });
+        }
+        if (sourceProjection !== undefined) {
+            issues.push({
+                path: `${path}.artifactAccess`,
+                message: 'cannot be "none" when sourceProjection is declared',
+            });
+        }
+    }
 }
 function validateRequiredReads(value, path, sourceIds, issues) {
     if (value === undefined)

package/dist/compiler.js

@@ -2,6 +2,7 @@ import { readFile } from "node:fs/promises";
 import { dirname, resolve } from "node:path";
 import { loadAgentByName } from "./agents.js";
 import { DYNAMIC_OUTPUT_PROFILES } from "./dynamic-profiles.js";
+import { stringifyPromptJson } from "./prompt-json.js";
 import { compileRole } from "./roles.js";
 import { classifyToolCapability, effectiveToolClassification, providersForSelectedTools, resolveToolSelection, TOOL_NAME_PATTERN, toolAllowedByAuthorityCeiling, toolNameForSpec, } from "./tool-metadata.js";
 import { WorkflowValidationError, WORKFLOW_RUN_TYPE, } from "./types.js";
@@ -102,7 +103,13 @@ function lowerArtifactGraphFrom(from) {
         typeof from === "object" &&
         !Array.isArray(from) &&
         typeof from.source === "string") {
-        return { stage: from.source, path: from.path };
+        return {
+            stage: from.source,
+            path: from.path,
+            ...(from.streaming !== undefined
+                ? { streaming: from.streaming }
+                : {}),
+        };
     }
     return from;
 }
@@ -127,10 +134,23 @@ function appendWorkflowOutputInstructions(prompt, stage) {
             : "Use schema `stage-control-v1` unless the workflow asks for a more specific control schema.",
         "Put detailed prose, reasoning, and evidence discussion in <analysis> only.",
         "Put structured evidence pointers in <refs> as a JSON array; use [] if none.",
+        ...partialOutputInstructions(stage.output?.partial?.paths),
     ]
         .filter(Boolean)
         .join("\n\n");
 }
+function partialOutputInstructions(paths) {
+    if (!paths || paths.length === 0)
+        return [];
+    return [
+        "# Workflow Partial Output Protocol (optional)",
+        `If a complete stable array item is ready before your final answer for one of these control paths (${paths.join(", ")}), you may emit a partial-control section before the final output:`,
+        '<partial-control>{"schema":"workflow-partial-output-v1","path":"$.items","items":[{"id":"stable-id","...":"..."}]}</partial-control>',
+        "Use the actual declared path, not the example path, and include only items that are final/stable enough to appear unchanged in your final <control> at that path.",
+        "Every partial item must be the exact JSON object that will appear in the final array and must include a stable non-empty string `id`; never revise or withdraw a published partial item.",
+        "If an item might change, do not publish it partially; wait for the final workflow output. The final answer must still include the normal <control>, <analysis>, and <refs> sections exactly once.",
+    ];
+}
 function artifactGraphTaskMetadata(stage, specDir) {
     const controlSchema = stage.output?.controlSchema;
     return {
@@ -144,8 +164,12 @@ function artifactGraphTaskMetadata(stage, specDir) {
                 ? resolve(specDir, controlSchema)
                 : undefined,
             maxDigestChars: stage.output?.maxDigestChars,
+            partial: stage.output?.partial
+                ? { paths: [...stage.output.partial.paths] }
+                : undefined,
         },
         requiredReads: stage.inputPolicy?.requiredReads ?? [],
+        artifactAccess: stage.inputPolicy?.artifactAccess ?? "enabled",
         sourceProjection: stage.sourceProjection,
     };
 }
@@ -457,7 +481,7 @@ async function compileArtifactGraphPlan(spec, options) {
         typeof workflowInput === "object" &&
         !Array.isArray(workflowInput) &&
         Object.keys(workflowInput).length > 0
-        ? `# Workflow Input\n\n${JSON.stringify(workflowInput, null, 2)}`
+        ? `# Workflow Input\n\n${stringifyPromptJson(workflowInput)}`
         : "";
     const runtimeOverrides = options.runtimeOverrides;
     const runtimeDefaults = options.runtimeDefaults;
@@ -538,15 +562,26 @@ async function compileArtifactGraphPlan(spec, options) {
         const injectRuntimeTaskInPrompt = (runtimeStageKind !== "foreach" && injectTask) ||
             (runtimeStageKind === "foreach" && optInInjectRuntimeTask);
         const normalizedPrompt = String(prompt ?? "").replace(/\$\{item\}/g, "the relevant item from the dependency context");
-        const compiledPrompt = [
-            injectRuntimeTaskInPrompt && options.task
-                ? `# Task\n\n${options.task}`
-                : undefined,
-            workflowInputText || undefined,
-            `# Workflow Stage\n\nstage=${stage.id}\ntype=${runtimeStageKind}`,
-            `# Instructions\n\n${normalizedPrompt}`,
-            roleText || undefined,
-        ]
+        const instructionText = `# Instructions\n\n${normalizedPrompt}`;
+        const stageText = `# Workflow Stage\n\nstage=${stage.id}\ntype=${runtimeStageKind}`;
+        const taskText = injectRuntimeTaskInPrompt && options.task
+            ? `# Task\n\n${options.task}`
+            : undefined;
+        const compiledPrompt = (runtimeStageKind === "foreach"
+            ? [
+                taskText,
+                workflowInputText || undefined,
+                stageText,
+                roleText || undefined,
+                instructionText,
+            ]
+            : [
+                taskText,
+                workflowInputText || undefined,
+                stageText,
+                instructionText,
+                roleText || undefined,
+            ])
             .filter(Boolean)
             .join("\n\n");
         const toolSelection = resolveToolSelection([spec.defaults?.tools, stage.tools], stageAgent.tools);

package/dist/dynamic-decision.d.ts

@@ -97,6 +97,7 @@ export interface DynamicDecisionArtifactWriteResult {
     hash?: string;
 }
 export declare function validateDynamicDecision(value: unknown, context?: DynamicDecisionValidationContext): DynamicDecisionValidationResult;
+export declare function assertValidDynamicDecision(value: unknown, context?: DynamicDecisionValidationContext): NormalizedDynamicDecision;
 export declare function hashDynamicDecision(value: unknown): string;
 export declare function dynamicLoopSignature(decision: NormalizedDynamicDecision): string;
 export declare function writeDynamicDecisionArtifacts(input: DynamicDecisionArtifactWriteInput): Promise<DynamicDecisionArtifactWriteResult>;

package/dist/dynamic-decision.js

@@ -121,6 +121,13 @@ export function validateDynamicDecision(value, context = {}) {
         hash: hashDynamicDecision(decision),
     };
 }
+export function assertValidDynamicDecision(value, context = {}) {
+    const result = validateDynamicDecision(value, context);
+    if (!result.ok || !result.decision) {
+        throw new Error(`invalid dynamic decision: ${result.errors.join("; ")}`);
+    }
+    return result.decision;
+}
 export function hashDynamicDecision(value) {
     return createHash("sha256")
         .update(stableStringify(toJsonNormalizedValue(value)))

package/dist/dynamic-generated-task-runtime.js

@@ -4,6 +4,7 @@ import { DynamicControllerBudgetBlocked } from "./dynamic-controller-errors.js";
 import { isDynamicOutputProfile, } from "./dynamic-profiles.js";
 import { readOrRebuildDynamicState } from "./dynamic-state.js";
 import { sanitizeTaskId } from "./engine-run-graph.js";
+import { compactStrings } from "./strings.js";
 import { fromProjectPath, isTerminalTaskStatus, readJson } from "./store.js";
 import { classifyToolCapability, effectiveToolClassification, providersForSelectedTools, toolAllowedByAuthorityCeiling, } from "./tool-metadata.js";
 import { resolveWorkflowRuntime, selectWorkflowRuntime, } from "./workflow-runtime.js";
@@ -173,6 +174,7 @@ export async function buildDynamicGeneratedCompiledTask(input) {
                 maxDigestChars: DYNAMIC_OUTPUT_MAX_DIGEST_CHARS,
             },
             requiredReads: input.request.requiredReads,
+            artifactAccess: "enabled",
             sourceProjection: dynamicInputSourceProjection(input.request.inputs),
         },
         dynamicGenerated: {
@@ -630,5 +632,5 @@ function dynamicOutputProfileInstructions(outputProfile) {
     return `# Dynamic Output Profile: ${outputProfile}\nEmit control JSON suitable for this output profile and surface gaps/blockers explicitly.`;
 }
 function uniqueStrings(values) {
-    return [...new Set(values.filter((value) => value.trim().length > 0))];
+    return compactStrings(values, { trim: false, dropWhitespaceOnly: true });
 }

package/dist/dynamic-profiles.d.ts

@@ -5,3 +5,4 @@ export declare const DYNAMIC_TERMINAL_OUTPUT_PROFILES: readonly ["synthesis_v1"]
 export declare function isDynamicOutputProfile(value: unknown): value is DynamicOutputProfile;
 export declare function isExtractableDynamicOutputProfile(value: unknown): value is (typeof DYNAMIC_EXTRACTABLE_OUTPUT_PROFILES)[number];
 export declare function isTerminalDynamicOutputProfile(value: unknown): value is (typeof DYNAMIC_TERMINAL_OUTPUT_PROFILES)[number];
+export declare function dynamicOutputProfileValues(): DynamicOutputProfile[];

package/dist/dynamic-profiles.js

@@ -26,3 +26,6 @@ export function isExtractableDynamicOutputProfile(value) {
 export function isTerminalDynamicOutputProfile(value) {
     return typeof value === "string" && TERMINAL_OUTPUT_PROFILE_SET.has(value);
 }
+export function dynamicOutputProfileValues() {
+    return [...DYNAMIC_OUTPUT_PROFILES];
+}

package/dist/engine-run-graph.d.ts

@@ -10,6 +10,8 @@ export declare function compiledTaskSpecId(task: CompiledTask): string;
 export declare function loopStageIdSet(compiledFlow: CompiledWorkflow): Set<string>;
 export declare function nextTaskRecordIndex(run: WorkflowRunRecord): number;
 export declare function dependenciesReady(compiledTask: CompiledTask, bySpecId: Map<string, WorkflowTaskRunRecord>, compiledFlow: CompiledWorkflow): boolean;
+export declare function foreachStreamingEnabled(compiledTask: CompiledTask): boolean;
+export declare function foreachStreamingMinChunk(compiledTask: CompiledTask): number;
 export declare function buildForeachGeneratedTasks(template: CompiledTask, runtimeTask: string | undefined, items: unknown[]): {
     tasks: CompiledTask[];
     error?: string;

package/dist/engine-run-graph.js

@@ -95,9 +95,14 @@ export function reconcileDynamicGeneratedRunRecords(cwd, run, compiledFlow) {
 export function reconcileForeachGeneratedRunRecords(cwd, run, compiledFlow) {
     let changed = false;
     const compiledSpecIds = new Set(compiledFlow.tasks.map((task) => compiledTaskSpecId(task)));
+    const compiledTaskBySpecId = new Map(compiledFlow.tasks.map((task) => [compiledTaskSpecId(task), task]));
     const placeholderToGeneratedSpecIds = new Map();
+    const streamingPlaceholderSpecIds = new Set();
     for (const compiledTask of compiledFlow.tasks) {
         const specId = compiledTaskSpecId(compiledTask);
+        if (compiledTask.foreach && foreachStreamingEnabled(compiledTask)) {
+            streamingPlaceholderSpecIds.add(specId);
+        }
         const placeholderSpecId = foreachGeneratedPlaceholderSpecId(compiledTask, compiledFlow, specId);
         if (!placeholderSpecId)
             continue;
@@ -117,6 +122,12 @@ export function reconcileForeachGeneratedRunRecords(cwd, run, compiledFlow) {
         const generatedSpecIds = placeholderToGeneratedSpecIds.get(task.specId);
         let placeholderSpecId = foreachGeneratedPlaceholderSpecId(task, compiledFlow, task.specId);
         if (generatedSpecIds && !placeholderSpecId) {
+            const compiledTask = compiledTaskBySpecId.get(task.specId);
+            if (compiledTask?.foreach &&
+                streamingPlaceholderSpecIds.has(task.specId)) {
+                filteredRunTasks.push(task);
+                continue;
+            }
             if (generatedSpecIds.includes(task.specId)) {
                 placeholderSpecId = task.specId;
                 task.foreachGenerated = { placeholderSpecId };
@@ -183,7 +194,7 @@ export function reconcileForeachGeneratedRunRecords(cwd, run, compiledFlow) {
     for (const task of reordered) {
         if (!task.dependsOn)
             continue;
-        const replaced = replaceForeachGeneratedDependencies(task.dependsOn, placeholderToGeneratedSpecIds);
+        const replaced = replaceForeachGeneratedDependencies(task.dependsOn, placeholderToGeneratedSpecIds, streamingPlaceholderSpecIds);
         if (!sameStringList(task.dependsOn, replaced)) {
             task.dependsOn = replaced;
             changed = true;
@@ -261,12 +272,15 @@ function foreachPlaceholderSpecId(compiledFlow, stageId) {
         return undefined;
     return `${stageId}.item`;
 }
-function replaceForeachGeneratedDependencies(dependsOn, placeholderToGeneratedSpecIds) {
+function replaceForeachGeneratedDependencies(dependsOn, placeholderToGeneratedSpecIds, keepPlaceholderSpecIds = new Set()) {
     const replaced = [];
     for (const dep of dependsOn) {
         const generatedSpecIds = placeholderToGeneratedSpecIds.get(dep);
-        if (generatedSpecIds)
+        if (generatedSpecIds) {
+            if (keepPlaceholderSpecIds.has(dep))
+                replaced.push(dep);
             replaced.push(...generatedSpecIds);
+        }
         else
             replaced.push(dep);
     }
@@ -341,6 +355,29 @@ export function dependenciesReady(compiledTask, bySpecId, compiledFlow) {
     if (deps.length === 0)
         return true;
     const partial = stageSourcePolicy(compiledFlow, compiledTask.stageId ?? "") === "partial";
+    if (foreachStreamingEnabled(compiledTask)) {
+        let completedDependencyReady = false;
+        let runningDependencyMayHavePartialItems = false;
+        let allKnownDependenciesTerminal = true;
+        for (const dep of deps) {
+            const status = bySpecId.get(dep)?.status;
+            if (status === "completed") {
+                completedDependencyReady = true;
+                continue;
+            }
+            if (status && isTerminalTaskStatus(status)) {
+                if (!partial)
+                    return false;
+                continue;
+            }
+            if (status === "running")
+                runningDependencyMayHavePartialItems = true;
+            allKnownDependenciesTerminal = false;
+        }
+        return (completedDependencyReady ||
+            runningDependencyMayHavePartialItems ||
+            allKnownDependenciesTerminal);
+    }
     return deps.every((dep) => {
         const status = bySpecId.get(dep)?.status;
         if (status === "completed")
@@ -350,6 +387,18 @@ export function dependenciesReady(compiledTask, bySpecId, compiledFlow) {
         return false;
     });
 }
+export function foreachStreamingEnabled(compiledTask) {
+    const streaming = compiledTask.foreach?.from?.streaming;
+    return Boolean(streaming &&
+        typeof streaming === "object" &&
+        streaming.enabled === true);
+}
+export function foreachStreamingMinChunk(compiledTask) {
+    const value = compiledTask.foreach?.from?.streaming?.minChunk;
+    return typeof value === "number" && Number.isFinite(value) && value > 0
+        ? Math.floor(value)
+        : 1;
+}
 export function buildForeachGeneratedTasks(template, runtimeTask, items) {
     const seen = new Set();
     const tasks = [];
@@ -368,9 +417,10 @@ export function buildForeachGeneratedTasks(template, runtimeTask, items) {
             template.foreach.injectRuntimeTask && runtimeTask
                 ? `# Task\n\n${runtimeTask}`
                 : undefined,
-            `# Workflow Stage\n\nstage=${template.stageId}\ntype=foreach\nitem=${taskId}`,
-            `# Instructions\n\n${instructions}`,
+            `# Workflow Stage\n\nstage=${template.stageId}\ntype=foreach`,
             template.foreach.roleText || undefined,
+            `# Workflow Item\n\nitem=${taskId}`,
+            `# Instructions\n\n${instructions}`,
         ]
             .filter(Boolean)
             .join("\n\n");