keystone-cli 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -14
- package/package.json +1 -1
- package/src/commands/init.ts +6 -0
- package/src/db/dynamic-state-manager.test.ts +319 -0
- package/src/db/dynamic-state-manager.ts +411 -0
- package/src/db/workflow-db.ts +64 -0
- package/src/parser/schema.ts +34 -1
- package/src/parser/workflow-parser.test.ts +3 -4
- package/src/parser/workflow-parser.ts +3 -62
- package/src/runner/executors/dynamic-executor.test.ts +613 -0
- package/src/runner/executors/dynamic-executor.ts +718 -0
- package/src/runner/executors/dynamic-types.ts +69 -0
- package/src/runner/step-executor.ts +20 -0
- package/src/templates/dynamic-demo.yaml +31 -0
- package/src/templates/scaffolding/decompose-problem.yaml +1 -1
- package/src/templates/scaffolding/dynamic-decompose.yaml +39 -0
- package/src/utils/topo-sort.ts +47 -0
|
@@ -0,0 +1,718 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dynamic Step Executor
|
|
3
|
+
*
|
|
4
|
+
* Enables LLM-driven workflow orchestration where an agent generates
|
|
5
|
+
* a sequence of steps at runtime that are then executed dynamically.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { DynamicStateManager } from '../../db/dynamic-state-manager.ts';
|
|
9
|
+
import type { WorkflowDb } from '../../db/workflow-db.ts';
|
|
10
|
+
import type { ExpressionContext } from '../../expression/evaluator.ts';
|
|
11
|
+
import { ExpressionEvaluator } from '../../expression/evaluator.ts';
|
|
12
|
+
import type { DynamicStep, LlmStep, Step } from '../../parser/schema.ts';
|
|
13
|
+
import type { Logger } from '../../utils/logger.ts';
|
|
14
|
+
import { topologicalSort } from '../../utils/topo-sort.ts';
|
|
15
|
+
import type { WorkflowEvent } from '../events.ts';
|
|
16
|
+
import type { MCPManager } from '../mcp-manager.ts';
|
|
17
|
+
import type { DynamicPlan, DynamicStepState, GeneratedStep } from './dynamic-types.ts';
|
|
18
|
+
import { executeHumanStep } from './human-executor.ts';
|
|
19
|
+
import { executeLlmStep } from './llm-executor.ts';
|
|
20
|
+
import type { StepExecutorOptions, StepResult } from './types.ts';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Schema for generated step definitions from the supervisor LLM
|
|
24
|
+
*/
|
|
25
|
+
export const DYNAMIC_STEP_OUTPUT_SCHEMA = {
|
|
26
|
+
type: 'object',
|
|
27
|
+
properties: {
|
|
28
|
+
workflow_id: { type: 'string', description: 'Unique identifier for this workflow instance' },
|
|
29
|
+
steps: {
|
|
30
|
+
type: 'array',
|
|
31
|
+
items: {
|
|
32
|
+
type: 'object',
|
|
33
|
+
properties: {
|
|
34
|
+
id: { type: 'string', description: 'Unique step identifier' },
|
|
35
|
+
name: { type: 'string', description: 'Human-readable step name' },
|
|
36
|
+
type: {
|
|
37
|
+
type: 'string',
|
|
38
|
+
enum: ['llm', 'shell', 'workflow', 'file', 'request'],
|
|
39
|
+
description: 'Step type to execute',
|
|
40
|
+
},
|
|
41
|
+
agent: { type: 'string', description: 'Agent to use for llm steps' },
|
|
42
|
+
prompt: { type: 'string', description: 'Prompt for llm steps' },
|
|
43
|
+
run: { type: 'string', description: 'Command for shell steps' },
|
|
44
|
+
path: { type: 'string', description: 'Path for workflow/file steps' },
|
|
45
|
+
op: {
|
|
46
|
+
type: 'string',
|
|
47
|
+
enum: ['read', 'write', 'append'],
|
|
48
|
+
description: 'Operation for file steps',
|
|
49
|
+
},
|
|
50
|
+
content: { type: 'string', description: 'Content for file write/append' },
|
|
51
|
+
needs: {
|
|
52
|
+
type: 'array',
|
|
53
|
+
items: { type: 'string' },
|
|
54
|
+
description: 'Step IDs this step depends on',
|
|
55
|
+
},
|
|
56
|
+
inputs: {
|
|
57
|
+
type: 'object',
|
|
58
|
+
additionalProperties: true,
|
|
59
|
+
description: 'Inputs to pass to the step',
|
|
60
|
+
},
|
|
61
|
+
allowStepFailure: {
|
|
62
|
+
type: 'boolean',
|
|
63
|
+
description: 'Whether to continue if this specific step fails',
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
required: ['id', 'name', 'type'],
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
notes: { type: 'string', description: 'Any additional notes about the plan' },
|
|
70
|
+
},
|
|
71
|
+
required: ['steps'],
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Build a supervisor prompt from the step configuration
|
|
76
|
+
*/
|
|
77
|
+
function buildSupervisorPrompt(
|
|
78
|
+
step: DynamicStep,
|
|
79
|
+
context: ExpressionContext,
|
|
80
|
+
failureContext?: string
|
|
81
|
+
): string {
|
|
82
|
+
const goal = ExpressionEvaluator.evaluateString(step.goal, context);
|
|
83
|
+
const contextText = step.context ? ExpressionEvaluator.evaluateString(step.context, context) : '';
|
|
84
|
+
|
|
85
|
+
// Build template descriptions if provided
|
|
86
|
+
let templateInfo = '';
|
|
87
|
+
if (step.templates && Object.keys(step.templates).length > 0) {
|
|
88
|
+
templateInfo = `\n\nAvailable specialized agents:\n${Object.entries(step.templates)
|
|
89
|
+
.map(([role, agent]) => `- ${role}: ${agent}`)
|
|
90
|
+
.join('\n')}`;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Build library patterns if provided
|
|
94
|
+
let libraryInfo = '';
|
|
95
|
+
if (step.library && step.library.length > 0) {
|
|
96
|
+
libraryInfo = `\n\nAvailable step patterns in your library:\n${step.library
|
|
97
|
+
.map((p) => `- ${p.name}: ${p.description}`)
|
|
98
|
+
.join(
|
|
99
|
+
'\n'
|
|
100
|
+
)}\n\nYou can use these patterns as inspiration or incorporate their logic into your plan.`;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
return `You are a workflow supervisor. Your job is to break down a goal into executable steps
|
|
104
|
+
and delegate to specialized agents.
|
|
105
|
+
|
|
106
|
+
## Goal
|
|
107
|
+
${goal}
|
|
108
|
+
|
|
109
|
+
## Context
|
|
110
|
+
${contextText || 'None provided'}
|
|
111
|
+
${templateInfo}${libraryInfo}
|
|
112
|
+
|
|
113
|
+
## Instructions
|
|
114
|
+
1. Analyze the goal and determine what steps are needed
|
|
115
|
+
2. For each step, specify:
|
|
116
|
+
- A unique id (lowercase, no spaces)
|
|
117
|
+
- A descriptive name
|
|
118
|
+
- The type (llm, shell, workflow, file, or request)
|
|
119
|
+
- For llm steps: which agent and what prompt
|
|
120
|
+
- For shell steps: what command to run
|
|
121
|
+
- For file steps: path, op (read/write/append), and content (if write/append)
|
|
122
|
+
- Dependencies on other steps (needs array)
|
|
123
|
+
- allowStepFailure (optional boolean, default false)
|
|
124
|
+
|
|
125
|
+
3. Order steps logically - steps can run in parallel if they don't depend on each other. Specify 'needs' for strict sequencing.
|
|
126
|
+
4. Keep the plan minimal but complete.
|
|
127
|
+
${failureContext ? `\n5. **FAILURE RECOVERY**: Some steps failed in the previous attempt. Analyze the errors below and generate a plan to fix the issues:\n\n${failureContext}\n` : ''}
|
|
128
|
+
|
|
129
|
+
Return a JSON object with the steps array. Each step should be independently executable.`;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Convert a generated step definition into an executable Step
|
|
134
|
+
*/
|
|
135
|
+
function convertToExecutableStep(
|
|
136
|
+
generated: GeneratedStep,
|
|
137
|
+
parentStepId: string,
|
|
138
|
+
allowInsecure?: boolean
|
|
139
|
+
): Step {
|
|
140
|
+
const baseProps = {
|
|
141
|
+
id: `${parentStepId}_${generated.id}`,
|
|
142
|
+
needs: generated.needs?.map((n) => `${parentStepId}_${n}`) || [],
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
switch (generated.type) {
|
|
146
|
+
case 'llm':
|
|
147
|
+
return {
|
|
148
|
+
...baseProps,
|
|
149
|
+
type: 'llm' as const,
|
|
150
|
+
agent: generated.agent || 'software-engineer',
|
|
151
|
+
prompt: generated.prompt || '',
|
|
152
|
+
maxIterations: 10, // Default for generated LLM steps
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
case 'shell':
|
|
156
|
+
return {
|
|
157
|
+
...baseProps,
|
|
158
|
+
type: 'shell' as const,
|
|
159
|
+
run: generated.run || 'echo "No command specified"',
|
|
160
|
+
allowInsecure: allowInsecure ?? false,
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
case 'workflow':
|
|
164
|
+
return {
|
|
165
|
+
...baseProps,
|
|
166
|
+
type: 'workflow' as const,
|
|
167
|
+
path: generated.path || '',
|
|
168
|
+
inputs: generated.inputs as Record<string, string> | undefined,
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
case 'file':
|
|
172
|
+
return {
|
|
173
|
+
...baseProps,
|
|
174
|
+
type: 'file' as const,
|
|
175
|
+
path: generated.path || '',
|
|
176
|
+
op: (generated.op as any) || (generated.inputs?.op as any) || 'read',
|
|
177
|
+
content: generated.content || (generated.inputs?.content as string),
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
case 'request':
|
|
181
|
+
return {
|
|
182
|
+
...baseProps,
|
|
183
|
+
type: 'request' as const,
|
|
184
|
+
allowInsecure: allowInsecure ?? false,
|
|
185
|
+
url: generated.path || '',
|
|
186
|
+
method: 'GET' as const,
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
default:
|
|
190
|
+
// Fallback to an echo shell step for unknown types
|
|
191
|
+
return {
|
|
192
|
+
...baseProps,
|
|
193
|
+
type: 'shell' as const,
|
|
194
|
+
run: `echo "Unknown step type: ${generated.type}"`,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Execute a dynamic step
|
|
201
|
+
*
|
|
202
|
+
* This is the core orchestrator that:
|
|
203
|
+
* 1. Calls the supervisor LLM to generate a plan
|
|
204
|
+
* 2. Converts the plan into executable steps
|
|
205
|
+
* 3. Executes steps in dependency order
|
|
206
|
+
* 4. Tracks state for resumability
|
|
207
|
+
*/
|
|
208
|
+
export async function executeDynamicStep(
|
|
209
|
+
step: DynamicStep,
|
|
210
|
+
context: ExpressionContext,
|
|
211
|
+
executeStepFn: (step: Step, context: ExpressionContext) => Promise<StepResult>,
|
|
212
|
+
logger: Logger,
|
|
213
|
+
options: StepExecutorOptions & {
|
|
214
|
+
stateManager?: DynamicStateManager;
|
|
215
|
+
loadState?: (stepId: string) => Promise<DynamicStepState | null>;
|
|
216
|
+
saveState?: (stepId: string, state: DynamicStepState) => Promise<void>;
|
|
217
|
+
executeLlmStep?: typeof executeLlmStep;
|
|
218
|
+
executeHumanStep?: typeof executeHumanStep;
|
|
219
|
+
}
|
|
220
|
+
): Promise<StepResult> {
|
|
221
|
+
const { runId, db, abortSignal } = options;
|
|
222
|
+
const stateManager = options.stateManager || (db ? new DynamicStateManager(db) : null);
|
|
223
|
+
|
|
224
|
+
const { state, dbState } = await initializeState(step, runId, stateManager, options.loadState);
|
|
225
|
+
const isResuming =
|
|
226
|
+
state.status !== 'completed' &&
|
|
227
|
+
state.status !== 'planning' &&
|
|
228
|
+
state.generatedPlan.steps.length > 0;
|
|
229
|
+
|
|
230
|
+
logger.log(` 🎯 Dynamic step: ${step.id} (${isResuming ? 'resuming' : 'starting'})`);
|
|
231
|
+
|
|
232
|
+
try {
|
|
233
|
+
while (state.status !== 'completed' && state.status !== 'failed') {
|
|
234
|
+
if (abortSignal?.aborted) throw new Error('Dynamic step execution canceled');
|
|
235
|
+
|
|
236
|
+
if (state.status === 'planning') {
|
|
237
|
+
await handlePlanningPhase(
|
|
238
|
+
step,
|
|
239
|
+
context,
|
|
240
|
+
state,
|
|
241
|
+
dbState,
|
|
242
|
+
stateManager,
|
|
243
|
+
executeStepFn,
|
|
244
|
+
logger,
|
|
245
|
+
options
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
if (state.status === 'awaiting_confirmation') {
|
|
250
|
+
await handleConfirmationPhase(step, context, state, dbState, stateManager, logger, options);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
if (state.status === 'executing') {
|
|
254
|
+
await handleExecutionPhase(
|
|
255
|
+
step,
|
|
256
|
+
context,
|
|
257
|
+
state,
|
|
258
|
+
dbState,
|
|
259
|
+
stateManager,
|
|
260
|
+
executeStepFn,
|
|
261
|
+
logger,
|
|
262
|
+
options
|
|
263
|
+
);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
return buildFinalResult(state);
|
|
268
|
+
} catch (error) {
|
|
269
|
+
return await handleExecutionError(step, state, dbState, stateManager, options.saveState, error);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Initialize state for dynamic step execution
|
|
275
|
+
*/
|
|
276
|
+
async function initializeState(
|
|
277
|
+
step: DynamicStep,
|
|
278
|
+
runId: string | undefined,
|
|
279
|
+
stateManager: DynamicStateManager | null,
|
|
280
|
+
loadState?: (stepId: string) => Promise<DynamicStepState | null>
|
|
281
|
+
): Promise<{ state: DynamicStepState; dbState: DynamicStepState | null }> {
|
|
282
|
+
let state: DynamicStepState = {
|
|
283
|
+
workflowId: runId || `dynamic-${Date.now()}`,
|
|
284
|
+
generatedPlan: { steps: [] },
|
|
285
|
+
stepResults: new Map(),
|
|
286
|
+
currentStepIndex: 0,
|
|
287
|
+
status: 'planning',
|
|
288
|
+
startedAt: new Date().toISOString(),
|
|
289
|
+
replanCount: 0,
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
let dbState: DynamicStepState | null = null;
|
|
293
|
+
|
|
294
|
+
if (stateManager) {
|
|
295
|
+
if (!runId) throw new Error('runId is required when using stateManager');
|
|
296
|
+
dbState = await stateManager.load(runId, step.id);
|
|
297
|
+
if (dbState?.id) {
|
|
298
|
+
const stepResults = await stateManager.getStepResultsMap(dbState.id);
|
|
299
|
+
state = {
|
|
300
|
+
workflowId: dbState.workflowId || dbState.runId || runId || 'unknown',
|
|
301
|
+
generatedPlan: dbState.generatedPlan,
|
|
302
|
+
stepResults: new Map(
|
|
303
|
+
Array.from(stepResults.entries()).map(([k, v]) => [k, v as StepResult])
|
|
304
|
+
),
|
|
305
|
+
currentStepIndex: dbState.currentStepIndex,
|
|
306
|
+
status: dbState.status as any,
|
|
307
|
+
startedAt: dbState.startedAt,
|
|
308
|
+
completedAt: dbState.completedAt,
|
|
309
|
+
error: dbState.error,
|
|
310
|
+
replanCount: (dbState as any).replanCount || 0,
|
|
311
|
+
};
|
|
312
|
+
} else {
|
|
313
|
+
dbState = await stateManager.create({ runId, stepId: step.id, workflowId: state.workflowId });
|
|
314
|
+
}
|
|
315
|
+
} else if (loadState) {
|
|
316
|
+
const loaded = await loadState(step.id);
|
|
317
|
+
if (loaded) state = loaded;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
return { state, dbState };
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* Phase 1: Planning
|
|
325
|
+
*/
|
|
326
|
+
async function handlePlanningPhase(
|
|
327
|
+
step: DynamicStep,
|
|
328
|
+
context: ExpressionContext,
|
|
329
|
+
state: DynamicStepState,
|
|
330
|
+
dbState: DynamicStepState | null,
|
|
331
|
+
stateManager: DynamicStateManager | null,
|
|
332
|
+
executeStepFn: (step: Step, context: ExpressionContext) => Promise<StepResult>,
|
|
333
|
+
logger: Logger,
|
|
334
|
+
options: StepExecutorOptions & {
|
|
335
|
+
stateManager?: DynamicStateManager;
|
|
336
|
+
saveState?: (stepId: string, state: DynamicStepState) => Promise<void>;
|
|
337
|
+
executeLlmStep?: typeof executeLlmStep;
|
|
338
|
+
}
|
|
339
|
+
) {
|
|
340
|
+
const { runId, emitEvent, workflowName, abortSignal, mcpManager, workflowDir } = options;
|
|
341
|
+
const runLlmStep = options.executeLlmStep || executeLlmStep;
|
|
342
|
+
|
|
343
|
+
logger.log(
|
|
344
|
+
state.replanCount > 0
|
|
345
|
+
? ` 📋 Re-planning attempt ${state.replanCount}/${step.maxReplans}...`
|
|
346
|
+
: ' 📋 Generating execution plan...'
|
|
347
|
+
);
|
|
348
|
+
|
|
349
|
+
let failureContext = '';
|
|
350
|
+
if (state.replanCount > 0) {
|
|
351
|
+
failureContext = Array.from(state.stepResults.entries())
|
|
352
|
+
.filter(([_, res]) => res.status === 'failed')
|
|
353
|
+
.map(([id, res]) => `- Step "${id}" failed: ${res.error}`)
|
|
354
|
+
.join('\n');
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
const supervisorPrompt = step.prompt
|
|
358
|
+
? ExpressionEvaluator.evaluateString(step.prompt, context)
|
|
359
|
+
: buildSupervisorPrompt(step, context, failureContext);
|
|
360
|
+
|
|
361
|
+
const llmStep: LlmStep = {
|
|
362
|
+
id: `${step.id}_supervisor_${state.replanCount}`,
|
|
363
|
+
type: 'llm',
|
|
364
|
+
agent: step.supervisor || step.agent || 'keystone-architect',
|
|
365
|
+
provider: step.provider,
|
|
366
|
+
model: step.model,
|
|
367
|
+
prompt: supervisorPrompt,
|
|
368
|
+
outputSchema: step.outputSchema ?? DYNAMIC_STEP_OUTPUT_SCHEMA,
|
|
369
|
+
maxIterations: step.maxIterations || 5,
|
|
370
|
+
useStandardTools: false,
|
|
371
|
+
needs: [],
|
|
372
|
+
};
|
|
373
|
+
|
|
374
|
+
const planResult = await runLlmStep(
|
|
375
|
+
llmStep,
|
|
376
|
+
context,
|
|
377
|
+
executeStepFn,
|
|
378
|
+
logger,
|
|
379
|
+
mcpManager,
|
|
380
|
+
workflowDir,
|
|
381
|
+
abortSignal,
|
|
382
|
+
undefined,
|
|
383
|
+
emitEvent,
|
|
384
|
+
workflowName && runId ? { runId, workflow: workflowName } : undefined
|
|
385
|
+
);
|
|
386
|
+
|
|
387
|
+
if (planResult.status !== 'success') {
|
|
388
|
+
state.status = 'failed';
|
|
389
|
+
state.error = planResult.error || 'Plan generation failed';
|
|
390
|
+
if (stateManager && dbState && dbState.id)
|
|
391
|
+
await stateManager.finish(dbState.id, 'failed', state.error);
|
|
392
|
+
throw new Error(state.error);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
state.generatedPlan = planResult.output as DynamicPlan;
|
|
396
|
+
state.status = step.confirmPlan ? 'awaiting_confirmation' : 'executing';
|
|
397
|
+
|
|
398
|
+
logger.log(` 📋 Plan generated with ${state.generatedPlan.steps.length} steps:`);
|
|
399
|
+
for (const s of state.generatedPlan.steps) {
|
|
400
|
+
const deps = s.needs?.length ? ` (needs: ${s.needs.join(', ')})` : '';
|
|
401
|
+
logger.log(` - [${s.type}] ${s.name}${deps}`);
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
if (stateManager && dbState && dbState.id) {
|
|
405
|
+
await stateManager.setPlan(dbState.id, state.generatedPlan, state.status);
|
|
406
|
+
} else if (options.saveState) {
|
|
407
|
+
await options.saveState(step.id, state);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* Phase 1.5: Confirmation
|
|
413
|
+
*/
|
|
414
|
+
async function handleConfirmationPhase(
|
|
415
|
+
step: DynamicStep,
|
|
416
|
+
context: ExpressionContext,
|
|
417
|
+
state: DynamicStepState,
|
|
418
|
+
dbState: DynamicStepState | null,
|
|
419
|
+
stateManager: DynamicStateManager | null,
|
|
420
|
+
logger: Logger,
|
|
421
|
+
options: StepExecutorOptions & {
|
|
422
|
+
stateManager?: DynamicStateManager;
|
|
423
|
+
saveState?: (stepId: string, state: DynamicStepState) => Promise<void>;
|
|
424
|
+
executeHumanStep?: typeof executeHumanStep;
|
|
425
|
+
}
|
|
426
|
+
) {
|
|
427
|
+
const { abortSignal } = options;
|
|
428
|
+
const planJson = JSON.stringify(state.generatedPlan, null, 2);
|
|
429
|
+
const message = `Please review and confirm the generated plan:\n\n${planJson}\n\nType 'yes' to confirm or provide a modified JSON plan:`;
|
|
430
|
+
|
|
431
|
+
const humanStep: any = { id: `${step.id}_confirm`, type: 'human', message, inputType: 'text' };
|
|
432
|
+
const confirmResult = await (options.executeHumanStep || executeHumanStep)(
|
|
433
|
+
humanStep,
|
|
434
|
+
context,
|
|
435
|
+
logger,
|
|
436
|
+
abortSignal
|
|
437
|
+
);
|
|
438
|
+
|
|
439
|
+
if (confirmResult.status === 'success') {
|
|
440
|
+
const response = (confirmResult.output as string).trim().toLowerCase();
|
|
441
|
+
if (response === 'yes' || response === 'y' || response === 'true' || response === '') {
|
|
442
|
+
logger.log(' ✓ Plan confirmed');
|
|
443
|
+
} else {
|
|
444
|
+
try {
|
|
445
|
+
const modifiedPlan = JSON.parse(response) as DynamicPlan;
|
|
446
|
+
if (modifiedPlan.steps && Array.isArray(modifiedPlan.steps)) {
|
|
447
|
+
state.generatedPlan = modifiedPlan;
|
|
448
|
+
logger.log(' ✓ Using modified plan');
|
|
449
|
+
}
|
|
450
|
+
} catch (e) {
|
|
451
|
+
logger.error(
|
|
452
|
+
` ⚠️ Invalid plan JSON. Proceeding with original. Error: ${e instanceof Error ? e.message : String(e)}`
|
|
453
|
+
);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
state.status = 'executing';
|
|
457
|
+
if (stateManager && dbState && dbState.id) {
|
|
458
|
+
await stateManager.setPlan(dbState.id, state.generatedPlan, 'executing');
|
|
459
|
+
} else if (options.saveState) {
|
|
460
|
+
await options.saveState(step.id, state);
|
|
461
|
+
}
|
|
462
|
+
} else {
|
|
463
|
+
state.status = 'failed';
|
|
464
|
+
state.error = confirmResult.error || 'Confirmation failed';
|
|
465
|
+
throw new Error(state.error);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Phase 2: Execution
|
|
471
|
+
*/
|
|
472
|
+
async function handleExecutionPhase(
|
|
473
|
+
step: DynamicStep,
|
|
474
|
+
context: ExpressionContext,
|
|
475
|
+
state: DynamicStepState,
|
|
476
|
+
dbState: DynamicStepState | null,
|
|
477
|
+
stateManager: DynamicStateManager | null,
|
|
478
|
+
executeStepFn: (step: Step, context: ExpressionContext) => Promise<StepResult>,
|
|
479
|
+
logger: Logger,
|
|
480
|
+
options: StepExecutorOptions & {
|
|
481
|
+
stateManager?: DynamicStateManager;
|
|
482
|
+
saveState?: (stepId: string, state: DynamicStepState) => Promise<void>;
|
|
483
|
+
}
|
|
484
|
+
) {
|
|
485
|
+
const { abortSignal, runId, workflowName, emitEvent, saveState } = options;
|
|
486
|
+
|
|
487
|
+
// Detect circular dependencies and validate plan
|
|
488
|
+
topologicalSort(state.generatedPlan.steps);
|
|
489
|
+
|
|
490
|
+
const currentStepIds = new Set(state.generatedPlan.steps.map((s) => s.id));
|
|
491
|
+
const dynamicContext = {
|
|
492
|
+
...context,
|
|
493
|
+
dynamic: {
|
|
494
|
+
plan: state.generatedPlan,
|
|
495
|
+
results: Object.fromEntries(
|
|
496
|
+
Array.from(state.stepResults.entries()).filter(([id]) => currentStepIds.has(id))
|
|
497
|
+
),
|
|
498
|
+
},
|
|
499
|
+
} as ExpressionContext;
|
|
500
|
+
|
|
501
|
+
const completed = new Set<string>();
|
|
502
|
+
const running = new Set<string>();
|
|
503
|
+
const failed = new Set<string>();
|
|
504
|
+
const resultsMap = new Map<string, StepResult>();
|
|
505
|
+
|
|
506
|
+
for (const [id, res] of state.stepResults.entries()) {
|
|
507
|
+
if (!currentStepIds.has(id)) continue;
|
|
508
|
+
resultsMap.set(id, res);
|
|
509
|
+
if (res.status === 'success') completed.add(id);
|
|
510
|
+
else if (res.status === 'failed') failed.add(id);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const maxConcurrency =
|
|
514
|
+
typeof step.concurrency === 'number'
|
|
515
|
+
? step.concurrency
|
|
516
|
+
: Number.parseInt(ExpressionEvaluator.evaluateString(step.concurrency as string, context)) ||
|
|
517
|
+
1;
|
|
518
|
+
logger.log(` 🚀 Starting parallel execution (concurrency: ${maxConcurrency})`);
|
|
519
|
+
|
|
520
|
+
while (completed.size + failed.size < state.generatedPlan.steps.length) {
|
|
521
|
+
if (abortSignal?.aborted) throw new Error('Dynamic step execution canceled');
|
|
522
|
+
|
|
523
|
+
const hasCriticalFailure = Array.from(failed).some((fid) => {
|
|
524
|
+
const s = state.generatedPlan.steps.find((x) => x.id === fid);
|
|
525
|
+
return !(s?.allowStepFailure ?? step.allowStepFailure ?? false);
|
|
526
|
+
});
|
|
527
|
+
if (hasCriticalFailure) break;
|
|
528
|
+
|
|
529
|
+
const ready = state.generatedPlan.steps.filter(
|
|
530
|
+
(s) =>
|
|
531
|
+
!completed.has(s.id) &&
|
|
532
|
+
!running.has(s.id) &&
|
|
533
|
+
!failed.has(s.id) &&
|
|
534
|
+
(s.needs || []).every((depId) => completed.has(depId))
|
|
535
|
+
);
|
|
536
|
+
|
|
537
|
+
if (ready.length === 0 && running.size === 0) {
|
|
538
|
+
const uncompleted = state.generatedPlan.steps.filter(
|
|
539
|
+
(s) => !completed.has(s.id) && !failed.has(s.id)
|
|
540
|
+
);
|
|
541
|
+
if (uncompleted.length > 0) {
|
|
542
|
+
state.status = 'failed';
|
|
543
|
+
state.error = `Dependency deadlock: ${uncompleted.length} steps remain but none are ready.`;
|
|
544
|
+
break;
|
|
545
|
+
}
|
|
546
|
+
break;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
if (ready.length > 0 && running.size < maxConcurrency) {
|
|
550
|
+
const toStart = ready.slice(0, maxConcurrency - running.size);
|
|
551
|
+
for (const genStep of toStart) {
|
|
552
|
+
running.add(genStep.id);
|
|
553
|
+
(async () => {
|
|
554
|
+
try {
|
|
555
|
+
const i = state.generatedPlan.steps.indexOf(genStep);
|
|
556
|
+
logger.log(
|
|
557
|
+
` ⚡ [${i + 1}/${state.generatedPlan.steps.length}] Executing step: ${genStep.name}`
|
|
558
|
+
);
|
|
559
|
+
|
|
560
|
+
const executableStep = convertToExecutableStep(genStep, step.id, step.allowInsecure);
|
|
561
|
+
const stepContext = {
|
|
562
|
+
...dynamicContext,
|
|
563
|
+
steps: {
|
|
564
|
+
...(dynamicContext.steps || {}),
|
|
565
|
+
...Object.fromEntries(
|
|
566
|
+
Array.from(resultsMap.entries()).map(([id, res]) => [
|
|
567
|
+
`${step.id}_${id}`,
|
|
568
|
+
{ output: res.output },
|
|
569
|
+
])
|
|
570
|
+
),
|
|
571
|
+
},
|
|
572
|
+
} as ExpressionContext;
|
|
573
|
+
|
|
574
|
+
if (emitEvent && runId && workflowName) {
|
|
575
|
+
emitEvent({
|
|
576
|
+
type: 'step.start',
|
|
577
|
+
timestamp: new Date().toISOString(),
|
|
578
|
+
runId,
|
|
579
|
+
workflow: workflowName,
|
|
580
|
+
stepId: executableStep.id,
|
|
581
|
+
stepType: executableStep.type,
|
|
582
|
+
phase: 'main',
|
|
583
|
+
stepIndex: i + 1,
|
|
584
|
+
totalSteps: state.generatedPlan.steps.length,
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
const res = await executeStepFn(executableStep, stepContext);
|
|
589
|
+
resultsMap.set(genStep.id, res);
|
|
590
|
+
state.stepResults.set(genStep.id, res);
|
|
591
|
+
|
|
592
|
+
if (stateManager && dbState && dbState.id) {
|
|
593
|
+
await stateManager.completeStep(dbState.id, genStep.id, res);
|
|
594
|
+
await stateManager.updateProgress(dbState.id, completed.size + failed.size + 1);
|
|
595
|
+
} else if (saveState) {
|
|
596
|
+
await saveState(step.id, state);
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
if (emitEvent && runId && workflowName) {
|
|
600
|
+
emitEvent({
|
|
601
|
+
type: 'step.end',
|
|
602
|
+
timestamp: new Date().toISOString(),
|
|
603
|
+
runId,
|
|
604
|
+
workflow: workflowName,
|
|
605
|
+
stepId: executableStep.id,
|
|
606
|
+
stepType: executableStep.type,
|
|
607
|
+
phase: 'main',
|
|
608
|
+
status: res.status as any,
|
|
609
|
+
stepIndex: i + 1,
|
|
610
|
+
totalSteps: state.generatedPlan.steps.length,
|
|
611
|
+
});
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
if (res.status === 'success') {
|
|
615
|
+
completed.add(genStep.id);
|
|
616
|
+
} else {
|
|
617
|
+
failed.add(genStep.id);
|
|
618
|
+
if (!(genStep.allowStepFailure ?? step.allowStepFailure ?? false)) {
|
|
619
|
+
state.status = 'failed';
|
|
620
|
+
state.error = `Step "${genStep.name}" failed: ${res.error}`;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
} catch (err) {
|
|
624
|
+
const failRes: StepResult = { status: 'failed', error: String(err), output: {} };
|
|
625
|
+
resultsMap.set(genStep.id, failRes);
|
|
626
|
+
state.stepResults.set(genStep.id, failRes);
|
|
627
|
+
failed.add(genStep.id);
|
|
628
|
+
state.status = 'failed';
|
|
629
|
+
state.error = `Step "${genStep.name}" crashed: ${err}`;
|
|
630
|
+
} finally {
|
|
631
|
+
running.delete(genStep.id);
|
|
632
|
+
}
|
|
633
|
+
})();
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
const allSatisfied = Array.from(resultsMap.entries()).every(
|
|
640
|
+
([id, r]) =>
|
|
641
|
+
r.status === 'success' ||
|
|
642
|
+
(r.status === 'failed' &&
|
|
643
|
+
(state.generatedPlan.steps.find((s) => s.id === id)?.allowStepFailure ??
|
|
644
|
+
step.allowStepFailure ??
|
|
645
|
+
false))
|
|
646
|
+
);
|
|
647
|
+
|
|
648
|
+
if (allSatisfied) {
|
|
649
|
+
state.status = 'completed';
|
|
650
|
+
state.error = undefined;
|
|
651
|
+
} else if (state.replanCount < step.maxReplans) {
|
|
652
|
+
logger.warn(
|
|
653
|
+
` ⚠️ Execution failed. Attempting self-correction (${state.replanCount + 1}/${step.maxReplans})...`
|
|
654
|
+
);
|
|
655
|
+
state.replanCount++;
|
|
656
|
+
state.status = 'planning';
|
|
657
|
+
state.error = undefined;
|
|
658
|
+
} else {
|
|
659
|
+
state.status = 'failed';
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
if (stateManager && dbState && dbState.id) {
|
|
663
|
+
if (state.status === 'completed' || state.status === 'failed')
|
|
664
|
+
await stateManager.finish(dbState.id, state.status, state.error);
|
|
665
|
+
else await stateManager.updateStatus(dbState.id, state.status);
|
|
666
|
+
} else if (saveState) {
|
|
667
|
+
await saveState(step.id, state);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/**
|
|
672
|
+
* Handle errors during execution
|
|
673
|
+
*/
|
|
674
|
+
async function handleExecutionError(
|
|
675
|
+
step: DynamicStep,
|
|
676
|
+
state: DynamicStepState,
|
|
677
|
+
dbState: DynamicStepState | null,
|
|
678
|
+
stateManager: DynamicStateManager | null,
|
|
679
|
+
saveState: ((stepId: string, state: DynamicStepState) => Promise<void>) | undefined,
|
|
680
|
+
error: any
|
|
681
|
+
): Promise<StepResult> {
|
|
682
|
+
state.status = 'failed';
|
|
683
|
+
state.error = error instanceof Error ? error.message : String(error);
|
|
684
|
+
if (stateManager && dbState && dbState.id) {
|
|
685
|
+
await stateManager.finish(dbState.id, 'failed', state.error);
|
|
686
|
+
} else if (saveState) {
|
|
687
|
+
await saveState(step.id, state);
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
return {
|
|
691
|
+
output: {
|
|
692
|
+
plan: state.generatedPlan,
|
|
693
|
+
results: Object.fromEntries(state.stepResults),
|
|
694
|
+
replans: state.replanCount,
|
|
695
|
+
},
|
|
696
|
+
status: 'failed',
|
|
697
|
+
error: state.error,
|
|
698
|
+
};
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
/**
|
|
702
|
+
* Build final result object
|
|
703
|
+
*/
|
|
704
|
+
function buildFinalResult(state: DynamicStepState): StepResult {
|
|
705
|
+
const results = Object.fromEntries(state.stepResults);
|
|
706
|
+
const summary = {
|
|
707
|
+
total: state.generatedPlan.steps.length,
|
|
708
|
+
succeeded: Array.from(state.stepResults.values()).filter((r) => r.status === 'success').length,
|
|
709
|
+
failed: Array.from(state.stepResults.values()).filter((r) => r.status === 'failed').length,
|
|
710
|
+
replans: state.replanCount,
|
|
711
|
+
};
|
|
712
|
+
|
|
713
|
+
return {
|
|
714
|
+
output: { plan: state.generatedPlan, results, summary },
|
|
715
|
+
status: state.status === 'completed' ? 'success' : 'failed',
|
|
716
|
+
error: state.error,
|
|
717
|
+
};
|
|
718
|
+
}
|