keystone-cli 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +288 -24
- package/package.json +8 -4
- package/src/cli.ts +538 -419
- package/src/commands/doc.ts +31 -0
- package/src/commands/event.ts +29 -0
- package/src/commands/graph.ts +37 -0
- package/src/commands/index.ts +14 -0
- package/src/commands/init.ts +185 -0
- package/src/commands/run.ts +124 -0
- package/src/commands/schema.ts +40 -0
- package/src/commands/utils.ts +78 -0
- package/src/commands/validate.ts +111 -0
- package/src/db/memory-db.ts +50 -2
- package/src/db/workflow-db.test.ts +314 -0
- package/src/db/workflow-db.ts +810 -210
- package/src/expression/evaluator-audit.test.ts +4 -2
- package/src/expression/evaluator.test.ts +14 -1
- package/src/expression/evaluator.ts +166 -19
- package/src/parser/config-schema.ts +18 -0
- package/src/parser/schema.ts +153 -22
- package/src/parser/test-schema.ts +6 -6
- package/src/parser/workflow-parser.test.ts +24 -0
- package/src/parser/workflow-parser.ts +65 -3
- package/src/runner/auto-heal.test.ts +5 -6
- package/src/runner/blueprint-executor.test.ts +2 -2
- package/src/runner/debug-repl.test.ts +5 -8
- package/src/runner/debug-repl.ts +59 -16
- package/src/runner/durable-timers.test.ts +11 -2
- package/src/runner/engine-executor.test.ts +1 -1
- package/src/runner/events.ts +57 -0
- package/src/runner/executors/artifact-executor.ts +166 -0
- package/src/runner/{blueprint-executor.ts → executors/blueprint-executor.ts} +15 -7
- package/src/runner/{engine-executor.ts → executors/engine-executor.ts} +55 -7
- package/src/runner/executors/file-executor.test.ts +48 -0
- package/src/runner/executors/file-executor.ts +324 -0
- package/src/runner/{foreach-executor.ts → executors/foreach-executor.ts} +168 -80
- package/src/runner/executors/human-executor.ts +144 -0
- package/src/runner/executors/join-executor.ts +75 -0
- package/src/runner/executors/llm-executor.ts +1266 -0
- package/src/runner/executors/memory-executor.ts +71 -0
- package/src/runner/executors/plan-executor.ts +104 -0
- package/src/runner/executors/request-executor.ts +265 -0
- package/src/runner/executors/script-executor.ts +43 -0
- package/src/runner/executors/shell-executor.ts +403 -0
- package/src/runner/executors/subworkflow-executor.ts +114 -0
- package/src/runner/executors/types.ts +69 -0
- package/src/runner/executors/wait-executor.ts +59 -0
- package/src/runner/join-scheduling.test.ts +197 -0
- package/src/runner/llm-adapter-runtime.test.ts +209 -0
- package/src/runner/llm-adapter.test.ts +419 -24
- package/src/runner/llm-adapter.ts +414 -17
- package/src/runner/llm-clarification.test.ts +2 -1
- package/src/runner/llm-executor.test.ts +532 -17
- package/src/runner/mcp-client-audit.test.ts +1 -2
- package/src/runner/mcp-client.ts +136 -46
- package/src/runner/mcp-manager.test.ts +4 -0
- package/src/runner/mcp-server.test.ts +58 -0
- package/src/runner/mcp-server.ts +26 -0
- package/src/runner/memoization.test.ts +190 -0
- package/src/runner/optimization-runner.ts +4 -9
- package/src/runner/quality-gate.test.ts +69 -0
- package/src/runner/reflexion.test.ts +6 -17
- package/src/runner/resource-pool.ts +102 -14
- package/src/runner/services/context-builder.ts +144 -0
- package/src/runner/services/secret-manager.ts +105 -0
- package/src/runner/services/workflow-validator.ts +131 -0
- package/src/runner/shell-executor.test.ts +28 -4
- package/src/runner/standard-tools-ast.test.ts +196 -0
- package/src/runner/standard-tools-execution.test.ts +27 -0
- package/src/runner/standard-tools-integration.test.ts +6 -10
- package/src/runner/standard-tools.ts +339 -102
- package/src/runner/step-executor.test.ts +216 -4
- package/src/runner/step-executor.ts +69 -941
- package/src/runner/stream-utils.ts +7 -3
- package/src/runner/test-harness.ts +20 -1
- package/src/runner/timeout.test.ts +10 -0
- package/src/runner/timeout.ts +11 -2
- package/src/runner/tool-integration.test.ts +1 -1
- package/src/runner/wait-step.test.ts +102 -0
- package/src/runner/workflow-runner.test.ts +208 -15
- package/src/runner/workflow-runner.ts +890 -818
- package/src/runner/workflow-scheduler.ts +75 -0
- package/src/runner/workflow-state.ts +269 -0
- package/src/runner/workflow-subflows.test.ts +13 -12
- package/src/scripts/generate-schemas.ts +16 -0
- package/src/templates/agents/explore.md +1 -0
- package/src/templates/agents/general.md +1 -0
- package/src/templates/agents/handoff-router.md +14 -0
- package/src/templates/agents/handoff-specialist.md +15 -0
- package/src/templates/agents/keystone-architect.md +13 -44
- package/src/templates/agents/my-agent.md +1 -0
- package/src/templates/agents/software-engineer.md +1 -0
- package/src/templates/agents/summarizer.md +1 -0
- package/src/templates/agents/test-agent.md +1 -0
- package/src/templates/agents/tester.md +1 -0
- package/src/templates/{basic-inputs.yaml → basics/basic-inputs.yaml} +2 -0
- package/src/templates/{basic-shell.yaml → basics/basic-shell.yaml} +2 -1
- package/src/templates/{full-feature-demo.yaml → basics/full-feature-demo.yaml} +2 -0
- package/src/templates/{stop-watch.yaml → basics/stop-watch.yaml} +1 -0
- package/src/templates/{child-rollback.yaml → control-flow/child-rollback.yaml} +1 -0
- package/src/templates/{cleanup-finally.yaml → control-flow/cleanup-finally.yaml} +1 -0
- package/src/templates/{fan-out-fan-in.yaml → control-flow/fan-out-fan-in.yaml} +3 -0
- package/src/templates/control-flow/idempotency-example.yaml +30 -0
- package/src/templates/{loop-parallel.yaml → control-flow/loop-parallel.yaml} +3 -0
- package/src/templates/{parent-rollback.yaml → control-flow/parent-rollback.yaml} +1 -0
- package/src/templates/{retry-policy.yaml → control-flow/retry-policy.yaml} +3 -0
- package/src/templates/features/artifact-example.yaml +39 -0
- package/src/templates/{engine-example.yaml → features/engine-example.yaml} +1 -0
- package/src/templates/{human-interaction.yaml → features/human-interaction.yaml} +1 -0
- package/src/templates/{llm-agent.yaml → features/llm-agent.yaml} +1 -0
- package/src/templates/{memory-service.yaml → features/memory-service.yaml} +2 -0
- package/src/templates/{robust-automation.yaml → features/robust-automation.yaml} +3 -0
- package/src/templates/features/script-example.yaml +27 -0
- package/src/templates/patterns/agent-handoff.yaml +53 -0
- package/src/templates/{approval-process.yaml → patterns/approval-process.yaml} +1 -0
- package/src/templates/{batch-processor.yaml → patterns/batch-processor.yaml} +2 -0
- package/src/templates/{composition-child.yaml → patterns/composition-child.yaml} +1 -0
- package/src/templates/{composition-parent.yaml → patterns/composition-parent.yaml} +1 -0
- package/src/templates/{data-pipeline.yaml → patterns/data-pipeline.yaml} +2 -0
- package/src/templates/{decompose-implement.yaml → scaffolding/decompose-implement.yaml} +1 -0
- package/src/templates/{decompose-problem.yaml → scaffolding/decompose-problem.yaml} +1 -0
- package/src/templates/{decompose-research.yaml → scaffolding/decompose-research.yaml} +1 -0
- package/src/templates/{decompose-review.yaml → scaffolding/decompose-review.yaml} +1 -0
- package/src/templates/{dev.yaml → scaffolding/dev.yaml} +1 -0
- package/src/templates/scaffolding/review-loop.yaml +97 -0
- package/src/templates/{scaffold-feature.yaml → scaffolding/scaffold-feature.yaml} +2 -0
- package/src/templates/{scaffold-generate.yaml → scaffolding/scaffold-generate.yaml} +1 -0
- package/src/templates/{scaffold-plan.yaml → scaffolding/scaffold-plan.yaml} +1 -0
- package/src/templates/testing/invalid.yaml +6 -0
- package/src/ui/dashboard.tsx +191 -33
- package/src/utils/auth-manager.test.ts +337 -0
- package/src/utils/auth-manager.ts +157 -61
- package/src/utils/blueprint-utils.ts +4 -6
- package/src/utils/config-loader.test.ts +2 -0
- package/src/utils/config-loader.ts +12 -3
- package/src/utils/constants.ts +76 -0
- package/src/utils/container.ts +63 -0
- package/src/utils/context-injector.test.ts +200 -0
- package/src/utils/context-injector.ts +244 -0
- package/src/utils/doc-generator.ts +85 -0
- package/src/utils/env-filter.ts +45 -0
- package/src/utils/json-parser.test.ts +12 -0
- package/src/utils/json-parser.ts +30 -5
- package/src/utils/logger.ts +12 -1
- package/src/utils/mermaid.ts +4 -0
- package/src/utils/paths.ts +52 -1
- package/src/utils/process-sandbox-worker.test.ts +46 -0
- package/src/utils/process-sandbox.ts +227 -14
- package/src/utils/redactor.test.ts +11 -6
- package/src/utils/redactor.ts +25 -9
- package/src/utils/sandbox.ts +3 -0
- package/src/utils/workflow-registry.test.ts +2 -2
- package/src/runner/llm-executor.ts +0 -638
- package/src/runner/shell-executor.ts +0 -366
- package/src/templates/invalid.yaml +0 -5
|
@@ -82,6 +82,30 @@ steps:
|
|
|
82
82
|
expect(workflow.steps.length).toBeGreaterThan(0);
|
|
83
83
|
});
|
|
84
84
|
|
|
85
|
+
test('should expand matrix strategy into foreach', () => {
|
|
86
|
+
const content = `
|
|
87
|
+
name: matrix-workflow
|
|
88
|
+
steps:
|
|
89
|
+
- id: test_matrix
|
|
90
|
+
type: shell
|
|
91
|
+
run: echo test
|
|
92
|
+
strategy:
|
|
93
|
+
matrix:
|
|
94
|
+
node: [18, 20]
|
|
95
|
+
os: [ubuntu, macos]
|
|
96
|
+
`;
|
|
97
|
+
const filePath = join(tempDir, 'matrix.yaml');
|
|
98
|
+
writeFileSync(filePath, content);
|
|
99
|
+
const workflow = WorkflowParser.loadWorkflow(filePath);
|
|
100
|
+
const step = workflow.steps[0] as { foreach?: string; strategy?: unknown };
|
|
101
|
+
expect(step.foreach).toBeDefined();
|
|
102
|
+
const items = JSON.parse(step.foreach || '[]') as Array<Record<string, unknown>>;
|
|
103
|
+
expect(items).toHaveLength(4);
|
|
104
|
+
expect(items[0]).toHaveProperty('node');
|
|
105
|
+
expect(items[0]).toHaveProperty('os');
|
|
106
|
+
expect(step.strategy).toBeUndefined();
|
|
107
|
+
});
|
|
108
|
+
|
|
85
109
|
test('should throw on invalid schema', () => {
|
|
86
110
|
const content = `
|
|
87
111
|
name: invalid
|
|
@@ -22,6 +22,9 @@ export class WorkflowParser {
|
|
|
22
22
|
const workflow = WorkflowSchema.parse(raw);
|
|
23
23
|
const workflowDir = dirname(path);
|
|
24
24
|
|
|
25
|
+
// Expand matrix strategies into foreach items
|
|
26
|
+
WorkflowParser.applyMatrixStrategies(workflow);
|
|
27
|
+
|
|
25
28
|
// Resolve implicit dependencies from expressions
|
|
26
29
|
WorkflowParser.resolveImplicitDependencies(workflow);
|
|
27
30
|
|
|
@@ -31,6 +34,9 @@ export class WorkflowParser {
|
|
|
31
34
|
// Validate agents exist
|
|
32
35
|
WorkflowParser.validateAgents(workflow, workflowDir);
|
|
33
36
|
|
|
37
|
+
// Validate artifact steps
|
|
38
|
+
WorkflowParser.validateArtifacts(workflow);
|
|
39
|
+
|
|
34
40
|
// Validate errors block
|
|
35
41
|
WorkflowParser.validateErrors(workflow);
|
|
36
42
|
|
|
@@ -77,6 +83,43 @@ export class WorkflowParser {
|
|
|
77
83
|
}
|
|
78
84
|
}
|
|
79
85
|
|
|
86
|
+
/**
|
|
87
|
+
* Expand step.strategy.matrix into foreach expressions.
|
|
88
|
+
*/
|
|
89
|
+
private static applyMatrixStrategies(workflow: Workflow): void {
|
|
90
|
+
const allSteps = [...workflow.steps, ...(workflow.errors || []), ...(workflow.finally || [])];
|
|
91
|
+
for (const step of allSteps) {
|
|
92
|
+
if (!step.strategy?.matrix) continue;
|
|
93
|
+
|
|
94
|
+
if (step.foreach) {
|
|
95
|
+
throw new Error(`Step "${step.id}" cannot use both foreach and strategy.matrix`);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const matrix = step.strategy.matrix;
|
|
99
|
+
const keys = Object.keys(matrix);
|
|
100
|
+
if (keys.length === 0) {
|
|
101
|
+
throw new Error(`Step "${step.id}" matrix must define at least one axis`);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
let combos: Array<Record<string, unknown>> = [{}];
|
|
105
|
+
for (const key of keys) {
|
|
106
|
+
const values = matrix[key];
|
|
107
|
+
if (!Array.isArray(values) || values.length === 0) {
|
|
108
|
+
throw new Error(`Step "${step.id}" matrix axis "${key}" must have at least one value`);
|
|
109
|
+
}
|
|
110
|
+
combos = combos.flatMap((combo) =>
|
|
111
|
+
values.map((value) => ({
|
|
112
|
+
...combo,
|
|
113
|
+
[key]: value,
|
|
114
|
+
}))
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
step.foreach = JSON.stringify(combos);
|
|
119
|
+
step.strategy = undefined;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
80
123
|
/**
|
|
81
124
|
* Automatically detect step dependencies from expressions
|
|
82
125
|
*/
|
|
@@ -172,6 +215,22 @@ export class WorkflowParser {
|
|
|
172
215
|
}
|
|
173
216
|
}
|
|
174
217
|
|
|
218
|
+
/**
|
|
219
|
+
* Validate artifact steps have the required fields for their operation.
|
|
220
|
+
*/
|
|
221
|
+
private static validateArtifacts(workflow: Workflow): void {
|
|
222
|
+
const allSteps = [...workflow.steps, ...(workflow.errors || []), ...(workflow.finally || [])];
|
|
223
|
+
for (const step of allSteps) {
|
|
224
|
+
if (step.type !== 'artifact') continue;
|
|
225
|
+
if (step.op === 'upload' && (!step.paths || step.paths.length === 0)) {
|
|
226
|
+
throw new Error(`Artifact step "${step.id}" requires paths for upload`);
|
|
227
|
+
}
|
|
228
|
+
if (step.op === 'download' && !step.path) {
|
|
229
|
+
throw new Error(`Artifact step "${step.id}" requires path for download`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
175
234
|
/**
|
|
176
235
|
* Validate finally block
|
|
177
236
|
*/
|
|
@@ -244,7 +303,8 @@ export class WorkflowParser {
|
|
|
244
303
|
|
|
245
304
|
// Validate all dependencies exist before sorting
|
|
246
305
|
for (const step of workflow.steps) {
|
|
247
|
-
|
|
306
|
+
const needs = step.needs || [];
|
|
307
|
+
for (const dep of needs) {
|
|
248
308
|
if (!stepMap.has(dep)) {
|
|
249
309
|
throw new Error(`Step "${step.id}" depends on non-existent step "${dep}"`);
|
|
250
310
|
}
|
|
@@ -254,13 +314,15 @@ export class WorkflowParser {
|
|
|
254
314
|
// Calculate in-degree
|
|
255
315
|
// In-degree = number of dependencies a step has
|
|
256
316
|
for (const step of workflow.steps) {
|
|
257
|
-
|
|
317
|
+
const needs = step.needs || [];
|
|
318
|
+
inDegree.set(step.id, needs.length);
|
|
258
319
|
}
|
|
259
320
|
|
|
260
321
|
// Build reverse dependency map for O(1) lookups instead of O(n)
|
|
261
322
|
const dependents = new Map<string, string[]>();
|
|
262
323
|
for (const step of workflow.steps) {
|
|
263
|
-
|
|
324
|
+
const needs = step.needs || [];
|
|
325
|
+
for (const dep of needs) {
|
|
264
326
|
if (!dependents.has(dep)) dependents.set(dep, []);
|
|
265
327
|
dependents.get(dep)?.push(step.id);
|
|
266
328
|
}
|
|
@@ -29,8 +29,7 @@ describe('WorkflowRunner Auto-Heal', () => {
|
|
|
29
29
|
dbPath: ':memory:',
|
|
30
30
|
});
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
const db = (runner as any).db;
|
|
32
|
+
const db = (runner as unknown as { db: any }).db;
|
|
34
33
|
await db.createRun(runner.runId, workflow.name, {});
|
|
35
34
|
|
|
36
35
|
const spy = jest.spyOn(StepExecutor, 'executeStep');
|
|
@@ -44,8 +43,7 @@ describe('WorkflowRunner Auto-Heal', () => {
|
|
|
44
43
|
}
|
|
45
44
|
|
|
46
45
|
if (step.id === 'fail-step') {
|
|
47
|
-
|
|
48
|
-
if ((step as any).run === 'echo "fixed"') {
|
|
46
|
+
if ((step as unknown as { run: string }).run === 'echo "fixed"') {
|
|
49
47
|
return { status: 'success', output: 'fixed' };
|
|
50
48
|
}
|
|
51
49
|
return { status: 'failed', output: null, error: 'Command failed' };
|
|
@@ -54,8 +52,9 @@ describe('WorkflowRunner Auto-Heal', () => {
|
|
|
54
52
|
return { status: 'failed', output: null, error: 'Unknown step' };
|
|
55
53
|
});
|
|
56
54
|
|
|
57
|
-
|
|
58
|
-
|
|
55
|
+
await (
|
|
56
|
+
runner as unknown as { executeStepWithForeach: (step: Step) => Promise<void> }
|
|
57
|
+
).executeStepWithForeach(workflow.steps[0]);
|
|
59
58
|
|
|
60
59
|
expect(spy).toHaveBeenCalledTimes(3);
|
|
61
60
|
|
|
@@ -4,8 +4,8 @@ import * as path from 'node:path';
|
|
|
4
4
|
import type { ExpressionContext } from '../expression/evaluator';
|
|
5
5
|
import type { Blueprint, BlueprintStep, Step } from '../parser/schema';
|
|
6
6
|
import type { Logger } from '../utils/logger';
|
|
7
|
-
import { executeBlueprintStep } from './blueprint-executor';
|
|
8
|
-
import type { executeLlmStep } from './llm-executor';
|
|
7
|
+
import { executeBlueprintStep } from './executors/blueprint-executor.ts';
|
|
8
|
+
import type { executeLlmStep } from './executors/llm-executor.ts';
|
|
9
9
|
import type { StepResult } from './step-executor';
|
|
10
10
|
|
|
11
11
|
describe('BlueprintExecutor', () => {
|
|
@@ -96,10 +96,9 @@ describe('DebugRepl', () => {
|
|
|
96
96
|
await new Promise((r) => setTimeout(r, 10));
|
|
97
97
|
|
|
98
98
|
expect(mockLogger.log).toHaveBeenCalled();
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
);
|
|
99
|
+
const lastCall = (
|
|
100
|
+
(mockLogger.log as unknown as { mock: { calls: any[][] } }).mock.calls as any[][]
|
|
101
|
+
).find((call: any[]) => String(call[0]).includes('foo'));
|
|
103
102
|
expect(lastCall?.[0]).toContain('bar');
|
|
104
103
|
input.write('exit\n');
|
|
105
104
|
});
|
|
@@ -233,8 +232,7 @@ describe('DebugRepl', () => {
|
|
|
233
232
|
const repl = new DebugRepl(mockContext, mockStep, mockError, mockLogger, input, output);
|
|
234
233
|
|
|
235
234
|
const spySpawnSync = spyOn(cp, 'spawnSync').mockImplementation(
|
|
236
|
-
|
|
237
|
-
() => ({ error: null, status: 0 }) as any
|
|
235
|
+
() => ({ error: null, status: 0 }) as unknown as cp.SpawnSyncReturns<Buffer>
|
|
238
236
|
);
|
|
239
237
|
const spyWriteFileSync = spyOn(fs, 'writeFileSync').mockImplementation(() => {});
|
|
240
238
|
const updatedStep = { ...mockStep, run: 'echo "fixed"' };
|
|
@@ -276,8 +274,7 @@ describe('DebugRepl', () => {
|
|
|
276
274
|
const repl = new DebugRepl(mockContext, mockStep, mockError, mockLogger, input, output);
|
|
277
275
|
|
|
278
276
|
const spySpawnSync = spyOn(cp, 'spawnSync').mockImplementation(
|
|
279
|
-
|
|
280
|
-
() => ({ error: null, status: 0 }) as any
|
|
277
|
+
() => ({ error: null, status: 0 }) as unknown as cp.SpawnSyncReturns<Buffer>
|
|
281
278
|
);
|
|
282
279
|
const spyWriteFileSync = spyOn(fs, 'writeFileSync').mockImplementation(() => {});
|
|
283
280
|
const spyReadFileSync = spyOn(fs, 'readFileSync').mockImplementation(
|
package/src/runner/debug-repl.ts
CHANGED
|
@@ -12,9 +12,16 @@ import { ConsoleLogger, type Logger } from '../utils/logger.ts';
|
|
|
12
12
|
|
|
13
13
|
export type DebugAction =
|
|
14
14
|
| { type: 'retry'; modifiedStep?: Step }
|
|
15
|
+
| { type: 'continue'; modifiedStep?: Step }
|
|
15
16
|
| { type: 'skip' }
|
|
16
17
|
| { type: 'continue_failure' }; // Default behavior (exit debug mode, let it fail)
|
|
17
18
|
|
|
19
|
+
export type DebugReplMode = 'error' | 'breakpoint';
|
|
20
|
+
|
|
21
|
+
export interface DebugReplOptions {
|
|
22
|
+
mode?: DebugReplMode;
|
|
23
|
+
}
|
|
24
|
+
|
|
18
25
|
export class DebugRepl {
|
|
19
26
|
constructor(
|
|
20
27
|
private context: ExpressionContext,
|
|
@@ -22,21 +29,35 @@ export class DebugRepl {
|
|
|
22
29
|
private error: unknown,
|
|
23
30
|
private logger: Logger = new ConsoleLogger(),
|
|
24
31
|
private inputStream: NodeJS.ReadableStream = process.stdin,
|
|
25
|
-
private outputStream: NodeJS.WritableStream = process.stdout
|
|
32
|
+
private outputStream: NodeJS.WritableStream = process.stdout,
|
|
33
|
+
private options: DebugReplOptions = {}
|
|
26
34
|
) {}
|
|
27
35
|
|
|
28
36
|
public async start(): Promise<DebugAction> {
|
|
29
|
-
this.
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
const mode = this.options.mode || 'error';
|
|
38
|
+
|
|
39
|
+
if (mode === 'breakpoint') {
|
|
40
|
+
this.logger.log(`\n⛔ Breakpoint hit before step '${this.step.id}'.`);
|
|
41
|
+
this.logger.log('\nEntering Debug Mode. Available commands:');
|
|
42
|
+
this.logger.log(' > context (view current inputs/outputs involved in this step)');
|
|
43
|
+
this.logger.log(' > continue (run the step, optionally with edited definition)');
|
|
44
|
+
this.logger.log(' > edit (edit the step definition in your $EDITOR)');
|
|
45
|
+
this.logger.log(' > skip (skip this step and proceed)');
|
|
46
|
+
this.logger.log(' > eval <code> (run JS expression against context)');
|
|
47
|
+
this.logger.log(' > exit (continue without changes)');
|
|
48
|
+
} else {
|
|
49
|
+
this.logger.error(`\n❌ Step '${this.step.id}' failed.`);
|
|
50
|
+
this.logger.error(
|
|
51
|
+
` Error: ${this.error instanceof Error ? this.error.message : String(this.error)}`
|
|
52
|
+
);
|
|
53
|
+
this.logger.log('\nEntering Debug Mode. Available commands:');
|
|
54
|
+
this.logger.log(' > context (view current inputs/outputs involved in this step)');
|
|
55
|
+
this.logger.log(' > retry (re-run step, optionally with edited definition)');
|
|
56
|
+
this.logger.log(' > edit (edit the step definition in your $EDITOR)');
|
|
57
|
+
this.logger.log(' > skip (skip this step and proceed)');
|
|
58
|
+
this.logger.log(' > eval <code> (run JS expression against context)');
|
|
59
|
+
this.logger.log(' > exit (resume failure/exit)');
|
|
60
|
+
}
|
|
40
61
|
|
|
41
62
|
const rl = readline.createInterface({
|
|
42
63
|
input: this.inputStream,
|
|
@@ -74,8 +95,21 @@ export class DebugRepl {
|
|
|
74
95
|
break;
|
|
75
96
|
|
|
76
97
|
case 'retry':
|
|
77
|
-
|
|
78
|
-
|
|
98
|
+
if (mode === 'breakpoint') {
|
|
99
|
+
resolveOnce({ type: 'continue', modifiedStep: this.step });
|
|
100
|
+
rl.close();
|
|
101
|
+
} else {
|
|
102
|
+
resolveOnce({ type: 'retry', modifiedStep: this.step });
|
|
103
|
+
rl.close();
|
|
104
|
+
}
|
|
105
|
+
break;
|
|
106
|
+
|
|
107
|
+
case 'continue':
|
|
108
|
+
case 'run':
|
|
109
|
+
if (mode === 'breakpoint') {
|
|
110
|
+
resolveOnce({ type: 'continue', modifiedStep: this.step });
|
|
111
|
+
rl.close();
|
|
112
|
+
}
|
|
79
113
|
break;
|
|
80
114
|
|
|
81
115
|
case 'skip':
|
|
@@ -85,7 +119,11 @@ export class DebugRepl {
|
|
|
85
119
|
|
|
86
120
|
case 'exit':
|
|
87
121
|
case 'quit':
|
|
88
|
-
|
|
122
|
+
if (mode === 'breakpoint') {
|
|
123
|
+
resolveOnce({ type: 'continue', modifiedStep: this.step });
|
|
124
|
+
} else {
|
|
125
|
+
resolveOnce({ type: 'continue_failure' });
|
|
126
|
+
}
|
|
89
127
|
rl.close();
|
|
90
128
|
break;
|
|
91
129
|
|
|
@@ -125,7 +163,12 @@ export class DebugRepl {
|
|
|
125
163
|
break;
|
|
126
164
|
}
|
|
127
165
|
|
|
128
|
-
|
|
166
|
+
const terminalCommands =
|
|
167
|
+
mode === 'breakpoint'
|
|
168
|
+
? new Set(['retry', 'continue', 'run', 'skip', 'exit', 'quit'])
|
|
169
|
+
: new Set(['retry', 'skip', 'exit', 'quit']);
|
|
170
|
+
|
|
171
|
+
if (!terminalCommands.has(cmd)) {
|
|
129
172
|
rl.prompt();
|
|
130
173
|
}
|
|
131
174
|
});
|
|
@@ -174,10 +174,17 @@ describe('Durable Timers Integration', () => {
|
|
|
174
174
|
}
|
|
175
175
|
|
|
176
176
|
// Manually backdate the timer in the DB to simulate elapsed time
|
|
177
|
-
const pastDate = new Date(Date.now() -
|
|
177
|
+
const pastDate = new Date(Date.now() - 10000).toISOString();
|
|
178
178
|
const { Database } = require('bun:sqlite');
|
|
179
179
|
const sqlite = new Database(dbPath);
|
|
180
180
|
sqlite.prepare('UPDATE durable_timers SET wake_at = ? WHERE id = ?').run(pastDate, timer.id);
|
|
181
|
+
|
|
182
|
+
// Also need to update the step_executions output, as WorkflowState hydrates from there
|
|
183
|
+
const newOutput = JSON.stringify({ durable: true, wakeAt: pastDate, durationMs: 120000 });
|
|
184
|
+
sqlite
|
|
185
|
+
.prepare('UPDATE step_executions SET output = ? WHERE run_id = ? AND step_id = ?')
|
|
186
|
+
.run(newOutput, runId, 'wait');
|
|
187
|
+
|
|
181
188
|
sqlite.close();
|
|
182
189
|
|
|
183
190
|
const resumeRunner = new WorkflowRunner(sleepWorkflow, {
|
|
@@ -192,7 +199,9 @@ describe('Durable Timers Integration', () => {
|
|
|
192
199
|
expect(run?.status).toBe(WorkflowStatus.SUCCESS);
|
|
193
200
|
|
|
194
201
|
const steps = await db.getStepsByRun(runId);
|
|
195
|
-
|
|
202
|
+
const waitStep = steps.find((s) => s.step_id === 'wait' && s.status === StepStatus.SUCCESS);
|
|
203
|
+
expect(waitStep).toBeDefined();
|
|
204
|
+
expect(waitStep?.status).toBe(StepStatus.SUCCESS);
|
|
196
205
|
|
|
197
206
|
const finalTimer = await db.getTimer(timer.id);
|
|
198
207
|
expect(finalTimer?.completed_at).not.toBeNull();
|
|
@@ -3,7 +3,7 @@ import { mkdirSync, rmSync } from 'node:fs';
|
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import type { EngineStep } from '../parser/schema';
|
|
6
|
-
import { executeEngineStep } from './engine-executor';
|
|
6
|
+
import { executeEngineStep } from './executors/engine-executor.ts';
|
|
7
7
|
|
|
8
8
|
// Helper to create a minimal valid EngineStep for testing
|
|
9
9
|
const createStep = (overrides: Partial<EngineStep>): EngineStep =>
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { StepStatusType, WorkflowStatusType } from '../types/status.ts';
|
|
2
|
+
|
|
3
|
+
export type StepPhase = 'main' | 'errors' | 'finally';
|
|
4
|
+
|
|
5
|
+
export type WorkflowEvent =
|
|
6
|
+
| {
|
|
7
|
+
type: 'workflow.start';
|
|
8
|
+
timestamp: string;
|
|
9
|
+
runId: string;
|
|
10
|
+
workflow: string;
|
|
11
|
+
inputs?: Record<string, unknown>;
|
|
12
|
+
}
|
|
13
|
+
| {
|
|
14
|
+
type: 'step.start';
|
|
15
|
+
timestamp: string;
|
|
16
|
+
runId: string;
|
|
17
|
+
workflow: string;
|
|
18
|
+
stepId: string;
|
|
19
|
+
stepType: string;
|
|
20
|
+
phase: StepPhase;
|
|
21
|
+
stepIndex?: number;
|
|
22
|
+
totalSteps?: number;
|
|
23
|
+
}
|
|
24
|
+
| {
|
|
25
|
+
type: 'step.end';
|
|
26
|
+
timestamp: string;
|
|
27
|
+
runId: string;
|
|
28
|
+
workflow: string;
|
|
29
|
+
stepId: string;
|
|
30
|
+
stepType: string;
|
|
31
|
+
phase: StepPhase;
|
|
32
|
+
status: StepStatusType;
|
|
33
|
+
durationMs?: number;
|
|
34
|
+
error?: string;
|
|
35
|
+
stepIndex?: number;
|
|
36
|
+
totalSteps?: number;
|
|
37
|
+
}
|
|
38
|
+
| {
|
|
39
|
+
type: 'llm.thought';
|
|
40
|
+
timestamp: string;
|
|
41
|
+
runId: string;
|
|
42
|
+
workflow: string;
|
|
43
|
+
stepId: string;
|
|
44
|
+
content: string;
|
|
45
|
+
source: 'thinking' | 'reasoning';
|
|
46
|
+
}
|
|
47
|
+
| {
|
|
48
|
+
type: 'workflow.complete';
|
|
49
|
+
timestamp: string;
|
|
50
|
+
runId: string;
|
|
51
|
+
workflow: string;
|
|
52
|
+
status: WorkflowStatusType;
|
|
53
|
+
outputs?: Record<string, unknown>;
|
|
54
|
+
error?: string;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export type EventHandler = (event: WorkflowEvent) => void;
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { globSync } from 'glob';
|
|
4
|
+
import type { ExpressionContext } from '../../expression/evaluator.ts';
|
|
5
|
+
import { ExpressionEvaluator } from '../../expression/evaluator.ts';
|
|
6
|
+
import type { ArtifactStep } from '../../parser/schema.ts';
|
|
7
|
+
import type { Logger } from '../../utils/logger.ts';
|
|
8
|
+
import type { StepResult } from './types.ts';
|
|
9
|
+
|
|
10
|
+
function normalizePath(rawPath: string): string {
|
|
11
|
+
const trimmed = rawPath.trim();
|
|
12
|
+
return trimmed.length > 0 ? trimmed : '.';
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function assertWithinBaseDir(
|
|
16
|
+
baseDir: string,
|
|
17
|
+
targetPath: string,
|
|
18
|
+
allowOutsideCwd?: boolean,
|
|
19
|
+
label = 'Path'
|
|
20
|
+
): void {
|
|
21
|
+
if (allowOutsideCwd) return;
|
|
22
|
+
const realBase = fs.realpathSync(baseDir);
|
|
23
|
+
const normalizedPath = normalizePath(targetPath);
|
|
24
|
+
const resolvedPath = path.resolve(baseDir, normalizedPath);
|
|
25
|
+
|
|
26
|
+
let current = resolvedPath;
|
|
27
|
+
while (current !== path.dirname(current) && !fs.existsSync(current)) {
|
|
28
|
+
current = path.dirname(current);
|
|
29
|
+
}
|
|
30
|
+
const realTarget = fs.existsSync(current) ? fs.realpathSync(current) : current;
|
|
31
|
+
const relativePath = path.relative(realBase, realTarget);
|
|
32
|
+
if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) {
|
|
33
|
+
throw new Error(
|
|
34
|
+
`Access denied: ${label} '${normalizedPath}' resolves outside the working directory. Use 'allowOutsideCwd: true' to override.`
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function resolveSafeRelativePath(baseDir: string, absolutePath: string): string {
|
|
40
|
+
const relativeToBase = path.relative(baseDir, absolutePath);
|
|
41
|
+
if (!relativeToBase.startsWith('..') && !path.isAbsolute(relativeToBase)) {
|
|
42
|
+
return relativeToBase;
|
|
43
|
+
}
|
|
44
|
+
const root = path.parse(absolutePath).root;
|
|
45
|
+
return path.relative(root, absolutePath);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Execute an artifact step (upload/download)
|
|
50
|
+
*/
|
|
51
|
+
export async function executeArtifactStep(
|
|
52
|
+
step: ArtifactStep,
|
|
53
|
+
context: ExpressionContext,
|
|
54
|
+
logger: Logger,
|
|
55
|
+
options: {
|
|
56
|
+
artifactRoot?: string;
|
|
57
|
+
workflowDir?: string;
|
|
58
|
+
runId?: string;
|
|
59
|
+
abortSignal?: AbortSignal;
|
|
60
|
+
}
|
|
61
|
+
): Promise<StepResult> {
|
|
62
|
+
if (options.abortSignal?.aborted) {
|
|
63
|
+
throw new Error('Artifact operation aborted');
|
|
64
|
+
}
|
|
65
|
+
const baseDir = options.workflowDir || process.cwd();
|
|
66
|
+
const rawName = ExpressionEvaluator.evaluateString(step.name, context);
|
|
67
|
+
if (typeof rawName !== 'string' || rawName.trim().length === 0) {
|
|
68
|
+
throw new Error('Artifact name must be a non-empty string');
|
|
69
|
+
}
|
|
70
|
+
const sanitizedName = rawName.replace(/[^a-zA-Z0-9._-]/g, '_');
|
|
71
|
+
if (sanitizedName !== rawName) {
|
|
72
|
+
logger.warn(
|
|
73
|
+
`⚠️ Artifact name "${rawName}" contained unsafe characters. Using "${sanitizedName}".`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const artifactRoot = options.artifactRoot || path.join(process.cwd(), '.keystone', 'artifacts');
|
|
78
|
+
const runDir = options.runId ? path.join(artifactRoot, options.runId) : artifactRoot;
|
|
79
|
+
|
|
80
|
+
if (!fs.existsSync(runDir)) {
|
|
81
|
+
fs.mkdirSync(runDir, { recursive: true });
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const artifactPath = path.join(runDir, sanitizedName);
|
|
85
|
+
|
|
86
|
+
if (step.op === 'upload') {
|
|
87
|
+
const patterns = (
|
|
88
|
+
step.paths && step.paths.length > 0 ? step.paths : step.path ? [step.path] : []
|
|
89
|
+
).map((value) => ExpressionEvaluator.evaluateString(value, context));
|
|
90
|
+
if (patterns.length === 0) {
|
|
91
|
+
throw new Error('Artifact upload requires at least one path');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const matchedFiles = new Set<string>();
|
|
95
|
+
for (const pattern of patterns) {
|
|
96
|
+
const matches = globSync(pattern, {
|
|
97
|
+
cwd: baseDir,
|
|
98
|
+
absolute: true,
|
|
99
|
+
dot: true,
|
|
100
|
+
nodir: true,
|
|
101
|
+
});
|
|
102
|
+
for (const match of matches) {
|
|
103
|
+
matchedFiles.add(match);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (matchedFiles.size === 0) {
|
|
108
|
+
throw new Error(`No files matched for artifact "${rawName}"`);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
await fs.promises.rm(artifactPath, { recursive: true, force: true });
|
|
112
|
+
fs.mkdirSync(artifactPath, { recursive: true });
|
|
113
|
+
|
|
114
|
+
const files: string[] = [];
|
|
115
|
+
for (const filePath of matchedFiles) {
|
|
116
|
+
if (options.abortSignal?.aborted) {
|
|
117
|
+
throw new Error('Artifact upload aborted');
|
|
118
|
+
}
|
|
119
|
+
assertWithinBaseDir(baseDir, filePath, step.allowOutsideCwd);
|
|
120
|
+
const relativePath = resolveSafeRelativePath(baseDir, filePath);
|
|
121
|
+
const destination = path.join(artifactPath, relativePath);
|
|
122
|
+
const relativeToArtifact = path.relative(artifactPath, destination);
|
|
123
|
+
if (relativeToArtifact.startsWith('..') || path.isAbsolute(relativeToArtifact)) {
|
|
124
|
+
throw new Error(`Artifact path escape detected for "${relativePath}"`);
|
|
125
|
+
}
|
|
126
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
127
|
+
await fs.promises.copyFile(filePath, destination);
|
|
128
|
+
files.push(relativePath);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return {
|
|
132
|
+
output: {
|
|
133
|
+
name: sanitizedName,
|
|
134
|
+
op: 'upload',
|
|
135
|
+
artifactPath,
|
|
136
|
+
files,
|
|
137
|
+
fileCount: files.length,
|
|
138
|
+
},
|
|
139
|
+
status: 'success',
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
// download
|
|
143
|
+
if (!step.path) {
|
|
144
|
+
throw new Error(`Artifact download requires a destination path for "${rawName}"`);
|
|
145
|
+
}
|
|
146
|
+
const dest = ExpressionEvaluator.evaluateString(step.path, context);
|
|
147
|
+
const destPath = path.isAbsolute(dest) ? dest : path.join(baseDir, dest);
|
|
148
|
+
assertWithinBaseDir(baseDir, destPath, step.allowOutsideCwd);
|
|
149
|
+
|
|
150
|
+
if (!fs.existsSync(artifactPath)) {
|
|
151
|
+
throw new Error(`Artifact not found for download: ${sanitizedName}`);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// ensure dest dir exists
|
|
155
|
+
const destDir = path.dirname(destPath);
|
|
156
|
+
if (!fs.existsSync(destDir)) {
|
|
157
|
+
fs.mkdirSync(destDir, { recursive: true });
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
await fs.promises.cp(artifactPath, destPath, { recursive: true, force: true });
|
|
161
|
+
|
|
162
|
+
return {
|
|
163
|
+
output: { name: sanitizedName, path: destPath, op: 'download', artifactPath },
|
|
164
|
+
status: 'success',
|
|
165
|
+
};
|
|
166
|
+
}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { mkdirSync } from 'node:fs';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
|
-
import type { ExpressionContext } from '
|
|
4
|
-
import type { Blueprint, BlueprintStep, LlmStep, Step } from '
|
|
5
|
-
import { BlueprintUtils } from '
|
|
6
|
-
import type { Logger } from '
|
|
3
|
+
import type { ExpressionContext } from '../../expression/evaluator.ts';
|
|
4
|
+
import type { Blueprint, BlueprintStep, LlmStep, Step } from '../../parser/schema.ts';
|
|
5
|
+
import { BlueprintUtils } from '../../utils/blueprint-utils.ts';
|
|
6
|
+
import type { Logger } from '../../utils/logger.ts';
|
|
7
|
+
import type { WorkflowEvent } from '../events.ts';
|
|
8
|
+
import type { MCPManager } from '../mcp-manager.ts';
|
|
7
9
|
import { executeLlmStep } from './llm-executor.ts';
|
|
8
|
-
import type {
|
|
9
|
-
import type { StepResult } from './step-executor.ts';
|
|
10
|
+
import type { StepResult } from './types.ts';
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
13
|
* Execute a blueprint step
|
|
@@ -23,6 +24,8 @@ export async function executeBlueprintStep(
|
|
|
23
24
|
runId?: string;
|
|
24
25
|
artifactRoot?: string;
|
|
25
26
|
executeLlmStep?: typeof executeLlmStep;
|
|
27
|
+
emitEvent?: (event: WorkflowEvent) => void;
|
|
28
|
+
workflowName?: string;
|
|
26
29
|
}
|
|
27
30
|
): Promise<StepResult> {
|
|
28
31
|
const {
|
|
@@ -32,6 +35,8 @@ export async function executeBlueprintStep(
|
|
|
32
35
|
runId,
|
|
33
36
|
artifactRoot,
|
|
34
37
|
executeLlmStep: injected,
|
|
38
|
+
emitEvent,
|
|
39
|
+
workflowName,
|
|
35
40
|
} = options;
|
|
36
41
|
const runLlmStep = injected || executeLlmStep;
|
|
37
42
|
|
|
@@ -123,7 +128,10 @@ export async function executeBlueprintStep(
|
|
|
123
128
|
logger,
|
|
124
129
|
mcpManager,
|
|
125
130
|
workflowDir,
|
|
126
|
-
abortSignal
|
|
131
|
+
abortSignal,
|
|
132
|
+
undefined,
|
|
133
|
+
emitEvent,
|
|
134
|
+
workflowName ? { runId, workflow: workflowName } : undefined
|
|
127
135
|
);
|
|
128
136
|
|
|
129
137
|
if (llmResult.status !== 'success') {
|