@pixelbyte-software/pixcode 1.42.1 → 1.42.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{index-C97kIvXz.js → index-BnaWRV1a.js} +182 -182
- package/dist/index.html +1 -1
- package/dist-server/server/modules/orchestration/tasks/orchestration-task.service.js +86 -0
- package/dist-server/server/modules/orchestration/tasks/orchestration-task.service.js.map +1 -1
- package/dist-server/server/modules/orchestration/tasks/task-run-graph.js +158 -0
- package/dist-server/server/modules/orchestration/tasks/task-run-graph.js.map +1 -0
- package/dist-server/server/modules/orchestration/workflows/workflow-fallback-policy.js +114 -0
- package/dist-server/server/modules/orchestration/workflows/workflow-fallback-policy.js.map +1 -0
- package/dist-server/server/modules/orchestration/workflows/workflow-replay.js +177 -0
- package/dist-server/server/modules/orchestration/workflows/workflow-replay.js.map +1 -0
- package/dist-server/server/modules/orchestration/workflows/workflow-runner.js +53 -7
- package/dist-server/server/modules/orchestration/workflows/workflow-runner.js.map +1 -1
- package/dist-server/server/modules/orchestration/workflows/workflow-trace.js +74 -0
- package/dist-server/server/modules/orchestration/workflows/workflow-trace.js.map +1 -1
- package/dist-server/server/modules/orchestration/workflows/workflow.routes.js +88 -0
- package/dist-server/server/modules/orchestration/workflows/workflow.routes.js.map +1 -1
- package/dist-server/server/routes/taskmaster.js +93 -25
- package/dist-server/server/routes/taskmaster.js.map +1 -1
- package/package.json +1 -1
- package/scripts/smoke/taskmaster-run-graph.mjs +55 -0
- package/scripts/smoke/workflow-fallback-replay.mjs +56 -0
- package/server/modules/orchestration/tasks/orchestration-task.service.ts +94 -0
- package/server/modules/orchestration/tasks/orchestration-task.types.ts +10 -0
- package/server/modules/orchestration/tasks/task-run-graph.ts +219 -0
- package/server/modules/orchestration/workflows/workflow-fallback-policy.ts +161 -0
- package/server/modules/orchestration/workflows/workflow-replay.ts +254 -0
- package/server/modules/orchestration/workflows/workflow-runner.ts +112 -7
- package/server/modules/orchestration/workflows/workflow-trace.ts +76 -0
- package/server/modules/orchestration/workflows/workflow.routes.ts +107 -0
- package/server/modules/orchestration/workflows/workflow.types.ts +5 -0
- package/server/routes/taskmaster.js +90 -23
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
Workflow,
|
|
3
|
+
WorkflowNode,
|
|
4
|
+
WorkflowNodeRun,
|
|
5
|
+
WorkflowRun,
|
|
6
|
+
} from '@/modules/orchestration/workflows/workflow.types.js';
|
|
7
|
+
import { redactTraceText } from '@/modules/orchestration/workflows/workflow-trace.js';
|
|
8
|
+
|
|
9
|
+
export const PIXCODE_REPLAY_PROTOCOL = 'pixcode.workflow-replay.v1';
|
|
10
|
+
|
|
11
|
+
export type WorkflowReplayScope = 'run' | 'node';
|
|
12
|
+
export type WorkflowReplaySafetyKind = 'file-write' | 'shell' | 'network';
|
|
13
|
+
|
|
14
|
+
export interface WorkflowReplayOperation {
|
|
15
|
+
kind: WorkflowReplaySafetyKind;
|
|
16
|
+
nodeId?: string;
|
|
17
|
+
summary: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface WorkflowReplayPlan {
|
|
21
|
+
protocol: typeof PIXCODE_REPLAY_PROTOCOL;
|
|
22
|
+
sourceRunId: string;
|
|
23
|
+
sourceWorkflowId: string;
|
|
24
|
+
scope: WorkflowReplayScope;
|
|
25
|
+
fromNodeId?: string;
|
|
26
|
+
selectedNodeIds: string[];
|
|
27
|
+
requiresApproval: boolean;
|
|
28
|
+
approvalReasons: string[];
|
|
29
|
+
destructiveOperations: WorkflowReplayOperation[];
|
|
30
|
+
limitations: string[];
|
|
31
|
+
input: string;
|
|
32
|
+
workflow: Workflow;
|
|
33
|
+
metadata: Record<string, unknown>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function safeNodeId(value: string): string {
|
|
37
|
+
return value.replace(/[^a-zA-Z0-9_]+/g, '_').slice(0, 48) || 'node';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function readRecord(value: unknown): Record<string, unknown> | undefined {
|
|
41
|
+
return value && typeof value === 'object' ? value as Record<string, unknown> : undefined;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function visibleNodes(run: WorkflowRun): WorkflowNodeRun[] {
|
|
45
|
+
return run.nodeRuns.filter((node) => !node.internal);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function defaultReplayNode(run: WorkflowRun): WorkflowNodeRun | undefined {
|
|
49
|
+
return visibleNodes(run).find((node) => node.status === 'failed')
|
|
50
|
+
?? [...visibleNodes(run)].reverse().find((node) => node.status !== 'skipped')
|
|
51
|
+
?? visibleNodes(run)[0];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function selectReplayNodes(run: WorkflowRun, scope: WorkflowReplayScope, fromNodeId?: string): WorkflowNodeRun[] {
|
|
55
|
+
const nodes = visibleNodes(run);
|
|
56
|
+
if (scope === 'run') return nodes;
|
|
57
|
+
|
|
58
|
+
const requested = fromNodeId
|
|
59
|
+
? nodes.find((node) => node.nodeId === fromNodeId)
|
|
60
|
+
: defaultReplayNode(run);
|
|
61
|
+
return requested ? [requested] : [];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function compact(value: string | undefined, run: WorkflowRun, maxLength = 1_200): string | undefined {
|
|
65
|
+
return redactTraceText(value, run, maxLength);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function nodeTraceSummary(run: WorkflowRun, node: WorkflowNodeRun): string {
|
|
69
|
+
const artifactTypes = (node.artifacts ?? []).map((artifact) => artifact.type).filter(Boolean);
|
|
70
|
+
return [
|
|
71
|
+
`Step: ${node.agentLabel || node.nodeId}`,
|
|
72
|
+
`Node id: ${node.nodeId}`,
|
|
73
|
+
`Status: ${node.status}`,
|
|
74
|
+
node.stage ? `Stage: ${node.stage}` : undefined,
|
|
75
|
+
node.adapterId ? `Adapter: ${node.adapterId}` : undefined,
|
|
76
|
+
node.model ? `Model: ${node.model}` : undefined,
|
|
77
|
+
node.error ? `Error: ${compact(node.error, run, 800)}` : undefined,
|
|
78
|
+
artifactTypes.length > 0 ? `Artifacts: ${artifactTypes.join(', ')}` : undefined,
|
|
79
|
+
node.outputText ? `Output excerpt:\n${compact(node.outputText, run)}` : undefined,
|
|
80
|
+
].filter(Boolean).join('\n');
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function replayTraceSummary(run: WorkflowRun, nodes: WorkflowNodeRun[]): string {
|
|
84
|
+
return nodes.map((node) => nodeTraceSummary(run, node)).join('\n\n---\n\n');
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function pushOperation(
|
|
88
|
+
operations: WorkflowReplayOperation[],
|
|
89
|
+
kind: WorkflowReplaySafetyKind,
|
|
90
|
+
nodeId: string | undefined,
|
|
91
|
+
summary: string,
|
|
92
|
+
): void {
|
|
93
|
+
if (operations.some((operation) =>
|
|
94
|
+
operation.kind === kind && operation.nodeId === nodeId && operation.summary === summary,
|
|
95
|
+
)) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
operations.push({ kind, nodeId, summary });
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function detectReplayOperations(run: WorkflowRun, nodes: WorkflowNodeRun[]): WorkflowReplayOperation[] {
|
|
102
|
+
const operations: WorkflowReplayOperation[] = [];
|
|
103
|
+
|
|
104
|
+
for (const node of nodes) {
|
|
105
|
+
for (const artifact of node.artifacts ?? []) {
|
|
106
|
+
if (artifact.type === 'file-diff') {
|
|
107
|
+
pushOperation(operations, 'file-write', node.nodeId, 'Prior step produced a file diff artifact.');
|
|
108
|
+
}
|
|
109
|
+
if (artifact.type === 'command-output') {
|
|
110
|
+
pushOperation(operations, 'shell', node.nodeId, 'Prior step produced command output.');
|
|
111
|
+
}
|
|
112
|
+
const text = [artifact.text, artifact.data ? JSON.stringify(artifact.data) : undefined]
|
|
113
|
+
.filter(Boolean)
|
|
114
|
+
.join('\n')
|
|
115
|
+
.toLocaleLowerCase('en');
|
|
116
|
+
if (/https?:\/\/|curl |wget |gh |npm publish|npm install|git push|ssh /u.test(text)) {
|
|
117
|
+
pushOperation(operations, 'network', node.nodeId, 'Prior artifact references a network-capable operation.');
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const text = [node.outputText, node.error, node.promptPreview].filter(Boolean).join('\n').toLocaleLowerCase('en');
|
|
122
|
+
if (/apply_patch|write file|file write|modified files|changed files/u.test(text)) {
|
|
123
|
+
pushOperation(operations, 'file-write', node.nodeId, 'Prior step text references file-write activity.');
|
|
124
|
+
}
|
|
125
|
+
if (/shell|command|terminal|npm run|node |python |php |go test|cargo |make |exit code/u.test(text)) {
|
|
126
|
+
pushOperation(operations, 'shell', node.nodeId, 'Prior step text references shell execution.');
|
|
127
|
+
}
|
|
128
|
+
if (/https?:\/\/|curl |wget |gh |npm publish|npm install|git push|ssh |network/u.test(text)) {
|
|
129
|
+
pushOperation(operations, 'network', node.nodeId, 'Prior step text references a network-capable operation.');
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return operations;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function replayNodeFromRunNode(
|
|
137
|
+
node: WorkflowNodeRun,
|
|
138
|
+
index: number,
|
|
139
|
+
previousReplayNodeId: string | undefined,
|
|
140
|
+
traceSummary: string,
|
|
141
|
+
limitations: string[],
|
|
142
|
+
requiresApproval: boolean,
|
|
143
|
+
): WorkflowNode {
|
|
144
|
+
const replayNodeId = `replay_${index + 1}_${safeNodeId(node.nodeId)}`;
|
|
145
|
+
return {
|
|
146
|
+
id: replayNodeId,
|
|
147
|
+
adapterId: node.adapterId || 'claude-code',
|
|
148
|
+
agentInstanceId: node.agentInstanceId,
|
|
149
|
+
agentLabel: node.agentLabel ? `${node.agentLabel} Replay` : 'Replay agent',
|
|
150
|
+
assignment: node.assignment ? `Replay: ${node.assignment}` : `Replay source node ${node.nodeId}`,
|
|
151
|
+
stage: node.stage ? `replay_${node.stage}` : 'replay',
|
|
152
|
+
model: node.model,
|
|
153
|
+
permissionMode: node.permissionMode === 'bypassPermissions' ? 'default' : node.permissionMode,
|
|
154
|
+
timeoutMs: node.timeoutMs,
|
|
155
|
+
inputs: previousReplayNodeId ? [previousReplayNodeId] : [],
|
|
156
|
+
output: 'both',
|
|
157
|
+
onFail: 'abort',
|
|
158
|
+
prompt: [
|
|
159
|
+
'This is a Pixcode workflow replay run.',
|
|
160
|
+
`Replay protocol: ${PIXCODE_REPLAY_PROTOCOL}`,
|
|
161
|
+
`Source node: ${node.nodeId}`,
|
|
162
|
+
requiresApproval
|
|
163
|
+
? 'Replay safety review found prior shell, network, or file-write activity. Do not repeat any such action unless the current CLI permission flow asks for and receives user approval.'
|
|
164
|
+
: 'Replay safety review did not find prior shell, network, or file-write artifacts, but still avoid destructive actions unless they are required and approved.',
|
|
165
|
+
'Use the trace summary to continue from the failure or inspect the run. Do not expose secrets, local-only paths, raw tool protocol, or irrelevant logs.',
|
|
166
|
+
`Known limitations:\n- ${limitations.join('\n- ')}`,
|
|
167
|
+
`Trace summary:\n${traceSummary}`,
|
|
168
|
+
`Original step prompt:\n${node.promptPreview || '(No source prompt was stored.)'}`,
|
|
169
|
+
].join('\n\n'),
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function buildWorkflowReplayPlan(
|
|
174
|
+
run: WorkflowRun,
|
|
175
|
+
options: {
|
|
176
|
+
scope?: WorkflowReplayScope;
|
|
177
|
+
fromNodeId?: string;
|
|
178
|
+
} = {},
|
|
179
|
+
): WorkflowReplayPlan {
|
|
180
|
+
const scope = options.scope ?? 'node';
|
|
181
|
+
const nodes = selectReplayNodes(run, scope, options.fromNodeId);
|
|
182
|
+
if (nodes.length === 0) {
|
|
183
|
+
throw new Error('No replayable workflow steps were found.');
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const limitations = [
|
|
187
|
+
'Replay uses stored run traces, prompt previews, messages, and artifacts; it cannot reproduce hidden provider state.',
|
|
188
|
+
'Replay reconstructs selected steps as a new workflow run instead of mutating the source run.',
|
|
189
|
+
'Shell, network, and file-write actions stay under the current CLI permission flow and require explicit replay approval when detected.',
|
|
190
|
+
];
|
|
191
|
+
const destructiveOperations = detectReplayOperations(run, nodes);
|
|
192
|
+
const requiresApproval = destructiveOperations.length > 0;
|
|
193
|
+
const traceSummary = replayTraceSummary(run, nodes);
|
|
194
|
+
const replayNodes = nodes.reduce<WorkflowNode[]>((accumulator, node, index) => {
|
|
195
|
+
const previousReplayNodeId = accumulator[accumulator.length - 1]?.id;
|
|
196
|
+
accumulator.push(replayNodeFromRunNode(
|
|
197
|
+
node,
|
|
198
|
+
index,
|
|
199
|
+
scope === 'run' ? previousReplayNodeId : undefined,
|
|
200
|
+
traceSummary,
|
|
201
|
+
limitations,
|
|
202
|
+
requiresApproval,
|
|
203
|
+
));
|
|
204
|
+
return accumulator;
|
|
205
|
+
}, []);
|
|
206
|
+
const settings = readRecord(run.metadata?.settings) ?? {};
|
|
207
|
+
const replayMetadata = {
|
|
208
|
+
protocol: PIXCODE_REPLAY_PROTOCOL,
|
|
209
|
+
sourceRunId: run.id,
|
|
210
|
+
sourceWorkflowId: run.workflowId,
|
|
211
|
+
scope,
|
|
212
|
+
fromNodeId: options.fromNodeId,
|
|
213
|
+
selectedNodeIds: nodes.map((node) => node.nodeId),
|
|
214
|
+
requiresApproval,
|
|
215
|
+
destructiveOperations,
|
|
216
|
+
limitations,
|
|
217
|
+
createdAt: Date.now(),
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
return {
|
|
221
|
+
protocol: PIXCODE_REPLAY_PROTOCOL,
|
|
222
|
+
sourceRunId: run.id,
|
|
223
|
+
sourceWorkflowId: run.workflowId,
|
|
224
|
+
scope,
|
|
225
|
+
fromNodeId: options.fromNodeId,
|
|
226
|
+
selectedNodeIds: nodes.map((node) => node.nodeId),
|
|
227
|
+
requiresApproval,
|
|
228
|
+
approvalReasons: destructiveOperations.map((operation) =>
|
|
229
|
+
`${operation.kind}${operation.nodeId ? ` in ${operation.nodeId}` : ''}: ${operation.summary}`,
|
|
230
|
+
),
|
|
231
|
+
destructiveOperations,
|
|
232
|
+
limitations,
|
|
233
|
+
input: [
|
|
234
|
+
`Replay ${scope === 'run' ? 'full workflow run' : 'workflow step'} from source run ${run.id}.`,
|
|
235
|
+
run.input ? `Original request:\n${compact(run.input, run, 2_000)}` : undefined,
|
|
236
|
+
].filter(Boolean).join('\n\n'),
|
|
237
|
+
workflow: {
|
|
238
|
+
id: `${run.workflowId}_replay`,
|
|
239
|
+
name: `Replay ${run.workflowId}`,
|
|
240
|
+
description: 'Replay generated from stored Pixcode workflow trace data.',
|
|
241
|
+
trigger: 'manual',
|
|
242
|
+
nodes: replayNodes,
|
|
243
|
+
},
|
|
244
|
+
metadata: {
|
|
245
|
+
...run.metadata,
|
|
246
|
+
workflowName: `Replay: ${String(run.metadata?.workflowName ?? run.workflowId)}`,
|
|
247
|
+
replay: replayMetadata,
|
|
248
|
+
settings: {
|
|
249
|
+
...settings,
|
|
250
|
+
replayMode: true,
|
|
251
|
+
},
|
|
252
|
+
},
|
|
253
|
+
};
|
|
254
|
+
}
|
|
@@ -16,6 +16,11 @@ import {
|
|
|
16
16
|
buildWorkflowContextPacket,
|
|
17
17
|
formatContextPacketForPrompt,
|
|
18
18
|
} from '@/modules/orchestration/workflows/context-packet.js';
|
|
19
|
+
import {
|
|
20
|
+
type WorkflowFallbackTrigger,
|
|
21
|
+
classifyWorkflowFailure,
|
|
22
|
+
resolveWorkflowFallbackDecision,
|
|
23
|
+
} from '@/modules/orchestration/workflows/workflow-fallback-policy.js';
|
|
19
24
|
import {
|
|
20
25
|
type ResolvedWorkspaceTarget,
|
|
21
26
|
resolveWorkflowWorkspace,
|
|
@@ -23,6 +28,7 @@ import {
|
|
|
23
28
|
workspaceTargetMetadata,
|
|
24
29
|
} from '@/modules/orchestration/workflows/workspace-target.js';
|
|
25
30
|
import { workflowStore } from '@/modules/orchestration/workflows/workflow-store.js';
|
|
31
|
+
import { orchestrationTaskService } from '@/modules/orchestration/tasks/orchestration-task.service.js';
|
|
26
32
|
// @ts-ignore — plain-JS service
|
|
27
33
|
import {
|
|
28
34
|
getDefaultProviderModel,
|
|
@@ -1123,6 +1129,8 @@ function nodeRunFromNode(node: WorkflowNode): WorkflowNodeRun {
|
|
|
1123
1129
|
timeoutMs: node.timeoutMs,
|
|
1124
1130
|
stage: node.stage,
|
|
1125
1131
|
internal: node.internal,
|
|
1132
|
+
fallbackTrigger: node.fallbackTrigger,
|
|
1133
|
+
fallbackSourceNodeId: node.fallbackSourceNodeId,
|
|
1126
1134
|
status: 'queued',
|
|
1127
1135
|
};
|
|
1128
1136
|
}
|
|
@@ -1208,7 +1216,7 @@ class WorkflowRunner {
|
|
|
1208
1216
|
const runtimeWorkflow = expandWorkflowForRun(workflow, metadata);
|
|
1209
1217
|
validateWorkflow(runtimeWorkflow);
|
|
1210
1218
|
const workspaceTarget = resolveWorkflowWorkspace(metadata);
|
|
1211
|
-
const runMetadata = {
|
|
1219
|
+
const runMetadata: Record<string, unknown> = {
|
|
1212
1220
|
...metadata,
|
|
1213
1221
|
projectPath: workspaceTarget.projectPath,
|
|
1214
1222
|
selectedProjectPath: workspaceTarget.selectedProjectPath,
|
|
@@ -1225,6 +1233,10 @@ class WorkflowRunner {
|
|
|
1225
1233
|
metadata: runMetadata,
|
|
1226
1234
|
};
|
|
1227
1235
|
workflowStore.setRun(run);
|
|
1236
|
+
const orchestrationTaskId = readString(runMetadata.orchestrationTaskId);
|
|
1237
|
+
if (orchestrationTaskId) {
|
|
1238
|
+
orchestrationTaskService.linkWorkflowRun(orchestrationTaskId, run);
|
|
1239
|
+
}
|
|
1228
1240
|
void this.execute(runtimeWorkflow, run);
|
|
1229
1241
|
return run;
|
|
1230
1242
|
}
|
|
@@ -1276,7 +1288,12 @@ class WorkflowRunner {
|
|
|
1276
1288
|
return readAgentAssignments(run.metadata).find((agent) => agent.instanceId === fallbackAgentInstanceId);
|
|
1277
1289
|
}
|
|
1278
1290
|
|
|
1279
|
-
private createFallbackNode(
|
|
1291
|
+
private createFallbackNode(
|
|
1292
|
+
node: WorkflowNode,
|
|
1293
|
+
fallbackAgent: AgentAssignment,
|
|
1294
|
+
reason: string,
|
|
1295
|
+
fallbackTrigger: WorkflowFallbackTrigger,
|
|
1296
|
+
): WorkflowNode {
|
|
1280
1297
|
const fallbackSuffix = safeNodeId(fallbackAgent.instanceId, 'fallback');
|
|
1281
1298
|
return {
|
|
1282
1299
|
...node,
|
|
@@ -1289,9 +1306,12 @@ class WorkflowRunner {
|
|
|
1289
1306
|
model: fallbackAgent.model,
|
|
1290
1307
|
permissionMode: fallbackAgent.permissionMode,
|
|
1291
1308
|
toolsSettings: fallbackAgent.toolsSettings,
|
|
1309
|
+
fallbackTrigger,
|
|
1310
|
+
fallbackSourceNodeId: node.id,
|
|
1292
1311
|
prompt: [
|
|
1293
1312
|
'The previous CLI agent failed on this orchestration step.',
|
|
1294
1313
|
`Failed step: ${node.agentLabel || node.id}`,
|
|
1314
|
+
`Fallback trigger: ${fallbackTrigger}`,
|
|
1295
1315
|
`Failure: ${reason}`,
|
|
1296
1316
|
'Take over the same assignment as the backup CLI. Use the original goal and upstream context.',
|
|
1297
1317
|
'Do not repeat unrelated work; complete the failed step and report what you did.',
|
|
@@ -1301,6 +1321,32 @@ class WorkflowRunner {
|
|
|
1301
1321
|
};
|
|
1302
1322
|
}
|
|
1303
1323
|
|
|
1324
|
+
private recordFallbackSkipped(
|
|
1325
|
+
run: WorkflowRun,
|
|
1326
|
+
node: WorkflowNode,
|
|
1327
|
+
reason: string,
|
|
1328
|
+
fallbackTrigger: WorkflowFallbackTrigger,
|
|
1329
|
+
skippedReason: string,
|
|
1330
|
+
): void {
|
|
1331
|
+
const fallbackSkippedEvents = Array.isArray(run.metadata?.fallbackSkippedEvents)
|
|
1332
|
+
? run.metadata.fallbackSkippedEvents
|
|
1333
|
+
: [];
|
|
1334
|
+
run.metadata = {
|
|
1335
|
+
...run.metadata,
|
|
1336
|
+
fallbackSkippedEvents: [
|
|
1337
|
+
...fallbackSkippedEvents,
|
|
1338
|
+
{
|
|
1339
|
+
nodeId: node.id,
|
|
1340
|
+
trigger: fallbackTrigger,
|
|
1341
|
+
reason,
|
|
1342
|
+
skippedReason,
|
|
1343
|
+
createdAt: Date.now(),
|
|
1344
|
+
},
|
|
1345
|
+
],
|
|
1346
|
+
};
|
|
1347
|
+
workflowStore.setRun(run);
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1304
1350
|
private async runFallbackAfterFailure(
|
|
1305
1351
|
node: WorkflowNode,
|
|
1306
1352
|
workflow: Workflow,
|
|
@@ -1309,9 +1355,29 @@ class WorkflowRunner {
|
|
|
1309
1355
|
started: Set<string>,
|
|
1310
1356
|
completed: Set<string>,
|
|
1311
1357
|
reason: string,
|
|
1358
|
+
trigger?: WorkflowFallbackTrigger,
|
|
1312
1359
|
): Promise<boolean> {
|
|
1360
|
+
const fallbackTrigger = classifyWorkflowFailure(reason, trigger);
|
|
1313
1361
|
const fallbackAgent = this.fallbackAgentFor(run, node);
|
|
1314
1362
|
if (!fallbackAgent) {
|
|
1363
|
+
this.recordFallbackSkipped(run, node, reason, fallbackTrigger, 'No fallback agent is configured for this run.');
|
|
1364
|
+
return false;
|
|
1365
|
+
}
|
|
1366
|
+
const decision = resolveWorkflowFallbackDecision({
|
|
1367
|
+
run,
|
|
1368
|
+
node,
|
|
1369
|
+
reason,
|
|
1370
|
+
trigger: fallbackTrigger,
|
|
1371
|
+
fallbackAgentInstanceId: fallbackAgent.instanceId,
|
|
1372
|
+
});
|
|
1373
|
+
if (!decision.shouldFallback) {
|
|
1374
|
+
this.recordFallbackSkipped(
|
|
1375
|
+
run,
|
|
1376
|
+
node,
|
|
1377
|
+
reason,
|
|
1378
|
+
decision.trigger,
|
|
1379
|
+
decision.skippedReason ?? 'Fallback policy skipped this failure.',
|
|
1380
|
+
);
|
|
1315
1381
|
return false;
|
|
1316
1382
|
}
|
|
1317
1383
|
if (workflow.nodes.length + 1 > 64) {
|
|
@@ -1323,7 +1389,7 @@ class WorkflowRunner {
|
|
|
1323
1389
|
return false;
|
|
1324
1390
|
}
|
|
1325
1391
|
|
|
1326
|
-
let fallbackNode = this.createFallbackNode(node, fallbackAgent, reason);
|
|
1392
|
+
let fallbackNode = this.createFallbackNode(node, fallbackAgent, reason, decision.trigger);
|
|
1327
1393
|
let collision = 1;
|
|
1328
1394
|
while (workflow.nodes.some((candidate) => candidate.id === fallbackNode.id)) {
|
|
1329
1395
|
collision += 1;
|
|
@@ -1357,6 +1423,8 @@ class WorkflowRunner {
|
|
|
1357
1423
|
nodeId: node.id,
|
|
1358
1424
|
fallbackNodeId: fallbackNode.id,
|
|
1359
1425
|
fallbackAgentInstanceId: fallbackAgent.instanceId,
|
|
1426
|
+
trigger: decision.trigger,
|
|
1427
|
+
policy: decision.policy,
|
|
1360
1428
|
reason,
|
|
1361
1429
|
startedAt: Date.now(),
|
|
1362
1430
|
},
|
|
@@ -1528,6 +1596,7 @@ class WorkflowRunner {
|
|
|
1528
1596
|
} finally {
|
|
1529
1597
|
run.finishedAt = run.finishedAt ?? Date.now();
|
|
1530
1598
|
workflowStore.setRun(run);
|
|
1599
|
+
orchestrationTaskService.updateFromWorkflowRun(run);
|
|
1531
1600
|
notifyWorkflowRunFinished(run);
|
|
1532
1601
|
this.cancelingRuns.delete(run.id);
|
|
1533
1602
|
}
|
|
@@ -1658,7 +1727,16 @@ class WorkflowRunner {
|
|
|
1658
1727
|
workflowStore.setRun(run);
|
|
1659
1728
|
return;
|
|
1660
1729
|
}
|
|
1661
|
-
if (await this.runFallbackAfterFailure(
|
|
1730
|
+
if (await this.runFallbackAfterFailure(
|
|
1731
|
+
node,
|
|
1732
|
+
workflow,
|
|
1733
|
+
run,
|
|
1734
|
+
outputs,
|
|
1735
|
+
started,
|
|
1736
|
+
completed,
|
|
1737
|
+
nodeRun.error,
|
|
1738
|
+
'provider_failure',
|
|
1739
|
+
)) {
|
|
1662
1740
|
return;
|
|
1663
1741
|
}
|
|
1664
1742
|
if (node.onFail === 'continue') {
|
|
@@ -1710,7 +1788,16 @@ class WorkflowRunner {
|
|
|
1710
1788
|
workflowStore.setRun(run);
|
|
1711
1789
|
return;
|
|
1712
1790
|
}
|
|
1713
|
-
if (await this.runFallbackAfterFailure(
|
|
1791
|
+
if (await this.runFallbackAfterFailure(
|
|
1792
|
+
node,
|
|
1793
|
+
workflow,
|
|
1794
|
+
run,
|
|
1795
|
+
outputs,
|
|
1796
|
+
started,
|
|
1797
|
+
completed,
|
|
1798
|
+
nodeRun.error,
|
|
1799
|
+
'timeout',
|
|
1800
|
+
)) {
|
|
1714
1801
|
return;
|
|
1715
1802
|
}
|
|
1716
1803
|
if (node.onFail === 'continue') {
|
|
@@ -1744,7 +1831,16 @@ class WorkflowRunner {
|
|
|
1744
1831
|
nodeRun.status = 'failed';
|
|
1745
1832
|
nodeRun.error = visibleHandoffError;
|
|
1746
1833
|
workflowStore.setRun(run);
|
|
1747
|
-
if (await this.runFallbackAfterFailure(
|
|
1834
|
+
if (await this.runFallbackAfterFailure(
|
|
1835
|
+
node,
|
|
1836
|
+
workflow,
|
|
1837
|
+
run,
|
|
1838
|
+
outputs,
|
|
1839
|
+
started,
|
|
1840
|
+
completed,
|
|
1841
|
+
visibleHandoffError,
|
|
1842
|
+
'invalid_output',
|
|
1843
|
+
)) {
|
|
1748
1844
|
return;
|
|
1749
1845
|
}
|
|
1750
1846
|
if (node.onFail === 'continue') {
|
|
@@ -1783,7 +1879,16 @@ class WorkflowRunner {
|
|
|
1783
1879
|
workflowStore.setRun(run);
|
|
1784
1880
|
return;
|
|
1785
1881
|
}
|
|
1786
|
-
if (await this.runFallbackAfterFailure(
|
|
1882
|
+
if (await this.runFallbackAfterFailure(
|
|
1883
|
+
node,
|
|
1884
|
+
workflow,
|
|
1885
|
+
run,
|
|
1886
|
+
outputs,
|
|
1887
|
+
started,
|
|
1888
|
+
completed,
|
|
1889
|
+
nodeRun.error,
|
|
1890
|
+
classifyWorkflowFailure(`${nodeRun.error}\n${nodeRun.outputText ?? ''}`),
|
|
1891
|
+
)) {
|
|
1787
1892
|
return;
|
|
1788
1893
|
}
|
|
1789
1894
|
if (node.onFail === 'continue') {
|
|
@@ -22,6 +22,10 @@ function readString(value: unknown): string | undefined {
|
|
|
22
22
|
return typeof value === 'string' && value.trim() ? value : undefined;
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
function readRecord(value: unknown): Record<string, unknown> | undefined {
|
|
26
|
+
return value && typeof value === 'object' ? value as Record<string, unknown> : undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
25
29
|
function redactionValues(run: WorkflowRun): string[] {
|
|
26
30
|
const metadata = run.metadata ?? {};
|
|
27
31
|
const workspaceTarget = metadata.workspaceTarget && typeof metadata.workspaceTarget === 'object'
|
|
@@ -140,6 +144,78 @@ export function buildWorkflowTrace(run: WorkflowRun): WorkflowTraceEvent[] {
|
|
|
140
144
|
},
|
|
141
145
|
});
|
|
142
146
|
|
|
147
|
+
const replay = readRecord(run.metadata?.replay);
|
|
148
|
+
if (replay) {
|
|
149
|
+
pushEvent(events, {
|
|
150
|
+
id: traceId([run.id, 'replay']),
|
|
151
|
+
type: 'run',
|
|
152
|
+
severity: replay.requiresApproval ? 'warning' : 'info',
|
|
153
|
+
status: run.status,
|
|
154
|
+
timestamp: run.startedAt + 0.25,
|
|
155
|
+
actor: 'Pixcode',
|
|
156
|
+
title: 'Workflow replay prepared',
|
|
157
|
+
titleKey: 'workflow.trace.replay',
|
|
158
|
+
summary: redactTraceText([
|
|
159
|
+
`Source run: ${readString(replay.sourceRunId) ?? 'unknown'}`,
|
|
160
|
+
`Scope: ${readString(replay.scope) ?? 'unknown'}`,
|
|
161
|
+
Array.isArray(replay.selectedNodeIds) ? `Selected steps: ${replay.selectedNodeIds.join(', ')}` : undefined,
|
|
162
|
+
replay.requiresApproval ? 'Replay required approval for prior shell, network, or file-write activity.' : undefined,
|
|
163
|
+
].filter(Boolean).join('\n'), run),
|
|
164
|
+
metadata: replay,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const fallbackEvents = Array.isArray(run.metadata?.fallbackEvents)
|
|
169
|
+
? run.metadata.fallbackEvents
|
|
170
|
+
: [];
|
|
171
|
+
fallbackEvents.forEach((event, index) => {
|
|
172
|
+
const record = readRecord(event);
|
|
173
|
+
if (!record) return;
|
|
174
|
+
pushEvent(events, {
|
|
175
|
+
id: traceId([run.id, 'fallback', index]),
|
|
176
|
+
type: 'node',
|
|
177
|
+
severity: 'warning',
|
|
178
|
+
status: 'submitted',
|
|
179
|
+
timestamp: typeof record.startedAt === 'number' ? record.startedAt : run.startedAt + 0.5 + index,
|
|
180
|
+
actor: 'Pixcode',
|
|
181
|
+
nodeId: readString(record.nodeId),
|
|
182
|
+
title: 'Fallback agent started',
|
|
183
|
+
titleKey: 'workflow.trace.fallback',
|
|
184
|
+
summary: redactTraceText([
|
|
185
|
+
`Trigger: ${readString(record.trigger) ?? 'unknown'}`,
|
|
186
|
+
`Source node: ${readString(record.nodeId) ?? 'unknown'}`,
|
|
187
|
+
`Fallback node: ${readString(record.fallbackNodeId) ?? 'unknown'}`,
|
|
188
|
+
readString(record.reason) ? `Reason: ${readString(record.reason)}` : undefined,
|
|
189
|
+
].filter(Boolean).join('\n'), run),
|
|
190
|
+
metadata: record,
|
|
191
|
+
});
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
const fallbackSkippedEvents = Array.isArray(run.metadata?.fallbackSkippedEvents)
|
|
195
|
+
? run.metadata.fallbackSkippedEvents
|
|
196
|
+
: [];
|
|
197
|
+
fallbackSkippedEvents.forEach((event, index) => {
|
|
198
|
+
const record = readRecord(event);
|
|
199
|
+
if (!record) return;
|
|
200
|
+
pushEvent(events, {
|
|
201
|
+
id: traceId([run.id, 'fallback-skipped', index]),
|
|
202
|
+
type: 'node',
|
|
203
|
+
severity: 'info',
|
|
204
|
+
status: 'skipped',
|
|
205
|
+
timestamp: typeof record.createdAt === 'number' ? record.createdAt : run.startedAt + 0.75 + index,
|
|
206
|
+
actor: 'Pixcode',
|
|
207
|
+
nodeId: readString(record.nodeId),
|
|
208
|
+
title: 'Fallback skipped',
|
|
209
|
+
titleKey: 'workflow.trace.fallback',
|
|
210
|
+
summary: redactTraceText([
|
|
211
|
+
`Trigger: ${readString(record.trigger) ?? 'unknown'}`,
|
|
212
|
+
`Skipped: ${readString(record.skippedReason) ?? 'policy did not allow fallback'}`,
|
|
213
|
+
readString(record.reason) ? `Reason: ${readString(record.reason)}` : undefined,
|
|
214
|
+
].filter(Boolean).join('\n'), run),
|
|
215
|
+
metadata: record,
|
|
216
|
+
});
|
|
217
|
+
});
|
|
218
|
+
|
|
143
219
|
run.nodeRuns.forEach((node, index) => {
|
|
144
220
|
const base = eventBase(node);
|
|
145
221
|
const timestamp = nodeTimestamp(run, node, index);
|
|
@@ -2,6 +2,10 @@ import type { Router } from 'express';
|
|
|
2
2
|
import express from 'express';
|
|
3
3
|
|
|
4
4
|
import { workflowRunner } from '@/modules/orchestration/workflows/workflow-runner.js';
|
|
5
|
+
import {
|
|
6
|
+
type WorkflowReplayScope,
|
|
7
|
+
buildWorkflowReplayPlan,
|
|
8
|
+
} from '@/modules/orchestration/workflows/workflow-replay.js';
|
|
5
9
|
import { workflowStore } from '@/modules/orchestration/workflows/workflow-store.js';
|
|
6
10
|
import { buildWorkflowTrace } from '@/modules/orchestration/workflows/workflow-trace.js';
|
|
7
11
|
import { findPixcodeAppRoot } from '@/modules/orchestration/workflows/workspace-target.js';
|
|
@@ -45,6 +49,30 @@ function readRequestUserId(req: express.Request): string | number | null {
|
|
|
45
49
|
return user?.id ?? user?.userId ?? null;
|
|
46
50
|
}
|
|
47
51
|
|
|
52
|
+
function readReplayScope(value: unknown): WorkflowReplayScope {
|
|
53
|
+
return value === 'run' ? 'run' : 'node';
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function readOptionalString(value: unknown): string | undefined {
|
|
57
|
+
return typeof value === 'string' && value.trim() ? value.trim() : undefined;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function readBooleanFlag(value: unknown): boolean {
|
|
61
|
+
return value === true || value === 'true' || value === '1';
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function replayOptions(req: express.Request): {
|
|
65
|
+
scope: WorkflowReplayScope;
|
|
66
|
+
fromNodeId?: string;
|
|
67
|
+
approveReplay: boolean;
|
|
68
|
+
} {
|
|
69
|
+
return {
|
|
70
|
+
scope: readReplayScope(req.body?.scope ?? req.query.scope),
|
|
71
|
+
fromNodeId: readOptionalString(req.body?.fromNodeId ?? req.query.fromNodeId),
|
|
72
|
+
approveReplay: readBooleanFlag(req.body?.approveReplay ?? req.query.approveReplay),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
48
76
|
function sendRunSnapshot(res: express.Response, runId: string): boolean {
|
|
49
77
|
const run = workflowStore.getRun(runId);
|
|
50
78
|
if (!run) {
|
|
@@ -134,6 +162,85 @@ export function createWorkflowRouter(): Router {
|
|
|
134
162
|
});
|
|
135
163
|
});
|
|
136
164
|
|
|
165
|
+
router.get('/workflows/runs/:runId/replay-plan', (req, res) => {
|
|
166
|
+
const run = workflowStore.getRun(req.params.runId);
|
|
167
|
+
if (!run) {
|
|
168
|
+
res.status(404).json({ error: { code: 'RUN_NOT_FOUND', message: req.params.runId } });
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
try {
|
|
173
|
+
const options = replayOptions(req);
|
|
174
|
+
res.json({
|
|
175
|
+
replayPlan: buildWorkflowReplayPlan(run, {
|
|
176
|
+
scope: options.scope,
|
|
177
|
+
fromNodeId: options.fromNodeId,
|
|
178
|
+
}),
|
|
179
|
+
});
|
|
180
|
+
} catch (error) {
|
|
181
|
+
res.status(400).json({
|
|
182
|
+
error: {
|
|
183
|
+
code: 'REPLAY_PLAN_INVALID',
|
|
184
|
+
message: error instanceof Error ? error.message : String(error),
|
|
185
|
+
},
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
router.post('/workflows/runs/:runId/replay', (req, res) => {
|
|
191
|
+
const run = workflowStore.getRun(req.params.runId);
|
|
192
|
+
if (!run) {
|
|
193
|
+
res.status(404).json({ error: { code: 'RUN_NOT_FOUND', message: req.params.runId } });
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
try {
|
|
198
|
+
const options = replayOptions(req);
|
|
199
|
+
const replayPlan = buildWorkflowReplayPlan(run, {
|
|
200
|
+
scope: options.scope,
|
|
201
|
+
fromNodeId: options.fromNodeId,
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
if (replayPlan.requiresApproval && !options.approveReplay) {
|
|
205
|
+
res.status(409).json({
|
|
206
|
+
error: {
|
|
207
|
+
code: 'REPLAY_APPROVAL_REQUIRED',
|
|
208
|
+
message: 'Replay requires explicit approval because prior shell, network, or file-write activity was detected.',
|
|
209
|
+
},
|
|
210
|
+
replayPlan,
|
|
211
|
+
});
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const replayRun = workflowRunner.start(
|
|
216
|
+
replayPlan.workflow,
|
|
217
|
+
replayPlan.input,
|
|
218
|
+
{
|
|
219
|
+
...replayPlan.metadata,
|
|
220
|
+
userId: readRequestUserId(req) ?? run.metadata?.userId,
|
|
221
|
+
replay: {
|
|
222
|
+
...(replayPlan.metadata.replay && typeof replayPlan.metadata.replay === 'object'
|
|
223
|
+
? replayPlan.metadata.replay as Record<string, unknown>
|
|
224
|
+
: {}),
|
|
225
|
+
approved: options.approveReplay,
|
|
226
|
+
approvedAt: options.approveReplay ? Date.now() : undefined,
|
|
227
|
+
},
|
|
228
|
+
},
|
|
229
|
+
);
|
|
230
|
+
res.status(202).json({
|
|
231
|
+
run: replayRun,
|
|
232
|
+
replayPlan,
|
|
233
|
+
});
|
|
234
|
+
} catch (error) {
|
|
235
|
+
res.status(400).json({
|
|
236
|
+
error: {
|
|
237
|
+
code: 'REPLAY_START_FAILED',
|
|
238
|
+
message: error instanceof Error ? error.message : String(error),
|
|
239
|
+
},
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
|
|
137
244
|
router.get('/workflows/runs/:runId', (req, res) => {
|
|
138
245
|
const run = workflowStore.getRun(req.params.runId);
|
|
139
246
|
if (!run) {
|