ai-maestro 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/LICENSE +15 -0
- package/README.md +727 -0
- package/bin/maestro.mjs +246 -0
- package/package.json +29 -0
- package/src/checkpoint.mjs +102 -0
- package/src/codex-diagnostics.mjs +682 -0
- package/src/codex-home-inspect.mjs +252 -0
- package/src/codex-home.mjs +258 -0
- package/src/commands.mjs +809 -0
- package/src/config.mjs +11 -0
- package/src/debug.mjs +164 -0
- package/src/encoding-guard.mjs +125 -0
- package/src/engines/ai-router-engine.mjs +37 -0
- package/src/engines/codex-engine.mjs +21 -0
- package/src/engines/engine.mjs +21 -0
- package/src/engines/registry.mjs +29 -0
- package/src/files.mjs +65 -0
- package/src/format.mjs +132 -0
- package/src/lock.mjs +439 -0
- package/src/memory-log.mjs +18 -0
- package/src/memory.mjs +1 -0
- package/src/orchestration/attempt-chain-runtime.mjs +627 -0
- package/src/orchestration/budget-manager.mjs +121 -0
- package/src/orchestration/capability-registry.mjs +108 -0
- package/src/orchestration/consolidator.mjs +772 -0
- package/src/orchestration/delegation-executor.mjs +262 -0
- package/src/orchestration/delegation-manager.mjs +116 -0
- package/src/orchestration/deployment-intent.mjs +36 -0
- package/src/orchestration/engine-history.mjs +110 -0
- package/src/orchestration/engine-policy.mjs +73 -0
- package/src/orchestration/engine-selector.mjs +187 -0
- package/src/orchestration/execution-context.mjs +45 -0
- package/src/orchestration/failure-classifier.mjs +136 -0
- package/src/orchestration/failure-evidence.mjs +237 -0
- package/src/orchestration/fallback-chain-lock.mjs +217 -0
- package/src/orchestration/fallback-chain-trail.mjs +218 -0
- package/src/orchestration/fallback-executor.mjs +146 -0
- package/src/orchestration/fallback-graph.mjs +106 -0
- package/src/orchestration/fallback-recommendation.mjs +73 -0
- package/src/orchestration/file-conflict-detector.mjs +126 -0
- package/src/orchestration/host-validation.mjs +241 -0
- package/src/orchestration/orchestration-loop.mjs +1971 -0
- package/src/orchestration/orchestration-runtime.mjs +1019 -0
- package/src/orchestration/orchestration-scheduler.mjs +135 -0
- package/src/orchestration/orchestration-trail.mjs +192 -0
- package/src/orchestration/preflight.mjs +212 -0
- package/src/orchestration/provider-health.mjs +566 -0
- package/src/orchestration/provider-router.mjs +352 -0
- package/src/orchestration/rc-check-adapters.mjs +817 -0
- package/src/orchestration/rc-check.mjs +544 -0
- package/src/orchestration/rc-policy.mjs +200 -0
- package/src/orchestration/runtime-gate.mjs +146 -0
- package/src/orchestration/sector-managers.mjs +215 -0
- package/src/orchestration/self-hosting-canary.mjs +231 -0
- package/src/orchestration/self-hosting-readiness.mjs +244 -0
- package/src/orchestration/spec-planner.mjs +877 -0
- package/src/orchestration/subtask-planner.mjs +176 -0
- package/src/orchestration/task-classifier.mjs +241 -0
- package/src/orchestration/verifier.mjs +1608 -0
- package/src/orchestration-commands.mjs +279 -0
- package/src/planner.mjs +38 -0
- package/src/project.mjs +1 -0
- package/src/run-diagnostics.mjs +641 -0
- package/src/runner.mjs +243 -0
- package/src/safety.mjs +116 -0
- package/src/schema.mjs +371 -0
- package/src/session-commands.mjs +521 -0
- package/src/shell.mjs +93 -0
- package/src/smart-planner.mjs +249 -0
- package/src/task-commands.mjs +1182 -0
- package/src/tasks.mjs +134 -0
- package/src/ui/commands.mjs +76 -0
- package/src/ui/events.mjs +45 -0
- package/src/ui/public/app.js +600 -0
- package/src/ui/public/index.html +88 -0
- package/src/ui/public/style.css +460 -0
- package/src/ui/server.mjs +112 -0
- package/src/ui/state.mjs +504 -0
- package/src/usage.mjs +178 -0
- package/src/workspace-diff.mjs +228 -0
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
|
|
4
|
+
import { CONFIG } from '../config.mjs';
|
|
5
|
+
import { redactObject } from '../format.mjs';
|
|
6
|
+
|
|
7
|
+
export const FALLBACK_CHAIN_LOCK_FILE = path.join('locks', 'fallback-chain-global.lock.json');
|
|
8
|
+
|
|
9
|
+
let activeLockHandle = null;
|
|
10
|
+
|
|
11
|
+
function lockPath(projectRoot = process.cwd()) {
|
|
12
|
+
return path.join(projectRoot, CONFIG.maestroPath, FALLBACK_CHAIN_LOCK_FILE);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function isPidAlive(pid) {
|
|
16
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
17
|
+
try {
|
|
18
|
+
process.kill(pid, 0);
|
|
19
|
+
return true;
|
|
20
|
+
} catch (error) {
|
|
21
|
+
return error && error.code === 'EPERM';
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async function readLock(filePath) {
|
|
26
|
+
try {
|
|
27
|
+
return JSON.parse(await fs.readFile(filePath, 'utf-8'));
|
|
28
|
+
} catch (error) {
|
|
29
|
+
if (error.code === 'ENOENT') return null;
|
|
30
|
+
if (error instanceof SyntaxError) {
|
|
31
|
+
return { corrupted: true, error: error.message };
|
|
32
|
+
}
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function makeLock({ command = 'run-task --failover', taskId = null } = {}) {
|
|
38
|
+
return redactObject({
|
|
39
|
+
schemaVersion: 1,
|
|
40
|
+
lockType: 'fallback-chain-global',
|
|
41
|
+
pid: process.pid,
|
|
42
|
+
command,
|
|
43
|
+
taskId: taskId == null ? null : String(taskId),
|
|
44
|
+
acquiredAt: new Date().toISOString(),
|
|
45
|
+
lockId: process.pid + '-' + Date.now().toString(36) + '-' + Math.random().toString(36).slice(2, 10)
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function createExclusive(filePath, lock) {
|
|
50
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
51
|
+
const handle = await fs.open(filePath, 'wx');
|
|
52
|
+
try {
|
|
53
|
+
await handle.writeFile(JSON.stringify(lock, null, 2) + '\n', { encoding: 'utf8' });
|
|
54
|
+
} finally {
|
|
55
|
+
await handle.close();
|
|
56
|
+
}
|
|
57
|
+
return lock;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async function createRecoveryClaim(filePath) {
|
|
61
|
+
const claimPath = filePath + '.recovery';
|
|
62
|
+
const claim = redactObject({
|
|
63
|
+
schemaVersion: 1,
|
|
64
|
+
lockType: 'fallback-chain-global-recovery',
|
|
65
|
+
pid: process.pid,
|
|
66
|
+
acquiredAt: new Date().toISOString(),
|
|
67
|
+
claimId: process.pid + '-' + Date.now().toString(36) + '-' + Math.random().toString(36).slice(2, 10)
|
|
68
|
+
});
|
|
69
|
+
const handle = await fs.open(claimPath, 'wx');
|
|
70
|
+
try {
|
|
71
|
+
await handle.writeFile(JSON.stringify(claim, null, 2) + '\n', { encoding: 'utf8' });
|
|
72
|
+
} finally {
|
|
73
|
+
await handle.close();
|
|
74
|
+
}
|
|
75
|
+
return { claimPath, claim };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async function releaseRecoveryClaim(claimHandle) {
|
|
79
|
+
if (!claimHandle) return;
|
|
80
|
+
try {
|
|
81
|
+
await fs.unlink(claimHandle.claimPath);
|
|
82
|
+
} catch (error) {
|
|
83
|
+
if (error.code !== 'ENOENT') throw error;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export async function acquireFallbackChainLock(options = {}) {
|
|
88
|
+
if (activeLockHandle) {
|
|
89
|
+
return {
|
|
90
|
+
acquired: false,
|
|
91
|
+
outcome: 'recursive_acquisition_blocked',
|
|
92
|
+
activeLock: activeLockHandle.lock
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
const filePath = options.filePath || lockPath(options.projectRoot);
|
|
96
|
+
const lock = makeLock(options);
|
|
97
|
+
try {
|
|
98
|
+
const created = await createExclusive(filePath, lock);
|
|
99
|
+
activeLockHandle = { filePath, lock: created };
|
|
100
|
+
return { acquired: true, outcome: 'acquired', lockHandle: activeLockHandle };
|
|
101
|
+
} catch (error) {
|
|
102
|
+
if (error.code !== 'EEXIST') throw error;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const existing = await readLock(filePath);
|
|
106
|
+
if (!existing || existing.corrupted) {
|
|
107
|
+
return {
|
|
108
|
+
acquired: false,
|
|
109
|
+
outcome: existing && existing.corrupted ? 'lock_corrupted' : 'held_by_unknown',
|
|
110
|
+
activeLock: existing
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
if (isPidAlive(existing.pid)) {
|
|
114
|
+
return { acquired: false, outcome: 'held_by_active', activeLock: existing };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
let claimHandle = null;
|
|
118
|
+
try {
|
|
119
|
+
claimHandle = await createRecoveryClaim(filePath);
|
|
120
|
+
} catch (error) {
|
|
121
|
+
if (error.code === 'EEXIST') {
|
|
122
|
+
return { acquired: false, outcome: 'recovery_in_progress', activeLock: existing };
|
|
123
|
+
}
|
|
124
|
+
throw error;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
try {
|
|
128
|
+
const current = await readLock(filePath);
|
|
129
|
+
if (!current || current.corrupted) {
|
|
130
|
+
return {
|
|
131
|
+
acquired: false,
|
|
132
|
+
outcome: current && current.corrupted ? 'lock_corrupted' : 'held_by_unknown',
|
|
133
|
+
activeLock: current
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
if (isPidAlive(current.pid)) {
|
|
137
|
+
return { acquired: false, outcome: 'held_by_active', activeLock: current };
|
|
138
|
+
}
|
|
139
|
+
await fs.unlink(filePath);
|
|
140
|
+
if (typeof options.afterOrphanLockUnlinked === 'function') {
|
|
141
|
+
await options.afterOrphanLockUnlinked({ filePath, previousLock: current, lock });
|
|
142
|
+
}
|
|
143
|
+
let recovered;
|
|
144
|
+
try {
|
|
145
|
+
recovered = await createExclusive(filePath, lock);
|
|
146
|
+
} catch (error) {
|
|
147
|
+
if (error.code === 'EEXIST') {
|
|
148
|
+
const activeLock = await readLock(filePath);
|
|
149
|
+
return {
|
|
150
|
+
acquired: false,
|
|
151
|
+
outcome: 'lost_recovery_race',
|
|
152
|
+
recoveredFromPid: current.pid || null,
|
|
153
|
+
activeLock: activeLock || current
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
throw error;
|
|
157
|
+
}
|
|
158
|
+
activeLockHandle = { filePath, lock: recovered, recoveredFromPid: current.pid || null };
|
|
159
|
+
return {
|
|
160
|
+
acquired: true,
|
|
161
|
+
outcome: 'recovered_and_acquired',
|
|
162
|
+
recoveredFromPid: current.pid || null,
|
|
163
|
+
activeLock: current,
|
|
164
|
+
lockHandle: activeLockHandle
|
|
165
|
+
};
|
|
166
|
+
} finally {
|
|
167
|
+
await releaseRecoveryClaim(claimHandle);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export async function releaseFallbackChainLock(lockHandle) {
|
|
172
|
+
if (!lockHandle) return false;
|
|
173
|
+
const filePath = lockHandle.filePath;
|
|
174
|
+
try {
|
|
175
|
+
const current = await readLock(filePath);
|
|
176
|
+
if (current && current.pid === lockHandle.lock.pid && current.lockId === lockHandle.lock.lockId) {
|
|
177
|
+
await fs.unlink(filePath);
|
|
178
|
+
return true;
|
|
179
|
+
}
|
|
180
|
+
return false;
|
|
181
|
+
} catch (error) {
|
|
182
|
+
if (error.code === 'ENOENT') return false;
|
|
183
|
+
throw error;
|
|
184
|
+
} finally {
|
|
185
|
+
if (activeLockHandle && activeLockHandle.lock.lockId === lockHandle.lock.lockId) {
|
|
186
|
+
activeLockHandle = null;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export async function withFallbackChainLock(fn, options = {}) {
|
|
192
|
+
const lock = await acquireFallbackChainLock(options);
|
|
193
|
+
if (!lock.acquired) {
|
|
194
|
+
return lock;
|
|
195
|
+
}
|
|
196
|
+
try {
|
|
197
|
+
const result = await fn(lock);
|
|
198
|
+
return { ...result, lockOutcome: lock.outcome, recoveredFromPid: lock.recoveredFromPid || null };
|
|
199
|
+
} finally {
|
|
200
|
+
await releaseFallbackChainLock(lock.lockHandle);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export async function inspectFallbackChainLock(options = {}) {
|
|
205
|
+
const filePath = options.filePath || lockPath(options.projectRoot);
|
|
206
|
+
const lock = await readLock(filePath);
|
|
207
|
+
return {
|
|
208
|
+
exists: !!lock,
|
|
209
|
+
lock,
|
|
210
|
+
heldByActive: !!(lock && !lock.corrupted && isPidAlive(lock.pid)),
|
|
211
|
+
corrupted: !!(lock && lock.corrupted)
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
export const fallbackChainLockPaths = {
|
|
216
|
+
lockPath
|
|
217
|
+
};
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
|
|
4
|
+
import { CONFIG } from '../config.mjs';
|
|
5
|
+
import { redactCredentials, redactObject } from '../format.mjs';
|
|
6
|
+
import { stampSchemaVersion } from '../schema.mjs';
|
|
7
|
+
import { EVIDENCE_SUMMARY_LIMIT } from './failure-evidence.mjs';
|
|
8
|
+
|
|
9
|
+
export const FALLBACK_CHAIN_EVENTS_FILE = 'FALLBACK_CHAIN_EVENTS.jsonl';
|
|
10
|
+
|
|
11
|
+
export const FALLBACK_CHAIN_TERMINAL_EVENTS = new Set([
|
|
12
|
+
'fallback_chain_completed',
|
|
13
|
+
'fallback_chain_exhausted',
|
|
14
|
+
'fallback_blocked_by_workspace',
|
|
15
|
+
'fallback_blocked_by_budget',
|
|
16
|
+
'fallback_needs_human'
|
|
17
|
+
]);
|
|
18
|
+
|
|
19
|
+
const REQUIRED_EVENT_FIELDS = [
|
|
20
|
+
'eventId',
|
|
21
|
+
'timestamp',
|
|
22
|
+
'taskId',
|
|
23
|
+
'orchestrationId',
|
|
24
|
+
'fallbackChainId',
|
|
25
|
+
'runId',
|
|
26
|
+
'attemptId',
|
|
27
|
+
'attemptNumber',
|
|
28
|
+
'engineId',
|
|
29
|
+
'healthTargetId',
|
|
30
|
+
'failureClass',
|
|
31
|
+
'decision',
|
|
32
|
+
'reason',
|
|
33
|
+
'budgetBefore',
|
|
34
|
+
'budgetAfter',
|
|
35
|
+
'workspaceSafety',
|
|
36
|
+
'previousAttemptId',
|
|
37
|
+
'selectedCandidate',
|
|
38
|
+
'rejectedCandidates',
|
|
39
|
+
'schemaVersion'
|
|
40
|
+
];
|
|
41
|
+
|
|
42
|
+
function trailPath(projectRoot = process.cwd()) {
|
|
43
|
+
return path.join(projectRoot, CONFIG.maestroPath, FALLBACK_CHAIN_EVENTS_FILE);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function makeEventId(eventType) {
|
|
47
|
+
return 'fallback-chain-' + process.pid + '-' + Date.now().toString(36) + '-' + eventType + '-' + Math.random().toString(36).slice(2, 8);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function normalizeText(value) {
|
|
51
|
+
const redacted = redactCredentials(value == null ? '' : String(value));
|
|
52
|
+
return redacted.length > EVIDENCE_SUMMARY_LIMIT ? redacted.slice(0, EVIDENCE_SUMMARY_LIMIT) : redacted;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function normalizeReason(reason) {
|
|
56
|
+
return Array.isArray(reason) ? reason.map(normalizeText) : [normalizeText(reason)].filter(Boolean);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function scrubEvent(input = {}) {
|
|
60
|
+
const event = {
|
|
61
|
+
eventId: input.eventId || makeEventId(input.eventType || 'event'),
|
|
62
|
+
eventType: input.eventType || 'fallback_event',
|
|
63
|
+
timestamp: input.timestamp || new Date().toISOString(),
|
|
64
|
+
taskId: input.taskId == null ? null : Number(input.taskId),
|
|
65
|
+
orchestrationId: input.orchestrationId == null ? null : String(input.orchestrationId),
|
|
66
|
+
fallbackChainId: input.fallbackChainId == null ? null : String(input.fallbackChainId),
|
|
67
|
+
runId: input.runId == null ? null : String(input.runId),
|
|
68
|
+
attemptId: input.attemptId == null ? null : String(input.attemptId),
|
|
69
|
+
attemptNumber: Number.isInteger(input.attemptNumber) ? input.attemptNumber : null,
|
|
70
|
+
engineId: input.engineId == null ? null : String(input.engineId),
|
|
71
|
+
healthTargetId: input.healthTargetId == null ? null : String(input.healthTargetId),
|
|
72
|
+
failureClass: input.failureClass == null ? null : String(input.failureClass),
|
|
73
|
+
decision: input.decision == null ? null : String(input.decision),
|
|
74
|
+
reason: normalizeReason(input.reason),
|
|
75
|
+
budgetBefore: input.budgetBefore || null,
|
|
76
|
+
budgetAfter: input.budgetAfter || null,
|
|
77
|
+
workspaceSafety: input.workspaceSafety || { workspaceChanged: null },
|
|
78
|
+
previousAttemptId: input.previousAttemptId == null ? null : String(input.previousAttemptId),
|
|
79
|
+
selectedCandidate: input.selectedCandidate || null,
|
|
80
|
+
rejectedCandidates: Array.isArray(input.rejectedCandidates) ? input.rejectedCandidates : [],
|
|
81
|
+
deadline: input.deadline == null ? null : String(input.deadline),
|
|
82
|
+
visitedEngineIds: Array.isArray(input.visitedEngineIds) ? input.visitedEngineIds.map(String) : [],
|
|
83
|
+
visitedHealthTargets: Array.isArray(input.visitedHealthTargets) ? input.visitedHealthTargets.map(String) : [],
|
|
84
|
+
originatingRunId: input.originatingRunId == null ? null : String(input.originatingRunId),
|
|
85
|
+
terminalOutcome: input.terminalOutcome == null ? null : String(input.terminalOutcome),
|
|
86
|
+
reasonChain: Array.isArray(input.reasonChain) ? input.reasonChain.map(normalizeText) : []
|
|
87
|
+
};
|
|
88
|
+
for (const forbidden of ['prompt', 'stdout', 'stderr', 'fullStdout', 'fullStderr', 'headers', 'authorization', 'cookie']) {
|
|
89
|
+
delete event[forbidden];
|
|
90
|
+
}
|
|
91
|
+
return redactObject(stampSchemaVersion(event));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function validateFallbackChainEvent(event) {
|
|
95
|
+
const errors = [];
|
|
96
|
+
if (!event || typeof event !== 'object' || Array.isArray(event)) {
|
|
97
|
+
return { ok: false, errors: ['event must be an object'] };
|
|
98
|
+
}
|
|
99
|
+
for (const field of REQUIRED_EVENT_FIELDS) {
|
|
100
|
+
if (!(field in event)) errors.push(field + ' is required');
|
|
101
|
+
}
|
|
102
|
+
if (typeof event.eventType !== 'string') errors.push('eventType is required');
|
|
103
|
+
if (event.schemaVersion !== 1) errors.push('schemaVersion must be 1');
|
|
104
|
+
return { ok: errors.length === 0, errors };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export async function appendFallbackChainEvent(input = {}, options = {}) {
|
|
108
|
+
const filePath = options.filePath || trailPath(options.projectRoot);
|
|
109
|
+
const event = scrubEvent(input);
|
|
110
|
+
const result = validateFallbackChainEvent(event);
|
|
111
|
+
if (!result.ok) {
|
|
112
|
+
throw new Error('Invalid fallback chain event: ' + result.errors.join('; '));
|
|
113
|
+
}
|
|
114
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
115
|
+
await fs.appendFile(filePath, JSON.stringify(event) + '\n', { encoding: 'utf8' });
|
|
116
|
+
return event;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export async function readFallbackChainEvents(options = {}) {
|
|
120
|
+
const filePath = options.filePath || trailPath(options.projectRoot);
|
|
121
|
+
try {
|
|
122
|
+
const content = await fs.readFile(filePath, 'utf-8');
|
|
123
|
+
return content.split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line));
|
|
124
|
+
} catch (error) {
|
|
125
|
+
if (error.code === 'ENOENT') return [];
|
|
126
|
+
throw error;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export function groupChainsById(events = []) {
|
|
131
|
+
const groups = new Map();
|
|
132
|
+
for (const event of events) {
|
|
133
|
+
if (!event || !event.fallbackChainId) continue;
|
|
134
|
+
const key = String(event.fallbackChainId);
|
|
135
|
+
if (!groups.has(key)) groups.set(key, []);
|
|
136
|
+
groups.get(key).push(event);
|
|
137
|
+
}
|
|
138
|
+
for (const group of groups.values()) {
|
|
139
|
+
group.sort((a, b) => String(a.timestamp || '').localeCompare(String(b.timestamp || '')));
|
|
140
|
+
}
|
|
141
|
+
return groups;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export function replayFallbackChain(events = []) {
|
|
145
|
+
const ordered = [...events].sort((a, b) => String(a.timestamp || '').localeCompare(String(b.timestamp || '')));
|
|
146
|
+
const visitedEngineIds = new Set();
|
|
147
|
+
const visitedHealthTargets = new Set();
|
|
148
|
+
const reasonChain = [];
|
|
149
|
+
let latest = null;
|
|
150
|
+
let attemptNumber = 0;
|
|
151
|
+
let previousAttemptId = null;
|
|
152
|
+
let fallbackChainId = null;
|
|
153
|
+
let originatingRunId = null;
|
|
154
|
+
let deadline = null;
|
|
155
|
+
let selectedCandidate = null;
|
|
156
|
+
let inProgressAttempt = null;
|
|
157
|
+
|
|
158
|
+
for (const event of ordered) {
|
|
159
|
+
latest = event;
|
|
160
|
+
fallbackChainId = fallbackChainId || event.fallbackChainId || null;
|
|
161
|
+
originatingRunId = originatingRunId || event.originatingRunId || event.fallbackChainId || null;
|
|
162
|
+
deadline = deadline || event.deadline || null;
|
|
163
|
+
if (Number.isInteger(event.attemptNumber)) {
|
|
164
|
+
attemptNumber = Math.max(attemptNumber, event.attemptNumber);
|
|
165
|
+
}
|
|
166
|
+
if (event.previousAttemptId != null) previousAttemptId = event.attemptId || event.runId || event.previousAttemptId;
|
|
167
|
+
if (event.engineId) visitedEngineIds.add(String(event.engineId));
|
|
168
|
+
if (event.healthTargetId) visitedHealthTargets.add(String(event.healthTargetId));
|
|
169
|
+
for (const engineId of event.visitedEngineIds || []) visitedEngineIds.add(String(engineId));
|
|
170
|
+
for (const targetId of event.visitedHealthTargets || []) visitedHealthTargets.add(String(targetId));
|
|
171
|
+
for (const reason of event.reason || []) reasonChain.push(normalizeText(reason));
|
|
172
|
+
if (event.selectedCandidate) selectedCandidate = event.selectedCandidate;
|
|
173
|
+
if (event.eventType === 'fallback_attempt_started') {
|
|
174
|
+
selectedCandidate = null;
|
|
175
|
+
inProgressAttempt = event;
|
|
176
|
+
}
|
|
177
|
+
if (event.eventType === 'fallback_attempt_finished') {
|
|
178
|
+
inProgressAttempt = null;
|
|
179
|
+
previousAttemptId = event.attemptId || event.runId || previousAttemptId;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const terminal = !!(latest && FALLBACK_CHAIN_TERMINAL_EVENTS.has(latest.eventType));
|
|
184
|
+
return {
|
|
185
|
+
fallbackChainId,
|
|
186
|
+
originatingRunId,
|
|
187
|
+
taskId: latest ? latest.taskId : null,
|
|
188
|
+
latestEvent: latest,
|
|
189
|
+
terminal,
|
|
190
|
+
attemptNumber,
|
|
191
|
+
previousAttemptId,
|
|
192
|
+
deadline,
|
|
193
|
+
visitedEngineIds: [...visitedEngineIds],
|
|
194
|
+
visitedHealthTargets: [...visitedHealthTargets],
|
|
195
|
+
reasonChain,
|
|
196
|
+
selectedCandidate,
|
|
197
|
+
inProgressAttempt,
|
|
198
|
+
events: ordered
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export async function findLatestFallbackChainForTask(taskId, options = {}) {
|
|
203
|
+
const events = await readFallbackChainEvents(options);
|
|
204
|
+
const taskEvents = events.filter(event => String(event.taskId) === String(taskId));
|
|
205
|
+
const groups = groupChainsById(taskEvents);
|
|
206
|
+
let latest = null;
|
|
207
|
+
for (const eventsForChain of groups.values()) {
|
|
208
|
+
const replayed = replayFallbackChain(eventsForChain);
|
|
209
|
+
if (!latest || String(replayed.latestEvent.timestamp || '').localeCompare(String(latest.latestEvent.timestamp || '')) > 0) {
|
|
210
|
+
latest = replayed;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
return latest;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
export const fallbackChainTrailPaths = {
|
|
217
|
+
trailPath
|
|
218
|
+
};
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
// src/orchestration/fallback-executor.mjs
|
|
2
|
+
|
|
3
|
+
export function planFallback(input) {
|
|
4
|
+
const { failure, context, budget, policy } = input;
|
|
5
|
+
const { category, retryable } = failure;
|
|
6
|
+
const { attemptNumber, maxAttempts } = context;
|
|
7
|
+
|
|
8
|
+
// needs_human is a statement about the underlying problem, not about how
|
|
9
|
+
// many attempts/budget remain — the failure itself says a human must
|
|
10
|
+
// decide, so this is checked before maxAttempts/policy and is always
|
|
11
|
+
// "allowed" (asking a human is never an unsafe action to permit).
|
|
12
|
+
if (category === 'needs_human') {
|
|
13
|
+
return {
|
|
14
|
+
action: 'ask_human',
|
|
15
|
+
successorEngine: null,
|
|
16
|
+
successorRole: null,
|
|
17
|
+
reason: ['Failure category "needs_human" always asks a human, regardless of attempts/budget.'],
|
|
18
|
+
allowed: true,
|
|
19
|
+
requiresHuman: true,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
let action = 'none';
|
|
24
|
+
let successorEngine = null;
|
|
25
|
+
let successorRole = null;
|
|
26
|
+
const reason = [];
|
|
27
|
+
let allowed = true; // Overall flag if the current plan is allowed
|
|
28
|
+
let requiresHuman = false;
|
|
29
|
+
|
|
30
|
+
// Rule: Max attempts exceeded - highest precedence block
|
|
31
|
+
if (attemptNumber >= maxAttempts) {
|
|
32
|
+
action = 'block';
|
|
33
|
+
allowed = false;
|
|
34
|
+
requiresHuman = true; // Human needs to know it's blocked
|
|
35
|
+
reason.push(`Max attempts (${maxAttempts}) exceeded.`);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Rule: Policy forbids paid engines (only if not already hard-blocked)
|
|
39
|
+
if (allowed && policy && policy.allowPaid === false && budget && budget.isPaid) {
|
|
40
|
+
action = 'block';
|
|
41
|
+
allowed = false;
|
|
42
|
+
requiresHuman = true;
|
|
43
|
+
reason.push('Policy forbids paid engines.');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Rule: Policy forbids direct OpenAI (only if not already hard-blocked)
|
|
47
|
+
if (allowed && policy && policy.allowDirectOpenAI === false && context && context.selectedEngine && context.selectedEngine.id === 'openai') {
|
|
48
|
+
action = 'block';
|
|
49
|
+
allowed = false;
|
|
50
|
+
requiresHuman = true;
|
|
51
|
+
reason.push('Policy forbids direct OpenAI calls.');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Rule: Policy forbids direct Claude (only if not already hard-blocked)
|
|
55
|
+
if (allowed && policy && policy.allowDirectClaude === false && context && context.selectedEngine && context.selectedEngine.id === 'claude') {
|
|
56
|
+
action = 'block';
|
|
57
|
+
allowed = false;
|
|
58
|
+
requiresHuman = true;
|
|
59
|
+
reason.push('Policy forbids direct Claude calls.');
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
// If a hard block was applied due to max attempts or policy, prioritize asking human about the block
|
|
64
|
+
if (!allowed && action === 'block') {
|
|
65
|
+
// No further actions, just return the block.
|
|
66
|
+
return {
|
|
67
|
+
action,
|
|
68
|
+
successorEngine,
|
|
69
|
+
successorRole,
|
|
70
|
+
reason,
|
|
71
|
+
allowed,
|
|
72
|
+
requiresHuman,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
// Fallback logic based on failure category (only if not already hard-blocked)
|
|
78
|
+
if (allowed) {
|
|
79
|
+
switch (category) {
|
|
80
|
+
case 'network_error':
|
|
81
|
+
case 'rate_limit':
|
|
82
|
+
case 'quota_exceeded':
|
|
83
|
+
if (retryable) {
|
|
84
|
+
action = 'retry';
|
|
85
|
+
reason.push(`Failure category "${category}" is retryable.`);
|
|
86
|
+
} else {
|
|
87
|
+
action = 'ask_human';
|
|
88
|
+
requiresHuman = true;
|
|
89
|
+
reason.push(`Failure category "${category}" is not retryable, asking human.`);
|
|
90
|
+
}
|
|
91
|
+
break;
|
|
92
|
+
case 'missing_tool':
|
|
93
|
+
case 'syntax_failure':
|
|
94
|
+
case 'test_failure':
|
|
95
|
+
case 'windows_shell_mismatch':
|
|
96
|
+
case 'unsupported_tool_call':
|
|
97
|
+
action = 'switch_engine'; // Or suggest a fix if context allows
|
|
98
|
+
reason.push(`Failure category "${category}" suggests switching engine or fixing.`);
|
|
99
|
+
break;
|
|
100
|
+
case 'nested_sandbox_process_failure':
|
|
101
|
+
action = 'verify';
|
|
102
|
+
reason.push('Worker validation was blocked by nested sandbox process creation; use allowlisted host validation if policy permits.');
|
|
103
|
+
break;
|
|
104
|
+
case 'authentication_error':
|
|
105
|
+
case 'policy_violation':
|
|
106
|
+
case 'budget_exceeded':
|
|
107
|
+
case 'needs_human': // This will be handled by the shouldAskHuman flag below
|
|
108
|
+
action = 'ask_human';
|
|
109
|
+
requiresHuman = true;
|
|
110
|
+
reason.push(`Failure category "${category}" requires human intervention.`);
|
|
111
|
+
break;
|
|
112
|
+
case 'unknown_failure':
|
|
113
|
+
if (retryable) {
|
|
114
|
+
action = 'retry';
|
|
115
|
+
reason.push('Unknown failure, attempting retry.');
|
|
116
|
+
} else {
|
|
117
|
+
action = 'ask_human';
|
|
118
|
+
requiresHuman = true;
|
|
119
|
+
reason.push('Unknown failure and not retryable, asking human.');
|
|
120
|
+
}
|
|
121
|
+
break;
|
|
122
|
+
default:
|
|
123
|
+
action = 'ask_human';
|
|
124
|
+
requiresHuman = true;
|
|
125
|
+
reason.push(`No specific fallback plan for category "${category}", asking human.`);
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Note: shouldAskHuman from the classifier is informational here, not an
|
|
131
|
+
// override — categories with a more specific action (e.g. missing_tool ->
|
|
132
|
+
// switch_engine) keep that action. Any category NOT explicitly handled by
|
|
133
|
+
// the switch above already falls through to the 'default' branch, which
|
|
134
|
+
// asks a human, so no separate blanket override is needed (and one would
|
|
135
|
+
// incorrectly stomp switch_engine/retry decisions for categories the
|
|
136
|
+
// classifier also flags as shouldAskHuman, like missing_tool).
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
action,
|
|
140
|
+
successorEngine,
|
|
141
|
+
successorRole,
|
|
142
|
+
reason,
|
|
143
|
+
allowed,
|
|
144
|
+
requiresHuman,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import { CONFIG } from '../config.mjs';
|
|
3
|
+
import { readJsonSafe } from '../files.mjs';
|
|
4
|
+
import { FAILURE_CATEGORIES } from '../schema.mjs';
|
|
5
|
+
import { isNestedSandboxProcessFailure } from './failure-classifier.mjs';
|
|
6
|
+
|
|
7
|
+
const fallbackGraphPath = path.join(CONFIG.maestroPath, 'fallback-graph.json');
|
|
8
|
+
|
|
9
|
+
const DEFAULT_RULE = { action: 'escalate-to-manager', maxAttempts: 0, reason: 'No rule configured for this failure category; escalate rather than guess.' };
|
|
10
|
+
|
|
11
|
+
export function getFallbackGraphPath() {
|
|
12
|
+
return fallbackGraphPath;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export async function loadFallbackGraph() {
|
|
16
|
+
return readJsonSafe(fallbackGraphPath, { schemaVersion: 1, rules: {} });
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// Deterministic, text/signal-based classification. This never calls a model
|
|
20
|
+
// to decide "why did this fail" — see project principle "Claude pensa
|
|
21
|
+
// somente quando necessário." `context` fields are all optional; provide
|
|
22
|
+
// whichever are known for a given failure and this returns the first
|
|
23
|
+
// matching category, defaulting to 'unknown_failure'.
|
|
24
|
+
export function classifyFailure(context = {}) {
|
|
25
|
+
const {
|
|
26
|
+
preflightDecision,
|
|
27
|
+
missingCapabilities = [],
|
|
28
|
+
blockedBySafety,
|
|
29
|
+
syntaxValidationFailed,
|
|
30
|
+
testFailed,
|
|
31
|
+
lowConfidence,
|
|
32
|
+
exitCode,
|
|
33
|
+
stderr = '',
|
|
34
|
+
stdout = ''
|
|
35
|
+
} = context;
|
|
36
|
+
|
|
37
|
+
const text = String(stderr) + '\n' + String(stdout);
|
|
38
|
+
|
|
39
|
+
if (isNestedSandboxProcessFailure(text)) return 'nested_sandbox_process_failure';
|
|
40
|
+
if (blockedBySafety) return 'policy_violation';
|
|
41
|
+
|
|
42
|
+
if (preflightDecision === 'NEEDS_CAPABILITY') {
|
|
43
|
+
const toolKeys = new Set(['shell', 'filesystemWrite', 'browser', 'vision', 'testExecution']);
|
|
44
|
+
const missingTool = missingCapabilities.some(key => toolKeys.has(key));
|
|
45
|
+
return missingTool ? 'missing_tool' : 'insufficient_capability';
|
|
46
|
+
}
|
|
47
|
+
if (preflightDecision === 'NEEDS_CONTEXT') return 'context_too_small';
|
|
48
|
+
if (preflightDecision === 'UNAVAILABLE') return 'provider_unavailable';
|
|
49
|
+
|
|
50
|
+
if (syntaxValidationFailed) return 'syntax_failure';
|
|
51
|
+
if (testFailed) return 'test_failure';
|
|
52
|
+
if (lowConfidence) return 'low_confidence';
|
|
53
|
+
|
|
54
|
+
if (/rate[\s-]?limit|too many requests|\b429\b/i.test(text)) return 'rate_limit';
|
|
55
|
+
if (/quota|insufficient[_\s]?(credit|balance)|out of credits/i.test(text)) return 'quota_exceeded';
|
|
56
|
+
if (/ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ECONNRESET|network error|fetch failed/i.test(text)) return 'network_error';
|
|
57
|
+
if (/\b401\b|unauthorized|invalid api key|authentication failed/i.test(text)) return 'authentication_error';
|
|
58
|
+
if (/\b5\d\d\b|service unavailable|bad gateway|provider (is )?down/i.test(text)) return 'provider_unavailable';
|
|
59
|
+
if (/unsupported call|invalid[_\s]?output|malformed (json|output)/i.test(text)) return 'invalid_output';
|
|
60
|
+
|
|
61
|
+
if (typeof exitCode === 'number' && exitCode !== 0) return 'unknown_failure';
|
|
62
|
+
|
|
63
|
+
return 'unknown_failure';
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export async function resolveFallbackRule(category) {
|
|
67
|
+
const graph = await loadFallbackGraph();
|
|
68
|
+
const rules = graph.rules || {};
|
|
69
|
+
return rules[category] || DEFAULT_RULE;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Turns an action string from fallback-graph.json into a concrete hint for
|
|
73
|
+
// whatever calls engine-selector.mjs next. This is intentionally data, not
|
|
74
|
+
// behavior — the delegation manager / task runner decides what to actually
|
|
75
|
+
// do with the hint.
|
|
76
|
+
const ACTION_HINTS = {
|
|
77
|
+
'try-equivalent-provider': { excludeCurrentEngine: true, requireSameCapabilityClass: true },
|
|
78
|
+
'retry-then-successor': { retrySameEngineFirst: true, thenExcludeCurrentEngine: true },
|
|
79
|
+
'switch-provider': { excludeCurrentEngine: true, excludeSameProvider: true },
|
|
80
|
+
'block-engine': { excludeCurrentEngine: true, blockEnginePermanently: true },
|
|
81
|
+
'require-capability': { excludeCurrentEngine: true, requireMissingTools: true },
|
|
82
|
+
'escalate-capability': { excludeCurrentEngine: true, requireHigherCapability: true },
|
|
83
|
+
'escalate-context': { excludeCurrentEngine: true, requireLargerContext: true },
|
|
84
|
+
'call-verifier': { callVerifierFirst: true },
|
|
85
|
+
'escalate-to-manager': { escalateToManager: true }
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
export async function planFallback(context = {}) {
|
|
89
|
+
const category = context.category || classifyFailure(context);
|
|
90
|
+
const rule = await resolveFallbackRule(category);
|
|
91
|
+
const hint = ACTION_HINTS[rule.action] || {};
|
|
92
|
+
return {
|
|
93
|
+
category,
|
|
94
|
+
action: rule.action,
|
|
95
|
+
maxAttempts: typeof rule.maxAttempts === 'number' ? rule.maxAttempts : 0,
|
|
96
|
+
retryDelayMs: rule.retryDelayMs || 0,
|
|
97
|
+
reason: rule.reason || '',
|
|
98
|
+
hint
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function isKnownFailureCategory(category) {
|
|
103
|
+
return FAILURE_CATEGORIES.has(category);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export { FAILURE_CATEGORIES };
|