agents-relay 2.0.139 → 2.0.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters.js +10 -1
- package/dist/chatgpt-platform-fallback.js +56 -0
- package/dist/execution-event-identity.js +15 -0
- package/dist/reconciler-review.js +21 -2
- package/dist/reconciler-runtime.js +31 -34
- package/dist/reconciler.js +10 -6
- package/dist/worker-rejection.js +45 -0
- package/dist/worker-routing.js +5 -8
- package/dist/workflow/contracts.js +4 -0
- package/dist/workflow/definitions-v2.js +1 -0
- package/dist/workflow/runtime.js +17 -0
- package/package.json +5 -2
- package/workflow/task-lifecycle.mmd +11 -0
package/dist/adapters.js
CHANGED
|
@@ -6,6 +6,7 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
|
|
6
6
|
import { withTestInjection } from './e2e-injection.js';
|
|
7
7
|
import { taskReferencePrompt } from './task-context.js';
|
|
8
8
|
import { spawnOwnedProcess, terminateOwnedProcess, ownedProcessTerminationReason } from './owned-process.js';
|
|
9
|
+
import { PreSubmissionRejection } from './worker-rejection.js';
|
|
9
10
|
export function buildCodexArgs(route, input) {
|
|
10
11
|
const args = ['exec', '--json'];
|
|
11
12
|
if (route.profile)
|
|
@@ -202,8 +203,14 @@ function rawBrowserWorkspaceCommand(cli, args, stdin, signal) {
|
|
|
202
203
|
try {
|
|
203
204
|
const envelope = parseWorkspaceJson(stdout);
|
|
204
205
|
const result = envelope.result;
|
|
205
|
-
if (result?.ok === false && typeof result.code === 'string' && typeof result.message === 'string')
|
|
206
|
+
if (result?.ok === false && typeof result.code === 'string' && typeof result.message === 'string') {
|
|
206
207
|
reason = `${result.code}: ${result.message}`;
|
|
208
|
+
const rejection = PreSubmissionRejection.fromPlatformResponse(result);
|
|
209
|
+
if (rejection) {
|
|
210
|
+
reject(rejection);
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
207
214
|
}
|
|
208
215
|
catch { /* Preserve legacy error handling for other CLI failures. */ }
|
|
209
216
|
reject(new Error(reason));
|
|
@@ -387,6 +394,8 @@ export class ChatGptAdapter {
|
|
|
387
394
|
throw error;
|
|
388
395
|
if (error instanceof StartAckTimeout)
|
|
389
396
|
throw error;
|
|
397
|
+
if (error instanceof PreSubmissionRejection)
|
|
398
|
+
throw error;
|
|
390
399
|
throw new LaunchFailure(error instanceof Error ? error.message : String(error));
|
|
391
400
|
}
|
|
392
401
|
})();
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { eligibleWorkerCandidates, workerCandidates } from './worker-routing.js';
|
|
2
|
+
import { routeExecution } from './execution-routing.js';
|
|
3
|
+
import { eventFor } from './events.js';
|
|
4
|
+
export function codexFallbackRouting(task, adapters, registered, usage) {
|
|
5
|
+
const requirements = task.workerRequirements ?? { capabilities: [] };
|
|
6
|
+
const excludedWorkers = new Set((task.routingExclusions ?? []).map(item => item.workerId).filter((id) => Boolean(id)));
|
|
7
|
+
const excludedAdapters = new Set((task.routingExclusions ?? []).map(item => item.adapter).filter((adapter) => Boolean(adapter)));
|
|
8
|
+
const candidates = eligibleWorkerCandidates(workerCandidates(adapters, registered, usage), requirements, { strictCapabilities: true })
|
|
9
|
+
.filter(candidate => candidate.adapter === 'codex' && !excludedWorkers.has(candidate.workerId) && !excludedAdapters.has(candidate.adapter));
|
|
10
|
+
const excludedRoutes = (task.routingExclusions ?? []).map(item => ({ provider: item.provider, model: item.model, ...(item.profile ? { profile: item.profile } : {}) }));
|
|
11
|
+
return { candidates, excludedRoutes };
|
|
12
|
+
}
|
|
13
|
+
export async function codexFallbackCompatible(task, adapters, registered, usage) {
|
|
14
|
+
if (task.adapter !== 'chatgpt' || task.chatgptThreadDelivery || task.workerRequirements?.runtime === 'chatgpt')
|
|
15
|
+
return false;
|
|
16
|
+
const requirements = task.workerRequirements ?? { capabilities: [] };
|
|
17
|
+
const { candidates, excludedRoutes } = codexFallbackRouting(task, adapters, registered, usage);
|
|
18
|
+
if (candidates.length === 0)
|
|
19
|
+
return false;
|
|
20
|
+
try {
|
|
21
|
+
await routeExecution(candidates, requirements, { registered, usage, taskFacts: { task }, excludedRoutes });
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
export async function quarantineUnavailableCodexFallback(transitions, emit, jobId, taskId, error) {
|
|
29
|
+
const transition = await transitions.apply(jobId, taskId, 'worker.platform_failure', {
|
|
30
|
+
eligible: (_job, task) => task.state === 'QUEUED' && task.adapter === 'chatgpt' && task.platformFailure?.safeToReroute === true,
|
|
31
|
+
facts: () => ({ platformFallbackSafe: false }),
|
|
32
|
+
mutate: (_job, task) => { task.error = `Codex fallback route became unavailable: ${error instanceof Error ? error.message : String(error)}`; },
|
|
33
|
+
});
|
|
34
|
+
if (transition?.decision.accepted)
|
|
35
|
+
await emit(eventFor(jobId, taskId, transition.task.parentTaskId, 'task.blocked', 'blocked', transition.task.error ?? 'Codex fallback route unavailable', 'user', { reason: transition.task.failureReason, phase: 'fallback-routing' }));
|
|
36
|
+
}
|
|
37
|
+
export function persistChatGptPlatformFailure(task, original, executionId, classification, evidence, safeToReroute) {
|
|
38
|
+
const route = original?.routing;
|
|
39
|
+
const workerId = original?.workerSelection?.workerId ?? 'chatgpt';
|
|
40
|
+
const at = new Date().toISOString();
|
|
41
|
+
task.platformFailure = {
|
|
42
|
+
classification,
|
|
43
|
+
executionId: executionId ?? 'unknown',
|
|
44
|
+
route: { workerId, adapter: 'chatgpt', provider: route?.provider ?? 'unknown', model: route?.model ?? 'unknown', ...(route?.profile ? { profile: route.profile } : {}) },
|
|
45
|
+
evidence,
|
|
46
|
+
safeToReroute,
|
|
47
|
+
at,
|
|
48
|
+
};
|
|
49
|
+
if (!task.routingExclusions?.some(item => item.workerId === workerId && item.adapter === 'chatgpt')) {
|
|
50
|
+
task.routingExclusions = [...(task.routingExclusions ?? []), {
|
|
51
|
+
...task.platformFailure.route,
|
|
52
|
+
reason: classification,
|
|
53
|
+
at,
|
|
54
|
+
}];
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export function executionEventSource(jobId, taskId, executionId) { return `job/${jobId}/task/${taskId}/execution/${executionId}`; }
|
|
2
|
+
export function eventSourceId(event) { return typeof event.source?.id === 'string' ? event.source.id : undefined; }
|
|
3
|
+
export function executionIdFromSource(sourceId, jobId, taskId) {
|
|
4
|
+
const prefix = `job/${jobId}/task/${taskId}/execution/`;
|
|
5
|
+
return sourceId?.startsWith(prefix) ? sourceId.slice(prefix.length) || undefined : undefined;
|
|
6
|
+
}
|
|
7
|
+
const terminalExecutionClaims = new Set();
|
|
8
|
+
export function claimTerminalExecution(sourceId) {
|
|
9
|
+
terminalExecutionClaims.add(sourceId);
|
|
10
|
+
const cleanup = setTimeout(() => terminalExecutionClaims.delete(sourceId), 60000);
|
|
11
|
+
cleanup.unref();
|
|
12
|
+
}
|
|
13
|
+
export function terminalExecutionClaimed(jobId, taskId, executionId) {
|
|
14
|
+
return terminalExecutionClaims.has(executionEventSource(jobId, taskId, executionId));
|
|
15
|
+
}
|
|
@@ -11,6 +11,7 @@ import { createTaskEntity } from './workflow/entity-creation.js';
|
|
|
11
11
|
import { localSourceHead, localWorkingHead } from './local-git-merge.js';
|
|
12
12
|
import { chatgptThreadRequirements, resolveReviewerThreadDestination, resolveImplementationTaskThreadDestination, prepareReviewerThreadDelivery } from './reviewer-thread.js';
|
|
13
13
|
import { reviewGenerationMatches } from './review-generation-fence.js';
|
|
14
|
+
import { codexFallbackRouting, quarantineUnavailableCodexFallback } from './chatgpt-platform-fallback.js';
|
|
14
15
|
export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
15
16
|
async currentHeadSha(job) {
|
|
16
17
|
// GitHub PR head facts are audit-only. Source identity is local Git.
|
|
@@ -268,7 +269,14 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
268
269
|
return { task, routed: false };
|
|
269
270
|
const usage = this.options.usageSnapshotProvider ? await this.options.usageSnapshotProvider() : undefined;
|
|
270
271
|
const registered = await this.store.listAgents();
|
|
271
|
-
const
|
|
272
|
+
const codexFallback = task.platformFailure?.safeToReroute === true && task.adapter === 'chatgpt';
|
|
273
|
+
const fallbackRouting = codexFallback ? codexFallbackRouting(task, this.options.adapters, registered, usage) : undefined;
|
|
274
|
+
const excludedWorkers = new Set((task.routingExclusions ?? []).map(item => item.workerId).filter((id) => Boolean(id)));
|
|
275
|
+
const excludedAdapters = new Set((task.routingExclusions ?? []).map(item => item.adapter).filter((adapter) => Boolean(adapter)));
|
|
276
|
+
const candidates = fallbackRouting?.candidates ?? workerCandidates(this.options.adapters, registered, usage)
|
|
277
|
+
.filter(candidate => !excludedWorkers.has(candidate.workerId))
|
|
278
|
+
.filter(candidate => !excludedAdapters.has(candidate.adapter))
|
|
279
|
+
.filter(candidate => (implementationThread || reviewerThread) ? candidate.adapter === 'chatgpt' : true);
|
|
272
280
|
const runtimeSummary = this.options.runtimeSummaryProvider?.();
|
|
273
281
|
const requirements = (implementationThread || reviewerThread) ? chatgptThreadRequirements(task.workerRequirements) : (task.workerRequirements ?? { capabilities: [] });
|
|
274
282
|
const taskFacts = {
|
|
@@ -296,7 +304,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
296
304
|
const parentExcludedRoutes = parentRouteFailedBeforeStart && parent?.routing
|
|
297
305
|
? [{ provider: parent.routing.provider, model: parent.routing.model, profile: parent.routing.profile }]
|
|
298
306
|
: [];
|
|
299
|
-
const taskExcludedRoutes = (task.routingExclusions ?? []).map(item => ({ provider: item.provider, model: item.model, profile: item.profile }));
|
|
307
|
+
const taskExcludedRoutes = fallbackRouting?.excludedRoutes ?? (task.routingExclusions ?? []).map(item => ({ provider: item.provider, model: item.model, profile: item.profile }));
|
|
300
308
|
const excludedRoutes = [...taskExcludedRoutes, ...parentExcludedRoutes];
|
|
301
309
|
const routed = await routeExecution(candidates, requirements, {
|
|
302
310
|
decisionEngine: this.options.decisionEngine,
|
|
@@ -320,6 +328,17 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
320
328
|
await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'model.selected', 'queued', 'Selected ' + task.routing.model + ' · thinking ' + (task.routing.reasoning ?? 'unknown'), 'user', { routing: task.routing }, { agent: 'relay', component: 'agents-relay' }));
|
|
321
329
|
return { task, routed: true };
|
|
322
330
|
}
|
|
331
|
+
async routeTaskForExecutionFailClosed(job, task) {
|
|
332
|
+
try {
|
|
333
|
+
return await this.routeTaskForExecution(job, task);
|
|
334
|
+
}
|
|
335
|
+
catch (error) {
|
|
336
|
+
if (task.adapter !== 'chatgpt' || !task.platformFailure?.safeToReroute)
|
|
337
|
+
throw error;
|
|
338
|
+
await quarantineUnavailableCodexFallback(this.taskTransitions, event => this.emit(event), job.id, task.id, error);
|
|
339
|
+
return undefined;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
323
342
|
async applyReviewEvent(event) {
|
|
324
343
|
const job = await this.store.load(event.job_id);
|
|
325
344
|
const task = job.tasks.find(item => item.id === event.task_id && item.agentName === 'reviewer');
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { normalizeReviewerTerminalEvent } from './review-terminal-normalization.js';
|
|
2
|
+
import { executionEventSource, eventSourceId, executionIdFromSource, claimTerminalExecution, terminalExecutionClaimed } from './execution-event-identity.js';
|
|
3
|
+
export { executionEventSource } from './execution-event-identity.js';
|
|
2
4
|
import { applyJobEffectOutcomes, applyTaskEffectOutcomes } from './workflow-effect-failures.js';
|
|
3
|
-
import { classifyWorkerRejection, reroutableWorkerRejections } from './worker-rejection.js';
|
|
5
|
+
import { classifyChatGptPlatformFailure, classifyWorkerRejection, reroutableWorkerRejections } from './worker-rejection.js';
|
|
4
6
|
import { applyTaskRetryAction } from './task-actions.js';
|
|
5
7
|
import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
|
|
6
8
|
import { eventFor } from './events.js';
|
|
@@ -12,24 +14,10 @@ import { createWorkerStartAck } from './worker-start-ack.js';
|
|
|
12
14
|
import { monitorExecutionProcess } from './execution-process-tracker.js';
|
|
13
15
|
import { HostInteractiveInvocationCoordinator, hostInvocationPorts } from './interactive-invocation.js';
|
|
14
16
|
import { resumeInteractiveTask } from './interactive-resume.js';
|
|
15
|
-
|
|
16
|
-
function eventSourceId(event) { return typeof event.source?.id === 'string' ? event.source.id : undefined; }
|
|
17
|
-
function executionIdFromSource(sourceId, jobId, taskId) {
|
|
18
|
-
const prefix = `job/${jobId}/task/${taskId}/execution/`;
|
|
19
|
-
return sourceId?.startsWith(prefix) ? sourceId.slice(prefix.length) || undefined : undefined;
|
|
20
|
-
}
|
|
21
|
-
const terminalExecutionClaims = new Set();
|
|
17
|
+
import { codexFallbackCompatible, persistChatGptPlatformFailure } from './chatgpt-platform-fallback.js';
|
|
22
18
|
const activeExecutionOwners = new Map();
|
|
23
19
|
import { withExecutionBoundary } from './execution-boundary.js';
|
|
24
20
|
export { withExecutionBoundary } from './execution-boundary.js';
|
|
25
|
-
function claimTerminalExecution(sourceId) {
|
|
26
|
-
terminalExecutionClaims.add(sourceId);
|
|
27
|
-
const cleanup = setTimeout(() => terminalExecutionClaims.delete(sourceId), 60000);
|
|
28
|
-
cleanup.unref();
|
|
29
|
-
}
|
|
30
|
-
function terminalExecutionClaimed(jobId, taskId, executionId) {
|
|
31
|
-
return terminalExecutionClaims.has(executionEventSource(jobId, taskId, executionId));
|
|
32
|
-
}
|
|
33
21
|
export class ReconcilerRuntimeBase {
|
|
34
22
|
store;
|
|
35
23
|
options;
|
|
@@ -271,7 +259,11 @@ export class ReconcilerRuntimeBase {
|
|
|
271
259
|
}
|
|
272
260
|
return true;
|
|
273
261
|
}
|
|
274
|
-
async failLaunch(jobId, taskId, executionId,
|
|
262
|
+
async failLaunch(jobId, taskId, executionId, failure, phase = 'launch') {
|
|
263
|
+
const message = failure instanceof Error ? failure.message : String(failure);
|
|
264
|
+
const currentJob = await this.store.load(jobId);
|
|
265
|
+
const currentTask = currentJob.tasks.find(task => task.id === taskId);
|
|
266
|
+
const platformFailure = currentTask?.adapter === 'chatgpt' ? classifyChatGptPlatformFailure(failure, phase) : undefined;
|
|
275
267
|
const failureReason = phase === 'start-ack' ? 'worker_start_timeout'
|
|
276
268
|
: /BROWSER_WORKSPACE_UNAVAILABLE|browser-workspace runtime dependency unavailable|browser-workspace session start returned no session_id|no close frame received or sent/i.test(message)
|
|
277
269
|
? 'browser_workspace_unavailable'
|
|
@@ -282,13 +274,20 @@ export class ReconcilerRuntimeBase {
|
|
|
282
274
|
this.launchFailureClaims.add(claim);
|
|
283
275
|
const cleanup = setTimeout(() => this.launchFailureClaims.delete(claim), 60000);
|
|
284
276
|
cleanup.unref();
|
|
285
|
-
const workflowEvent = phase === 'start-ack' ? 'worker.start_ack_failed' : 'worker.delivery_failed';
|
|
286
|
-
const
|
|
277
|
+
const workflowEvent = platformFailure ? 'worker.platform_failure' : phase === 'start-ack' ? 'worker.start_ack_failed' : 'worker.delivery_failed';
|
|
278
|
+
const usage = this.options.usageSnapshotProvider ? await this.options.usageSnapshotProvider() : undefined;
|
|
279
|
+
const platformWorkerCompatible = Boolean(platformFailure?.safeToReroute && currentTask && !(currentTask.agentName === 'reviewer' && this.options.reviewer?.threadId)
|
|
280
|
+
&& await codexFallbackCompatible(currentTask, this.options.adapters, await this.store.listAgents(), usage));
|
|
281
|
+
const transition = await this.taskTransitions.apply(jobId, taskId, platformFailure ? 'worker.platform_failure' : workflowEvent, {
|
|
287
282
|
eligible: (_job, task) => task.executionId === executionId,
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
283
|
+
facts: (_job, task) => ({ platformFallbackSafe: task.state === 'QUEUED' && Boolean(platformFailure?.safeToReroute && platformWorkerCompatible && task.attempt < task.maxAttempts) }),
|
|
284
|
+
mutate: (_job, task, decision) => {
|
|
285
|
+
if (platformFailure)
|
|
286
|
+
persistChatGptPlatformFailure(task, currentTask, executionId, platformFailure.classification, platformFailure.evidence, platformFailure.safeToReroute);
|
|
287
|
+
task.error = decision.target === 'QUEUED' ? null : `Worker ${phase} failed: ${message}`;
|
|
288
|
+
task.failureReason = decision.target === 'BLOCKED'
|
|
289
|
+
? (task.platformFailure?.safeToReroute && currentTask?.state === 'QUEUED' ? 'worker_platform_fallback_unavailable' : 'worker_outcome_unknown')
|
|
290
|
+
: failureReason;
|
|
292
291
|
task.leaseOwner = null;
|
|
293
292
|
task.leaseExpiresAt = null;
|
|
294
293
|
},
|
|
@@ -299,21 +298,19 @@ export class ReconcilerRuntimeBase {
|
|
|
299
298
|
await this.emit(eventFor(jobId, taskId, transition.task.parentTaskId, 'task.process.delivery_failed', 'failed', `Worker ${phase} reported after lifecycle moved to ${transition.from}: ${message}`, 'orchestrator', { phase, authoritative: false, executionId, workflowEvent }, { agent: 'relay', component: 'agents-relay' }));
|
|
300
299
|
return;
|
|
301
300
|
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
current.workerSelection = undefined;
|
|
309
|
-
await this.store.saveTask(current);
|
|
310
|
-
runtimeRetry = current;
|
|
301
|
+
if (executionId) {
|
|
302
|
+
const live = this.live.get(taskId);
|
|
303
|
+
if (live?.executionId === executionId) {
|
|
304
|
+
live.controller.abort();
|
|
305
|
+
live.execution.cancel();
|
|
306
|
+
this.clearExecution(taskId, executionId);
|
|
311
307
|
}
|
|
312
308
|
}
|
|
313
|
-
const
|
|
309
|
+
const runtimeRetry = transition.decision.target === 'QUEUED' ? transition.task : null;
|
|
310
|
+
const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Worker ${phase} failed: ${message}`, 'user', { phase, reason: transition.task.failureReason ?? failureReason, workflowEvent, ...(transition.task.platformFailure ? { platformFailure: transition.task.platformFailure } : {}) }, { agent: 'relay', component: 'agents-relay', watchdog: true });
|
|
314
311
|
this.detach(this.emit(event), jobId, taskId, 'worker failure event');
|
|
315
312
|
if (runtimeRetry) {
|
|
316
|
-
this.detach(this.emit(eventFor(jobId, taskId, runtimeRetry.parentTaskId, 'task.
|
|
313
|
+
this.detach(this.emit(eventFor(jobId, taskId, runtimeRetry.parentTaskId, 'task.rerouting', 'queued', 'ChatGPT platform rejection recorded; routing the same Task through Execution Router', 'orchestrator', { previousFailure: transition.task.platformFailure?.classification, executionId, attempt: runtimeRetry.attempt, maxAttempts: runtimeRetry.maxAttempts })), jobId, taskId, 'runtime retry event');
|
|
317
314
|
this.detach(this.eventCoordinator.accept({ jobId, reason: 'browser-workspace-runtime-retry' }), jobId, taskId, 'runtime retry reconcile wake');
|
|
318
315
|
return;
|
|
319
316
|
}
|
package/dist/reconciler.js
CHANGED
|
@@ -10,6 +10,7 @@ import { dependencyReady, schedule, transitionTaskEvent } from './scheduler.js';
|
|
|
10
10
|
import { applyTaskRetryAction } from './task-actions.js';
|
|
11
11
|
import { LaunchFailure, StartAckTimeout } from './adapters.js';
|
|
12
12
|
import { eventFor } from './events.js';
|
|
13
|
+
import { PreSubmissionRejection } from './worker-rejection.js';
|
|
13
14
|
import { createPlanRound, materializePlannerChild, pendingPlan } from './planner.js';
|
|
14
15
|
import { retryPendingPlannerUnblocks } from './planner-runtime.js';
|
|
15
16
|
import { dispatchPlannerOrchestration, plannerOrchestrationFacts } from './workflow/planner-orchestration-v2.js';
|
|
@@ -373,7 +374,10 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
373
374
|
await bindPlannedReviewHead(this.store, freshTask, await this.currentHeadSha(freshJob));
|
|
374
375
|
const implementationThread = resolveImplementationTaskThreadDestination(freshJob, freshTask);
|
|
375
376
|
if ((!freshTask.routing || ((implementationThread || (freshTask.agentName === 'reviewer' && resolveReviewerThreadDestination(freshJob, this.options.reviewer?.threadId))) && freshTask.adapter !== 'chatgpt')) && freshTask.adapter !== 'orchestrator') {
|
|
376
|
-
|
|
377
|
+
const routed = await this.routeTaskForExecutionFailClosed(freshJob, freshTask);
|
|
378
|
+
if (!routed)
|
|
379
|
+
return;
|
|
380
|
+
freshTask = routed.task;
|
|
377
381
|
}
|
|
378
382
|
if (['codex', 'chatgpt'].includes(freshTask.adapter) && !freshTask.routing) {
|
|
379
383
|
throw new Error('Task ' + freshTask.id + ' requires routing metadata before execution claim');
|
|
@@ -509,10 +513,10 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
509
513
|
catch (error) {
|
|
510
514
|
await startAckSetup?.stop();
|
|
511
515
|
terminalWait?.cancel();
|
|
512
|
-
await this.failLaunch(job.id, task.id, task.executionId, error
|
|
516
|
+
await this.failLaunch(job.id, task.id, task.executionId, error);
|
|
513
517
|
return;
|
|
514
518
|
}
|
|
515
|
-
void (execution.launched ?? Promise.resolve()).then(() => Promise.all([recordThreadDelivery(this.store, job.id, task.id, durableExecutionId, workerTask), this.emit(eventFor(job.id, task.id, task.parentTaskId, 'task.process.launched', 'running', `Task ${task.id} process launched`, 'user', { adapter: adapter.id, model: task.routing?.model, pid: execution.pid ?? null, executionId: durableExecutionId }, { agent: 'relay', component: 'agents-relay' }))])).catch(error => this.failLaunch(job.id, task.id, durableExecutionId, error
|
|
519
|
+
void (execution.launched ?? Promise.resolve()).then(() => Promise.all([recordThreadDelivery(this.store, job.id, task.id, durableExecutionId, workerTask), this.emit(eventFor(job.id, task.id, task.parentTaskId, 'task.process.launched', 'running', `Task ${task.id} process launched`, 'user', { adapter: adapter.id, model: task.routing?.model, pid: execution.pid ?? null, executionId: durableExecutionId }, { agent: 'relay', component: 'agents-relay' }))])).catch(error => this.failLaunch(job.id, task.id, durableExecutionId, error, 'launch'));
|
|
516
520
|
if (startAckSetup)
|
|
517
521
|
void startAckSetup.ack.finally(() => { void startAckSetup.stop(); }).catch(() => { });
|
|
518
522
|
this.live.set(task.id, { jobId: job.id, taskId: task.id, executionId: durableExecutionId, execution, controller, launchBarrier: startedAt, workerStarted: false, processSettled: false });
|
|
@@ -529,9 +533,9 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
529
533
|
}, Math.max(1, task.timeoutMs));
|
|
530
534
|
this.terminalTimers.set(task.id, terminalTimer);
|
|
531
535
|
this.detach(execution.promise.then(result => this.runtimeSettled(job.id, task.id, durableExecutionId, result, null), error => error instanceof StartAckTimeout
|
|
532
|
-
? this.failLaunch(job.id, task.id, durableExecutionId, error
|
|
533
|
-
: error instanceof LaunchFailure
|
|
534
|
-
? this.failLaunch(job.id, task.id, durableExecutionId, error
|
|
536
|
+
? this.failLaunch(job.id, task.id, durableExecutionId, error, 'start-ack')
|
|
537
|
+
: error instanceof LaunchFailure || error instanceof PreSubmissionRejection
|
|
538
|
+
? this.failLaunch(job.id, task.id, durableExecutionId, error, 'delivery')
|
|
535
539
|
: this.runtimeSettled(job.id, task.id, durableExecutionId, null, error instanceof Error ? error.message : String(error))), job.id, task.id, 'worker runtime completion');
|
|
536
540
|
}
|
|
537
541
|
async applyPlannerTree(jobId, planId, executionId, proposal) {
|
package/dist/worker-rejection.js
CHANGED
|
@@ -9,3 +9,48 @@ export function classifyWorkerRejection(message) {
|
|
|
9
9
|
return 'worker_process_failed';
|
|
10
10
|
}
|
|
11
11
|
export const reroutableWorkerRejections = new Set(['rate_limit_exceeded', 'model_not_found', 'provider_unavailable']);
|
|
12
|
+
/** Constructed only from a platform response that explicitly says no submission occurred. */
|
|
13
|
+
export class PreSubmissionRejection extends Error {
|
|
14
|
+
classification;
|
|
15
|
+
platformEvidence;
|
|
16
|
+
constructor(classification, platformEvidence) {
|
|
17
|
+
super(platformEvidence);
|
|
18
|
+
this.classification = classification;
|
|
19
|
+
this.platformEvidence = platformEvidence;
|
|
20
|
+
this.name = 'PreSubmissionRejection';
|
|
21
|
+
}
|
|
22
|
+
static fromPlatformResponse(response) {
|
|
23
|
+
if (!response || typeof response !== 'object' || Array.isArray(response))
|
|
24
|
+
return undefined;
|
|
25
|
+
const result = response;
|
|
26
|
+
if (result.status !== 'rejected' || result.submitted !== false || typeof result.code !== 'string' || typeof result.message !== 'string')
|
|
27
|
+
return undefined;
|
|
28
|
+
const classifications = {
|
|
29
|
+
MODEL_UNAVAILABLE: 'model_unavailable',
|
|
30
|
+
MODEL_NOT_FOUND: 'model_unavailable',
|
|
31
|
+
MCP_LOGIN_REQUIRED: 'mcp_login_required',
|
|
32
|
+
PROJECT_SETUP_UNAVAILABLE: 'project_setup_unavailable',
|
|
33
|
+
BROWSER_WORKSPACE_UNAVAILABLE: 'browser_workspace_unavailable',
|
|
34
|
+
};
|
|
35
|
+
const classification = classifications[result.code];
|
|
36
|
+
if (!classification)
|
|
37
|
+
return undefined;
|
|
38
|
+
const platformEvidence = JSON.stringify({ status: result.status, submitted: result.submitted, code: result.code, message: result.message });
|
|
39
|
+
return new PreSubmissionRejection(classification, platformEvidence);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Only positive pre-submission rejection evidence permits another route to launch. */
|
|
43
|
+
export function classifyChatGptPlatformFailure(error, phase) {
|
|
44
|
+
if (error instanceof PreSubmissionRejection)
|
|
45
|
+
return {
|
|
46
|
+
classification: error.classification,
|
|
47
|
+
safeToReroute: true,
|
|
48
|
+
evidence: error.platformEvidence,
|
|
49
|
+
};
|
|
50
|
+
const evidence = error instanceof Error ? error.message : String(error);
|
|
51
|
+
if (phase === 'start-ack')
|
|
52
|
+
return { classification: 'start_ack_timeout', safeToReroute: false, evidence };
|
|
53
|
+
if (phase === 'launch')
|
|
54
|
+
return { classification: 'launch_failure', safeToReroute: false, evidence };
|
|
55
|
+
return { classification: 'outcome_unknown', safeToReroute: false, evidence };
|
|
56
|
+
}
|
package/dist/worker-routing.js
CHANGED
|
@@ -74,22 +74,19 @@ export function workerCandidates(adapters, registered, snapshot) {
|
|
|
74
74
|
});
|
|
75
75
|
});
|
|
76
76
|
}
|
|
77
|
-
|
|
78
|
-
* Worker legality is intentionally narrow. Descriptive capability labels and provider
|
|
79
|
-
* quota are facts for decide(), not hard gates. An explicit runtime is retained as a
|
|
80
|
-
* compatibility/caller-owned constraint, and an explicitly offline worker is illegal.
|
|
81
|
-
*/
|
|
82
|
-
export function satisfiesWorker(candidate, requirements) {
|
|
77
|
+
export function satisfiesWorker(candidate, requirements, options = {}) {
|
|
83
78
|
if (candidate.availability === 'offline')
|
|
84
79
|
return false;
|
|
85
80
|
if (requirements.runtime && candidate.runtime !== requirements.runtime)
|
|
86
81
|
return false;
|
|
82
|
+
if (options.strictCapabilities && (requirements.capabilities ?? []).some(capability => !candidate.capabilities.includes(capability)))
|
|
83
|
+
return false;
|
|
87
84
|
if (candidate.adapter === 'chatgpt' && candidate.browserWorkspaceCapacity?.available === 0)
|
|
88
85
|
return false;
|
|
89
86
|
return true;
|
|
90
87
|
}
|
|
91
|
-
export function eligibleWorkerCandidates(candidates, requirements) {
|
|
92
|
-
return candidates.filter(candidate => satisfiesWorker(candidate, requirements));
|
|
88
|
+
export function eligibleWorkerCandidates(candidates, requirements, options = {}) {
|
|
89
|
+
return candidates.filter(candidate => satisfiesWorker(candidate, requirements, options));
|
|
93
90
|
}
|
|
94
91
|
function deterministicWorker(eligible) {
|
|
95
92
|
return eligible.find(candidate => candidate.adapter === 'codex') ?? eligible[0];
|
|
@@ -40,6 +40,7 @@ export const WORKFLOW_EVENT_NAMES = [
|
|
|
40
40
|
'watchdog.failed',
|
|
41
41
|
'worker.delivery_failed',
|
|
42
42
|
'worker.start_ack_failed',
|
|
43
|
+
'worker.platform_failure',
|
|
43
44
|
];
|
|
44
45
|
export const WORKFLOW_FACT_NAMES = [
|
|
45
46
|
'dependenciesSatisfied',
|
|
@@ -59,6 +60,7 @@ export const WORKFLOW_FACT_NAMES = [
|
|
|
59
60
|
'implementationComplete',
|
|
60
61
|
'headAvailable',
|
|
61
62
|
'mergeCapable',
|
|
63
|
+
'platformFallbackSafe',
|
|
62
64
|
];
|
|
63
65
|
export const WORKFLOW_MUTATION_NAMES = [
|
|
64
66
|
'mutation.clearError',
|
|
@@ -68,6 +70,8 @@ export const WORKFLOW_MUTATION_NAMES = [
|
|
|
68
70
|
'mutation.markLeaseExpired',
|
|
69
71
|
'mutation.markStartupOrphanedExhausted',
|
|
70
72
|
'mutation.markPlannerUnavailable',
|
|
73
|
+
'mutation.preparePlatformFallback',
|
|
74
|
+
'mutation.quarantinePlatformFailure',
|
|
71
75
|
];
|
|
72
76
|
export const WORKFLOW_EFFECT_NAMES = [
|
|
73
77
|
'effect.launchTask',
|
|
@@ -28,6 +28,7 @@ export const taskLifecycleV2 = load('task-lifecycle', TASK_STATES, {
|
|
|
28
28
|
troubleshooterRecovery: boolGuard('troubleshooterRecovery'),
|
|
29
29
|
retryAvailable: boolGuard('retryAvailable'),
|
|
30
30
|
autonomousJob: boolGuard('autonomousJob'),
|
|
31
|
+
platformFallbackSafe: boolGuard('platformFallbackSafe'),
|
|
31
32
|
});
|
|
32
33
|
export const jobLifecycleV2 = load('job-lifecycle', JOB_STATES, {
|
|
33
34
|
notMerged: boolGuard('notMerged'),
|
package/dist/workflow/runtime.js
CHANGED
|
@@ -34,6 +34,23 @@ export function applyTaskMutation(task, mutation) {
|
|
|
34
34
|
task.error = 'Orphaned after relayd restart after maximum attempts';
|
|
35
35
|
if (mutation === 'mutation.markPlannerUnavailable')
|
|
36
36
|
task.error = 'No objective planner configured';
|
|
37
|
+
if (mutation === 'mutation.preparePlatformFallback') {
|
|
38
|
+
task.error = null;
|
|
39
|
+
task.failureReason = undefined;
|
|
40
|
+
task.routing = undefined;
|
|
41
|
+
task.workerSelection = undefined;
|
|
42
|
+
task.leaseOwner = null;
|
|
43
|
+
task.leaseExpiresAt = null;
|
|
44
|
+
task.executionId = null;
|
|
45
|
+
}
|
|
46
|
+
if (mutation === 'mutation.quarantinePlatformFailure') {
|
|
47
|
+
task.routing = undefined;
|
|
48
|
+
task.workerSelection = undefined;
|
|
49
|
+
task.leaseOwner = null;
|
|
50
|
+
task.leaseExpiresAt = null;
|
|
51
|
+
task.executionId = null;
|
|
52
|
+
task.failureReason = task.platformFailure?.safeToReroute ? 'worker_platform_fallback_unavailable' : 'worker_outcome_unknown';
|
|
53
|
+
}
|
|
37
54
|
}
|
|
38
55
|
export function applyTaskGraphTarget(task, target, timestamp = new Date().toISOString()) {
|
|
39
56
|
const previous = Date.parse(task.updatedAt), candidate = Date.parse(timestamp);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agents-relay",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.141",
|
|
4
4
|
"description": "Durable async agent jobs coordinated through GitHub pull requests",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -18,7 +18,10 @@
|
|
|
18
18
|
"dev": "tsx src/cli.ts",
|
|
19
19
|
"test:publish": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 --test-force-exit test/*.test.js",
|
|
20
20
|
"test:fast": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=2 test/workflow-lifecycle.test.js test/workflow-v2-lifecycle-parity.test.js test/workflow-effects.test.js test/relay-store.test.js test/local-authority.test.js test/local-jobid-routing.test.js test/local-sqlite-jobid.test.js test/no-github.test.js test/scheduled-jobs.test.js test/reconcile-coordinator.test.js test/service-publication.test.js test/github-webhook.test.js",
|
|
21
|
-
"test:full": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 test/*.test.js",
|
|
21
|
+
"test:full": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 test/*.test.js && npm run test:verification:l3",
|
|
22
|
+
"test:verification:l1": "node --test test/verification/harness.test.mjs",
|
|
23
|
+
"test:verification:l3": "npm run build && node --test test/verification/harness.test.mjs test/verification/l3-sqlite.test.mjs test/verification/l3-ingress.test.mjs test/verification/l3-replay.test.mjs",
|
|
24
|
+
"test:verification": "npm run test:verification:l1 && npm run test:verification:l3",
|
|
22
25
|
"e2e": "node e2e/runner/run.mjs"
|
|
23
26
|
},
|
|
24
27
|
"devDependencies": {
|
|
@@ -7,6 +7,7 @@ blocked["state: BLOCKED"]
|
|
|
7
7
|
succeeded["state: SUCCEEDED"]
|
|
8
8
|
failed["state: FAILED"]
|
|
9
9
|
cancelled["state: CANCELLED"]
|
|
10
|
+
platformFallbackSafe["guard: platformFallbackSafe"]
|
|
10
11
|
|
|
11
12
|
depSat["guard: dependenciesSatisfied"]
|
|
12
13
|
depUnsat["guard: dependenciesUnsatisfied"]
|
|
@@ -124,6 +125,13 @@ queued -->|on: effect.launchTask.failed| failed
|
|
|
124
125
|
queued -->|on: watchdog.failed| failed
|
|
125
126
|
queued -->|on: worker.delivery_failed| failed
|
|
126
127
|
queued -->|on: worker.start_ack_failed| failed
|
|
128
|
+
queued -->|on: worker.platform_failure| platformFallbackSafe
|
|
129
|
+
platformFallbackSafe -->|true| platformFallbackClearFailure
|
|
130
|
+
platformFallbackClearFailure["action: mutation.preparePlatformFallback"]
|
|
131
|
+
platformFallbackClearFailure --> queued
|
|
132
|
+
platformFallbackSafe -->|false| platformFallbackQuarantine
|
|
133
|
+
platformFallbackQuarantine["action: mutation.quarantinePlatformFailure"]
|
|
134
|
+
platformFallbackQuarantine --> blocked
|
|
127
135
|
queued -->|on: planner.unavailable| plannerUnavailableMark
|
|
128
136
|
plannerUnavailableMark --> failed
|
|
129
137
|
queued -->|on: execution.orphaned| orphanRetryAvailable
|
|
@@ -150,6 +158,9 @@ running -->|on: task.completed| completeClearError
|
|
|
150
158
|
completeClearError --> completeClearReason
|
|
151
159
|
completeClearReason --> succeeded
|
|
152
160
|
running -->|on: task.failed| failed
|
|
161
|
+
running -->|on: worker.platform_failure| runningPlatformFailureQuarantine
|
|
162
|
+
runningPlatformFailureQuarantine["action: mutation.quarantinePlatformFailure"]
|
|
163
|
+
runningPlatformFailureQuarantine --> blocked
|
|
153
164
|
running -->|on: watchdog.failed| failed
|
|
154
165
|
running -->|on: planner.failed| plannerRetryAvailable
|
|
155
166
|
running -->|on: execution.orphaned| orphanRetryAvailable
|