agents-relay 2.0.150 → 2.0.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/local-git-merge.js +15 -4
- package/dist/local-merge-delivery.js +11 -7
- package/dist/managed-worker-input.js +2 -1
- package/dist/pool.js +1 -1
- package/dist/projected-store.js +6 -1
- package/dist/projection-failure-recovery.js +52 -0
- package/dist/projection-recovery-runtime.js +121 -0
- package/dist/projection-recovery.js +101 -0
- package/dist/reconciler-review.js +12 -6
- package/dist/reconciler-runtime.js +0 -1
- package/dist/reconciler.js +2 -2
- package/dist/relay-store.js +1 -1
- package/dist/relay-workflow-operations.js +43 -1
- package/dist/relayd.js +21 -12
- package/dist/review-generation-fence.js +2 -1
- package/dist/store-github.js +16 -0
- package/dist/task-comments.js +12 -2
- package/dist/task-context.js +18 -1
- package/dist/task-recovery.js +15 -2
- package/dist/troubleshooter.js +2 -2
- package/dist/workflow/definitions-v2.js +14 -2
- package/dist/workflow/graph-v2.js +1 -3
- package/dist/workflow/projection-conflict-v2.js +28 -11
- package/package.json +2 -2
- package/skills/agents-relay/SKILL.md +1 -1
- package/skills/agents-relay/agents/troubleshooter.agent.md +3 -3
- package/skills/agents-relay/contracts/event-contract.md +1 -1
- package/skills/agents-relay/contracts/task-job-contract.md +7 -0
- package/skills/agents-relay/contracts/task-recovery-flows.md +18 -1
- package/workflow/projection-conflict.mmd +53 -2
package/dist/local-git-merge.js
CHANGED
|
@@ -24,10 +24,21 @@ export function localSourceHead(repoPath, sourceRef) {
|
|
|
24
24
|
export function requiresLocalGitMerge(job) {
|
|
25
25
|
return Boolean(job.localMerge) || job.tasks.some(task => task.output?.kind === 'task_pr');
|
|
26
26
|
}
|
|
27
|
-
export function
|
|
28
|
-
return
|
|
29
|
-
|
|
30
|
-
|
|
27
|
+
export function localMergeFacts(job, headSha, now = Date.now) {
|
|
28
|
+
return {
|
|
29
|
+
jobActive: job.state === 'ACTIVE',
|
|
30
|
+
autoMergeEnabled: job.autoMerge !== false,
|
|
31
|
+
schedulePresent: Boolean(job.schedule),
|
|
32
|
+
implementationComplete: implementationComplete(job),
|
|
33
|
+
completionGateSatisfied: completionGateSatisfied(job),
|
|
34
|
+
reviewSatisfied: job.reviewEnabled === false || job.review?.outcome === 'approved',
|
|
35
|
+
localMergeRequired: requiresLocalGitMerge(job),
|
|
36
|
+
localMergeSucceeded: job.localMerge?.status === 'succeeded',
|
|
37
|
+
localMergePresent: Boolean(job.localMerge),
|
|
38
|
+
reviewHeadCurrent: job.reviewEnabled === false || job.review?.headSha === headSha,
|
|
39
|
+
mergeAttemptReady: Boolean(job.localMerge && job.localMerge.status !== 'succeeded'
|
|
40
|
+
&& (!job.localMerge.nextAttemptAt || Date.parse(job.localMerge.nextAttemptAt) <= now())),
|
|
41
|
+
};
|
|
31
42
|
}
|
|
32
43
|
export function localMergeCommitMessage(request, sourceSha) {
|
|
33
44
|
const lines = ['Agents Relay local merge', '', `Agents-Relay-Source-SHA: ${sourceSha}`];
|
|
@@ -1,22 +1,25 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { eventFor } from './events.js';
|
|
3
3
|
import { transitionJobEvent } from './scheduler.js';
|
|
4
|
-
import { deliverLocalMerge,
|
|
4
|
+
import { deliverLocalMerge, localMergeFacts } from './local-git-merge.js';
|
|
5
5
|
import { errorMessage } from './observability.js';
|
|
6
|
+
import { decideJobLifecycle } from './workflow/lifecycle.js';
|
|
6
7
|
/** Graph-selected durable local merge and completion; no checked-out worktree is ever changed. */
|
|
7
8
|
export async function reconcileLocalGitCompletion(store, finalState, headSha, noGitHub, projectPathForJob, emit, mergeEffect = deliverLocalMerge, now = Date.now) {
|
|
8
|
-
|
|
9
|
-
|
|
9
|
+
const facts = localMergeFacts(finalState, headSha, now);
|
|
10
|
+
if (decideJobLifecycle(finalState, 'local.completed', facts).target === 'COMPLETED') {
|
|
11
|
+
await transitionJobEvent(finalState, 'local.completed', facts);
|
|
10
12
|
await store.saveJob(finalState);
|
|
11
13
|
return true;
|
|
12
14
|
}
|
|
13
|
-
if (
|
|
14
|
-
finalState.localMerge && (!finalState.reviewEnabled || finalState.review?.headSha === headSha)) {
|
|
15
|
+
if (decideJobLifecycle(finalState, 'local.merge.requested', facts).actions.includes('effect.deliverLocalMerge')) {
|
|
15
16
|
const receipt = finalState.localMerge;
|
|
17
|
+
if (!receipt)
|
|
18
|
+
return false;
|
|
16
19
|
let activeReceipt = receipt;
|
|
17
20
|
if (receipt.status !== 'succeeded' &&
|
|
18
21
|
(!receipt.nextAttemptAt || Date.parse(receipt.nextAttemptAt) <= now())) {
|
|
19
|
-
await transitionJobEvent(finalState, 'local.merge.requested',
|
|
22
|
+
await transitionJobEvent(finalState, 'local.merge.requested', facts, async (actionId) => {
|
|
20
23
|
if (actionId !== 'effect.deliverLocalMerge')
|
|
21
24
|
throw new Error('Unsupported local merge Graph effect ' + actionId);
|
|
22
25
|
try {
|
|
@@ -75,7 +78,8 @@ export async function reconcileLocalGitCompletion(store, finalState, headSha, no
|
|
|
75
78
|
});
|
|
76
79
|
}
|
|
77
80
|
if (finalState.localMerge?.status === 'succeeded') {
|
|
78
|
-
|
|
81
|
+
const completionFacts = localMergeFacts(finalState, headSha, now);
|
|
82
|
+
await transitionJobEvent(finalState, 'local.completed', completionFacts);
|
|
79
83
|
await store.saveJob(finalState);
|
|
80
84
|
return true;
|
|
81
85
|
}
|
|
@@ -4,6 +4,7 @@ import { agentGraphPrompt } from './agent-graph.js';
|
|
|
4
4
|
import { createReviewChairExecutionPlan } from './review-chair.js';
|
|
5
5
|
import { taskTagWorkerPrompt } from './task-tag.js';
|
|
6
6
|
import { executionEventSource } from './reconciler-runtime.js';
|
|
7
|
+
import { taskAgentHistoryContext } from './task-context.js';
|
|
7
8
|
export async function managedWorkerInput(job, task, context, mode = 'production') {
|
|
8
9
|
const isLocalJob = Boolean(job.localMerge?.sourceRef);
|
|
9
10
|
const projectedReview = isLocalJob && task.agentName === 'reviewer' && job.prNumber > 0;
|
|
@@ -53,7 +54,7 @@ ${interactiveContext}Task attempt/execution: ${task.attempt}/${task.maxAttempts}
|
|
|
53
54
|
Dependencies: ${task.dependencies.length ? task.dependencies.join(', ') : 'none'}
|
|
54
55
|
${task.attempt > 1 || task.error ? `Retry summary: prior error=${task.error ?? 'none'}; prior generations=${(task.executionHistory ?? []).map(item => `${item.id}@${item.attempt}`).join(', ') || 'none'}` : 'Retry summary: first attempt'}
|
|
55
56
|
SQLite/RelayStore and the canonical Mermaid Graphs own lifecycle, merge, recovery, and completion decisions. Do not perform Relay-owned merge or completion actions from this worker.
|
|
56
|
-
${chairPlan ? `[Review Chair execution plan]\n${JSON.stringify(chairPlan, null, 2)}\n\nUse this runtime-discovered panel to create sibling durable specialist Tasks. Preserve each specialistTasks entry as review metadata, keep the target immutable, and synthesize structured results with synthesizeReview semantics before returning the unified revision plan.\n\n` : ''}Follow the resolved project instructions and Agents Relay/events-bus worker contracts.
|
|
57
|
+
${taskAgentHistoryContext(job, task)}\n\n${chairPlan ? `[Review Chair execution plan]\n${JSON.stringify(chairPlan, null, 2)}\n\nUse this runtime-discovered panel to create sibling durable specialist Tasks. Preserve each specialistTasks entry as review metadata, keep the target immutable, and synthesize structured results with synthesizeReview semantics before returning the unified revision plan.\n\n` : ''}Follow the resolved project instructions and Agents Relay/events-bus worker contracts.
|
|
57
58
|
[Output contract]
|
|
58
59
|
${output}
|
|
59
60
|
${taskTag}${mode === 'evaluation' ? `[Isolated prompt evaluation mode]
|
package/dist/pool.js
CHANGED
|
@@ -8,7 +8,7 @@ import { jobHasActiveExecution } from './job-execution-slot.js';
|
|
|
8
8
|
import { unblockNeedsNewPlan } from './unblock.js';
|
|
9
9
|
const RANK = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
10
10
|
export function compareJobPriority(a, b) { return RANK[a.priority ?? 'P2'] - RANK[b.priority ?? 'P2'] || a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id); }
|
|
11
|
-
export function runnableManagedJobs(items) { return items.filter(item => item.job.state === 'ACTIVE').map(item => item.job).sort(compareJobPriority); }
|
|
11
|
+
export function runnableManagedJobs(items) { return items.filter(item => item.job.state === 'ACTIVE' || item.job.projectionRecovery?.status === 'repairing').map(item => item.job).sort(compareJobPriority); }
|
|
12
12
|
export function needsLifecycleReconciliation(job, now = Date.now()) {
|
|
13
13
|
if (job.pause)
|
|
14
14
|
return job.tasks.some(task => Boolean(task.pausedExecution)) || Boolean(job.plans?.some(plan => Boolean(plan.pausedExecutionId)));
|
package/dist/projected-store.js
CHANGED
|
@@ -95,7 +95,7 @@ export class GitHubProjectionWriter {
|
|
|
95
95
|
let deferred = 0;
|
|
96
96
|
for (const item of items) {
|
|
97
97
|
try {
|
|
98
|
-
if (this.externalOnly && !['merge_pull_request', 'close_pull_request', 'update_pr_body', 'create_pull_request'].includes(item.operation)) {
|
|
98
|
+
if (this.externalOnly && !['merge_pull_request', 'close_pull_request', 'update_pr_body', 'create_pull_request', 'append_task_comment'].includes(item.operation)) {
|
|
99
99
|
// Drop legacy lifecycle projections; GitHub may not override SQLite truth.
|
|
100
100
|
await this.relay.completeProjection(item.id);
|
|
101
101
|
continue;
|
|
@@ -182,6 +182,11 @@ export class GitHubProjectionWriter {
|
|
|
182
182
|
case 'save_task':
|
|
183
183
|
await this.github.saveTask(payload.value);
|
|
184
184
|
return;
|
|
185
|
+
case 'append_task_comment':
|
|
186
|
+
if (!this.github.appendTaskHistoryComment)
|
|
187
|
+
throw new Error('GitHub projection target cannot append Task history comments');
|
|
188
|
+
await this.github.appendTaskHistoryComment(payload.value);
|
|
189
|
+
return;
|
|
185
190
|
case 'save_agent':
|
|
186
191
|
await this.github.saveAgent(payload.value);
|
|
187
192
|
return;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { eventFor } from './events.js';
|
|
2
|
+
import { beginProjectionRecovery } from './projection-recovery.js';
|
|
3
|
+
import { reconcileProjectionRecovery } from './projection-recovery-runtime.js';
|
|
4
|
+
import { SQLiteRelayStore } from './relay-store.js';
|
|
5
|
+
import { classifyFailure, errorMessage, observability } from './observability.js';
|
|
6
|
+
import { resolveManagedProjectPath } from './workspace.js';
|
|
7
|
+
export async function reconcileBindingProjectionRecovery(job, store, options) {
|
|
8
|
+
const recovery = await reconcileProjectionRecovery(job, store, options);
|
|
9
|
+
if (recovery && recovery.state === 'escalated')
|
|
10
|
+
return { skip: true };
|
|
11
|
+
if (recovery && !['retry', 'projected'].includes(recovery.state))
|
|
12
|
+
return { skip: true };
|
|
13
|
+
if (recovery?.state === 'retry' && !recovery.projectedMerge)
|
|
14
|
+
return { skip: true };
|
|
15
|
+
return { skip: false, projectedMerge: recovery?.projectedMerge };
|
|
16
|
+
}
|
|
17
|
+
export async function projectBindingLocalMerge(job, store, options) {
|
|
18
|
+
const recovery = await reconcileBindingProjectionRecovery(job, store, options);
|
|
19
|
+
if (recovery.skip)
|
|
20
|
+
return recovery;
|
|
21
|
+
return { skip: false, projectedMerge: recovery.projectedMerge ?? await options.project(job) };
|
|
22
|
+
}
|
|
23
|
+
export async function handleGitHubProjectionFailure(error, job, snapshotPath, options) {
|
|
24
|
+
const reason = errorMessage(error), classification = classifyFailure(error).classification;
|
|
25
|
+
if (job.state === 'COMPLETED' && job.localMerge?.status === 'succeeded') {
|
|
26
|
+
const recoveryStore = options.relayStores.get(`${snapshotPath}#${job.id}`)
|
|
27
|
+
?? new SQLiteRelayStore(snapshotPath, job.id);
|
|
28
|
+
options.relayStores.set(`${snapshotPath}#${job.id}`, recoveryStore);
|
|
29
|
+
const recoveryPath = options.repositoryPaths.get(job.repository)
|
|
30
|
+
?? await resolveManagedProjectPath(job.repository, options.explicitProjectPath, options.workspaceRoot);
|
|
31
|
+
const remoteSha = await options.gitText(recoveryPath, ['fetch', '--no-tags', 'origin', '+refs/heads/main:refs/remotes/origin/main'])
|
|
32
|
+
.then(() => options.gitText(recoveryPath, ['rev-parse', '--verify', 'refs/remotes/origin/main']))
|
|
33
|
+
.catch(() => '');
|
|
34
|
+
const recovery = await beginProjectionRecovery(recoveryStore, job.id, remoteSha, reason);
|
|
35
|
+
if (recovery === 'created') {
|
|
36
|
+
await options.reconcileLocalJob(job.id, 'projection.delivery_failed');
|
|
37
|
+
await options.eventBus?.publish(eventFor(job.id, 'projection', null, 'projection.recovery_requested', 'queued', `Projection recovery task created for origin/main ${remoteSha}`, 'orchestrator', { remoteSha, idempotencyKey: `${job.id}:${remoteSha}` }, { agent: 'relay', component: 'projection-conflict' })).catch(() => { });
|
|
38
|
+
}
|
|
39
|
+
else if (recovery === 'escalated') {
|
|
40
|
+
await options.eventBus?.publish(eventFor(job.id, 'projection', null, 'projection.recovery_escalated', 'failed', 'Projection recovery exhausted its bounded task retry budget', 'user', { idempotencyKey: `${job.id}:${remoteSha}` }, { agent: 'relay', component: 'projection-conflict' })).catch(() => { });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
if (!options.deduplicator.shouldReport(job.id, job.updatedAt, reason, classification))
|
|
44
|
+
return;
|
|
45
|
+
await options.eventBus?.publish(eventFor(job.id, 'projection', null, 'projection.delivery_failed', 'failed', `GitHub projection failed: ${errorMessage(error)}`, 'orchestrator', { operation: 'github.projectLocalMerge', error: reason }, { agent: 'relay', component: 'github-projection' }))?.catch(() => { });
|
|
46
|
+
observability().record({
|
|
47
|
+
stage: 'projection', event: 'github.binding_job_failed', level: 'warn', outcome: 'failed',
|
|
48
|
+
message: `GitHub binding/projection failed for ${job.id}: ${reason}`,
|
|
49
|
+
correlation: { repository: job.repository, prNumber: job.prNumber, jobId: job.id },
|
|
50
|
+
data: { classification, reason },
|
|
51
|
+
});
|
|
52
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { eventFor } from './events.js';
|
|
2
|
+
import { deliverLocalMerge, localSourceHead } from './local-git-merge.js';
|
|
3
|
+
import { dispatchWorkflowGraph } from './workflow/dispatcher.js';
|
|
4
|
+
import { projectionConflictV2 } from './workflow/projection-conflict-v2.js';
|
|
5
|
+
/** Executes only the retry or escalation selected by the projection-conflict graph. */
|
|
6
|
+
export async function reconcileProjectionRecovery(job, store, options) {
|
|
7
|
+
const recovery = job.projectionRecovery;
|
|
8
|
+
if (!recovery)
|
|
9
|
+
return null;
|
|
10
|
+
if (recovery.status === 'projected' || recovery.status === 'escalated')
|
|
11
|
+
return { state: recovery.status };
|
|
12
|
+
const repair = job.tasks.find(task => task.id === recovery.taskId);
|
|
13
|
+
const reviewTask = job.review?.taskId ? job.tasks.find(task => task.id === job.review?.taskId) : undefined;
|
|
14
|
+
const expectedReviewTaskId = job.review?.taskId;
|
|
15
|
+
const expectedReviewOutcome = job.review?.outcome;
|
|
16
|
+
const expectedReviewHeadSha = job.review?.headSha;
|
|
17
|
+
let projectedMerge;
|
|
18
|
+
let claimed = false;
|
|
19
|
+
let escalated = false;
|
|
20
|
+
let projectionCompleted = false;
|
|
21
|
+
const decision = await dispatchWorkflowGraph(projectionConflictV2, {
|
|
22
|
+
initialState: recovery.status, input: { transport: 'event', id: 'reconcile', payload: {} },
|
|
23
|
+
durableFacts: {
|
|
24
|
+
repairTaskState: repair?.state,
|
|
25
|
+
reviewTaskState: reviewTask?.state,
|
|
26
|
+
reviewOutcome: job.review?.outcome,
|
|
27
|
+
taskRetryAvailable: repair?.state !== 'FAILED',
|
|
28
|
+
attempts: recovery.attempts,
|
|
29
|
+
maxAttempts: recovery.maxAttempts,
|
|
30
|
+
}, graphName: 'projection-conflict', maxSteps: 1,
|
|
31
|
+
persistTransition: async (source, target) => {
|
|
32
|
+
if (target === 'retry') {
|
|
33
|
+
if (!store.mutateJob)
|
|
34
|
+
throw new Error('Atomic SQLite mutation is required for projection recovery attempts');
|
|
35
|
+
await store.mutateJob(job.id, current => {
|
|
36
|
+
const currentRecovery = current.projectionRecovery;
|
|
37
|
+
const currentRepair = current.tasks.find(task => task.id === currentRecovery?.taskId);
|
|
38
|
+
const currentReviewTask = current.review?.taskId
|
|
39
|
+
? current.tasks.find(task => task.id === current.review?.taskId) : undefined;
|
|
40
|
+
if (current.state !== 'COMPLETED' || currentRecovery?.key !== recovery.key
|
|
41
|
+
|| currentRecovery.taskId !== recovery.taskId || currentRecovery.status !== source
|
|
42
|
+
|| currentRecovery.attempts !== recovery.attempts || currentRepair?.state !== repair?.state
|
|
43
|
+
|| currentReviewTask?.id !== expectedReviewTaskId || current.review?.outcome !== expectedReviewOutcome
|
|
44
|
+
|| current.review?.headSha !== expectedReviewHeadSha)
|
|
45
|
+
return current;
|
|
46
|
+
claimed = true;
|
|
47
|
+
return { ...current, projectionRecovery: { ...currentRecovery, status: 'retry',
|
|
48
|
+
attempts: currentRecovery.attempts + 1, updatedAt: new Date().toISOString() } };
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
else if (target === 'escalated') {
|
|
52
|
+
if (!store.mutateJob)
|
|
53
|
+
throw new Error('Atomic SQLite mutation is required for projection recovery status');
|
|
54
|
+
await store.mutateJob(job.id, current => current.projectionRecovery?.key === recovery.key
|
|
55
|
+
&& current.projectionRecovery.taskId === recovery.taskId
|
|
56
|
+
&& current.projectionRecovery.status === source
|
|
57
|
+
&& current.projectionRecovery.attempts === recovery.attempts
|
|
58
|
+
? (escalated = true, { ...current, projectionRecovery: { ...current.projectionRecovery, status: 'escalated', updatedAt: new Date().toISOString() } })
|
|
59
|
+
: current);
|
|
60
|
+
}
|
|
61
|
+
else if (source !== target)
|
|
62
|
+
throw new Error(`Unexpected projection recovery transition ${source} -> ${target}`);
|
|
63
|
+
},
|
|
64
|
+
executeAction: async (actionId) => {
|
|
65
|
+
if (actionId === 'effect.projectLocalMerge') {
|
|
66
|
+
if (!claimed)
|
|
67
|
+
return {};
|
|
68
|
+
const localMerge = job.localMerge;
|
|
69
|
+
if (!localMerge)
|
|
70
|
+
throw new Error('Projection recovery requires a durable local merge receipt');
|
|
71
|
+
const sourceSha = localSourceHead(options.projectPath, localMerge.sourceRef);
|
|
72
|
+
if (job.review?.headSha !== sourceSha)
|
|
73
|
+
throw new Error('Projection recovery source changed after review');
|
|
74
|
+
const delivered = deliverLocalMerge(options.projectPath, { ...localMerge, sourceSha,
|
|
75
|
+
correlationJobId: job.id, displayPrNumber: job.prNumber > 0 ? job.prNumber : undefined });
|
|
76
|
+
await options.verifyObservedRemote(recovery.remoteSha);
|
|
77
|
+
const projectedJob = { ...job, localMerge: { ...localMerge, sourceSha: delivered.sourceSha,
|
|
78
|
+
mergedCommitSha: delivered.mergedCommitSha, status: 'succeeded', error: undefined,
|
|
79
|
+
nextAttemptAt: undefined, updatedAt: new Date().toISOString() } };
|
|
80
|
+
projectedMerge = await options.project(projectedJob);
|
|
81
|
+
let finalized = false;
|
|
82
|
+
const deliveredDecision = await dispatchWorkflowGraph(projectionConflictV2, {
|
|
83
|
+
initialState: 'retry', input: { transport: 'event', id: 'projection.delivered', payload: {} },
|
|
84
|
+
durableFacts: {}, graphName: 'projection-conflict', maxSteps: 1,
|
|
85
|
+
persistTransition: async (source, target) => {
|
|
86
|
+
if (source !== 'retry' || target !== 'projected' || !store.mutateJob)
|
|
87
|
+
return;
|
|
88
|
+
await store.mutateJob(job.id, current => {
|
|
89
|
+
const currentMerge = current.localMerge;
|
|
90
|
+
if (current.projectionRecovery?.key !== recovery.key
|
|
91
|
+
|| current.projectionRecovery.taskId !== recovery.taskId
|
|
92
|
+
|| current.projectionRecovery.status !== 'retry'
|
|
93
|
+
|| current.projectionRecovery.attempts !== recovery.attempts + 1
|
|
94
|
+
|| currentMerge?.sourceRef !== localMerge.sourceRef || currentMerge.targetRef !== localMerge.targetRef
|
|
95
|
+
|| currentMerge.status !== localMerge.status || currentMerge.sourceSha !== localMerge.sourceSha
|
|
96
|
+
|| currentMerge.mergedCommitSha !== localMerge.mergedCommitSha)
|
|
97
|
+
return current;
|
|
98
|
+
finalized = true;
|
|
99
|
+
return { ...current, localMerge: projectedJob.localMerge,
|
|
100
|
+
projectionRecovery: { ...current.projectionRecovery, status: 'projected', updatedAt: new Date().toISOString() },
|
|
101
|
+
updatedAt: new Date().toISOString() };
|
|
102
|
+
});
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
projectionCompleted = finalized;
|
|
106
|
+
if (!deliveredDecision.accepted || deliveredDecision.target !== 'projected' || !finalized)
|
|
107
|
+
projectedMerge = undefined;
|
|
108
|
+
return {};
|
|
109
|
+
}
|
|
110
|
+
if (actionId === 'projection.escalate') {
|
|
111
|
+
if (escalated)
|
|
112
|
+
await options.eventBus?.publish(eventFor(job.id, 'projection', null, 'projection.recovery_escalated', 'failed', 'Projection recovery exhausted its bounded retry path', 'user', { idempotencyKey: recovery.key, attempts: recovery.attempts }, { agent: 'relay', component: 'projection-conflict' })).catch(() => { });
|
|
113
|
+
return {};
|
|
114
|
+
}
|
|
115
|
+
if (actionId === 'projection.createRecoveryTask')
|
|
116
|
+
return {};
|
|
117
|
+
throw new Error(`Unexpected projection recovery action ${actionId}`);
|
|
118
|
+
},
|
|
119
|
+
});
|
|
120
|
+
return { state: projectionCompleted ? 'projected' : decision.accepted ? decision.target : '', projectedMerge };
|
|
121
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { createTaskEntity } from './workflow/entity-creation.js';
|
|
2
|
+
import { dispatchWorkflowGraph } from './workflow/dispatcher.js';
|
|
3
|
+
import { projectionConflictV2 } from './workflow/projection-conflict-v2.js';
|
|
4
|
+
import { taskOverallTimeoutMs } from './types.js';
|
|
5
|
+
const MAX_RECOVERY_ATTEMPTS = 2;
|
|
6
|
+
function taskFor(job, remoteSha, now) {
|
|
7
|
+
const key = `${job.id}:${remoteSha}`;
|
|
8
|
+
const id = `projection-recovery-${remoteSha}`;
|
|
9
|
+
return createTaskEntity(job, {
|
|
10
|
+
jobId: job.id, id, title: `Reconcile main divergence for ${job.title}`, kind: 'work',
|
|
11
|
+
priority: job.priority ?? 'P1', storySize: 'M', projectName: job.title,
|
|
12
|
+
agentName: 'implementation', parentTaskId: null, dependencies: [],
|
|
13
|
+
capabilities: ['implementation'], workerRequirements: { capabilities: ['implementation'] },
|
|
14
|
+
adapter: 'codex', input: [
|
|
15
|
+
`Projection recovery key: ${key}`,
|
|
16
|
+
`Observed origin/main SHA: ${remoteSha}`,
|
|
17
|
+
`Job ${job.id} is already COMPLETED; preserve that Job state.`,
|
|
18
|
+
'Inspect local main and origin/main, reconcile their divergence with a normal merge or rebase that preserves both histories, and never force-push.',
|
|
19
|
+
'Do not publish or rewrite refs. Record the resulting local main SHA and verify it contains the completed Job merge and the observed remote main history.',
|
|
20
|
+
'If the histories cannot be reconciled safely, stop and explain the exact blocker.',
|
|
21
|
+
].join('\n'), output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null,
|
|
22
|
+
attempt: 0, maxAttempts: 2, leaseOwner: null, leaseExpiresAt: null,
|
|
23
|
+
executionId: null, threadId: null, result: null, plannerResult: null, error: null,
|
|
24
|
+
timeoutMs: taskOverallTimeoutMs('M'), createdAt: now, updatedAt: now,
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
/** Select graph policy from SQLite facts, then atomically materialize its selected recovery action. */
|
|
28
|
+
export async function beginProjectionRecovery(store, jobId, remoteSha, error) {
|
|
29
|
+
if (!store.mutateJob)
|
|
30
|
+
throw new Error('Projection recovery requires atomic SQLite Job mutation');
|
|
31
|
+
let created = false;
|
|
32
|
+
let escalated = false;
|
|
33
|
+
const snapshot = await store.loadJob(jobId);
|
|
34
|
+
if (!snapshot)
|
|
35
|
+
return false;
|
|
36
|
+
const sameKey = snapshot.projectionRecovery?.key === `${jobId}:${remoteSha}`;
|
|
37
|
+
const expectedRecoveryKey = snapshot.projectionRecovery?.key ?? null;
|
|
38
|
+
const expectedAttempts = snapshot.projectionRecovery?.attempts ?? 0;
|
|
39
|
+
const expectedRecoveryStatus = snapshot.projectionRecovery?.status ?? null;
|
|
40
|
+
const expectedRecoveryRemoteSha = snapshot.projectionRecovery?.remoteSha ?? null;
|
|
41
|
+
const recoveryInProgress = snapshot.projectionRecovery?.status === 'repairing'
|
|
42
|
+
|| (snapshot.projectionRecovery?.status === 'retry' && snapshot.projectionRecovery.remoteSha === remoteSha);
|
|
43
|
+
const decision = await dispatchWorkflowGraph(projectionConflictV2, {
|
|
44
|
+
initialState: 'classify', input: { transport: 'event', id: 'projection.delivery_failed', payload: {} },
|
|
45
|
+
durableFacts: {
|
|
46
|
+
jobState: snapshot.state,
|
|
47
|
+
mergeSucceeded: snapshot.localMerge?.status === 'succeeded',
|
|
48
|
+
projectionStrategy: snapshot.localMerge?.projectionStrategy,
|
|
49
|
+
divergenceConfirmed: /non-fast-forward|fetch first|diverg|not a fast-forward|rejected/i.test(error),
|
|
50
|
+
remoteShaAvailable: /^[0-9a-f]{40,64}$/i.test(remoteSha),
|
|
51
|
+
sameIdempotencyKey: sameKey,
|
|
52
|
+
recoveryInProgress,
|
|
53
|
+
attempts: snapshot.projectionRecovery?.attempts ?? 0,
|
|
54
|
+
maxAttempts: MAX_RECOVERY_ATTEMPTS,
|
|
55
|
+
}, graphName: 'projection-conflict', maxSteps: 1,
|
|
56
|
+
executeAction: async (actionId) => {
|
|
57
|
+
if (actionId === 'projection.escalate') {
|
|
58
|
+
await store.mutateJob(jobId, current => {
|
|
59
|
+
if (current.state === 'COMPLETED'
|
|
60
|
+
&& (current.projectionRecovery?.key ?? null) === expectedRecoveryKey
|
|
61
|
+
&& (current.projectionRecovery?.attempts ?? 0) === expectedAttempts
|
|
62
|
+
&& current.projectionRecovery?.status === snapshot.projectionRecovery?.status
|
|
63
|
+
&& current.projectionRecovery) {
|
|
64
|
+
current.projectionRecovery = { ...current.projectionRecovery, key: `${jobId}:${remoteSha}`, remoteSha,
|
|
65
|
+
status: 'escalated', error, updatedAt: new Date().toISOString() };
|
|
66
|
+
escalated = true;
|
|
67
|
+
}
|
|
68
|
+
return current;
|
|
69
|
+
});
|
|
70
|
+
return {};
|
|
71
|
+
}
|
|
72
|
+
if (actionId !== 'projection.createRecoveryTask')
|
|
73
|
+
throw new Error(`Unexpected projection action ${actionId}`);
|
|
74
|
+
const now = new Date().toISOString();
|
|
75
|
+
await store.mutateJob(jobId, current => {
|
|
76
|
+
if (current.state !== 'COMPLETED' || current.localMerge?.status !== 'succeeded'
|
|
77
|
+
|| (current.projectionRecovery?.key ?? null) !== expectedRecoveryKey
|
|
78
|
+
|| (current.projectionRecovery?.attempts ?? 0) !== expectedAttempts
|
|
79
|
+
|| (current.projectionRecovery?.status ?? null) !== expectedRecoveryStatus
|
|
80
|
+
|| (current.projectionRecovery?.remoteSha ?? null) !== expectedRecoveryRemoteSha
|
|
81
|
+
|| current.projectionRecovery?.status === 'repairing'
|
|
82
|
+
|| current.projectionRecovery?.key === `${jobId}:${remoteSha}`)
|
|
83
|
+
return current;
|
|
84
|
+
const task = taskFor(current, remoteSha, now);
|
|
85
|
+
if (!current.tasks.some(item => item.id === task.id))
|
|
86
|
+
current.tasks.push(task);
|
|
87
|
+
const recovery = {
|
|
88
|
+
key: `${jobId}:${remoteSha}`, remoteSha, status: 'repairing', taskId: task.id,
|
|
89
|
+
attempts: current.projectionRecovery?.attempts ?? 0,
|
|
90
|
+
maxAttempts: MAX_RECOVERY_ATTEMPTS, error, updatedAt: now,
|
|
91
|
+
};
|
|
92
|
+
current.projectionRecovery = recovery;
|
|
93
|
+
current.updatedAt = now;
|
|
94
|
+
created = true;
|
|
95
|
+
return current;
|
|
96
|
+
});
|
|
97
|
+
return {};
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
return decision.accepted ? (created ? 'created' : escalated ? 'escalated' : false) : false;
|
|
101
|
+
}
|
|
@@ -18,6 +18,8 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
18
18
|
// immutable input SHA is already present in the authoritative Job.
|
|
19
19
|
if (this.options.noGitHub && this.options.localMergeEffect)
|
|
20
20
|
return job.headSha ?? job.review?.headSha ?? null;
|
|
21
|
+
if (this.options.headShaProvider)
|
|
22
|
+
return this.options.headShaProvider(job);
|
|
21
23
|
// GitHub PR head facts are audit-only. Source identity is local Git.
|
|
22
24
|
if (!this.options.projectPathForJob)
|
|
23
25
|
return job.headSha ?? job.review?.headSha ?? null;
|
|
@@ -37,7 +39,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
37
39
|
return {
|
|
38
40
|
unscheduled: !job.schedule,
|
|
39
41
|
reviewEnabled: job.reviewEnabled !== false,
|
|
40
|
-
jobActive: job.state === 'ACTIVE',
|
|
42
|
+
jobActive: job.state === 'ACTIVE' || job.projectionRecovery?.status === 'repairing',
|
|
41
43
|
implementationComplete: implementationComplete(job),
|
|
42
44
|
headAvailable: Boolean(headSha),
|
|
43
45
|
completionGateSatisfied: completionGateSatisfied(job),
|
|
@@ -141,7 +143,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
141
143
|
}, this.options.reviewer?.threadId);
|
|
142
144
|
const input = [
|
|
143
145
|
`Job ID: ${job.id}`,
|
|
144
|
-
`Review PR #${reviewPrNumber} at exactly head SHA ${headSha}.`,
|
|
146
|
+
reviewPrNumber > 0 ? `Review PR #${reviewPrNumber} at exactly head SHA ${headSha}.` : `Review the local Job artifact at exactly head SHA ${headSha}.`,
|
|
145
147
|
`Review head SHA: ${headSha}`,
|
|
146
148
|
'Job objective: ' + (job.objective ?? job.description ?? job.title),
|
|
147
149
|
'Use the selected reviewer agent contract as the authoritative review protocol.',
|
|
@@ -150,7 +152,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
150
152
|
'Do not treat historical failed attempts as current artifact defects; distinguish preserved baseline files from implementation.',
|
|
151
153
|
].join('\n\n');
|
|
152
154
|
const task = createTaskEntity(job, {
|
|
153
|
-
jobId: job.id, id, title: `
|
|
155
|
+
jobId: job.id, id, title: `Review ${headSha.slice(0, 8)}`, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'reviewer',
|
|
154
156
|
parentTaskId: null, dependencies: reviewGateTasks.map(item => item.id), capabilities: ['review'],
|
|
155
157
|
workerSelection: reviewerRouting.selection, adapter: reviewerRouting.candidate.adapter,
|
|
156
158
|
input,
|
|
@@ -355,7 +357,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
355
357
|
if (task.reviewHeadSha !== headSha)
|
|
356
358
|
return { applied: false, reconcile: false, reason: event.type + '.stale' };
|
|
357
359
|
const currentHeadSha = await this.currentHeadSha(job);
|
|
358
|
-
if (
|
|
360
|
+
if (currentHeadSha ? currentHeadSha !== headSha : Boolean(job.headSha && job.headSha !== headSha))
|
|
359
361
|
return { applied: false, reconcile: false, reason: event.type + '.stale' };
|
|
360
362
|
// Planner-created reviewer tasks predate the final-review Job gate. Bind
|
|
361
363
|
// their first trustworthy review event to PENDING before dispatching it.
|
|
@@ -388,8 +390,10 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
388
390
|
const now = event.timestamp || new Date().toISOString();
|
|
389
391
|
const id = 'review-fix-' + headSha.slice(0, 8);
|
|
390
392
|
if (!job.tasks.some(item => item.id === id)) {
|
|
391
|
-
const
|
|
392
|
-
const
|
|
393
|
+
const fixPrNumber = this.externalReviewPrNumber(job);
|
|
394
|
+
const fixTarget = fixPrNumber > 0 ? `PR #${fixPrNumber}` : `Job ${job.id} artifact`;
|
|
395
|
+
const input = 'Address all reviewer findings for ' + fixTarget + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
|
|
396
|
+
const fixTask = createTaskEntity(job, { jobId: job.id, id, title: `Fix review ${headSha.slice(0, 8)}`, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'implementation', parentTaskId: task.id, dependencies: [task.id], capabilities: ['implementation'], workerRequirements: { capabilities: [] }, adapter: 'codex', input, output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null, attempt: 0, maxAttempts: 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: 0, createdAt: now, updatedAt: now });
|
|
393
397
|
pendingFixTask = fixTask;
|
|
394
398
|
}
|
|
395
399
|
return {};
|
|
@@ -435,6 +439,8 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
435
439
|
if ((!current.review || (current.review.outcome === 'stale' && current.review.headSha !== headSha)) && freshTask.planningDecisionId) {
|
|
436
440
|
current.review = { outcome: 'pending', headSha, taskId: freshTask.id, evidence: null, updatedAt: reviewUpdatedAt };
|
|
437
441
|
}
|
|
442
|
+
if (currentHeadSha === headSha)
|
|
443
|
+
current.headSha = headSha;
|
|
438
444
|
freshTask.error = null;
|
|
439
445
|
if (freshDecision.target !== 'RUNNING')
|
|
440
446
|
freshTask.result = { summary: event.message || `Review ${event.type}`, data: event.data ?? {} };
|
|
@@ -224,7 +224,6 @@ export class ReconcilerRuntimeBase {
|
|
|
224
224
|
const outcome = await this.store.verifyWorkflowLaunch(event.job_id, event.task_id, transition.task.executionId);
|
|
225
225
|
if (outcome === 'Failed')
|
|
226
226
|
throw Error('worker start postcondition violated');
|
|
227
|
-
// Unknown remains durable and never authorizes repeating this generation.
|
|
228
227
|
}
|
|
229
228
|
if (live?.executionId === transition.task.executionId)
|
|
230
229
|
live.workerStarted = true;
|
package/dist/reconciler.js
CHANGED
|
@@ -114,7 +114,7 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
114
114
|
} });
|
|
115
115
|
return job;
|
|
116
116
|
}
|
|
117
|
-
if (job.state !== 'ACTIVE')
|
|
117
|
+
if (job.state !== 'ACTIVE' && job.projectionRecovery?.status !== 'repairing')
|
|
118
118
|
return job;
|
|
119
119
|
await recoverExpiredExecutions(job, now, this.executionRecoveryPorts(job));
|
|
120
120
|
await repairLegacyOrchestrators(jobId, now, { store: this.store, owner: this.options.owner, emit: event => this.emit(event) });
|
|
@@ -202,7 +202,7 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
202
202
|
await this.ensureFinalReview(finalState, headSha);
|
|
203
203
|
await this.dispatchExecutableTasks(jobId);
|
|
204
204
|
finalState = await this.store.load(jobId);
|
|
205
|
-
const localMergeCompleted = await reconcileLocalGitCompletion(this.store, finalState, headSha, this.options.noGitHub === true, this.options.projectPathForJob, event => this.emit(event), this.options.localMergeEffect, () => (this.options.now ?? new Date()).getTime());
|
|
205
|
+
const localMergeCompleted = await reconcileLocalGitCompletion(this.store, finalState, headSha, this.options.noGitHub === true, this.options.projectPathForJob, event => this.emit(event), this.options.localMergeEffect ?? this.options.localMergeDelivery, () => (this.options.now ?? new Date()).getTime());
|
|
206
206
|
await drainProductionWorkflowOutbox(this.store, this.options);
|
|
207
207
|
if (localMergeCompleted)
|
|
208
208
|
return this.store.load(jobId);
|
package/dist/relay-store.js
CHANGED
|
@@ -163,7 +163,7 @@ export class SQLiteRelayStore {
|
|
|
163
163
|
}
|
|
164
164
|
}
|
|
165
165
|
workflowOperations() {
|
|
166
|
-
return new SQLiteWorkflowOperations(this.db, (job, at) => this.writeJobSnapshot(job, at));
|
|
166
|
+
return new SQLiteWorkflowOperations(this.db, (job, at) => this.writeJobSnapshot(job, at), (request, now) => { this.enqueueProjectionInTransaction(request, now); });
|
|
167
167
|
}
|
|
168
168
|
async commitWorkflowMutationAndEvents(jobId, mutation, events) {
|
|
169
169
|
this.assertScope(jobId);
|
|
@@ -5,13 +5,45 @@ import { dispatchWorkflowGraphSync } from './workflow/dispatcher.js';
|
|
|
5
5
|
import { WorkflowActionLedger } from './workflow/action-ledger.js';
|
|
6
6
|
import { ConditionRegistry, booleanCondition } from './workflow/conditions.js';
|
|
7
7
|
import { DurableWorkflowPublication } from './workflow/durable-publication.js';
|
|
8
|
+
import { randomUUID } from 'node:crypto';
|
|
9
|
+
import { appendTaskComment } from './task-comments.js';
|
|
8
10
|
function monotonicJobTimestamp(previous, candidate = new Date().toISOString()) { const a = Date.parse(previous), b = Date.parse(candidate); return new Date(Math.max(Number.isFinite(a) ? a + 1 : Date.now(), Number.isFinite(b) ? b : Date.now())).toISOString(); }
|
|
11
|
+
function recordTaskStateHistory(current, next) {
|
|
12
|
+
const appended = [];
|
|
13
|
+
for (const task of next.tasks) {
|
|
14
|
+
const before = current.tasks.find(item => item.id === task.id);
|
|
15
|
+
if (!before)
|
|
16
|
+
continue;
|
|
17
|
+
for (const field of ['resolution', 'escalation']) {
|
|
18
|
+
const prior = before[field] ?? null, value = task[field] ?? null;
|
|
19
|
+
if (JSON.stringify(prior) === JSON.stringify(value) || value === null)
|
|
20
|
+
continue;
|
|
21
|
+
const resolution = field === 'resolution' ? value : null;
|
|
22
|
+
const escalation = field === 'escalation' ? value : null;
|
|
23
|
+
const kind = resolution ? 'resolution.changed'
|
|
24
|
+
: escalation?.status === 'responded' ? 'escalation.responded' : 'escalation.requested';
|
|
25
|
+
const action = resolution ? `Authoritative resolution changed: ${JSON.stringify(resolution)}`
|
|
26
|
+
: escalation?.status === 'responded'
|
|
27
|
+
? `Human response to escalation ${escalation.id}: ${escalation.response ?? ''}. Re-evaluate the historical resolution condition against original execution evidence.`
|
|
28
|
+
: `Authoritative escalation changed: ${JSON.stringify(escalation)}`;
|
|
29
|
+
appended.push(appendTaskComment(next, task.id, action, {
|
|
30
|
+
author: escalation?.status === 'responded' ? 'human' : 'relay', kind,
|
|
31
|
+
executionId: resolution ? before.executionId ?? task.executionId ?? undefined : escalation?.originalExecutionId ?? before.executionId ?? undefined,
|
|
32
|
+
transition: resolution ? `resolution.${resolution.state}` : `escalation.${escalation?.status}`,
|
|
33
|
+
idempotencyKey: `task-state/${next.id}/${task.id}/${field}/${randomUUID()}`,
|
|
34
|
+
}));
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return appended;
|
|
38
|
+
}
|
|
9
39
|
export class SQLiteWorkflowOperations {
|
|
10
40
|
db;
|
|
11
41
|
write;
|
|
12
|
-
|
|
42
|
+
enqueue;
|
|
43
|
+
constructor(db, write, enqueue) {
|
|
13
44
|
this.db = db;
|
|
14
45
|
this.write = write;
|
|
46
|
+
this.enqueue = enqueue;
|
|
15
47
|
}
|
|
16
48
|
async commitWorkflowMutationAndEvents(jobId, mutation, events) {
|
|
17
49
|
let committed;
|
|
@@ -21,15 +53,20 @@ export class SQLiteWorkflowOperations {
|
|
|
21
53
|
throw new Error(`Unknown Job ${jobId}`);
|
|
22
54
|
const current = normalizeLegacyJobPrNumber(JSON.parse(row.json));
|
|
23
55
|
current.tasks = current.tasks.map(task => normalizeLegacyTaskLifecycle(task, current.executionMode));
|
|
56
|
+
const previous = structuredClone(current);
|
|
24
57
|
const next = mutation(current);
|
|
25
58
|
if (next.id !== jobId)
|
|
26
59
|
throw new Error('Cannot change authoritative Job identity');
|
|
27
60
|
if (next.state !== current.state && ['COMPLETED', 'CANCELLED'].includes(current.state))
|
|
28
61
|
throw new Error('Cannot overwrite terminal authoritative state');
|
|
62
|
+
const history = recordTaskStateHistory(previous, next);
|
|
29
63
|
const beforeTime = Date.parse(current.updatedAt);
|
|
30
64
|
if (Number.isFinite(beforeTime) && !(Date.parse(next.updatedAt) > beforeTime))
|
|
31
65
|
next.updatedAt = new Date(beforeTime + 1).toISOString();
|
|
32
66
|
this.write(next, new Date().toISOString());
|
|
67
|
+
if (next.prNumber > 0)
|
|
68
|
+
for (const comment of history)
|
|
69
|
+
this.enqueue?.({ idempotencyKey: `append_task_comment:${comment.idempotencyKey ?? comment.id}`, operation: 'append_task_comment', payload: { operation: 'append_task_comment', value: comment }, priority: 20 }, new Date().toISOString());
|
|
33
70
|
committed = structuredClone(next);
|
|
34
71
|
}, () => events.map(item => ({ key: item.key, eventId: item.event.type, payload: item.event })));
|
|
35
72
|
return committed;
|
|
@@ -85,14 +122,19 @@ export class SQLiteWorkflowOperations {
|
|
|
85
122
|
throw new Error(`Local authoritative Job ${jobId} not found`);
|
|
86
123
|
const current = normalizeLegacyJobPrNumber(JSON.parse(row.json));
|
|
87
124
|
current.tasks = current.tasks.map(task => normalizeLegacyTaskLifecycle(task, current.executionMode));
|
|
125
|
+
const previous = structuredClone(current);
|
|
88
126
|
const before = JSON.stringify(current);
|
|
89
127
|
const next = mutation(current);
|
|
128
|
+
const history = recordTaskStateHistory(previous, next);
|
|
90
129
|
if (next.id !== jobId)
|
|
91
130
|
throw new Error('Cannot change authoritative Job identity');
|
|
92
131
|
if (JSON.stringify(next) !== before && next.updatedAt <= JSON.parse(before).updatedAt) {
|
|
93
132
|
next.updatedAt = monotonicJobTimestamp(JSON.parse(before).updatedAt);
|
|
94
133
|
}
|
|
95
134
|
this.write(next, new Date().toISOString());
|
|
135
|
+
if (next.prNumber > 0)
|
|
136
|
+
for (const comment of history)
|
|
137
|
+
this.enqueue?.({ idempotencyKey: `append_task_comment:${comment.idempotencyKey ?? comment.id}`, operation: 'append_task_comment', payload: { operation: 'append_task_comment', value: comment }, priority: 20 }, new Date().toISOString());
|
|
96
138
|
this.db.exec('COMMIT');
|
|
97
139
|
return structuredClone(next);
|
|
98
140
|
}
|
package/dist/relayd.js
CHANGED
|
@@ -20,6 +20,7 @@ import { projectAuthoritativeLocalMerge } from './local-merge-projection.js';
|
|
|
20
20
|
import { archiveDisplayPullRequest } from './github-pr-archive.js';
|
|
21
21
|
import { orderGitHubBindingCandidates } from './github-binding-order.js';
|
|
22
22
|
import { ProjectionFailureDeduplicator } from './projection-failure-dedupe.js';
|
|
23
|
+
import { handleGitHubProjectionFailure, projectBindingLocalMerge } from './projection-failure-recovery.js';
|
|
23
24
|
import { authoritativeBranchPushArgs } from './github-branch-projection.js';
|
|
24
25
|
import { githubAuthContext } from './github-auth.js';
|
|
25
26
|
import { DisabledGitHubClient, noGitHubMode } from './no-github.js';
|
|
@@ -320,8 +321,23 @@ export async function runDaemon(argv) {
|
|
|
320
321
|
const github = new GitHubStore(client, job.repository, prNumber, trusted);
|
|
321
322
|
projectionWriters.set(`job:${job.id}`, new GitHubProjectionWriter(relay, github, undefined, true, effect));
|
|
322
323
|
if (job.state === 'COMPLETED' && job.localMerge?.status === 'succeeded' && job.localMerge.projectionStrategy === 'target_ref_push') {
|
|
323
|
-
const
|
|
324
|
-
|
|
324
|
+
const projection = await projectBindingLocalMerge(job, relay, {
|
|
325
|
+
projectPath: canonical,
|
|
326
|
+
project: async (projectedJob) => withTestInjection('github.projectLocalMerge', await consumeMethodInjection(relay, job.repository, job.id, 'github.projectLocalMerge'), () => projectAuthoritativeLocalMerge(canonical, projectedJob)),
|
|
327
|
+
verifyObservedRemote: async (sha) => {
|
|
328
|
+
try {
|
|
329
|
+
await gitText(canonical, ['merge-base', '--is-ancestor', sha, 'refs/heads/main']);
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
throw new Error(`Projection recovery did not preserve observed origin/main ${sha} in local main`);
|
|
333
|
+
}
|
|
334
|
+
}, eventBus: bus,
|
|
335
|
+
});
|
|
336
|
+
if (projection.skip)
|
|
337
|
+
continue;
|
|
338
|
+
const projectedMerge = projection.projectedMerge;
|
|
339
|
+
if (!projectedMerge)
|
|
340
|
+
continue;
|
|
325
341
|
await bus?.publish(eventFor(job.id, 'projection', null, 'projection.delivered', 'succeeded', `GitHub local merge projection ${projectedMerge.status}`, 'orchestrator', { operation: 'github.projectLocalMerge', deliveryStatus: projectedMerge.status, mergedCommitSha: projectedMerge.localMergedCommitSha }, { agent: 'relay', component: 'github-projection' }));
|
|
326
342
|
// A review-only PR is never merged by GitHub. Close it after local main
|
|
327
343
|
// has been safely published, retaining the diff, comments and commit link.
|
|
@@ -339,16 +355,9 @@ export async function runDaemon(argv) {
|
|
|
339
355
|
}
|
|
340
356
|
}
|
|
341
357
|
catch (error) {
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
await bus?.publish(eventFor(discovered.id, 'projection', null, 'projection.delivery_failed', 'failed', `GitHub projection failed: ${errorMessage(error)}`, 'orchestrator', { operation: 'github.projectLocalMerge', error: reason }, { agent: 'relay', component: 'github-projection' }))?.catch(() => { });
|
|
346
|
-
observability().record({
|
|
347
|
-
stage: 'projection', event: 'github.binding_job_failed', level: 'warn', outcome: 'failed',
|
|
348
|
-
message: `GitHub binding/projection failed for ${discovered.id}: ${reason}`,
|
|
349
|
-
correlation: { repository: discovered.repository, prNumber: discovered.prNumber, jobId: discovered.id },
|
|
350
|
-
data: { classification, reason },
|
|
351
|
-
});
|
|
358
|
+
await handleGitHubProjectionFailure(error, discovered, snapshot.path, { relayStores, repositoryPaths,
|
|
359
|
+
explicitProjectPath, workspaceRoot, gitText, reconcileLocalJob: (jobId, reason) => pool.reconcileLocalJob(jobId, reason),
|
|
360
|
+
eventBus: bus, deduplicator: projectionFailureDeduplicator });
|
|
352
361
|
}
|
|
353
362
|
}
|
|
354
363
|
cachedOverviews = await inventoryFromLocal();
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/** Called inside SQLite mutateJob against the latest durable Job snapshot. */
|
|
2
2
|
export function reviewGenerationMatches(current, original, task, headSha) {
|
|
3
3
|
const fresh = current.tasks.find(item => item.id === task.id);
|
|
4
|
-
if (!fresh || fresh.reviewHeadSha !== headSha
|
|
4
|
+
if (!fresh || fresh.reviewHeadSha !== headSha
|
|
5
|
+
|| (current.state !== 'ACTIVE' && current.projectionRecovery?.status !== 'repairing')
|
|
5
6
|
|| (current.headSha && current.headSha !== headSha)
|
|
6
7
|
|| fresh.executionId !== task.executionId)
|
|
7
8
|
return false;
|
package/dist/store-github.js
CHANGED
|
@@ -258,6 +258,22 @@ export class GitHubStore {
|
|
|
258
258
|
throw new Error(`JOB_BINDING_CONFLICT: PR #${this.prNumber} contains trusted ${JOB_MARKER} markers for multiple job ids`);
|
|
259
259
|
}
|
|
260
260
|
async listAgents() { return (await this.list()).filter(comment => this.isTrusted(comment)).map(x => parseAgent(x.body)).filter((x) => x !== null); }
|
|
261
|
+
async appendTaskHistoryComment(comment) {
|
|
262
|
+
if (!comment.idempotencyKey || comment.jobId === undefined || comment.taskId === undefined)
|
|
263
|
+
throw new Error('Task history projection requires durable identity');
|
|
264
|
+
const marker = `<!-- agents-relay:task-comment:v1 key=${comment.idempotencyKey} -->`;
|
|
265
|
+
if ((await this.list()).some(existing => this.isTrusted(existing) && existing.body.includes(marker)))
|
|
266
|
+
return;
|
|
267
|
+
try {
|
|
268
|
+
await this.comment(`${marker}\n**Task ${comment.taskId} · ${comment.kind ?? 'history'}**\n\n${comment.text}`);
|
|
269
|
+
}
|
|
270
|
+
catch (error) {
|
|
271
|
+
// The server may have accepted the comment before the connection failed.
|
|
272
|
+
// Force a fresh read before the outbox retries this stable logical key.
|
|
273
|
+
this.clearReadSnapshot();
|
|
274
|
+
throw error;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
261
277
|
async comment(body) {
|
|
262
278
|
const pr = await this.prNode();
|
|
263
279
|
const data = await this.graphql('mutation($subjectId:ID!,$body:String!){addComment(input:{subjectId:$subjectId,body:$body}){commentEdge{node{id databaseId url body updatedAt author{login}}}}}', { subjectId: pr.id, body });
|
package/dist/task-comments.js
CHANGED
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
-
export function appendTaskComment(job, taskId, message,
|
|
2
|
+
export function appendTaskComment(job, taskId, message, options = 'cli') {
|
|
3
3
|
const task = job.tasks.find(item => item.id === taskId);
|
|
4
4
|
if (!task)
|
|
5
5
|
throw new Error(`Task ${taskId} not found`);
|
|
6
6
|
if (!message.trim())
|
|
7
7
|
throw new Error('task comment requires nonempty --message');
|
|
8
|
-
const
|
|
8
|
+
const config = typeof options === 'string' ? { author: options } : options;
|
|
9
|
+
const key = config.idempotencyKey;
|
|
10
|
+
const existing = key && task.comments?.find(comment => comment.idempotencyKey === key);
|
|
11
|
+
if (existing)
|
|
12
|
+
return existing;
|
|
13
|
+
const comment = {
|
|
14
|
+
id: key ?? randomUUID(), text: message.trim(), author: config.author ?? 'cli', createdAt: new Date().toISOString(),
|
|
15
|
+
jobId: job.id, taskId, ...(config.kind ? { kind: config.kind } : {}),
|
|
16
|
+
...(config.executionId ? { executionId: config.executionId } : {}),
|
|
17
|
+
...(config.transition ? { transition: config.transition } : {}), ...(key ? { idempotencyKey: key } : {}),
|
|
18
|
+
};
|
|
9
19
|
task.comments = [...(task.comments ?? []), comment];
|
|
10
20
|
return comment;
|
|
11
21
|
}
|
package/dist/task-context.js
CHANGED
|
@@ -7,10 +7,27 @@ export function taskExecutionContext(job, taskId, executionId) {
|
|
|
7
7
|
throw new Error(`Task ${taskId} not found`);
|
|
8
8
|
if (task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state))
|
|
9
9
|
throw new Error(`Stale Task execution ${taskId}/${executionId}; current state ${task.state}`);
|
|
10
|
+
const relevant = task.agentName === 'troubleshooter'
|
|
11
|
+
? job.tasks.filter(item => item.id === task.parentTaskId)
|
|
12
|
+
: task.agentName === 'reviewer' ? task.dependencies.map(id => job.tasks.find(item => item.id === id)).filter((item) => Boolean(item)) : [];
|
|
10
13
|
return { job: { id: job.id, repository: job.repository, title: job.title, objective: job.objective,
|
|
11
|
-
projectPath: job.projectPath, state: job.state }, task,
|
|
14
|
+
projectPath: job.projectPath, state: job.state }, task, currentState: task.state, resolution: task.resolution ?? null, escalation: task.escalation ?? null,
|
|
15
|
+
comments: task.comments ?? [], relevantTasks: relevant.map(item => ({ id: item.id, title: item.title, state: item.state, attempt: item.attempt, executionId: item.executionId, resolution: item.resolution ?? null, escalation: item.escalation ?? null, comments: item.comments ?? [], result: item.result, error: item.error, failureReason: item.failureReason })),
|
|
16
|
+
dependencies: task.dependencies.map(id => job.tasks.find(item => item.id === id) ?? { id, missing: true }),
|
|
12
17
|
plan: job.plans?.find(plan => plan.id === task.planningDecisionId) ?? null };
|
|
13
18
|
}
|
|
19
|
+
/** Runtime-only task history; durable task.input remains the stable objective. */
|
|
20
|
+
export function taskAgentHistoryContext(job, task) {
|
|
21
|
+
if (task.agentName !== 'troubleshooter' && task.agentName !== 'reviewer')
|
|
22
|
+
return '';
|
|
23
|
+
const related = task.agentName === 'troubleshooter'
|
|
24
|
+
? job.tasks.filter(item => item.id === task.parentTaskId)
|
|
25
|
+
: task.dependencies.map(id => job.tasks.find(item => item.id === id)).filter((item) => Boolean(item));
|
|
26
|
+
return `\n\n[Authoritative task state and append-only comments]\n${JSON.stringify({
|
|
27
|
+
currentTask: { id: task.id, state: task.state, attempt: task.attempt, executionId: task.executionId, resolution: task.resolution ?? null, escalation: task.escalation ?? null, comments: task.comments ?? [] },
|
|
28
|
+
relatedTasks: related.map(item => ({ id: item.id, title: item.title, state: item.state, attempt: item.attempt, executionId: item.executionId, resolution: item.resolution ?? null, escalation: item.escalation ?? null, comments: item.comments ?? [], result: item.result, error: item.error, failureReason: item.failureReason })),
|
|
29
|
+
}, null, 2)}\nTreat comments as supplementary history. Re-evaluate historical conditions using evidence from the original failing execution; a human response or prior condition alone does not prove recovery.`;
|
|
30
|
+
}
|
|
14
31
|
/** Browser new-turn envelope: the local Contract and durable Task Context are authoritative. */
|
|
15
32
|
export function taskReferencePrompt(repository, jobId, taskId, executionId, managedInput) {
|
|
16
33
|
if (managedInput !== undefined && !managedInput.includes('[Execution event contract — mandatory]'))
|
package/dist/task-recovery.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { randomUUID } from 'node:crypto';
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
2
|
import { eventFor } from './events.js';
|
|
3
3
|
import { tryTransitionTaskEvent } from './scheduler.js';
|
|
4
4
|
import { dispatchTaskResolutionSync, materializeResolution } from './workflow/task-resolution.js';
|
|
@@ -20,6 +20,11 @@ function validateRequest(request) {
|
|
|
20
20
|
if (!request.context.trim())
|
|
21
21
|
throw new Error('Escalation requires diagnostic context');
|
|
22
22
|
}
|
|
23
|
+
function escalationFingerprint(request, executionId) {
|
|
24
|
+
return createHash('sha256').update(JSON.stringify([
|
|
25
|
+
request.requiredAction.trim(), request.resolutionCondition.trim(), request.context.trim(), executionId ?? null,
|
|
26
|
+
])).digest('hex');
|
|
27
|
+
}
|
|
23
28
|
/** Relay-owned escalation: the recovery Task, never the failed original, becomes BLOCKED. */
|
|
24
29
|
export async function requestTroubleshooterEscalation(store, jobId, recoveryTaskId, request, now = new Date().toISOString()) {
|
|
25
30
|
validateRequest(request);
|
|
@@ -35,6 +40,11 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
|
|
|
35
40
|
const originalPlan = recovery.recoveryPlanId ? job.plans?.find(plan => plan.id === recovery.recoveryPlanId) : undefined;
|
|
36
41
|
if ((!original || original.state !== 'FAILED') && (!originalPlan || originalPlan.status !== 'FAILED'))
|
|
37
42
|
throw new Error(`Troubleshooter ${recoveryTaskId} has no current FAILED original Task or Plan`);
|
|
43
|
+
const fingerprint = escalationFingerprint(request, original?.executionId ?? originalPlan?.executionId);
|
|
44
|
+
if (recovery.escalation?.fingerprint === fingerprint) {
|
|
45
|
+
result = { task: structuredClone(recovery), escalation: structuredClone(recovery.escalation), duplicate: true };
|
|
46
|
+
return job;
|
|
47
|
+
}
|
|
38
48
|
if (recovery.escalation?.status === 'pending' && recovery.state === 'BLOCKED') {
|
|
39
49
|
result = { task: structuredClone(recovery), escalation: structuredClone(recovery.escalation), duplicate: true };
|
|
40
50
|
return job;
|
|
@@ -56,6 +66,7 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
|
|
|
56
66
|
throw new Error(`Troubleshooter ${recoveryTaskId} cannot be blocked`);
|
|
57
67
|
const escalation = {
|
|
58
68
|
id: randomUUID(), originalTaskId: original?.id, originalPlanId: originalPlan?.id,
|
|
69
|
+
fingerprint,
|
|
59
70
|
originalExecutionId: original?.executionId ?? undefined,
|
|
60
71
|
recoveryExecutionId: recovery.executionId ?? undefined, requiredAction: request.requiredAction.trim(),
|
|
61
72
|
resolutionCondition: request.resolutionCondition.trim(), context: request.context.trim(), status: 'pending',
|
|
@@ -84,6 +95,8 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
|
|
|
84
95
|
await store.saveTask(task); await store.saveJob(current); return current; });
|
|
85
96
|
if (!result)
|
|
86
97
|
throw new Error('Escalation did not produce a durable result');
|
|
98
|
+
result.task = structuredClone(job.tasks.find(task => task.id === recoveryTaskId) ?? result.task);
|
|
99
|
+
result.escalation = structuredClone(result.task.escalation ?? result.escalation);
|
|
87
100
|
return { job, ...result };
|
|
88
101
|
}
|
|
89
102
|
/** Authorized human response; only a blocked Troubleshooter may be resumed. */
|
|
@@ -102,7 +115,6 @@ export async function resumeTroubleshooter(store, jobId, recoveryTaskId, respons
|
|
|
102
115
|
if (!tryTransitionTaskEvent(task, 'task.unblocked', { autonomousJob: false }).accepted)
|
|
103
116
|
throw new Error(`Troubleshooter ${recoveryTaskId} cannot be resumed`);
|
|
104
117
|
task.escalation = { ...task.escalation, status: 'responded', response: response.trim(), respondedAt: now };
|
|
105
|
-
task.input += `\n\n## Human response (${now})\n${response.trim()}\n\nRe-validate the original failure against this response. Required resolution condition: ${task.escalation.resolutionCondition}`;
|
|
106
118
|
task.result = null;
|
|
107
119
|
task.error = null;
|
|
108
120
|
task.failureReason = undefined;
|
|
@@ -116,6 +128,7 @@ export async function resumeTroubleshooter(store, jobId, recoveryTaskId, respons
|
|
|
116
128
|
await store.saveTask(task); await store.saveJob(current); return current; });
|
|
117
129
|
if (!updated)
|
|
118
130
|
throw new Error('Troubleshooter was not resumed');
|
|
131
|
+
updated = structuredClone(job.tasks.find(item => item.id === recoveryTaskId) ?? updated);
|
|
119
132
|
return { job, task: updated };
|
|
120
133
|
}
|
|
121
134
|
export function hasPendingTroubleshooterEscalation(job, original) {
|
package/dist/troubleshooter.js
CHANGED
|
@@ -26,7 +26,7 @@ Retrieve this task's lifecycle event history before classifying the root cause o
|
|
|
26
26
|
GET http://127.0.0.1:8787/api/events?job=${encodeURIComponent(job.id)}&task=${encodeURIComponent(task.id)}&limit=200
|
|
27
27
|
Inspect event order, type/status, source, data/phase, duplicate events, missing expected lifecycle events, late events, and execution identifiers. If the event stream points to delivery/runtime failure, inspect the corresponding Relay/worker/tool logs too. Do not treat the final task.error string as the complete root cause.
|
|
28
28
|
|
|
29
|
-
Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state
|
|
29
|
+
Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state, append-only comments, task-scoped event history, and relevant runtime logs before retry. The final task.error string alone is never sufficient. Independently verify every human response against evidence from the exact original failing execution context; a response or historical resolution condition alone never proves recovery. Classify evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Select one graph-enabled resolution path: exact_original_retry, same_job_dependency, blocker_job_dependency, escalation_gate, or cancel_supersede. For confirmed adapter/model/provider failure, include --failure-kind, --reason, --evidence and --reroute in task recover; use --exclude-current-route only if the exact failed route should not be selected again. For semantic task failure use --failure-kind task_failure without rerouting. For uncertain failures use --failure-kind unknown and preserve routing unless evidence justifies rerouting. Request the existing agents-relay task recover or agents-relay escalation operation; submission alone is not recovery. Reload authoritative SQLite state and verify the requested resolution and exact-original retry postconditions before reporting success. Bound self-healing retries to the durable Task retry budget. For a shared worker/runtime/tool defect, use its owning repository's P0/self-job workflow and verify the durable fix before requesting recovery. Never use ordinary task retry from a Troubleshooter because manual retry intentionally supersedes active recovery. Do not manually choose a provider/model or create a replacement task. If diagnosis suggests a genuine human/external blocker, invoke the escalation command; do not publish task.blocked or implement a workflow branch procedurally. The canonical escalation graph invokes Neo escalation and owns the resolved/failed/needs_user branch; Relay applies the selected outcome. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
|
|
30
30
|
}
|
|
31
31
|
function belongsToCurrentRun(job, task) {
|
|
32
32
|
const slot = job.schedule?.currentRunSlot;
|
|
@@ -92,7 +92,7 @@ export function pendingTroubleshooterTasks(job) {
|
|
|
92
92
|
recoveries.push(createTaskEntity(job, {
|
|
93
93
|
jobId: job.id,
|
|
94
94
|
id,
|
|
95
|
-
title: `
|
|
95
|
+
title: `Recover task ${task.id}`,
|
|
96
96
|
recoveryForAttempt: task.attempt,
|
|
97
97
|
kind: 'work',
|
|
98
98
|
priority: 'P0',
|
|
@@ -43,8 +43,20 @@ export const jobLifecycleV2 = load('job-lifecycle', JOB_STATES, {
|
|
|
43
43
|
implementationComplete: boolGuard('implementationComplete'),
|
|
44
44
|
headAvailable: boolGuard('headAvailable'),
|
|
45
45
|
mergeCapable: boolGuard('mergeCapable'),
|
|
46
|
-
localCompletionAllowed:
|
|
47
|
-
|
|
46
|
+
localCompletionAllowed: (facts) => {
|
|
47
|
+
const values = facts.durableFacts;
|
|
48
|
+
return values.jobActive === true && values.autoMergeEnabled === true && values.schedulePresent !== true
|
|
49
|
+
&& values.implementationComplete === true && values.completionGateSatisfied === true
|
|
50
|
+
&& values.reviewSatisfied === true
|
|
51
|
+
&& (values.localMergeRequired !== true || values.localMergeSucceeded === true);
|
|
52
|
+
},
|
|
53
|
+
localMergeReady: (facts) => {
|
|
54
|
+
const values = facts.durableFacts;
|
|
55
|
+
return values.jobActive === true && values.autoMergeEnabled === true && values.schedulePresent !== true
|
|
56
|
+
&& values.implementationComplete === true && values.completionGateSatisfied === true
|
|
57
|
+
&& values.reviewSatisfied === true && values.localMergePresent === true
|
|
58
|
+
&& values.reviewHeadCurrent === true && values.mergeAttemptReady === true;
|
|
59
|
+
},
|
|
48
60
|
});
|
|
49
61
|
function topologyTargets(graph, from) {
|
|
50
62
|
const source = [...graph.nodes.values()].find(node => node.kind === 'state' && node.semanticId === from);
|
|
@@ -25,7 +25,7 @@ function parseLabel(raw, line) {
|
|
|
25
25
|
export function parseGraphV2(source) {
|
|
26
26
|
const lines = source.split(/\r?\n/);
|
|
27
27
|
// Mermaid YAML frontmatter is metadata only; never execute it as graph policy.
|
|
28
|
-
//
|
|
28
|
+
// Keep source line positions stable even when graph metadata grows.
|
|
29
29
|
let graphStart = 0;
|
|
30
30
|
if (lines[0]?.trim() === '---') {
|
|
31
31
|
let closing = -1;
|
|
@@ -34,8 +34,6 @@ export function parseGraphV2(source) {
|
|
|
34
34
|
closing = i;
|
|
35
35
|
break;
|
|
36
36
|
}
|
|
37
|
-
if (!/^(title|description):\s+[^\r\n]+$/.test(lines[i]))
|
|
38
|
-
fail(i + 1, 'unsupported graph frontmatter field');
|
|
39
37
|
}
|
|
40
38
|
if (closing < 0)
|
|
41
39
|
fail(1, 'unterminated graph frontmatter');
|
|
@@ -3,19 +3,36 @@ import { GraphRuntime } from './graph-v2.js';
|
|
|
3
3
|
import { dispatchWorkflowGraphSync } from './dispatcher.js';
|
|
4
4
|
const source = readFileSync(new URL('../../workflow/projection-conflict.mmd', import.meta.url), 'utf8');
|
|
5
5
|
const boolGuard = (name) => (facts) => facts.durableFacts[name] === true;
|
|
6
|
+
const guards = {
|
|
7
|
+
deterministicDefinitionConflict: boolGuard('deterministicDefinitionConflict'),
|
|
8
|
+
'projection.recoveryInProgress': boolGuard('recoveryInProgress'),
|
|
9
|
+
'projection.recoveryEligible': (facts) => {
|
|
10
|
+
const values = facts.durableFacts;
|
|
11
|
+
return values.jobState === 'COMPLETED' && values.mergeSucceeded === true
|
|
12
|
+
&& values.projectionStrategy === 'target_ref_push' && values.divergenceConfirmed === true
|
|
13
|
+
&& values.remoteShaAvailable === true
|
|
14
|
+
&& values.sameIdempotencyKey !== true && values.recoveryInProgress !== true
|
|
15
|
+
&& Number(values.attempts ?? 0) < Number(values.maxAttempts ?? 0);
|
|
16
|
+
},
|
|
17
|
+
'projection.recoveryExhausted': (facts) => {
|
|
18
|
+
const values = facts.durableFacts;
|
|
19
|
+
return values.jobState === 'COMPLETED' && values.divergenceConfirmed === true && values.remoteShaAvailable === true
|
|
20
|
+
&& values.sameIdempotencyKey !== true && values.recoveryInProgress !== true
|
|
21
|
+
&& Number(values.attempts ?? 0) >= Number(values.maxAttempts ?? 0);
|
|
22
|
+
},
|
|
23
|
+
'projection.retryAvailable': (facts) => facts.durableFacts.taskRetryAvailable === true
|
|
24
|
+
&& Number(facts.durableFacts.attempts ?? 0) < Number(facts.durableFacts.maxAttempts ?? 0),
|
|
25
|
+
'projection.repairReviewed': (facts) => facts.durableFacts.repairTaskState === 'SUCCEEDED' && facts.durableFacts.reviewOutcome === 'approved',
|
|
26
|
+
'projection.repairFailed': (facts) => facts.durableFacts.repairTaskState === 'FAILED' || facts.durableFacts.repairTaskState === 'BLOCKED'
|
|
27
|
+
|| facts.durableFacts.reviewOutcome === 'blocked' || facts.durableFacts.reviewTaskState === 'FAILED',
|
|
28
|
+
};
|
|
6
29
|
export const projectionConflictV2 = new GraphRuntime(source, {
|
|
7
|
-
states: new Set(['classify', 'retry', 'conflict']),
|
|
8
|
-
guards
|
|
9
|
-
actions: {},
|
|
10
|
-
events:
|
|
30
|
+
states: new Set(['classify', 'repairing', 'retry', 'conflict', 'escalated', 'projected']),
|
|
31
|
+
guards,
|
|
32
|
+
actions: Object.fromEntries(['projection.createRecoveryTask', 'effect.projectLocalMerge', 'projection.escalate'].map(id => [id, { id }])),
|
|
33
|
+
events: Object.fromEntries(['projection.failed', 'projection.delivery_failed', 'projection.delivered', 'reconcile'].map(id => [id, { id }])),
|
|
11
34
|
});
|
|
12
|
-
/**
|
|
13
|
-
* Core projection-failure disposition policy.
|
|
14
|
-
*
|
|
15
|
-
* A deterministic same-id/different-definition conflict stops consuming retry budget;
|
|
16
|
-
* any other failure keeps the normal backoff/retry path. This branch is owned by the
|
|
17
|
-
* canonical Mermaid graph, not by a procedural `if` in the projection writer.
|
|
18
|
-
*/
|
|
35
|
+
/** The decision is graph-owned; callers supply SQLite and inspected Git facts. */
|
|
19
36
|
export function decideProjectionFailure(facts) {
|
|
20
37
|
return dispatchWorkflowGraphSync(projectionConflictV2, {
|
|
21
38
|
initialState: 'classify',
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agents-relay",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.151",
|
|
4
4
|
"description": "Durable async agent jobs coordinated through GitHub pull requests",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"test:fast": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=2 test/workflow-lifecycle.test.js test/workflow-v2-lifecycle-parity.test.js test/workflow-effects.test.js test/relay-store.test.js test/local-authority.test.js test/local-jobid-routing.test.js test/local-sqlite-jobid.test.js test/no-github.test.js test/scheduled-jobs.test.js test/reconcile-coordinator.test.js test/service-publication.test.js test/github-webhook.test.js",
|
|
21
21
|
"test:full": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 test/*.test.js && npm run test:verification:l3",
|
|
22
22
|
"test:verification:l1": "npm run build && node --test test/verification/harness.test.mjs test/verification/contract.test.mjs test/verification/graph-inventory.test.mjs test/verification/dashboard-drilldown.test.mjs test/verification/evidence-matrix.test.mjs",
|
|
23
|
-
"test:verification:l3": "npm run build && node --test test/verification/harness.test.mjs test/verification/l3-sqlite.test.mjs test/verification/l3-ingress.test.mjs test/verification/l3-replay.test.mjs test/verification/objective-authority.test.mjs test/verification/l3-convergence.test.mjs test/verification/l3-review-convergence.test.mjs test/verification/l3-human-escalation.test.mjs test/verification/l3-escalation-notifier.test.mjs test/verification/l3-unknown-effect.test.mjs test/verification/l3-cli-actions.test.mjs test/verification/l3-planner-round.test.mjs test/verification/full-recovery-gates.test.mjs test/verification/production-wiring.test.mjs",
|
|
23
|
+
"test:verification:l3": "npm run build && node --test --test-concurrency=1 test/verification/harness.test.mjs test/verification/l3-sqlite.test.mjs test/verification/l3-ingress.test.mjs test/verification/l3-replay.test.mjs test/verification/objective-authority.test.mjs test/verification/l3-convergence.test.mjs test/verification/l3-review-convergence.test.mjs test/verification/l3-human-escalation.test.mjs test/verification/l3-escalation-notifier.test.mjs test/verification/l3-unknown-effect.test.mjs test/verification/l3-cli-actions.test.mjs test/verification/l3-planner-round.test.mjs test/verification/full-recovery-gates.test.mjs test/verification/production-wiring.test.mjs test/verification/l3-full-path.test.mjs test/verification/l3-projection-recovery.test.mjs",
|
|
24
24
|
"test:verification": "npm run test:verification:l1 && npm run test:verification:l2 && npm run test:verification:l3 && npm run test:verification:cross-a && node --test test/workflow-production-outbox.test.js && npm run test:verification:matrix",
|
|
25
25
|
"e2e": "node e2e/runner/run.mjs",
|
|
26
26
|
"test:verification:cross-a": "npm run build && RELAY_A_DIST=$PWD/dist node --test test/verification/action-ledger-cross-branch.test.mjs test/verification/outbox-cross-a.test.mjs",
|
|
@@ -54,7 +54,7 @@ The event must carry the exact durable `job_id` and `task_id`. Progress events a
|
|
|
54
54
|
Agents Relay maps those terminal events to durable state:
|
|
55
55
|
- `task.completed` → `SUCCEEDED`
|
|
56
56
|
- `task.failed` → `FAILED`
|
|
57
|
-
Relay may authoritatively move a Task to `BLOCKED` for an external condition and may resume it only through an authorized unblock operation. A human response resumes the blocked Troubleshooter, not the failed original Task.
|
|
57
|
+
Relay may authoritatively move a Task to `BLOCKED` for an external condition and may resume it only through an authorized unblock operation. A human response is an append-only Task Comment and resumes the blocked Troubleshooter, not the failed original Task. Task input remains stable; recovery agents receive current authoritative state and relevant comments as execution context.
|
|
58
58
|
|
|
59
59
|
A model-backed task must not launch without an event bus. If no terminal event arrives before the task timeout, Relay records an explicit timeout failure. A Codex process exit or ChatGPT submission receipt alone never changes the task to a terminal state.
|
|
60
60
|
|
|
@@ -17,7 +17,7 @@ Read and follow the shared Task / Job contract and orchestrator P0/self-job work
|
|
|
17
17
|
|
|
18
18
|
## Recovery loop
|
|
19
19
|
|
|
20
|
-
1. Retrieve the durable original Job and
|
|
20
|
+
1. Retrieve the durable original Job and Task from the authoritative Relay store with `agents-relay task context`; GitHub is only an optional projection and is never lifecycle authority.
|
|
21
21
|
2. **Mandatory before diagnosis:** retrieve the failed Task's task-scoped lifecycle event history. When the local Relay dashboard is available, use `GET http://127.0.0.1:8787/api/events?job=<job_id>&task=<task_id>&limit=200`. Inspect event order, type/status, source, data/phase, execution IDs, duplicates, missing expected lifecycle events, and late events. Then inspect task error/failureReason, worker selection, routing, attempt count, task input/output contract, PR state, and relevant runtime/service logs.
|
|
22
22
|
3. Classify the blocker:
|
|
23
23
|
- transient execution/delivery/lease failure;
|
|
@@ -32,8 +32,8 @@ Read and follow the shared Task / Job contract and orchestrator P0/self-job work
|
|
|
32
32
|
- For task-local failure, make the smallest durable repair in the original Job/PR when that is within scope.
|
|
33
33
|
- For a shared platform/runtime defect, first create or reuse the owning blocker P0/self-job. If it can be completed inside this recovery execution, fix/verify it and retry the original Task. If it cannot be completed now, register the original Task with `agents-relay blocker add` on that blocker Job and finish this Troubleshooter task; do not stay running while waiting for another Job.
|
|
34
34
|
- For a suspected genuine human/external blocker, do not publish `task.blocked` and do not branch on Neo's result yourself. Invoke `agents-relay escalation` with the original intent, blocker, and relevant context. The canonical graph owns the `resolved`, `failed`, and `needs_user` outcomes. When it returns `needs_user`, call Relay's `task escalate` operation with the concrete required action, resolution condition, and diagnostic context. Relay records the escalation and blocks this Troubleshooter; no worker remains running while waiting.
|
|
35
|
-
7. For an immediate repair, only after the blocker is removed
|
|
36
|
-
8. Verify durable evidence
|
|
35
|
+
7. For an immediate repair, only after the blocker is removed request `agents-relay task recover` with this Troubleshooter's recovery Task ID, then reconcile the original Job. Verify the SQLite postcondition: original Task state QUEUED, expected resolution and recovery correlation. An authorized human response is a Task Comment, not a rewrite of Task input. Retain historical conditions as provenance, independently verify the original failing execution context, and do not submit an identical escalation with no new evidence. Deferred repair uses the durable blocker relation: merge-mode blockers wake dependents after merge; release/deploy/runtime blockers require `--resolution verified` and `agents-relay blocker resolve` only after verification.
|
|
36
|
+
8. Verify durable evidence the original Task moved: at minimum QUEUED after retry, preferably a new execution / task.started / observable progress.
|
|
37
37
|
9. Publish this Troubleshooter Task's terminal event only after the recovery outcome is durable.
|
|
38
38
|
|
|
39
39
|
## Rules
|
|
@@ -25,7 +25,7 @@ A Reviewer must **not** publish unstructured `task.completed`: missing verdict o
|
|
|
25
25
|
|
|
26
26
|
Only a real user action merits escalation. Troubleshooter invokes the Relay escalation operation, specifying the required action, resolution condition, and diagnostic context. Relay durably records the human action and blocks **the Troubleshooter Task**, leaving the original failed Task `FAILED`. This is a Relay state transition, not a third agent terminal event.
|
|
27
27
|
|
|
28
|
-
When the human responds, the
|
|
28
|
+
When the human responds, Relay records the response as an append-only Task Comment and authoritatively unblocks the **Troubleshooter Task**. The stable Task input is unchanged. Troubleshooter receives current authoritative state and relevant comments, independently verifies the response against the original failing execution context, requests an authorized original Task recovery, and only then publishes its own `task.completed`. A repeated identical escalation without new evidence is idempotent. If unresolved, it continues diagnosis or escalates again; human assertion alone is not technical verification.
|
|
29
29
|
|
|
30
30
|
See [task-recovery-flows.md](task-recovery-flows.md) for unified architecture, overall flow, failure recovery, and post-blocked resumption.
|
|
31
31
|
|
|
@@ -32,6 +32,13 @@ A later Planner round must inspect the existing semantic tree and attach review,
|
|
|
32
32
|
|
|
33
33
|
A Task input records **what is known and what has already been decided before execution**. A Task result records **what execution discovered and what was actually done**.
|
|
34
34
|
|
|
35
|
+
After creation, keep `Task.input` stable as the objective and initial evidence.
|
|
36
|
+
Responses, resolution changes, and escalation history belong in append-only
|
|
37
|
+
Task Comments. SQLite atomically records typed comments with authoritative
|
|
38
|
+
resolution/escalation mutations; PR-bound Jobs project those comments through
|
|
39
|
+
the durable idempotent projection outbox. Comments supplement current state and
|
|
40
|
+
never replace `Task.resolution` or `Task.escalation`.
|
|
41
|
+
|
|
35
42
|
Task input should include, as applicable:
|
|
36
43
|
|
|
37
44
|
- goal and observed behavior;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Task Failure Recovery and Human Escalation — Agreed Target Design
|
|
2
2
|
|
|
3
|
-
**Status:**
|
|
3
|
+
**Status:** Current recovery contract. SQLite Task snapshots retain append-only typed comments alongside authoritative state.
|
|
4
4
|
|
|
5
5
|
## Principles and ownership
|
|
6
6
|
|
|
@@ -133,3 +133,20 @@ flowchart TD
|
|
|
133
133
|
- Align escalation graph, CLI, reconciler, and unblock commands so human intervention blocks **Troubleshooter**, not the original failed Task; preserve the original failure and correlated recovery context.
|
|
134
134
|
- Ensure the blocked Troubleshooter can resume after an authorized human unblock rather than being treated as a finished terminal execution.
|
|
135
135
|
- Add integrated checks: technical retry, blocker-Job handoff, human request/block, human unblock/resume, failed verification/re-escalation, stale/duplicate responses, and correct original-Task retry.
|
|
136
|
+
|
|
137
|
+
## Durable recovery history
|
|
138
|
+
|
|
139
|
+
`Task.input` remains the stable objective and original evidence. `Task.resolution` and
|
|
140
|
+
`Task.escalation` remain the current authoritative states; comments are append-only
|
|
141
|
+
history, not a second state authority. SQLite mutation transactions compare the
|
|
142
|
+
before/after authoritative values and append typed comments with Job, Task,
|
|
143
|
+
execution, transition, and idempotency identity in the same commit. Do not add a
|
|
144
|
+
`resolutionHistory` field.
|
|
145
|
+
|
|
146
|
+
Human responses are comments. Resuming a Troubleshooter must not rewrite its
|
|
147
|
+
input or append a stale “Required resolution condition” as new objective text.
|
|
148
|
+
The condition remains in escalation provenance. Reviewer and Troubleshooter
|
|
149
|
+
prompts receive current authoritative state and relevant task comments as runtime
|
|
150
|
+
context; Troubleshooters independently verify a response against evidence from
|
|
151
|
+
the original failing execution. Repeating an identical escalation with unchanged
|
|
152
|
+
evidence is idempotent.
|
|
@@ -1,15 +1,66 @@
|
|
|
1
1
|
---
|
|
2
2
|
title: Projection Conflict
|
|
3
|
-
description:
|
|
3
|
+
description: Recover local-main and origin-main divergence after a completed Job's GitHub projection fails.
|
|
4
|
+
name: projection-conflict
|
|
5
|
+
authority: SQLite Relay Job snapshot
|
|
6
|
+
trigger: projection.delivery_failed
|
|
7
|
+
idempotency: job-id plus observed origin/main SHA
|
|
8
|
+
actions:
|
|
9
|
+
- projection.createRecoveryTask
|
|
10
|
+
- projection.escalate
|
|
11
|
+
effects:
|
|
12
|
+
- effect.projectLocalMerge
|
|
13
|
+
guards:
|
|
14
|
+
- deterministicDefinitionConflict
|
|
15
|
+
- projection.recoveryEligible
|
|
16
|
+
- projection.recoveryExhausted
|
|
17
|
+
- projection.recoveryInProgress
|
|
18
|
+
- projection.retryAvailable
|
|
19
|
+
- projection.repairReviewed
|
|
20
|
+
- projection.repairFailed
|
|
4
21
|
---
|
|
5
22
|
flowchart LR
|
|
6
23
|
%% relay: initial classify
|
|
7
24
|
classify["state: classify"]
|
|
8
|
-
retry["state: retry"]
|
|
9
25
|
conflict["state: conflict"]
|
|
26
|
+
repairing["state: repairing"]
|
|
27
|
+
retry["state: retry"]
|
|
28
|
+
escalated["state: escalated"]
|
|
29
|
+
projected["state: projected"]
|
|
10
30
|
|
|
31
|
+
eligible["guard: projection.recoveryEligible"]
|
|
32
|
+
inProgress["guard: projection.recoveryInProgress"]
|
|
11
33
|
deterministic["guard: deterministicDefinitionConflict"]
|
|
34
|
+
exhausted["guard: projection.recoveryExhausted"]
|
|
35
|
+
retryAvailable["guard: projection.retryAvailable"]
|
|
36
|
+
reviewed["guard: projection.repairReviewed"]
|
|
37
|
+
repairFailed["guard: projection.repairFailed"]
|
|
38
|
+
createTask["action: projection.createRecoveryTask"]
|
|
39
|
+
retryProjection["action: effect.projectLocalMerge"]
|
|
40
|
+
escalate["action: projection.escalate"]
|
|
12
41
|
|
|
42
|
+
classify -->|on: projection.delivery_failed| inProgress
|
|
43
|
+
inProgress -->|true| classify
|
|
44
|
+
inProgress -->|false| eligible
|
|
13
45
|
classify -->|on: projection.failed| deterministic
|
|
14
46
|
deterministic -->|true| conflict
|
|
15
47
|
deterministic -->|false| retry
|
|
48
|
+
eligible -->|true| createTask
|
|
49
|
+
eligible -->|false| exhausted
|
|
50
|
+
exhausted -->|true| escalate
|
|
51
|
+
exhausted -->|false| classify
|
|
52
|
+
createTask --> repairing
|
|
53
|
+
|
|
54
|
+
repairing -->|on: reconcile| reviewed
|
|
55
|
+
retry -->|on: reconcile| retryAvailable
|
|
56
|
+
reviewed -->|true| retryAvailable
|
|
57
|
+
retryAvailable -->|true| retryProjection
|
|
58
|
+
retryAvailable -->|false| escalate
|
|
59
|
+
reviewed -->|false| repairFailed
|
|
60
|
+
repairFailed -->|true| escalate
|
|
61
|
+
repairFailed -->|false| repairing
|
|
62
|
+
retryProjection --> retry
|
|
63
|
+
escalate --> escalated
|
|
64
|
+
|
|
65
|
+
retry -->|on: projection.delivered| projected
|
|
66
|
+
retry -->|on: projection.delivery_failed| eligible
|