agents-relay 2.0.150 → 2.0.152

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,10 +24,21 @@ export function localSourceHead(repoPath, sourceRef) {
24
24
  export function requiresLocalGitMerge(job) {
25
25
  return Boolean(job.localMerge) || job.tasks.some(task => task.output?.kind === 'task_pr');
26
26
  }
27
- export function localMergeEligible(job) {
28
- return job.state === 'ACTIVE' && job.autoMerge !== false && !job.schedule &&
29
- implementationComplete(job) && completionGateSatisfied(job) &&
30
- (job.reviewEnabled === false || job.review?.outcome === 'approved');
27
+ export function localMergeFacts(job, headSha, now = Date.now) {
28
+ return {
29
+ jobActive: job.state === 'ACTIVE',
30
+ autoMergeEnabled: job.autoMerge !== false,
31
+ schedulePresent: Boolean(job.schedule),
32
+ implementationComplete: implementationComplete(job),
33
+ completionGateSatisfied: completionGateSatisfied(job),
34
+ reviewSatisfied: job.reviewEnabled === false || job.review?.outcome === 'approved',
35
+ localMergeRequired: requiresLocalGitMerge(job),
36
+ localMergeSucceeded: job.localMerge?.status === 'succeeded',
37
+ localMergePresent: Boolean(job.localMerge),
38
+ reviewHeadCurrent: job.reviewEnabled === false || job.review?.headSha === headSha,
39
+ mergeAttemptReady: Boolean(job.localMerge && job.localMerge.status !== 'succeeded'
40
+ && (!job.localMerge.nextAttemptAt || Date.parse(job.localMerge.nextAttemptAt) <= now())),
41
+ };
31
42
  }
32
43
  export function localMergeCommitMessage(request, sourceSha) {
33
44
  const lines = ['Agents Relay local merge', '', `Agents-Relay-Source-SHA: ${sourceSha}`];
@@ -1,22 +1,25 @@
1
1
  import { createHash } from 'node:crypto';
2
2
  import { eventFor } from './events.js';
3
3
  import { transitionJobEvent } from './scheduler.js';
4
- import { deliverLocalMerge, localMergeEligible, requiresLocalGitMerge } from './local-git-merge.js';
4
+ import { deliverLocalMerge, localMergeFacts } from './local-git-merge.js';
5
5
  import { errorMessage } from './observability.js';
6
+ import { decideJobLifecycle } from './workflow/lifecycle.js';
6
7
  /** Graph-selected durable local merge and completion; no checked-out worktree is ever changed. */
7
8
  export async function reconcileLocalGitCompletion(store, finalState, headSha, noGitHub, projectPathForJob, emit, mergeEffect = deliverLocalMerge, now = Date.now) {
8
- if (localMergeEligible(finalState) && !requiresLocalGitMerge(finalState)) {
9
- await transitionJobEvent(finalState, 'local.completed', { localCompletionAllowed: true });
9
+ const facts = localMergeFacts(finalState, headSha, now);
10
+ if (decideJobLifecycle(finalState, 'local.completed', facts).target === 'COMPLETED') {
11
+ await transitionJobEvent(finalState, 'local.completed', facts);
10
12
  await store.saveJob(finalState);
11
13
  return true;
12
14
  }
13
- if (localMergeEligible(finalState) &&
14
- finalState.localMerge && (!finalState.reviewEnabled || finalState.review?.headSha === headSha)) {
15
+ if (decideJobLifecycle(finalState, 'local.merge.requested', facts).actions.includes('effect.deliverLocalMerge')) {
15
16
  const receipt = finalState.localMerge;
17
+ if (!receipt)
18
+ return false;
16
19
  let activeReceipt = receipt;
17
20
  if (receipt.status !== 'succeeded' &&
18
21
  (!receipt.nextAttemptAt || Date.parse(receipt.nextAttemptAt) <= now())) {
19
- await transitionJobEvent(finalState, 'local.merge.requested', { localMergeReady: true }, async (actionId) => {
22
+ await transitionJobEvent(finalState, 'local.merge.requested', facts, async (actionId) => {
20
23
  if (actionId !== 'effect.deliverLocalMerge')
21
24
  throw new Error('Unsupported local merge Graph effect ' + actionId);
22
25
  try {
@@ -75,7 +78,8 @@ export async function reconcileLocalGitCompletion(store, finalState, headSha, no
75
78
  });
76
79
  }
77
80
  if (finalState.localMerge?.status === 'succeeded') {
78
- await transitionJobEvent(finalState, 'local.completed', { localCompletionAllowed: true });
81
+ const completionFacts = localMergeFacts(finalState, headSha, now);
82
+ await transitionJobEvent(finalState, 'local.completed', completionFacts);
79
83
  await store.saveJob(finalState);
80
84
  return true;
81
85
  }
@@ -4,6 +4,7 @@ import { agentGraphPrompt } from './agent-graph.js';
4
4
  import { createReviewChairExecutionPlan } from './review-chair.js';
5
5
  import { taskTagWorkerPrompt } from './task-tag.js';
6
6
  import { executionEventSource } from './reconciler-runtime.js';
7
+ import { taskAgentHistoryContext } from './task-context.js';
7
8
  export async function managedWorkerInput(job, task, context, mode = 'production') {
8
9
  const isLocalJob = Boolean(job.localMerge?.sourceRef);
9
10
  const projectedReview = isLocalJob && task.agentName === 'reviewer' && job.prNumber > 0;
@@ -53,7 +54,7 @@ ${interactiveContext}Task attempt/execution: ${task.attempt}/${task.maxAttempts}
53
54
  Dependencies: ${task.dependencies.length ? task.dependencies.join(', ') : 'none'}
54
55
  ${task.attempt > 1 || task.error ? `Retry summary: prior error=${task.error ?? 'none'}; prior generations=${(task.executionHistory ?? []).map(item => `${item.id}@${item.attempt}`).join(', ') || 'none'}` : 'Retry summary: first attempt'}
55
56
  SQLite/RelayStore and the canonical Mermaid Graphs own lifecycle, merge, recovery, and completion decisions. Do not perform Relay-owned merge or completion actions from this worker.
56
- ${chairPlan ? `[Review Chair execution plan]\n${JSON.stringify(chairPlan, null, 2)}\n\nUse this runtime-discovered panel to create sibling durable specialist Tasks. Preserve each specialistTasks entry as review metadata, keep the target immutable, and synthesize structured results with synthesizeReview semantics before returning the unified revision plan.\n\n` : ''}Follow the resolved project instructions and Agents Relay/events-bus worker contracts.
57
+ ${taskAgentHistoryContext(job, task)}\n\n${chairPlan ? `[Review Chair execution plan]\n${JSON.stringify(chairPlan, null, 2)}\n\nUse this runtime-discovered panel to create sibling durable specialist Tasks. Preserve each specialistTasks entry as review metadata, keep the target immutable, and synthesize structured results with synthesizeReview semantics before returning the unified revision plan.\n\n` : ''}Follow the resolved project instructions and Agents Relay/events-bus worker contracts.
57
58
  [Output contract]
58
59
  ${output}
59
60
  ${taskTag}${mode === 'evaluation' ? `[Isolated prompt evaluation mode]
package/dist/pool.js CHANGED
@@ -8,7 +8,7 @@ import { jobHasActiveExecution } from './job-execution-slot.js';
8
8
  import { unblockNeedsNewPlan } from './unblock.js';
9
9
  const RANK = { P0: 0, P1: 1, P2: 2, P3: 3 };
10
10
  export function compareJobPriority(a, b) { return RANK[a.priority ?? 'P2'] - RANK[b.priority ?? 'P2'] || a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id); }
11
- export function runnableManagedJobs(items) { return items.filter(item => item.job.state === 'ACTIVE').map(item => item.job).sort(compareJobPriority); }
11
+ export function runnableManagedJobs(items) { return items.filter(item => item.job.state === 'ACTIVE' || item.job.projectionRecovery?.status === 'repairing').map(item => item.job).sort(compareJobPriority); }
12
12
  export function needsLifecycleReconciliation(job, now = Date.now()) {
13
13
  if (job.pause)
14
14
  return job.tasks.some(task => Boolean(task.pausedExecution)) || Boolean(job.plans?.some(plan => Boolean(plan.pausedExecutionId)));
@@ -95,7 +95,7 @@ export class GitHubProjectionWriter {
95
95
  let deferred = 0;
96
96
  for (const item of items) {
97
97
  try {
98
- if (this.externalOnly && !['merge_pull_request', 'close_pull_request', 'update_pr_body', 'create_pull_request'].includes(item.operation)) {
98
+ if (this.externalOnly && !['merge_pull_request', 'close_pull_request', 'update_pr_body', 'create_pull_request', 'append_task_comment'].includes(item.operation)) {
99
99
  // Drop legacy lifecycle projections; GitHub may not override SQLite truth.
100
100
  await this.relay.completeProjection(item.id);
101
101
  continue;
@@ -182,6 +182,11 @@ export class GitHubProjectionWriter {
182
182
  case 'save_task':
183
183
  await this.github.saveTask(payload.value);
184
184
  return;
185
+ case 'append_task_comment':
186
+ if (!this.github.appendTaskHistoryComment)
187
+ throw new Error('GitHub projection target cannot append Task history comments');
188
+ await this.github.appendTaskHistoryComment(payload.value);
189
+ return;
185
190
  case 'save_agent':
186
191
  await this.github.saveAgent(payload.value);
187
192
  return;
@@ -0,0 +1,52 @@
1
+ import { eventFor } from './events.js';
2
+ import { beginProjectionRecovery } from './projection-recovery.js';
3
+ import { reconcileProjectionRecovery } from './projection-recovery-runtime.js';
4
+ import { SQLiteRelayStore } from './relay-store.js';
5
+ import { classifyFailure, errorMessage, observability } from './observability.js';
6
+ import { resolveManagedProjectPath } from './workspace.js';
7
+ export async function reconcileBindingProjectionRecovery(job, store, options) {
8
+ const recovery = await reconcileProjectionRecovery(job, store, options);
9
+ if (recovery && recovery.state === 'escalated')
10
+ return { skip: true };
11
+ if (recovery && !['retry', 'projected'].includes(recovery.state))
12
+ return { skip: true };
13
+ if (recovery?.state === 'retry' && !recovery.projectedMerge)
14
+ return { skip: true };
15
+ return { skip: false, projectedMerge: recovery?.projectedMerge };
16
+ }
17
+ export async function projectBindingLocalMerge(job, store, options) {
18
+ const recovery = await reconcileBindingProjectionRecovery(job, store, options);
19
+ if (recovery.skip)
20
+ return recovery;
21
+ return { skip: false, projectedMerge: recovery.projectedMerge ?? await options.project(job) };
22
+ }
23
+ export async function handleGitHubProjectionFailure(error, job, snapshotPath, options) {
24
+ const reason = errorMessage(error), classification = classifyFailure(error).classification;
25
+ if (job.state === 'COMPLETED' && job.localMerge?.status === 'succeeded') {
26
+ const recoveryStore = options.relayStores.get(`${snapshotPath}#${job.id}`)
27
+ ?? new SQLiteRelayStore(snapshotPath, job.id);
28
+ options.relayStores.set(`${snapshotPath}#${job.id}`, recoveryStore);
29
+ const recoveryPath = options.repositoryPaths.get(job.repository)
30
+ ?? await resolveManagedProjectPath(job.repository, options.explicitProjectPath, options.workspaceRoot);
31
+ const remoteSha = await options.gitText(recoveryPath, ['fetch', '--no-tags', 'origin', '+refs/heads/main:refs/remotes/origin/main'])
32
+ .then(() => options.gitText(recoveryPath, ['rev-parse', '--verify', 'refs/remotes/origin/main']))
33
+ .catch(() => '');
34
+ const recovery = await beginProjectionRecovery(recoveryStore, job.id, remoteSha, reason);
35
+ if (recovery === 'created') {
36
+ await options.reconcileLocalJob(job.id, 'projection.delivery_failed');
37
+ await options.eventBus?.publish(eventFor(job.id, 'projection', null, 'projection.recovery_requested', 'queued', `Projection recovery task created for origin/main ${remoteSha}`, 'orchestrator', { remoteSha, idempotencyKey: `${job.id}:${remoteSha}` }, { agent: 'relay', component: 'projection-conflict' })).catch(() => { });
38
+ }
39
+ else if (recovery === 'escalated') {
40
+ await options.eventBus?.publish(eventFor(job.id, 'projection', null, 'projection.recovery_escalated', 'failed', 'Projection recovery exhausted its bounded task retry budget', 'user', { idempotencyKey: `${job.id}:${remoteSha}` }, { agent: 'relay', component: 'projection-conflict' })).catch(() => { });
41
+ }
42
+ }
43
+ if (!options.deduplicator.shouldReport(job.id, job.updatedAt, reason, classification))
44
+ return;
45
+ await options.eventBus?.publish(eventFor(job.id, 'projection', null, 'projection.delivery_failed', 'failed', `GitHub projection failed: ${errorMessage(error)}`, 'orchestrator', { operation: 'github.projectLocalMerge', error: reason }, { agent: 'relay', component: 'github-projection' }))?.catch(() => { });
46
+ observability().record({
47
+ stage: 'projection', event: 'github.binding_job_failed', level: 'warn', outcome: 'failed',
48
+ message: `GitHub binding/projection failed for ${job.id}: ${reason}`,
49
+ correlation: { repository: job.repository, prNumber: job.prNumber, jobId: job.id },
50
+ data: { classification, reason },
51
+ });
52
+ }
@@ -0,0 +1,121 @@
1
+ import { eventFor } from './events.js';
2
+ import { deliverLocalMerge, localSourceHead } from './local-git-merge.js';
3
+ import { dispatchWorkflowGraph } from './workflow/dispatcher.js';
4
+ import { projectionConflictV2 } from './workflow/projection-conflict-v2.js';
5
+ /** Executes only the retry or escalation selected by the projection-conflict graph. */
6
+ export async function reconcileProjectionRecovery(job, store, options) {
7
+ const recovery = job.projectionRecovery;
8
+ if (!recovery)
9
+ return null;
10
+ if (recovery.status === 'projected' || recovery.status === 'escalated')
11
+ return { state: recovery.status };
12
+ const repair = job.tasks.find(task => task.id === recovery.taskId);
13
+ const reviewTask = job.review?.taskId ? job.tasks.find(task => task.id === job.review?.taskId) : undefined;
14
+ const expectedReviewTaskId = job.review?.taskId;
15
+ const expectedReviewOutcome = job.review?.outcome;
16
+ const expectedReviewHeadSha = job.review?.headSha;
17
+ let projectedMerge;
18
+ let claimed = false;
19
+ let escalated = false;
20
+ let projectionCompleted = false;
21
+ const decision = await dispatchWorkflowGraph(projectionConflictV2, {
22
+ initialState: recovery.status, input: { transport: 'event', id: 'reconcile', payload: {} },
23
+ durableFacts: {
24
+ repairTaskState: repair?.state,
25
+ reviewTaskState: reviewTask?.state,
26
+ reviewOutcome: job.review?.outcome,
27
+ taskRetryAvailable: repair?.state !== 'FAILED',
28
+ attempts: recovery.attempts,
29
+ maxAttempts: recovery.maxAttempts,
30
+ }, graphName: 'projection-conflict', maxSteps: 1,
31
+ persistTransition: async (source, target) => {
32
+ if (target === 'retry') {
33
+ if (!store.mutateJob)
34
+ throw new Error('Atomic SQLite mutation is required for projection recovery attempts');
35
+ await store.mutateJob(job.id, current => {
36
+ const currentRecovery = current.projectionRecovery;
37
+ const currentRepair = current.tasks.find(task => task.id === currentRecovery?.taskId);
38
+ const currentReviewTask = current.review?.taskId
39
+ ? current.tasks.find(task => task.id === current.review?.taskId) : undefined;
40
+ if (current.state !== 'COMPLETED' || currentRecovery?.key !== recovery.key
41
+ || currentRecovery.taskId !== recovery.taskId || currentRecovery.status !== source
42
+ || currentRecovery.attempts !== recovery.attempts || currentRepair?.state !== repair?.state
43
+ || currentReviewTask?.id !== expectedReviewTaskId || current.review?.outcome !== expectedReviewOutcome
44
+ || current.review?.headSha !== expectedReviewHeadSha)
45
+ return current;
46
+ claimed = true;
47
+ return { ...current, projectionRecovery: { ...currentRecovery, status: 'retry',
48
+ attempts: currentRecovery.attempts + 1, updatedAt: new Date().toISOString() } };
49
+ });
50
+ }
51
+ else if (target === 'escalated') {
52
+ if (!store.mutateJob)
53
+ throw new Error('Atomic SQLite mutation is required for projection recovery status');
54
+ await store.mutateJob(job.id, current => current.projectionRecovery?.key === recovery.key
55
+ && current.projectionRecovery.taskId === recovery.taskId
56
+ && current.projectionRecovery.status === source
57
+ && current.projectionRecovery.attempts === recovery.attempts
58
+ ? (escalated = true, { ...current, projectionRecovery: { ...current.projectionRecovery, status: 'escalated', updatedAt: new Date().toISOString() } })
59
+ : current);
60
+ }
61
+ else if (source !== target)
62
+ throw new Error(`Unexpected projection recovery transition ${source} -> ${target}`);
63
+ },
64
+ executeAction: async (actionId) => {
65
+ if (actionId === 'effect.projectLocalMerge') {
66
+ if (!claimed)
67
+ return {};
68
+ const localMerge = job.localMerge;
69
+ if (!localMerge)
70
+ throw new Error('Projection recovery requires a durable local merge receipt');
71
+ const sourceSha = localSourceHead(options.projectPath, localMerge.sourceRef);
72
+ if (job.review?.headSha !== sourceSha)
73
+ throw new Error('Projection recovery source changed after review');
74
+ const delivered = deliverLocalMerge(options.projectPath, { ...localMerge, sourceSha,
75
+ correlationJobId: job.id, displayPrNumber: job.prNumber > 0 ? job.prNumber : undefined });
76
+ await options.verifyObservedRemote(recovery.remoteSha);
77
+ const projectedJob = { ...job, localMerge: { ...localMerge, sourceSha: delivered.sourceSha,
78
+ mergedCommitSha: delivered.mergedCommitSha, status: 'succeeded', error: undefined,
79
+ nextAttemptAt: undefined, updatedAt: new Date().toISOString() } };
80
+ projectedMerge = await options.project(projectedJob);
81
+ let finalized = false;
82
+ const deliveredDecision = await dispatchWorkflowGraph(projectionConflictV2, {
83
+ initialState: 'retry', input: { transport: 'event', id: 'projection.delivered', payload: {} },
84
+ durableFacts: {}, graphName: 'projection-conflict', maxSteps: 1,
85
+ persistTransition: async (source, target) => {
86
+ if (source !== 'retry' || target !== 'projected' || !store.mutateJob)
87
+ return;
88
+ await store.mutateJob(job.id, current => {
89
+ const currentMerge = current.localMerge;
90
+ if (current.projectionRecovery?.key !== recovery.key
91
+ || current.projectionRecovery.taskId !== recovery.taskId
92
+ || current.projectionRecovery.status !== 'retry'
93
+ || current.projectionRecovery.attempts !== recovery.attempts + 1
94
+ || currentMerge?.sourceRef !== localMerge.sourceRef || currentMerge.targetRef !== localMerge.targetRef
95
+ || currentMerge.status !== localMerge.status || currentMerge.sourceSha !== localMerge.sourceSha
96
+ || currentMerge.mergedCommitSha !== localMerge.mergedCommitSha)
97
+ return current;
98
+ finalized = true;
99
+ return { ...current, localMerge: projectedJob.localMerge,
100
+ projectionRecovery: { ...current.projectionRecovery, status: 'projected', updatedAt: new Date().toISOString() },
101
+ updatedAt: new Date().toISOString() };
102
+ });
103
+ },
104
+ });
105
+ projectionCompleted = finalized;
106
+ if (!deliveredDecision.accepted || deliveredDecision.target !== 'projected' || !finalized)
107
+ projectedMerge = undefined;
108
+ return {};
109
+ }
110
+ if (actionId === 'projection.escalate') {
111
+ if (escalated)
112
+ await options.eventBus?.publish(eventFor(job.id, 'projection', null, 'projection.recovery_escalated', 'failed', 'Projection recovery exhausted its bounded retry path', 'user', { idempotencyKey: recovery.key, attempts: recovery.attempts }, { agent: 'relay', component: 'projection-conflict' })).catch(() => { });
113
+ return {};
114
+ }
115
+ if (actionId === 'projection.createRecoveryTask')
116
+ return {};
117
+ throw new Error(`Unexpected projection recovery action ${actionId}`);
118
+ },
119
+ });
120
+ return { state: projectionCompleted ? 'projected' : decision.accepted ? decision.target : '', projectedMerge };
121
+ }
@@ -0,0 +1,101 @@
1
+ import { createTaskEntity } from './workflow/entity-creation.js';
2
+ import { dispatchWorkflowGraph } from './workflow/dispatcher.js';
3
+ import { projectionConflictV2 } from './workflow/projection-conflict-v2.js';
4
+ import { taskOverallTimeoutMs } from './types.js';
5
+ const MAX_RECOVERY_ATTEMPTS = 2;
6
+ function taskFor(job, remoteSha, now) {
7
+ const key = `${job.id}:${remoteSha}`;
8
+ const id = `projection-recovery-${remoteSha}`;
9
+ return createTaskEntity(job, {
10
+ jobId: job.id, id, title: `Reconcile main divergence for ${job.title}`, kind: 'work',
11
+ priority: job.priority ?? 'P1', storySize: 'M', projectName: job.title,
12
+ agentName: 'implementation', parentTaskId: null, dependencies: [],
13
+ capabilities: ['implementation'], workerRequirements: { capabilities: ['implementation'] },
14
+ adapter: 'codex', input: [
15
+ `Projection recovery key: ${key}`,
16
+ `Observed origin/main SHA: ${remoteSha}`,
17
+ `Job ${job.id} is already COMPLETED; preserve that Job state.`,
18
+ 'Inspect local main and origin/main, reconcile their divergence with a normal merge or rebase that preserves both histories, and never force-push.',
19
+ 'Do not publish or rewrite refs. Record the resulting local main SHA and verify it contains the completed Job merge and the observed remote main history.',
20
+ 'If the histories cannot be reconciled safely, stop and explain the exact blocker.',
21
+ ].join('\n'), output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null,
22
+ attempt: 0, maxAttempts: 2, leaseOwner: null, leaseExpiresAt: null,
23
+ executionId: null, threadId: null, result: null, plannerResult: null, error: null,
24
+ timeoutMs: taskOverallTimeoutMs('M'), createdAt: now, updatedAt: now,
25
+ });
26
+ }
27
+ /** Select graph policy from SQLite facts, then atomically materialize its selected recovery action. */
28
+ export async function beginProjectionRecovery(store, jobId, remoteSha, error) {
29
+ if (!store.mutateJob)
30
+ throw new Error('Projection recovery requires atomic SQLite Job mutation');
31
+ let created = false;
32
+ let escalated = false;
33
+ const snapshot = await store.loadJob(jobId);
34
+ if (!snapshot)
35
+ return false;
36
+ const sameKey = snapshot.projectionRecovery?.key === `${jobId}:${remoteSha}`;
37
+ const expectedRecoveryKey = snapshot.projectionRecovery?.key ?? null;
38
+ const expectedAttempts = snapshot.projectionRecovery?.attempts ?? 0;
39
+ const expectedRecoveryStatus = snapshot.projectionRecovery?.status ?? null;
40
+ const expectedRecoveryRemoteSha = snapshot.projectionRecovery?.remoteSha ?? null;
41
+ const recoveryInProgress = snapshot.projectionRecovery?.status === 'repairing'
42
+ || (snapshot.projectionRecovery?.status === 'retry' && snapshot.projectionRecovery.remoteSha === remoteSha);
43
+ const decision = await dispatchWorkflowGraph(projectionConflictV2, {
44
+ initialState: 'classify', input: { transport: 'event', id: 'projection.delivery_failed', payload: {} },
45
+ durableFacts: {
46
+ jobState: snapshot.state,
47
+ mergeSucceeded: snapshot.localMerge?.status === 'succeeded',
48
+ projectionStrategy: snapshot.localMerge?.projectionStrategy,
49
+ divergenceConfirmed: /non-fast-forward|fetch first|diverg|not a fast-forward|rejected/i.test(error),
50
+ remoteShaAvailable: /^[0-9a-f]{40,64}$/i.test(remoteSha),
51
+ sameIdempotencyKey: sameKey,
52
+ recoveryInProgress,
53
+ attempts: snapshot.projectionRecovery?.attempts ?? 0,
54
+ maxAttempts: MAX_RECOVERY_ATTEMPTS,
55
+ }, graphName: 'projection-conflict', maxSteps: 1,
56
+ executeAction: async (actionId) => {
57
+ if (actionId === 'projection.escalate') {
58
+ await store.mutateJob(jobId, current => {
59
+ if (current.state === 'COMPLETED'
60
+ && (current.projectionRecovery?.key ?? null) === expectedRecoveryKey
61
+ && (current.projectionRecovery?.attempts ?? 0) === expectedAttempts
62
+ && current.projectionRecovery?.status === snapshot.projectionRecovery?.status
63
+ && current.projectionRecovery) {
64
+ current.projectionRecovery = { ...current.projectionRecovery, key: `${jobId}:${remoteSha}`, remoteSha,
65
+ status: 'escalated', error, updatedAt: new Date().toISOString() };
66
+ escalated = true;
67
+ }
68
+ return current;
69
+ });
70
+ return {};
71
+ }
72
+ if (actionId !== 'projection.createRecoveryTask')
73
+ throw new Error(`Unexpected projection action ${actionId}`);
74
+ const now = new Date().toISOString();
75
+ await store.mutateJob(jobId, current => {
76
+ if (current.state !== 'COMPLETED' || current.localMerge?.status !== 'succeeded'
77
+ || (current.projectionRecovery?.key ?? null) !== expectedRecoveryKey
78
+ || (current.projectionRecovery?.attempts ?? 0) !== expectedAttempts
79
+ || (current.projectionRecovery?.status ?? null) !== expectedRecoveryStatus
80
+ || (current.projectionRecovery?.remoteSha ?? null) !== expectedRecoveryRemoteSha
81
+ || current.projectionRecovery?.status === 'repairing'
82
+ || current.projectionRecovery?.key === `${jobId}:${remoteSha}`)
83
+ return current;
84
+ const task = taskFor(current, remoteSha, now);
85
+ if (!current.tasks.some(item => item.id === task.id))
86
+ current.tasks.push(task);
87
+ const recovery = {
88
+ key: `${jobId}:${remoteSha}`, remoteSha, status: 'repairing', taskId: task.id,
89
+ attempts: current.projectionRecovery?.attempts ?? 0,
90
+ maxAttempts: MAX_RECOVERY_ATTEMPTS, error, updatedAt: now,
91
+ };
92
+ current.projectionRecovery = recovery;
93
+ current.updatedAt = now;
94
+ created = true;
95
+ return current;
96
+ });
97
+ return {};
98
+ },
99
+ });
100
+ return decision.accepted ? (created ? 'created' : escalated ? 'escalated' : false) : false;
101
+ }
@@ -18,6 +18,8 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
18
18
  // immutable input SHA is already present in the authoritative Job.
19
19
  if (this.options.noGitHub && this.options.localMergeEffect)
20
20
  return job.headSha ?? job.review?.headSha ?? null;
21
+ if (this.options.headShaProvider)
22
+ return this.options.headShaProvider(job);
21
23
  // GitHub PR head facts are audit-only. Source identity is local Git.
22
24
  if (!this.options.projectPathForJob)
23
25
  return job.headSha ?? job.review?.headSha ?? null;
@@ -37,7 +39,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
37
39
  return {
38
40
  unscheduled: !job.schedule,
39
41
  reviewEnabled: job.reviewEnabled !== false,
40
- jobActive: job.state === 'ACTIVE',
42
+ jobActive: job.state === 'ACTIVE' || job.projectionRecovery?.status === 'repairing',
41
43
  implementationComplete: implementationComplete(job),
42
44
  headAvailable: Boolean(headSha),
43
45
  completionGateSatisfied: completionGateSatisfied(job),
@@ -141,7 +143,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
141
143
  }, this.options.reviewer?.threadId);
142
144
  const input = [
143
145
  `Job ID: ${job.id}`,
144
- `Review PR #${reviewPrNumber} at exactly head SHA ${headSha}.`,
146
+ reviewPrNumber > 0 ? `Review PR #${reviewPrNumber} at exactly head SHA ${headSha}.` : `Review the local Job artifact at exactly head SHA ${headSha}.`,
145
147
  `Review head SHA: ${headSha}`,
146
148
  'Job objective: ' + (job.objective ?? job.description ?? job.title),
147
149
  'Use the selected reviewer agent contract as the authoritative review protocol.',
@@ -150,7 +152,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
150
152
  'Do not treat historical failed attempts as current artifact defects; distinguish preserved baseline files from implementation.',
151
153
  ].join('\n\n');
152
154
  const task = createTaskEntity(job, {
153
- jobId: job.id, id, title: `Final review: ${job.title}`, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'reviewer',
155
+ jobId: job.id, id, title: `Review ${headSha.slice(0, 8)}`, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'reviewer',
154
156
  parentTaskId: null, dependencies: reviewGateTasks.map(item => item.id), capabilities: ['review'],
155
157
  workerSelection: reviewerRouting.selection, adapter: reviewerRouting.candidate.adapter,
156
158
  input,
@@ -355,7 +357,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
355
357
  if (task.reviewHeadSha !== headSha)
356
358
  return { applied: false, reconcile: false, reason: event.type + '.stale' };
357
359
  const currentHeadSha = await this.currentHeadSha(job);
358
- if ((currentHeadSha && currentHeadSha !== headSha) || (job.headSha && job.headSha !== headSha))
360
+ if (currentHeadSha ? currentHeadSha !== headSha : Boolean(job.headSha && job.headSha !== headSha))
359
361
  return { applied: false, reconcile: false, reason: event.type + '.stale' };
360
362
  // Planner-created reviewer tasks predate the final-review Job gate. Bind
361
363
  // their first trustworthy review event to PENDING before dispatching it.
@@ -388,8 +390,10 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
388
390
  const now = event.timestamp || new Date().toISOString();
389
391
  const id = 'review-fix-' + headSha.slice(0, 8);
390
392
  if (!job.tasks.some(item => item.id === id)) {
391
- const input = 'Address all reviewer findings for PR #' + this.externalReviewPrNumber(job) + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
392
- const fixTask = createTaskEntity(job, { jobId: job.id, id, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'implementation', parentTaskId: task.id, dependencies: [task.id], capabilities: ['implementation'], workerRequirements: { capabilities: [] }, adapter: 'codex', input, output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null, attempt: 0, maxAttempts: 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: 0, createdAt: now, updatedAt: now });
393
+ const fixPrNumber = this.externalReviewPrNumber(job);
394
+ const fixTarget = fixPrNumber > 0 ? `PR #${fixPrNumber}` : `Job ${job.id} artifact`;
395
+ const input = 'Address all reviewer findings for ' + fixTarget + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
396
+ const fixTask = createTaskEntity(job, { jobId: job.id, id, title: `Fix review ${headSha.slice(0, 8)}`, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'implementation', parentTaskId: task.id, dependencies: [task.id], capabilities: ['implementation'], workerRequirements: { capabilities: [] }, adapter: 'codex', input, output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null, attempt: 0, maxAttempts: 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: 0, createdAt: now, updatedAt: now });
393
397
  pendingFixTask = fixTask;
394
398
  }
395
399
  return {};
@@ -435,6 +439,8 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
435
439
  if ((!current.review || (current.review.outcome === 'stale' && current.review.headSha !== headSha)) && freshTask.planningDecisionId) {
436
440
  current.review = { outcome: 'pending', headSha, taskId: freshTask.id, evidence: null, updatedAt: reviewUpdatedAt };
437
441
  }
442
+ if (currentHeadSha === headSha)
443
+ current.headSha = headSha;
438
444
  freshTask.error = null;
439
445
  if (freshDecision.target !== 'RUNNING')
440
446
  freshTask.result = { summary: event.message || `Review ${event.type}`, data: event.data ?? {} };
@@ -224,7 +224,6 @@ export class ReconcilerRuntimeBase {
224
224
  const outcome = await this.store.verifyWorkflowLaunch(event.job_id, event.task_id, transition.task.executionId);
225
225
  if (outcome === 'Failed')
226
226
  throw Error('worker start postcondition violated');
227
- // Unknown remains durable and never authorizes repeating this generation.
228
227
  }
229
228
  if (live?.executionId === transition.task.executionId)
230
229
  live.workerStarted = true;
@@ -114,7 +114,7 @@ export class Reconciler extends ReconcilerReviewBase {
114
114
  } });
115
115
  return job;
116
116
  }
117
- if (job.state !== 'ACTIVE')
117
+ if (job.state !== 'ACTIVE' && job.projectionRecovery?.status !== 'repairing')
118
118
  return job;
119
119
  await recoverExpiredExecutions(job, now, this.executionRecoveryPorts(job));
120
120
  await repairLegacyOrchestrators(jobId, now, { store: this.store, owner: this.options.owner, emit: event => this.emit(event) });
@@ -202,7 +202,7 @@ export class Reconciler extends ReconcilerReviewBase {
202
202
  await this.ensureFinalReview(finalState, headSha);
203
203
  await this.dispatchExecutableTasks(jobId);
204
204
  finalState = await this.store.load(jobId);
205
- const localMergeCompleted = await reconcileLocalGitCompletion(this.store, finalState, headSha, this.options.noGitHub === true, this.options.projectPathForJob, event => this.emit(event), this.options.localMergeEffect, () => (this.options.now ?? new Date()).getTime());
205
+ const localMergeCompleted = await reconcileLocalGitCompletion(this.store, finalState, headSha, this.options.noGitHub === true, this.options.projectPathForJob, event => this.emit(event), this.options.localMergeEffect ?? this.options.localMergeDelivery, () => (this.options.now ?? new Date()).getTime());
206
206
  await drainProductionWorkflowOutbox(this.store, this.options);
207
207
  if (localMergeCompleted)
208
208
  return this.store.load(jobId);
@@ -163,7 +163,7 @@ export class SQLiteRelayStore {
163
163
  }
164
164
  }
165
165
  workflowOperations() {
166
- return new SQLiteWorkflowOperations(this.db, (job, at) => this.writeJobSnapshot(job, at));
166
+ return new SQLiteWorkflowOperations(this.db, (job, at) => this.writeJobSnapshot(job, at), (request, now) => { this.enqueueProjectionInTransaction(request, now); });
167
167
  }
168
168
  async commitWorkflowMutationAndEvents(jobId, mutation, events) {
169
169
  this.assertScope(jobId);
@@ -5,13 +5,45 @@ import { dispatchWorkflowGraphSync } from './workflow/dispatcher.js';
5
5
  import { WorkflowActionLedger } from './workflow/action-ledger.js';
6
6
  import { ConditionRegistry, booleanCondition } from './workflow/conditions.js';
7
7
  import { DurableWorkflowPublication } from './workflow/durable-publication.js';
8
+ import { randomUUID } from 'node:crypto';
9
+ import { appendTaskComment } from './task-comments.js';
8
10
  function monotonicJobTimestamp(previous, candidate = new Date().toISOString()) { const a = Date.parse(previous), b = Date.parse(candidate); return new Date(Math.max(Number.isFinite(a) ? a + 1 : Date.now(), Number.isFinite(b) ? b : Date.now())).toISOString(); }
11
+ function recordTaskStateHistory(current, next) {
12
+ const appended = [];
13
+ for (const task of next.tasks) {
14
+ const before = current.tasks.find(item => item.id === task.id);
15
+ if (!before)
16
+ continue;
17
+ for (const field of ['resolution', 'escalation']) {
18
+ const prior = before[field] ?? null, value = task[field] ?? null;
19
+ if (JSON.stringify(prior) === JSON.stringify(value) || value === null)
20
+ continue;
21
+ const resolution = field === 'resolution' ? value : null;
22
+ const escalation = field === 'escalation' ? value : null;
23
+ const kind = resolution ? 'resolution.changed'
24
+ : escalation?.status === 'responded' ? 'escalation.responded' : 'escalation.requested';
25
+ const action = resolution ? `Authoritative resolution changed: ${JSON.stringify(resolution)}`
26
+ : escalation?.status === 'responded'
27
+ ? `Human response to escalation ${escalation.id}: ${escalation.response ?? ''}. Re-evaluate the historical resolution condition against original execution evidence.`
28
+ : `Authoritative escalation changed: ${JSON.stringify(escalation)}`;
29
+ appended.push(appendTaskComment(next, task.id, action, {
30
+ author: escalation?.status === 'responded' ? 'human' : 'relay', kind,
31
+ executionId: resolution ? before.executionId ?? task.executionId ?? undefined : escalation?.originalExecutionId ?? before.executionId ?? undefined,
32
+ transition: resolution ? `resolution.${resolution.state}` : `escalation.${escalation?.status}`,
33
+ idempotencyKey: `task-state/${next.id}/${task.id}/${field}/${randomUUID()}`,
34
+ }));
35
+ }
36
+ }
37
+ return appended;
38
+ }
9
39
  export class SQLiteWorkflowOperations {
10
40
  db;
11
41
  write;
12
- constructor(db, write) {
42
+ enqueue;
43
+ constructor(db, write, enqueue) {
13
44
  this.db = db;
14
45
  this.write = write;
46
+ this.enqueue = enqueue;
15
47
  }
16
48
  async commitWorkflowMutationAndEvents(jobId, mutation, events) {
17
49
  let committed;
@@ -21,15 +53,20 @@ export class SQLiteWorkflowOperations {
21
53
  throw new Error(`Unknown Job ${jobId}`);
22
54
  const current = normalizeLegacyJobPrNumber(JSON.parse(row.json));
23
55
  current.tasks = current.tasks.map(task => normalizeLegacyTaskLifecycle(task, current.executionMode));
56
+ const previous = structuredClone(current);
24
57
  const next = mutation(current);
25
58
  if (next.id !== jobId)
26
59
  throw new Error('Cannot change authoritative Job identity');
27
60
  if (next.state !== current.state && ['COMPLETED', 'CANCELLED'].includes(current.state))
28
61
  throw new Error('Cannot overwrite terminal authoritative state');
62
+ const history = recordTaskStateHistory(previous, next);
29
63
  const beforeTime = Date.parse(current.updatedAt);
30
64
  if (Number.isFinite(beforeTime) && !(Date.parse(next.updatedAt) > beforeTime))
31
65
  next.updatedAt = new Date(beforeTime + 1).toISOString();
32
66
  this.write(next, new Date().toISOString());
67
+ if (next.prNumber > 0)
68
+ for (const comment of history)
69
+ this.enqueue?.({ idempotencyKey: `append_task_comment:${comment.idempotencyKey ?? comment.id}`, operation: 'append_task_comment', payload: { operation: 'append_task_comment', value: comment }, priority: 20 }, new Date().toISOString());
33
70
  committed = structuredClone(next);
34
71
  }, () => events.map(item => ({ key: item.key, eventId: item.event.type, payload: item.event })));
35
72
  return committed;
@@ -85,14 +122,19 @@ export class SQLiteWorkflowOperations {
85
122
  throw new Error(`Local authoritative Job ${jobId} not found`);
86
123
  const current = normalizeLegacyJobPrNumber(JSON.parse(row.json));
87
124
  current.tasks = current.tasks.map(task => normalizeLegacyTaskLifecycle(task, current.executionMode));
125
+ const previous = structuredClone(current);
88
126
  const before = JSON.stringify(current);
89
127
  const next = mutation(current);
128
+ const history = recordTaskStateHistory(previous, next);
90
129
  if (next.id !== jobId)
91
130
  throw new Error('Cannot change authoritative Job identity');
92
131
  if (JSON.stringify(next) !== before && next.updatedAt <= JSON.parse(before).updatedAt) {
93
132
  next.updatedAt = monotonicJobTimestamp(JSON.parse(before).updatedAt);
94
133
  }
95
134
  this.write(next, new Date().toISOString());
135
+ if (next.prNumber > 0)
136
+ for (const comment of history)
137
+ this.enqueue?.({ idempotencyKey: `append_task_comment:${comment.idempotencyKey ?? comment.id}`, operation: 'append_task_comment', payload: { operation: 'append_task_comment', value: comment }, priority: 20 }, new Date().toISOString());
96
138
  this.db.exec('COMMIT');
97
139
  return structuredClone(next);
98
140
  }
package/dist/relayd.js CHANGED
@@ -20,6 +20,7 @@ import { projectAuthoritativeLocalMerge } from './local-merge-projection.js';
20
20
  import { archiveDisplayPullRequest } from './github-pr-archive.js';
21
21
  import { orderGitHubBindingCandidates } from './github-binding-order.js';
22
22
  import { ProjectionFailureDeduplicator } from './projection-failure-dedupe.js';
23
+ import { handleGitHubProjectionFailure, projectBindingLocalMerge } from './projection-failure-recovery.js';
23
24
  import { authoritativeBranchPushArgs } from './github-branch-projection.js';
24
25
  import { githubAuthContext } from './github-auth.js';
25
26
  import { DisabledGitHubClient, noGitHubMode } from './no-github.js';
@@ -320,8 +321,23 @@ export async function runDaemon(argv) {
320
321
  const github = new GitHubStore(client, job.repository, prNumber, trusted);
321
322
  projectionWriters.set(`job:${job.id}`, new GitHubProjectionWriter(relay, github, undefined, true, effect));
322
323
  if (job.state === 'COMPLETED' && job.localMerge?.status === 'succeeded' && job.localMerge.projectionStrategy === 'target_ref_push') {
323
- const injection = await consumeMethodInjection(relay, job.repository, job.id, 'github.projectLocalMerge');
324
- const projectedMerge = await withTestInjection('github.projectLocalMerge', injection, () => projectAuthoritativeLocalMerge(canonical, job));
324
+ const projection = await projectBindingLocalMerge(job, relay, {
325
+ projectPath: canonical,
326
+ project: async (projectedJob) => withTestInjection('github.projectLocalMerge', await consumeMethodInjection(relay, job.repository, job.id, 'github.projectLocalMerge'), () => projectAuthoritativeLocalMerge(canonical, projectedJob)),
327
+ verifyObservedRemote: async (sha) => {
328
+ try {
329
+ await gitText(canonical, ['merge-base', '--is-ancestor', sha, 'refs/heads/main']);
330
+ }
331
+ catch {
332
+ throw new Error(`Projection recovery did not preserve observed origin/main ${sha} in local main`);
333
+ }
334
+ }, eventBus: bus,
335
+ });
336
+ if (projection.skip)
337
+ continue;
338
+ const projectedMerge = projection.projectedMerge;
339
+ if (!projectedMerge)
340
+ continue;
325
341
  await bus?.publish(eventFor(job.id, 'projection', null, 'projection.delivered', 'succeeded', `GitHub local merge projection ${projectedMerge.status}`, 'orchestrator', { operation: 'github.projectLocalMerge', deliveryStatus: projectedMerge.status, mergedCommitSha: projectedMerge.localMergedCommitSha }, { agent: 'relay', component: 'github-projection' }));
326
342
  // A review-only PR is never merged by GitHub. Close it after local main
327
343
  // has been safely published, retaining the diff, comments and commit link.
@@ -339,16 +355,9 @@ export async function runDaemon(argv) {
339
355
  }
340
356
  }
341
357
  catch (error) {
342
- const reason = errorMessage(error), classification = classifyFailure(error).classification;
343
- if (!projectionFailureDeduplicator.shouldReport(discovered.id, discovered.updatedAt, reason, classification))
344
- continue;
345
- await bus?.publish(eventFor(discovered.id, 'projection', null, 'projection.delivery_failed', 'failed', `GitHub projection failed: ${errorMessage(error)}`, 'orchestrator', { operation: 'github.projectLocalMerge', error: reason }, { agent: 'relay', component: 'github-projection' }))?.catch(() => { });
346
- observability().record({
347
- stage: 'projection', event: 'github.binding_job_failed', level: 'warn', outcome: 'failed',
348
- message: `GitHub binding/projection failed for ${discovered.id}: ${reason}`,
349
- correlation: { repository: discovered.repository, prNumber: discovered.prNumber, jobId: discovered.id },
350
- data: { classification, reason },
351
- });
358
+ await handleGitHubProjectionFailure(error, discovered, snapshot.path, { relayStores, repositoryPaths,
359
+ explicitProjectPath, workspaceRoot, gitText, reconcileLocalJob: (jobId, reason) => pool.reconcileLocalJob(jobId, reason),
360
+ eventBus: bus, deduplicator: projectionFailureDeduplicator });
352
361
  }
353
362
  }
354
363
  cachedOverviews = await inventoryFromLocal();
@@ -1,7 +1,8 @@
1
1
  /** Called inside SQLite mutateJob against the latest durable Job snapshot. */
2
2
  export function reviewGenerationMatches(current, original, task, headSha) {
3
3
  const fresh = current.tasks.find(item => item.id === task.id);
4
- if (!fresh || fresh.reviewHeadSha !== headSha || current.state !== 'ACTIVE'
4
+ if (!fresh || fresh.reviewHeadSha !== headSha
5
+ || (current.state !== 'ACTIVE' && current.projectionRecovery?.status !== 'repairing')
5
6
  || (current.headSha && current.headSha !== headSha)
6
7
  || fresh.executionId !== task.executionId)
7
8
  return false;
@@ -258,6 +258,22 @@ export class GitHubStore {
258
258
  throw new Error(`JOB_BINDING_CONFLICT: PR #${this.prNumber} contains trusted ${JOB_MARKER} markers for multiple job ids`);
259
259
  }
260
260
  async listAgents() { return (await this.list()).filter(comment => this.isTrusted(comment)).map(x => parseAgent(x.body)).filter((x) => x !== null); }
261
+ async appendTaskHistoryComment(comment) {
262
+ if (!comment.idempotencyKey || comment.jobId === undefined || comment.taskId === undefined)
263
+ throw new Error('Task history projection requires durable identity');
264
+ const marker = `<!-- agents-relay:task-comment:v1 key=${comment.idempotencyKey} -->`;
265
+ if ((await this.list()).some(existing => this.isTrusted(existing) && existing.body.includes(marker)))
266
+ return;
267
+ try {
268
+ await this.comment(`${marker}\n**Task ${comment.taskId} · ${comment.kind ?? 'history'}**\n\n${comment.text}`);
269
+ }
270
+ catch (error) {
271
+ // The server may have accepted the comment before the connection failed.
272
+ // Force a fresh read before the outbox retries this stable logical key.
273
+ this.clearReadSnapshot();
274
+ throw error;
275
+ }
276
+ }
261
277
  async comment(body) {
262
278
  const pr = await this.prNode();
263
279
  const data = await this.graphql('mutation($subjectId:ID!,$body:String!){addComment(input:{subjectId:$subjectId,body:$body}){commentEdge{node{id databaseId url body updatedAt author{login}}}}}', { subjectId: pr.id, body });
@@ -1,11 +1,21 @@
1
1
  import { randomUUID } from 'node:crypto';
2
- export function appendTaskComment(job, taskId, message, author = 'cli') {
2
+ export function appendTaskComment(job, taskId, message, options = 'cli') {
3
3
  const task = job.tasks.find(item => item.id === taskId);
4
4
  if (!task)
5
5
  throw new Error(`Task ${taskId} not found`);
6
6
  if (!message.trim())
7
7
  throw new Error('task comment requires nonempty --message');
8
- const comment = { id: randomUUID(), text: message.trim(), author, createdAt: new Date().toISOString() };
8
+ const config = typeof options === 'string' ? { author: options } : options;
9
+ const key = config.idempotencyKey;
10
+ const existing = key && task.comments?.find(comment => comment.idempotencyKey === key);
11
+ if (existing)
12
+ return existing;
13
+ const comment = {
14
+ id: key ?? randomUUID(), text: message.trim(), author: config.author ?? 'cli', createdAt: new Date().toISOString(),
15
+ jobId: job.id, taskId, ...(config.kind ? { kind: config.kind } : {}),
16
+ ...(config.executionId ? { executionId: config.executionId } : {}),
17
+ ...(config.transition ? { transition: config.transition } : {}), ...(key ? { idempotencyKey: key } : {}),
18
+ };
9
19
  task.comments = [...(task.comments ?? []), comment];
10
20
  return comment;
11
21
  }
@@ -7,10 +7,27 @@ export function taskExecutionContext(job, taskId, executionId) {
7
7
  throw new Error(`Task ${taskId} not found`);
8
8
  if (task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state))
9
9
  throw new Error(`Stale Task execution ${taskId}/${executionId}; current state ${task.state}`);
10
+ const relevant = task.agentName === 'troubleshooter'
11
+ ? job.tasks.filter(item => item.id === task.parentTaskId)
12
+ : task.agentName === 'reviewer' ? task.dependencies.map(id => job.tasks.find(item => item.id === id)).filter((item) => Boolean(item)) : [];
10
13
  return { job: { id: job.id, repository: job.repository, title: job.title, objective: job.objective,
11
- projectPath: job.projectPath, state: job.state }, task, comments: task.comments ?? [], dependencies: task.dependencies.map(id => job.tasks.find(item => item.id === id) ?? { id, missing: true }),
14
+ projectPath: job.projectPath, state: job.state }, task, currentState: task.state, resolution: task.resolution ?? null, escalation: task.escalation ?? null,
15
+ comments: task.comments ?? [], relevantTasks: relevant.map(item => ({ id: item.id, title: item.title, state: item.state, attempt: item.attempt, executionId: item.executionId, resolution: item.resolution ?? null, escalation: item.escalation ?? null, comments: item.comments ?? [], result: item.result, error: item.error, failureReason: item.failureReason })),
16
+ dependencies: task.dependencies.map(id => job.tasks.find(item => item.id === id) ?? { id, missing: true }),
12
17
  plan: job.plans?.find(plan => plan.id === task.planningDecisionId) ?? null };
13
18
  }
19
+ /** Runtime-only task history; durable task.input remains the stable objective. */
20
+ export function taskAgentHistoryContext(job, task) {
21
+ if (task.agentName !== 'troubleshooter' && task.agentName !== 'reviewer')
22
+ return '';
23
+ const related = task.agentName === 'troubleshooter'
24
+ ? job.tasks.filter(item => item.id === task.parentTaskId)
25
+ : task.dependencies.map(id => job.tasks.find(item => item.id === id)).filter((item) => Boolean(item));
26
+ return `\n\n[Authoritative task state and append-only comments]\n${JSON.stringify({
27
+ currentTask: { id: task.id, state: task.state, attempt: task.attempt, executionId: task.executionId, resolution: task.resolution ?? null, escalation: task.escalation ?? null, comments: task.comments ?? [] },
28
+ relatedTasks: related.map(item => ({ id: item.id, title: item.title, state: item.state, attempt: item.attempt, executionId: item.executionId, resolution: item.resolution ?? null, escalation: item.escalation ?? null, comments: item.comments ?? [], result: item.result, error: item.error, failureReason: item.failureReason })),
29
+ }, null, 2)}\nTreat comments as supplementary history. Re-evaluate historical conditions using evidence from the original failing execution; a human response or prior condition alone does not prove recovery.`;
30
+ }
14
31
  /** Browser new-turn envelope: the local Contract and durable Task Context are authoritative. */
15
32
  export function taskReferencePrompt(repository, jobId, taskId, executionId, managedInput) {
16
33
  if (managedInput !== undefined && !managedInput.includes('[Execution event contract — mandatory]'))
@@ -1,4 +1,4 @@
1
- import { randomUUID } from 'node:crypto';
1
+ import { createHash, randomUUID } from 'node:crypto';
2
2
  import { eventFor } from './events.js';
3
3
  import { tryTransitionTaskEvent } from './scheduler.js';
4
4
  import { dispatchTaskResolutionSync, materializeResolution } from './workflow/task-resolution.js';
@@ -20,6 +20,11 @@ function validateRequest(request) {
20
20
  if (!request.context.trim())
21
21
  throw new Error('Escalation requires diagnostic context');
22
22
  }
23
+ function escalationFingerprint(request, executionId) {
24
+ return createHash('sha256').update(JSON.stringify([
25
+ request.requiredAction.trim(), request.resolutionCondition.trim(), request.context.trim(), executionId ?? null,
26
+ ])).digest('hex');
27
+ }
23
28
  /** Relay-owned escalation: the recovery Task, never the failed original, becomes BLOCKED. */
24
29
  export async function requestTroubleshooterEscalation(store, jobId, recoveryTaskId, request, now = new Date().toISOString()) {
25
30
  validateRequest(request);
@@ -35,6 +40,11 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
35
40
  const originalPlan = recovery.recoveryPlanId ? job.plans?.find(plan => plan.id === recovery.recoveryPlanId) : undefined;
36
41
  if ((!original || original.state !== 'FAILED') && (!originalPlan || originalPlan.status !== 'FAILED'))
37
42
  throw new Error(`Troubleshooter ${recoveryTaskId} has no current FAILED original Task or Plan`);
43
+ const fingerprint = escalationFingerprint(request, original?.executionId ?? originalPlan?.executionId);
44
+ if (recovery.escalation?.fingerprint === fingerprint) {
45
+ result = { task: structuredClone(recovery), escalation: structuredClone(recovery.escalation), duplicate: true };
46
+ return job;
47
+ }
38
48
  if (recovery.escalation?.status === 'pending' && recovery.state === 'BLOCKED') {
39
49
  result = { task: structuredClone(recovery), escalation: structuredClone(recovery.escalation), duplicate: true };
40
50
  return job;
@@ -56,6 +66,7 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
56
66
  throw new Error(`Troubleshooter ${recoveryTaskId} cannot be blocked`);
57
67
  const escalation = {
58
68
  id: randomUUID(), originalTaskId: original?.id, originalPlanId: originalPlan?.id,
69
+ fingerprint,
59
70
  originalExecutionId: original?.executionId ?? undefined,
60
71
  recoveryExecutionId: recovery.executionId ?? undefined, requiredAction: request.requiredAction.trim(),
61
72
  resolutionCondition: request.resolutionCondition.trim(), context: request.context.trim(), status: 'pending',
@@ -84,6 +95,8 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
84
95
  await store.saveTask(task); await store.saveJob(current); return current; });
85
96
  if (!result)
86
97
  throw new Error('Escalation did not produce a durable result');
98
+ result.task = structuredClone(job.tasks.find(task => task.id === recoveryTaskId) ?? result.task);
99
+ result.escalation = structuredClone(result.task.escalation ?? result.escalation);
87
100
  return { job, ...result };
88
101
  }
89
102
  /** Authorized human response; only a blocked Troubleshooter may be resumed. */
@@ -102,7 +115,6 @@ export async function resumeTroubleshooter(store, jobId, recoveryTaskId, respons
102
115
  if (!tryTransitionTaskEvent(task, 'task.unblocked', { autonomousJob: false }).accepted)
103
116
  throw new Error(`Troubleshooter ${recoveryTaskId} cannot be resumed`);
104
117
  task.escalation = { ...task.escalation, status: 'responded', response: response.trim(), respondedAt: now };
105
- task.input += `\n\n## Human response (${now})\n${response.trim()}\n\nRe-validate the original failure against this response. Required resolution condition: ${task.escalation.resolutionCondition}`;
106
118
  task.result = null;
107
119
  task.error = null;
108
120
  task.failureReason = undefined;
@@ -116,6 +128,7 @@ export async function resumeTroubleshooter(store, jobId, recoveryTaskId, respons
116
128
  await store.saveTask(task); await store.saveJob(current); return current; });
117
129
  if (!updated)
118
130
  throw new Error('Troubleshooter was not resumed');
131
+ updated = structuredClone(job.tasks.find(item => item.id === recoveryTaskId) ?? updated);
119
132
  return { job, task: updated };
120
133
  }
121
134
  export function hasPendingTroubleshooterEscalation(job, original) {
@@ -26,7 +26,7 @@ Retrieve this task's lifecycle event history before classifying the root cause o
26
26
  GET http://127.0.0.1:8787/api/events?job=${encodeURIComponent(job.id)}&task=${encodeURIComponent(task.id)}&limit=200
27
27
  Inspect event order, type/status, source, data/phase, duplicate events, missing expected lifecycle events, late events, and execution identifiers. If the event stream points to delivery/runtime failure, inspect the corresponding Relay/worker/tool logs too. Do not treat the final task.error string as the complete root cause.
28
28
 
29
- Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. The final task.error string alone is never sufficient. Classify the evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Select exactly one graph-enabled resolution path: exact_original_retry, same_job_dependency, blocker_job_dependency (with merge or verified mode), escalation_gate, or cancel_supersede. For confirmed adapter/model/provider failure, include --failure-kind, --reason, --evidence and --reroute in task recover; use --exclude-current-route only if the exact failed route should not be selected again. For semantic task failure use --failure-kind task_failure without rerouting. For uncertain failures use --failure-kind unknown and preserve routing unless evidence justifies rerouting. The recovery transaction stores diagnosis, clears old routing only on request, and next execution automatically reroutes with the evidence and exclusions. Troubleshooter must then reactivate/retry this exact original task and verify progress. Use the dedicated agents-relay task recover path with --task-id ${task.id} and --recovery-task-id ${recoveryId}; never use ordinary task retry from a Troubleshooter because manual retry intentionally supersedes active recovery. Do not manually choose a provider/model or create a replacement task; no replacement_task action exists. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. If diagnosis suggests a genuine human/external blocker, do not publish task.blocked directly and do not implement the result branch procedurally. Call agents-relay escalation with the original intent, blocker, and relevant context. Pass any caller-defined notification plan through --on-needs-user/--on-needs-user-file; do not hard-code a channel. The canonical escalation graph invokes Neo escalation and owns the resolved/failed/needs_user branch; follow its selected outcome. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
29
+ Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state, append-only comments, task-scoped event history, and relevant runtime logs before retry. The final task.error string alone is never sufficient. Independently verify every human response against evidence from the exact original failing execution context; a response or historical resolution condition alone never proves recovery. Classify evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Select one graph-enabled resolution path: exact_original_retry, same_job_dependency, blocker_job_dependency, escalation_gate, or cancel_supersede. For confirmed adapter/model/provider failure, include --failure-kind, --reason, --evidence and --reroute in task recover; use --exclude-current-route only if the exact failed route should not be selected again. For semantic task failure use --failure-kind task_failure without rerouting. For uncertain failures use --failure-kind unknown and preserve routing unless evidence justifies rerouting. Request the existing agents-relay task recover or agents-relay escalation operation; submission alone is not recovery. Reload authoritative SQLite state and verify the requested resolution and exact-original retry postconditions before reporting success. Bound self-healing retries to the durable Task retry budget. For a shared worker/runtime/tool defect, use its owning repository's P0/self-job workflow and verify the durable fix before requesting recovery. Never use ordinary task retry from a Troubleshooter because manual retry intentionally supersedes active recovery. Do not manually choose a provider/model or create a replacement task. If diagnosis suggests a genuine human/external blocker, invoke the escalation command; do not publish task.blocked or implement a workflow branch procedurally. The canonical escalation graph invokes Neo escalation and owns the resolved/failed/needs_user branch; Relay applies the selected outcome. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
30
30
  }
31
31
  function belongsToCurrentRun(job, task) {
32
32
  const slot = job.schedule?.currentRunSlot;
@@ -92,7 +92,7 @@ export function pendingTroubleshooterTasks(job) {
92
92
  recoveries.push(createTaskEntity(job, {
93
93
  jobId: job.id,
94
94
  id,
95
- title: `Diagnose and recover failed task ${task.id}`,
95
+ title: `Recover task ${task.id}`,
96
96
  recoveryForAttempt: task.attempt,
97
97
  kind: 'work',
98
98
  priority: 'P0',
@@ -43,8 +43,20 @@ export const jobLifecycleV2 = load('job-lifecycle', JOB_STATES, {
43
43
  implementationComplete: boolGuard('implementationComplete'),
44
44
  headAvailable: boolGuard('headAvailable'),
45
45
  mergeCapable: boolGuard('mergeCapable'),
46
- localCompletionAllowed: boolGuard('localCompletionAllowed'),
47
- localMergeReady: boolGuard('localMergeReady'),
46
+ localCompletionAllowed: (facts) => {
47
+ const values = facts.durableFacts;
48
+ return values.jobActive === true && values.autoMergeEnabled === true && values.schedulePresent !== true
49
+ && values.implementationComplete === true && values.completionGateSatisfied === true
50
+ && values.reviewSatisfied === true
51
+ && (values.localMergeRequired !== true || values.localMergeSucceeded === true);
52
+ },
53
+ localMergeReady: (facts) => {
54
+ const values = facts.durableFacts;
55
+ return values.jobActive === true && values.autoMergeEnabled === true && values.schedulePresent !== true
56
+ && values.implementationComplete === true && values.completionGateSatisfied === true
57
+ && values.reviewSatisfied === true && values.localMergePresent === true
58
+ && values.reviewHeadCurrent === true && values.mergeAttemptReady === true;
59
+ },
48
60
  });
49
61
  function topologyTargets(graph, from) {
50
62
  const source = [...graph.nodes.values()].find(node => node.kind === 'state' && node.semanticId === from);
@@ -25,7 +25,7 @@ function parseLabel(raw, line) {
25
25
  export function parseGraphV2(source) {
26
26
  const lines = source.split(/\r?\n/);
27
27
  // Mermaid YAML frontmatter is metadata only; never execute it as graph policy.
28
- // Preserve original source line positions for diagnostics and coverage reports.
28
+ // Keep source line positions stable even when graph metadata grows.
29
29
  let graphStart = 0;
30
30
  if (lines[0]?.trim() === '---') {
31
31
  let closing = -1;
@@ -34,8 +34,6 @@ export function parseGraphV2(source) {
34
34
  closing = i;
35
35
  break;
36
36
  }
37
- if (!/^(title|description):\s+[^\r\n]+$/.test(lines[i]))
38
- fail(i + 1, 'unsupported graph frontmatter field');
39
37
  }
40
38
  if (closing < 0)
41
39
  fail(1, 'unterminated graph frontmatter');
@@ -3,19 +3,36 @@ import { GraphRuntime } from './graph-v2.js';
3
3
  import { dispatchWorkflowGraphSync } from './dispatcher.js';
4
4
  const source = readFileSync(new URL('../../workflow/projection-conflict.mmd', import.meta.url), 'utf8');
5
5
  const boolGuard = (name) => (facts) => facts.durableFacts[name] === true;
6
+ const guards = {
7
+ deterministicDefinitionConflict: boolGuard('deterministicDefinitionConflict'),
8
+ 'projection.recoveryInProgress': boolGuard('recoveryInProgress'),
9
+ 'projection.recoveryEligible': (facts) => {
10
+ const values = facts.durableFacts;
11
+ return values.jobState === 'COMPLETED' && values.mergeSucceeded === true
12
+ && values.projectionStrategy === 'target_ref_push' && values.divergenceConfirmed === true
13
+ && values.remoteShaAvailable === true
14
+ && values.sameIdempotencyKey !== true && values.recoveryInProgress !== true
15
+ && Number(values.attempts ?? 0) < Number(values.maxAttempts ?? 0);
16
+ },
17
+ 'projection.recoveryExhausted': (facts) => {
18
+ const values = facts.durableFacts;
19
+ return values.jobState === 'COMPLETED' && values.divergenceConfirmed === true && values.remoteShaAvailable === true
20
+ && values.sameIdempotencyKey !== true && values.recoveryInProgress !== true
21
+ && Number(values.attempts ?? 0) >= Number(values.maxAttempts ?? 0);
22
+ },
23
+ 'projection.retryAvailable': (facts) => facts.durableFacts.taskRetryAvailable === true
24
+ && Number(facts.durableFacts.attempts ?? 0) < Number(facts.durableFacts.maxAttempts ?? 0),
25
+ 'projection.repairReviewed': (facts) => facts.durableFacts.repairTaskState === 'SUCCEEDED' && facts.durableFacts.reviewOutcome === 'approved',
26
+ 'projection.repairFailed': (facts) => facts.durableFacts.repairTaskState === 'FAILED' || facts.durableFacts.repairTaskState === 'BLOCKED'
27
+ || facts.durableFacts.reviewOutcome === 'blocked' || facts.durableFacts.reviewTaskState === 'FAILED',
28
+ };
6
29
  export const projectionConflictV2 = new GraphRuntime(source, {
7
- states: new Set(['classify', 'retry', 'conflict']),
8
- guards: { deterministicDefinitionConflict: boolGuard('deterministicDefinitionConflict') },
9
- actions: {},
10
- events: { 'projection.failed': { id: 'projection.failed' } },
30
+ states: new Set(['classify', 'repairing', 'retry', 'conflict', 'escalated', 'projected']),
31
+ guards,
32
+ actions: Object.fromEntries(['projection.createRecoveryTask', 'effect.projectLocalMerge', 'projection.escalate'].map(id => [id, { id }])),
33
+ events: Object.fromEntries(['projection.failed', 'projection.delivery_failed', 'projection.delivered', 'reconcile'].map(id => [id, { id }])),
11
34
  });
12
- /**
13
- * Core projection-failure disposition policy.
14
- *
15
- * A deterministic same-id/different-definition conflict stops consuming retry budget;
16
- * any other failure keeps the normal backoff/retry path. This branch is owned by the
17
- * canonical Mermaid graph, not by a procedural `if` in the projection writer.
18
- */
35
+ /** The decision is graph-owned; callers supply SQLite and inspected Git facts. */
19
36
  export function decideProjectionFailure(facts) {
20
37
  return dispatchWorkflowGraphSync(projectionConflictV2, {
21
38
  initialState: 'classify',
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agents-relay",
3
- "version": "2.0.150",
3
+ "version": "2.0.152",
4
4
  "description": "Durable async agent jobs coordinated through GitHub pull requests",
5
5
  "repository": {
6
6
  "type": "git",
@@ -20,13 +20,14 @@
20
20
  "test:fast": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=2 test/workflow-lifecycle.test.js test/workflow-v2-lifecycle-parity.test.js test/workflow-effects.test.js test/relay-store.test.js test/local-authority.test.js test/local-jobid-routing.test.js test/local-sqlite-jobid.test.js test/no-github.test.js test/scheduled-jobs.test.js test/reconcile-coordinator.test.js test/service-publication.test.js test/github-webhook.test.js",
21
21
  "test:full": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 test/*.test.js && npm run test:verification:l3",
22
22
  "test:verification:l1": "npm run build && node --test test/verification/harness.test.mjs test/verification/contract.test.mjs test/verification/graph-inventory.test.mjs test/verification/dashboard-drilldown.test.mjs test/verification/evidence-matrix.test.mjs",
23
- "test:verification:l3": "npm run build && node --test test/verification/harness.test.mjs test/verification/l3-sqlite.test.mjs test/verification/l3-ingress.test.mjs test/verification/l3-replay.test.mjs test/verification/objective-authority.test.mjs test/verification/l3-convergence.test.mjs test/verification/l3-review-convergence.test.mjs test/verification/l3-human-escalation.test.mjs test/verification/l3-escalation-notifier.test.mjs test/verification/l3-unknown-effect.test.mjs test/verification/l3-cli-actions.test.mjs test/verification/l3-planner-round.test.mjs test/verification/full-recovery-gates.test.mjs test/verification/production-wiring.test.mjs",
23
+ "test:verification:l3": "npm run build && node --test --test-concurrency=1 test/verification/harness.test.mjs test/verification/l3-sqlite.test.mjs test/verification/l3-ingress.test.mjs test/verification/l3-replay.test.mjs test/verification/objective-authority.test.mjs test/verification/l3-convergence.test.mjs test/verification/l3-review-convergence.test.mjs test/verification/l3-human-escalation.test.mjs test/verification/l3-escalation-notifier.test.mjs test/verification/l3-unknown-effect.test.mjs test/verification/l3-cli-actions.test.mjs test/verification/l3-planner-round.test.mjs test/verification/full-recovery-gates.test.mjs test/verification/production-wiring.test.mjs test/verification/l3-full-path.test.mjs test/verification/l3-projection-recovery.test.mjs",
24
24
  "test:verification": "npm run test:verification:l1 && npm run test:verification:l2 && npm run test:verification:l3 && npm run test:verification:cross-a && node --test test/workflow-production-outbox.test.js && npm run test:verification:matrix",
25
25
  "e2e": "node e2e/runner/run.mjs",
26
26
  "test:verification:cross-a": "npm run build && RELAY_A_DIST=$PWD/dist node --test test/verification/action-ledger-cross-branch.test.mjs test/verification/outbox-cross-a.test.mjs",
27
27
  "test:verification:matrix": "node test/verification/recovery-report.mjs /tmp/agents-relay-job481-evidence",
28
28
  "test:verification:l2": "npm run build && node --test test/verification/l2/*.test.mjs",
29
29
  "test:verification:l2:live": "node test/verification/l2/live-provider.mjs",
30
+ "test:verification:l2:live-codex": "npm run build && node test/verification/l2/live-codex-agents.mjs",
30
31
  "test:l2:agents": "npm run build && node test/l2-agent-prompts/evaluate.mjs"
31
32
  },
32
33
  "devDependencies": {
@@ -54,7 +54,7 @@ The event must carry the exact durable `job_id` and `task_id`. Progress events a
54
54
  Agents Relay maps those terminal events to durable state:
55
55
  - `task.completed` → `SUCCEEDED`
56
56
  - `task.failed` → `FAILED`
57
- Relay may authoritatively move a Task to `BLOCKED` for an external condition and may resume it only through an authorized unblock operation. A human response resumes the blocked Troubleshooter, not the failed original Task.
57
+ Relay may authoritatively move a Task to `BLOCKED` for an external condition and may resume it only through an authorized unblock operation. A human response is an append-only Task Comment and resumes the blocked Troubleshooter, not the failed original Task. Task input remains stable; recovery agents receive current authoritative state and relevant comments as execution context.
58
58
 
59
59
  A model-backed task must not launch without an event bus. If no terminal event arrives before the task timeout, Relay records an explicit timeout failure. A Codex process exit or ChatGPT submission receipt alone never changes the task to a terminal state.
60
60
 
@@ -17,7 +17,7 @@ Read and follow the shared Task / Job contract and orchestrator P0/self-job work
17
17
 
18
18
  ## Recovery loop
19
19
 
20
- 1. Retrieve the durable original Job and failed/blocked Task from GitHub.
20
+ 1. Retrieve the durable original Job and Task from the authoritative Relay store with `agents-relay task context`; GitHub is only an optional projection and is never lifecycle authority.
21
21
  2. **Mandatory before diagnosis:** retrieve the failed Task's task-scoped lifecycle event history. When the local Relay dashboard is available, use `GET http://127.0.0.1:8787/api/events?job=<job_id>&task=<task_id>&limit=200`. Inspect event order, type/status, source, data/phase, execution IDs, duplicates, missing expected lifecycle events, and late events. Then inspect task error/failureReason, worker selection, routing, attempt count, task input/output contract, PR state, and relevant runtime/service logs.
22
22
  3. Classify the blocker:
23
23
  - transient execution/delivery/lease failure;
@@ -32,8 +32,8 @@ Read and follow the shared Task / Job contract and orchestrator P0/self-job work
32
32
  - For task-local failure, make the smallest durable repair in the original Job/PR when that is within scope.
33
33
  - For a shared platform/runtime defect, first create or reuse the owning blocker P0/self-job. If it can be completed inside this recovery execution, fix/verify it and retry the original Task. If it cannot be completed now, register the original Task with `agents-relay blocker add` on that blocker Job and finish this Troubleshooter task; do not stay running while waiting for another Job.
34
34
  - For a suspected genuine human/external blocker, do not publish `task.blocked` and do not branch on Neo's result yourself. Invoke `agents-relay escalation` with the original intent, blocker, and relevant context. The canonical graph owns the `resolved`, `failed`, and `needs_user` outcomes. When it returns `needs_user`, call Relay's `task escalate` operation with the concrete required action, resolution condition, and diagnostic context. Relay records the escalation and blocks this Troubleshooter; no worker remains running while waiting.
35
- 7. For an immediate repair, only after the blocker is removed run `agents-relay task recover` for the original Task with this Troubleshooter's recovery Task ID and reconcile its Job. An authorized human response resumes this same Troubleshooter with its diagnostic context; verify the action before requesting original-task recovery. For deferred repair, rely on the durable blocker relation: merge-mode blockers wake matching dependents after merge; release/deploy/runtime blockers use `--resolution verified` and `agents-relay blocker resolve` only after verification.
36
- 8. Verify durable evidence that the original Task moved: at minimum QUEUED after retry, preferably a new execution / task.started / observable progress.
35
+ 7. For an immediate repair, only after the blocker is removed request `agents-relay task recover` with this Troubleshooter's recovery Task ID, then reconcile the original Job. Verify the SQLite postcondition: original Task state QUEUED, expected resolution and recovery correlation. An authorized human response is a Task Comment, not a rewrite of Task input. Retain historical conditions as provenance, independently verify the original failing execution context, and do not submit an identical escalation with no new evidence. Deferred repair uses the durable blocker relation: merge-mode blockers wake dependents after merge; release/deploy/runtime blockers require `--resolution verified` and `agents-relay blocker resolve` only after verification.
36
+ 8. Verify durable evidence the original Task moved: at minimum QUEUED after retry, preferably a new execution / task.started / observable progress.
37
37
  9. Publish this Troubleshooter Task's terminal event only after the recovery outcome is durable.
38
38
 
39
39
  ## Rules
@@ -25,7 +25,7 @@ A Reviewer must **not** publish unstructured `task.completed`: missing verdict o
25
25
 
26
26
  Only a real user action merits escalation. Troubleshooter invokes the Relay escalation operation, specifying the required action, resolution condition, and diagnostic context. Relay durably records the human action and blocks **the Troubleshooter Task**, leaving the original failed Task `FAILED`. This is a Relay state transition, not a third agent terminal event.
27
27
 
28
- When the human responds, the **Troubleshooter Task** is authoritatively unblocked and resumed with its prior context plus the response. Troubleshooter independently verifies resolution, requests an authorized original Task recovery, and only then publishes its own `task.completed`. If unresolved, it continues diagnosis or escalates again; human assertion alone is not technical verification.
28
+ When the human responds, Relay records the response as an append-only Task Comment and authoritatively unblocks the **Troubleshooter Task**. The stable Task input is unchanged. Troubleshooter receives current authoritative state and relevant comments, independently verifies the response against the original failing execution context, requests an authorized original Task recovery, and only then publishes its own `task.completed`. A repeated identical escalation without new evidence is idempotent. If unresolved, it continues diagnosis or escalates again; human assertion alone is not technical verification.
29
29
 
30
30
  See [task-recovery-flows.md](task-recovery-flows.md) for unified architecture, overall flow, failure recovery, and post-blocked resumption.
31
31
 
@@ -32,6 +32,13 @@ A later Planner round must inspect the existing semantic tree and attach review,
32
32
 
33
33
  A Task input records **what is known and what has already been decided before execution**. A Task result records **what execution discovered and what was actually done**.
34
34
 
35
+ After creation, keep `Task.input` stable as the objective and initial evidence.
36
+ Responses, resolution changes, and escalation history belong in append-only
37
+ Task Comments. SQLite atomically records typed comments with authoritative
38
+ resolution/escalation mutations; PR-bound Jobs project those comments through
39
+ the durable idempotent projection outbox. Comments supplement current state and
40
+ never replace `Task.resolution` or `Task.escalation`.
41
+
35
42
  Task input should include, as applicable:
36
43
 
37
44
  - goal and observed behavior;
@@ -1,6 +1,6 @@
1
1
  # Task Failure Recovery and Human Escalation — Agreed Target Design
2
2
 
3
- **Status:** Design agreement; not an assertion of current implementation. This document describes two flows. Existing runtime, workflow graphs, and agent instructions may need changes to conform.
3
+ **Status:** Current recovery contract. SQLite Task snapshots retain append-only typed comments alongside authoritative state.
4
4
 
5
5
  ## Principles and ownership
6
6
 
@@ -133,3 +133,20 @@ flowchart TD
133
133
  - Align escalation graph, CLI, reconciler, and unblock commands so human intervention blocks **Troubleshooter**, not the original failed Task; preserve the original failure and correlated recovery context.
134
134
  - Ensure the blocked Troubleshooter can resume after an authorized human unblock rather than being treated as a finished terminal execution.
135
135
  - Add integrated checks: technical retry, blocker-Job handoff, human request/block, human unblock/resume, failed verification/re-escalation, stale/duplicate responses, and correct original-Task retry.
136
+
137
+ ## Durable recovery history
138
+
139
+ `Task.input` remains the stable objective and original evidence. `Task.resolution` and
140
+ `Task.escalation` remain the current authoritative states; comments are append-only
141
+ history, not a second state authority. SQLite mutation transactions compare the
142
+ before/after authoritative values and append typed comments with Job, Task,
143
+ execution, transition, and idempotency identity in the same commit. Do not add a
144
+ `resolutionHistory` field.
145
+
146
+ Human responses are comments. Resuming a Troubleshooter must not rewrite its
147
+ input or append a stale “Required resolution condition” as new objective text.
148
+ The condition remains in escalation provenance. Reviewer and Troubleshooter
149
+ prompts receive current authoritative state and relevant task comments as runtime
150
+ context; Troubleshooters independently verify a response against evidence from
151
+ the original failing execution. Repeating an identical escalation with unchanged
152
+ evidence is idempotent.
@@ -1,15 +1,66 @@
1
1
  ---
2
2
  title: Projection Conflict
3
- description: Detects and resolves conflicting workflow state projections against authoritative facts.
3
+ description: Recover local-main and origin-main divergence after a completed Job's GitHub projection fails.
4
+ name: projection-conflict
5
+ authority: SQLite Relay Job snapshot
6
+ trigger: projection.delivery_failed
7
+ idempotency: job-id plus observed origin/main SHA
8
+ actions:
9
+ - projection.createRecoveryTask
10
+ - projection.escalate
11
+ effects:
12
+ - effect.projectLocalMerge
13
+ guards:
14
+ - deterministicDefinitionConflict
15
+ - projection.recoveryEligible
16
+ - projection.recoveryExhausted
17
+ - projection.recoveryInProgress
18
+ - projection.retryAvailable
19
+ - projection.repairReviewed
20
+ - projection.repairFailed
4
21
  ---
5
22
  flowchart LR
6
23
  %% relay: initial classify
7
24
  classify["state: classify"]
8
- retry["state: retry"]
9
25
  conflict["state: conflict"]
26
+ repairing["state: repairing"]
27
+ retry["state: retry"]
28
+ escalated["state: escalated"]
29
+ projected["state: projected"]
10
30
 
31
+ eligible["guard: projection.recoveryEligible"]
32
+ inProgress["guard: projection.recoveryInProgress"]
11
33
  deterministic["guard: deterministicDefinitionConflict"]
34
+ exhausted["guard: projection.recoveryExhausted"]
35
+ retryAvailable["guard: projection.retryAvailable"]
36
+ reviewed["guard: projection.repairReviewed"]
37
+ repairFailed["guard: projection.repairFailed"]
38
+ createTask["action: projection.createRecoveryTask"]
39
+ retryProjection["action: effect.projectLocalMerge"]
40
+ escalate["action: projection.escalate"]
12
41
 
42
+ classify -->|on: projection.delivery_failed| inProgress
43
+ inProgress -->|true| classify
44
+ inProgress -->|false| eligible
13
45
  classify -->|on: projection.failed| deterministic
14
46
  deterministic -->|true| conflict
15
47
  deterministic -->|false| retry
48
+ eligible -->|true| createTask
49
+ eligible -->|false| exhausted
50
+ exhausted -->|true| escalate
51
+ exhausted -->|false| classify
52
+ createTask --> repairing
53
+
54
+ repairing -->|on: reconcile| reviewed
55
+ retry -->|on: reconcile| retryAvailable
56
+ reviewed -->|true| retryAvailable
57
+ retryAvailable -->|true| retryProjection
58
+ retryAvailable -->|false| escalate
59
+ reviewed -->|false| repairFailed
60
+ repairFailed -->|true| escalate
61
+ repairFailed -->|false| repairing
62
+ retryProjection --> retry
63
+ escalate --> escalated
64
+
65
+ retry -->|on: projection.delivered| projected
66
+ retry -->|on: projection.delivery_failed| eligible