agents-relay 2.0.149 → 2.0.151

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,101 @@
1
+ import { createTaskEntity } from './workflow/entity-creation.js';
2
+ import { dispatchWorkflowGraph } from './workflow/dispatcher.js';
3
+ import { projectionConflictV2 } from './workflow/projection-conflict-v2.js';
4
+ import { taskOverallTimeoutMs } from './types.js';
5
+ const MAX_RECOVERY_ATTEMPTS = 2;
6
+ function taskFor(job, remoteSha, now) {
7
+ const key = `${job.id}:${remoteSha}`;
8
+ const id = `projection-recovery-${remoteSha}`;
9
+ return createTaskEntity(job, {
10
+ jobId: job.id, id, title: `Reconcile main divergence for ${job.title}`, kind: 'work',
11
+ priority: job.priority ?? 'P1', storySize: 'M', projectName: job.title,
12
+ agentName: 'implementation', parentTaskId: null, dependencies: [],
13
+ capabilities: ['implementation'], workerRequirements: { capabilities: ['implementation'] },
14
+ adapter: 'codex', input: [
15
+ `Projection recovery key: ${key}`,
16
+ `Observed origin/main SHA: ${remoteSha}`,
17
+ `Job ${job.id} is already COMPLETED; preserve that Job state.`,
18
+ 'Inspect local main and origin/main, reconcile their divergence with a normal merge or rebase that preserves both histories, and never force-push.',
19
+ 'Do not publish or rewrite refs. Record the resulting local main SHA and verify it contains the completed Job merge and the observed remote main history.',
20
+ 'If the histories cannot be reconciled safely, stop and explain the exact blocker.',
21
+ ].join('\n'), output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null,
22
+ attempt: 0, maxAttempts: 2, leaseOwner: null, leaseExpiresAt: null,
23
+ executionId: null, threadId: null, result: null, plannerResult: null, error: null,
24
+ timeoutMs: taskOverallTimeoutMs('M'), createdAt: now, updatedAt: now,
25
+ });
26
+ }
27
+ /** Select graph policy from SQLite facts, then atomically materialize its selected recovery action. */
28
+ export async function beginProjectionRecovery(store, jobId, remoteSha, error) {
29
+ if (!store.mutateJob)
30
+ throw new Error('Projection recovery requires atomic SQLite Job mutation');
31
+ let created = false;
32
+ let escalated = false;
33
+ const snapshot = await store.loadJob(jobId);
34
+ if (!snapshot)
35
+ return false;
36
+ const sameKey = snapshot.projectionRecovery?.key === `${jobId}:${remoteSha}`;
37
+ const expectedRecoveryKey = snapshot.projectionRecovery?.key ?? null;
38
+ const expectedAttempts = snapshot.projectionRecovery?.attempts ?? 0;
39
+ const expectedRecoveryStatus = snapshot.projectionRecovery?.status ?? null;
40
+ const expectedRecoveryRemoteSha = snapshot.projectionRecovery?.remoteSha ?? null;
41
+ const recoveryInProgress = snapshot.projectionRecovery?.status === 'repairing'
42
+ || (snapshot.projectionRecovery?.status === 'retry' && snapshot.projectionRecovery.remoteSha === remoteSha);
43
+ const decision = await dispatchWorkflowGraph(projectionConflictV2, {
44
+ initialState: 'classify', input: { transport: 'event', id: 'projection.delivery_failed', payload: {} },
45
+ durableFacts: {
46
+ jobState: snapshot.state,
47
+ mergeSucceeded: snapshot.localMerge?.status === 'succeeded',
48
+ projectionStrategy: snapshot.localMerge?.projectionStrategy,
49
+ divergenceConfirmed: /non-fast-forward|fetch first|diverg|not a fast-forward|rejected/i.test(error),
50
+ remoteShaAvailable: /^[0-9a-f]{40,64}$/i.test(remoteSha),
51
+ sameIdempotencyKey: sameKey,
52
+ recoveryInProgress,
53
+ attempts: snapshot.projectionRecovery?.attempts ?? 0,
54
+ maxAttempts: MAX_RECOVERY_ATTEMPTS,
55
+ }, graphName: 'projection-conflict', maxSteps: 1,
56
+ executeAction: async (actionId) => {
57
+ if (actionId === 'projection.escalate') {
58
+ await store.mutateJob(jobId, current => {
59
+ if (current.state === 'COMPLETED'
60
+ && (current.projectionRecovery?.key ?? null) === expectedRecoveryKey
61
+ && (current.projectionRecovery?.attempts ?? 0) === expectedAttempts
62
+ && current.projectionRecovery?.status === snapshot.projectionRecovery?.status
63
+ && current.projectionRecovery) {
64
+ current.projectionRecovery = { ...current.projectionRecovery, key: `${jobId}:${remoteSha}`, remoteSha,
65
+ status: 'escalated', error, updatedAt: new Date().toISOString() };
66
+ escalated = true;
67
+ }
68
+ return current;
69
+ });
70
+ return {};
71
+ }
72
+ if (actionId !== 'projection.createRecoveryTask')
73
+ throw new Error(`Unexpected projection action ${actionId}`);
74
+ const now = new Date().toISOString();
75
+ await store.mutateJob(jobId, current => {
76
+ if (current.state !== 'COMPLETED' || current.localMerge?.status !== 'succeeded'
77
+ || (current.projectionRecovery?.key ?? null) !== expectedRecoveryKey
78
+ || (current.projectionRecovery?.attempts ?? 0) !== expectedAttempts
79
+ || (current.projectionRecovery?.status ?? null) !== expectedRecoveryStatus
80
+ || (current.projectionRecovery?.remoteSha ?? null) !== expectedRecoveryRemoteSha
81
+ || current.projectionRecovery?.status === 'repairing'
82
+ || current.projectionRecovery?.key === `${jobId}:${remoteSha}`)
83
+ return current;
84
+ const task = taskFor(current, remoteSha, now);
85
+ if (!current.tasks.some(item => item.id === task.id))
86
+ current.tasks.push(task);
87
+ const recovery = {
88
+ key: `${jobId}:${remoteSha}`, remoteSha, status: 'repairing', taskId: task.id,
89
+ attempts: current.projectionRecovery?.attempts ?? 0,
90
+ maxAttempts: MAX_RECOVERY_ATTEMPTS, error, updatedAt: now,
91
+ };
92
+ current.projectionRecovery = recovery;
93
+ current.updatedAt = now;
94
+ created = true;
95
+ return current;
96
+ });
97
+ return {};
98
+ },
99
+ });
100
+ return decision.accepted ? (created ? 'created' : escalated ? 'escalated' : false) : false;
101
+ }
@@ -18,6 +18,8 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
18
18
  // immutable input SHA is already present in the authoritative Job.
19
19
  if (this.options.noGitHub && this.options.localMergeEffect)
20
20
  return job.headSha ?? job.review?.headSha ?? null;
21
+ if (this.options.headShaProvider)
22
+ return this.options.headShaProvider(job);
21
23
  // GitHub PR head facts are audit-only. Source identity is local Git.
22
24
  if (!this.options.projectPathForJob)
23
25
  return job.headSha ?? job.review?.headSha ?? null;
@@ -37,7 +39,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
37
39
  return {
38
40
  unscheduled: !job.schedule,
39
41
  reviewEnabled: job.reviewEnabled !== false,
40
- jobActive: job.state === 'ACTIVE',
42
+ jobActive: job.state === 'ACTIVE' || job.projectionRecovery?.status === 'repairing',
41
43
  implementationComplete: implementationComplete(job),
42
44
  headAvailable: Boolean(headSha),
43
45
  completionGateSatisfied: completionGateSatisfied(job),
@@ -141,7 +143,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
141
143
  }, this.options.reviewer?.threadId);
142
144
  const input = [
143
145
  `Job ID: ${job.id}`,
144
- `Review PR #${reviewPrNumber} at exactly head SHA ${headSha}.`,
146
+ reviewPrNumber > 0 ? `Review PR #${reviewPrNumber} at exactly head SHA ${headSha}.` : `Review the local Job artifact at exactly head SHA ${headSha}.`,
145
147
  `Review head SHA: ${headSha}`,
146
148
  'Job objective: ' + (job.objective ?? job.description ?? job.title),
147
149
  'Use the selected reviewer agent contract as the authoritative review protocol.',
@@ -150,7 +152,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
150
152
  'Do not treat historical failed attempts as current artifact defects; distinguish preserved baseline files from implementation.',
151
153
  ].join('\n\n');
152
154
  const task = createTaskEntity(job, {
153
- jobId: job.id, id, title: `Final review: ${job.title}`, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'reviewer',
155
+ jobId: job.id, id, title: `Review ${headSha.slice(0, 8)}`, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'reviewer',
154
156
  parentTaskId: null, dependencies: reviewGateTasks.map(item => item.id), capabilities: ['review'],
155
157
  workerSelection: reviewerRouting.selection, adapter: reviewerRouting.candidate.adapter,
156
158
  input,
@@ -355,7 +357,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
355
357
  if (task.reviewHeadSha !== headSha)
356
358
  return { applied: false, reconcile: false, reason: event.type + '.stale' };
357
359
  const currentHeadSha = await this.currentHeadSha(job);
358
- if ((currentHeadSha && currentHeadSha !== headSha) || (job.headSha && job.headSha !== headSha))
360
+ if (currentHeadSha ? currentHeadSha !== headSha : Boolean(job.headSha && job.headSha !== headSha))
359
361
  return { applied: false, reconcile: false, reason: event.type + '.stale' };
360
362
  // Planner-created reviewer tasks predate the final-review Job gate. Bind
361
363
  // their first trustworthy review event to PENDING before dispatching it.
@@ -388,8 +390,10 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
388
390
  const now = event.timestamp || new Date().toISOString();
389
391
  const id = 'review-fix-' + headSha.slice(0, 8);
390
392
  if (!job.tasks.some(item => item.id === id)) {
391
- const input = 'Address all reviewer findings for PR #' + this.externalReviewPrNumber(job) + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
392
- const fixTask = createTaskEntity(job, { jobId: job.id, id, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'implementation', parentTaskId: task.id, dependencies: [task.id], capabilities: ['implementation'], workerRequirements: { capabilities: [] }, adapter: 'codex', input, output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null, attempt: 0, maxAttempts: 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: 0, createdAt: now, updatedAt: now });
393
+ const fixPrNumber = this.externalReviewPrNumber(job);
394
+ const fixTarget = fixPrNumber > 0 ? `PR #${fixPrNumber}` : `Job ${job.id} artifact`;
395
+ const input = 'Address all reviewer findings for ' + fixTarget + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
396
+ const fixTask = createTaskEntity(job, { jobId: job.id, id, title: `Fix review ${headSha.slice(0, 8)}`, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'implementation', parentTaskId: task.id, dependencies: [task.id], capabilities: ['implementation'], workerRequirements: { capabilities: [] }, adapter: 'codex', input, output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null, attempt: 0, maxAttempts: 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: 0, createdAt: now, updatedAt: now });
393
397
  pendingFixTask = fixTask;
394
398
  }
395
399
  return {};
@@ -435,6 +439,8 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
435
439
  if ((!current.review || (current.review.outcome === 'stale' && current.review.headSha !== headSha)) && freshTask.planningDecisionId) {
436
440
  current.review = { outcome: 'pending', headSha, taskId: freshTask.id, evidence: null, updatedAt: reviewUpdatedAt };
437
441
  }
442
+ if (currentHeadSha === headSha)
443
+ current.headSha = headSha;
438
444
  freshTask.error = null;
439
445
  if (freshDecision.target !== 'RUNNING')
440
446
  freshTask.result = { summary: event.message || `Review ${event.type}`, data: event.data ?? {} };
@@ -224,7 +224,6 @@ export class ReconcilerRuntimeBase {
224
224
  const outcome = await this.store.verifyWorkflowLaunch(event.job_id, event.task_id, transition.task.executionId);
225
225
  if (outcome === 'Failed')
226
226
  throw Error('worker start postcondition violated');
227
- // Unknown remains durable and never authorizes repeating this generation.
228
227
  }
229
228
  if (live?.executionId === transition.task.executionId)
230
229
  live.workerStarted = true;
@@ -114,7 +114,7 @@ export class Reconciler extends ReconcilerReviewBase {
114
114
  } });
115
115
  return job;
116
116
  }
117
- if (job.state !== 'ACTIVE')
117
+ if (job.state !== 'ACTIVE' && job.projectionRecovery?.status !== 'repairing')
118
118
  return job;
119
119
  await recoverExpiredExecutions(job, now, this.executionRecoveryPorts(job));
120
120
  await repairLegacyOrchestrators(jobId, now, { store: this.store, owner: this.options.owner, emit: event => this.emit(event) });
@@ -202,7 +202,7 @@ export class Reconciler extends ReconcilerReviewBase {
202
202
  await this.ensureFinalReview(finalState, headSha);
203
203
  await this.dispatchExecutableTasks(jobId);
204
204
  finalState = await this.store.load(jobId);
205
- const localMergeCompleted = await reconcileLocalGitCompletion(this.store, finalState, headSha, this.options.noGitHub === true, this.options.projectPathForJob, event => this.emit(event), this.options.localMergeEffect, () => (this.options.now ?? new Date()).getTime());
205
+ const localMergeCompleted = await reconcileLocalGitCompletion(this.store, finalState, headSha, this.options.noGitHub === true, this.options.projectPathForJob, event => this.emit(event), this.options.localMergeEffect ?? this.options.localMergeDelivery, () => (this.options.now ?? new Date()).getTime());
206
206
  await drainProductionWorkflowOutbox(this.store, this.options);
207
207
  if (localMergeCompleted)
208
208
  return this.store.load(jobId);
@@ -163,7 +163,7 @@ export class SQLiteRelayStore {
163
163
  }
164
164
  }
165
165
  workflowOperations() {
166
- return new SQLiteWorkflowOperations(this.db, (job, at) => this.writeJobSnapshot(job, at));
166
+ return new SQLiteWorkflowOperations(this.db, (job, at) => this.writeJobSnapshot(job, at), (request, now) => { this.enqueueProjectionInTransaction(request, now); });
167
167
  }
168
168
  async commitWorkflowMutationAndEvents(jobId, mutation, events) {
169
169
  this.assertScope(jobId);
@@ -5,13 +5,45 @@ import { dispatchWorkflowGraphSync } from './workflow/dispatcher.js';
5
5
  import { WorkflowActionLedger } from './workflow/action-ledger.js';
6
6
  import { ConditionRegistry, booleanCondition } from './workflow/conditions.js';
7
7
  import { DurableWorkflowPublication } from './workflow/durable-publication.js';
8
+ import { randomUUID } from 'node:crypto';
9
+ import { appendTaskComment } from './task-comments.js';
8
10
  function monotonicJobTimestamp(previous, candidate = new Date().toISOString()) { const a = Date.parse(previous), b = Date.parse(candidate); return new Date(Math.max(Number.isFinite(a) ? a + 1 : Date.now(), Number.isFinite(b) ? b : Date.now())).toISOString(); }
11
+ function recordTaskStateHistory(current, next) {
12
+ const appended = [];
13
+ for (const task of next.tasks) {
14
+ const before = current.tasks.find(item => item.id === task.id);
15
+ if (!before)
16
+ continue;
17
+ for (const field of ['resolution', 'escalation']) {
18
+ const prior = before[field] ?? null, value = task[field] ?? null;
19
+ if (JSON.stringify(prior) === JSON.stringify(value) || value === null)
20
+ continue;
21
+ const resolution = field === 'resolution' ? value : null;
22
+ const escalation = field === 'escalation' ? value : null;
23
+ const kind = resolution ? 'resolution.changed'
24
+ : escalation?.status === 'responded' ? 'escalation.responded' : 'escalation.requested';
25
+ const action = resolution ? `Authoritative resolution changed: ${JSON.stringify(resolution)}`
26
+ : escalation?.status === 'responded'
27
+ ? `Human response to escalation ${escalation.id}: ${escalation.response ?? ''}. Re-evaluate the historical resolution condition against original execution evidence.`
28
+ : `Authoritative escalation changed: ${JSON.stringify(escalation)}`;
29
+ appended.push(appendTaskComment(next, task.id, action, {
30
+ author: escalation?.status === 'responded' ? 'human' : 'relay', kind,
31
+ executionId: resolution ? before.executionId ?? task.executionId ?? undefined : escalation?.originalExecutionId ?? before.executionId ?? undefined,
32
+ transition: resolution ? `resolution.${resolution.state}` : `escalation.${escalation?.status}`,
33
+ idempotencyKey: `task-state/${next.id}/${task.id}/${field}/${randomUUID()}`,
34
+ }));
35
+ }
36
+ }
37
+ return appended;
38
+ }
9
39
  export class SQLiteWorkflowOperations {
10
40
  db;
11
41
  write;
12
- constructor(db, write) {
42
+ enqueue;
43
+ constructor(db, write, enqueue) {
13
44
  this.db = db;
14
45
  this.write = write;
46
+ this.enqueue = enqueue;
15
47
  }
16
48
  async commitWorkflowMutationAndEvents(jobId, mutation, events) {
17
49
  let committed;
@@ -21,15 +53,20 @@ export class SQLiteWorkflowOperations {
21
53
  throw new Error(`Unknown Job ${jobId}`);
22
54
  const current = normalizeLegacyJobPrNumber(JSON.parse(row.json));
23
55
  current.tasks = current.tasks.map(task => normalizeLegacyTaskLifecycle(task, current.executionMode));
56
+ const previous = structuredClone(current);
24
57
  const next = mutation(current);
25
58
  if (next.id !== jobId)
26
59
  throw new Error('Cannot change authoritative Job identity');
27
60
  if (next.state !== current.state && ['COMPLETED', 'CANCELLED'].includes(current.state))
28
61
  throw new Error('Cannot overwrite terminal authoritative state');
62
+ const history = recordTaskStateHistory(previous, next);
29
63
  const beforeTime = Date.parse(current.updatedAt);
30
64
  if (Number.isFinite(beforeTime) && !(Date.parse(next.updatedAt) > beforeTime))
31
65
  next.updatedAt = new Date(beforeTime + 1).toISOString();
32
66
  this.write(next, new Date().toISOString());
67
+ if (next.prNumber > 0)
68
+ for (const comment of history)
69
+ this.enqueue?.({ idempotencyKey: `append_task_comment:${comment.idempotencyKey ?? comment.id}`, operation: 'append_task_comment', payload: { operation: 'append_task_comment', value: comment }, priority: 20 }, new Date().toISOString());
33
70
  committed = structuredClone(next);
34
71
  }, () => events.map(item => ({ key: item.key, eventId: item.event.type, payload: item.event })));
35
72
  return committed;
@@ -85,14 +122,19 @@ export class SQLiteWorkflowOperations {
85
122
  throw new Error(`Local authoritative Job ${jobId} not found`);
86
123
  const current = normalizeLegacyJobPrNumber(JSON.parse(row.json));
87
124
  current.tasks = current.tasks.map(task => normalizeLegacyTaskLifecycle(task, current.executionMode));
125
+ const previous = structuredClone(current);
88
126
  const before = JSON.stringify(current);
89
127
  const next = mutation(current);
128
+ const history = recordTaskStateHistory(previous, next);
90
129
  if (next.id !== jobId)
91
130
  throw new Error('Cannot change authoritative Job identity');
92
131
  if (JSON.stringify(next) !== before && next.updatedAt <= JSON.parse(before).updatedAt) {
93
132
  next.updatedAt = monotonicJobTimestamp(JSON.parse(before).updatedAt);
94
133
  }
95
134
  this.write(next, new Date().toISOString());
135
+ if (next.prNumber > 0)
136
+ for (const comment of history)
137
+ this.enqueue?.({ idempotencyKey: `append_task_comment:${comment.idempotencyKey ?? comment.id}`, operation: 'append_task_comment', payload: { operation: 'append_task_comment', value: comment }, priority: 20 }, new Date().toISOString());
96
138
  this.db.exec('COMMIT');
97
139
  return structuredClone(next);
98
140
  }
package/dist/relayd.js CHANGED
@@ -20,6 +20,7 @@ import { projectAuthoritativeLocalMerge } from './local-merge-projection.js';
20
20
  import { archiveDisplayPullRequest } from './github-pr-archive.js';
21
21
  import { orderGitHubBindingCandidates } from './github-binding-order.js';
22
22
  import { ProjectionFailureDeduplicator } from './projection-failure-dedupe.js';
23
+ import { handleGitHubProjectionFailure, projectBindingLocalMerge } from './projection-failure-recovery.js';
23
24
  import { authoritativeBranchPushArgs } from './github-branch-projection.js';
24
25
  import { githubAuthContext } from './github-auth.js';
25
26
  import { DisabledGitHubClient, noGitHubMode } from './no-github.js';
@@ -320,8 +321,23 @@ export async function runDaemon(argv) {
320
321
  const github = new GitHubStore(client, job.repository, prNumber, trusted);
321
322
  projectionWriters.set(`job:${job.id}`, new GitHubProjectionWriter(relay, github, undefined, true, effect));
322
323
  if (job.state === 'COMPLETED' && job.localMerge?.status === 'succeeded' && job.localMerge.projectionStrategy === 'target_ref_push') {
323
- const injection = await consumeMethodInjection(relay, job.repository, job.id, 'github.projectLocalMerge');
324
- const projectedMerge = await withTestInjection('github.projectLocalMerge', injection, () => projectAuthoritativeLocalMerge(canonical, job));
324
+ const projection = await projectBindingLocalMerge(job, relay, {
325
+ projectPath: canonical,
326
+ project: async (projectedJob) => withTestInjection('github.projectLocalMerge', await consumeMethodInjection(relay, job.repository, job.id, 'github.projectLocalMerge'), () => projectAuthoritativeLocalMerge(canonical, projectedJob)),
327
+ verifyObservedRemote: async (sha) => {
328
+ try {
329
+ await gitText(canonical, ['merge-base', '--is-ancestor', sha, 'refs/heads/main']);
330
+ }
331
+ catch {
332
+ throw new Error(`Projection recovery did not preserve observed origin/main ${sha} in local main`);
333
+ }
334
+ }, eventBus: bus,
335
+ });
336
+ if (projection.skip)
337
+ continue;
338
+ const projectedMerge = projection.projectedMerge;
339
+ if (!projectedMerge)
340
+ continue;
325
341
  await bus?.publish(eventFor(job.id, 'projection', null, 'projection.delivered', 'succeeded', `GitHub local merge projection ${projectedMerge.status}`, 'orchestrator', { operation: 'github.projectLocalMerge', deliveryStatus: projectedMerge.status, mergedCommitSha: projectedMerge.localMergedCommitSha }, { agent: 'relay', component: 'github-projection' }));
326
342
  // A review-only PR is never merged by GitHub. Close it after local main
327
343
  // has been safely published, retaining the diff, comments and commit link.
@@ -339,16 +355,9 @@ export async function runDaemon(argv) {
339
355
  }
340
356
  }
341
357
  catch (error) {
342
- const reason = errorMessage(error), classification = classifyFailure(error).classification;
343
- if (!projectionFailureDeduplicator.shouldReport(discovered.id, discovered.updatedAt, reason, classification))
344
- continue;
345
- await bus?.publish(eventFor(discovered.id, 'projection', null, 'projection.delivery_failed', 'failed', `GitHub projection failed: ${errorMessage(error)}`, 'orchestrator', { operation: 'github.projectLocalMerge', error: reason }, { agent: 'relay', component: 'github-projection' }))?.catch(() => { });
346
- observability().record({
347
- stage: 'projection', event: 'github.binding_job_failed', level: 'warn', outcome: 'failed',
348
- message: `GitHub binding/projection failed for ${discovered.id}: ${reason}`,
349
- correlation: { repository: discovered.repository, prNumber: discovered.prNumber, jobId: discovered.id },
350
- data: { classification, reason },
351
- });
358
+ await handleGitHubProjectionFailure(error, discovered, snapshot.path, { relayStores, repositoryPaths,
359
+ explicitProjectPath, workspaceRoot, gitText, reconcileLocalJob: (jobId, reason) => pool.reconcileLocalJob(jobId, reason),
360
+ eventBus: bus, deduplicator: projectionFailureDeduplicator });
352
361
  }
353
362
  }
354
363
  cachedOverviews = await inventoryFromLocal();
@@ -1,7 +1,8 @@
1
1
  /** Called inside SQLite mutateJob against the latest durable Job snapshot. */
2
2
  export function reviewGenerationMatches(current, original, task, headSha) {
3
3
  const fresh = current.tasks.find(item => item.id === task.id);
4
- if (!fresh || fresh.reviewHeadSha !== headSha || current.state !== 'ACTIVE'
4
+ if (!fresh || fresh.reviewHeadSha !== headSha
5
+ || (current.state !== 'ACTIVE' && current.projectionRecovery?.status !== 'repairing')
5
6
  || (current.headSha && current.headSha !== headSha)
6
7
  || fresh.executionId !== task.executionId)
7
8
  return false;
@@ -258,6 +258,22 @@ export class GitHubStore {
258
258
  throw new Error(`JOB_BINDING_CONFLICT: PR #${this.prNumber} contains trusted ${JOB_MARKER} markers for multiple job ids`);
259
259
  }
260
260
  async listAgents() { return (await this.list()).filter(comment => this.isTrusted(comment)).map(x => parseAgent(x.body)).filter((x) => x !== null); }
261
+ async appendTaskHistoryComment(comment) {
262
+ if (!comment.idempotencyKey || comment.jobId === undefined || comment.taskId === undefined)
263
+ throw new Error('Task history projection requires durable identity');
264
+ const marker = `<!-- agents-relay:task-comment:v1 key=${comment.idempotencyKey} -->`;
265
+ if ((await this.list()).some(existing => this.isTrusted(existing) && existing.body.includes(marker)))
266
+ return;
267
+ try {
268
+ await this.comment(`${marker}\n**Task ${comment.taskId} · ${comment.kind ?? 'history'}**\n\n${comment.text}`);
269
+ }
270
+ catch (error) {
271
+ // The server may have accepted the comment before the connection failed.
272
+ // Force a fresh read before the outbox retries this stable logical key.
273
+ this.clearReadSnapshot();
274
+ throw error;
275
+ }
276
+ }
261
277
  async comment(body) {
262
278
  const pr = await this.prNode();
263
279
  const data = await this.graphql('mutation($subjectId:ID!,$body:String!){addComment(input:{subjectId:$subjectId,body:$body}){commentEdge{node{id databaseId url body updatedAt author{login}}}}}', { subjectId: pr.id, body });
@@ -1,11 +1,21 @@
1
1
  import { randomUUID } from 'node:crypto';
2
- export function appendTaskComment(job, taskId, message, author = 'cli') {
2
+ export function appendTaskComment(job, taskId, message, options = 'cli') {
3
3
  const task = job.tasks.find(item => item.id === taskId);
4
4
  if (!task)
5
5
  throw new Error(`Task ${taskId} not found`);
6
6
  if (!message.trim())
7
7
  throw new Error('task comment requires nonempty --message');
8
- const comment = { id: randomUUID(), text: message.trim(), author, createdAt: new Date().toISOString() };
8
+ const config = typeof options === 'string' ? { author: options } : options;
9
+ const key = config.idempotencyKey;
10
+ const existing = key && task.comments?.find(comment => comment.idempotencyKey === key);
11
+ if (existing)
12
+ return existing;
13
+ const comment = {
14
+ id: key ?? randomUUID(), text: message.trim(), author: config.author ?? 'cli', createdAt: new Date().toISOString(),
15
+ jobId: job.id, taskId, ...(config.kind ? { kind: config.kind } : {}),
16
+ ...(config.executionId ? { executionId: config.executionId } : {}),
17
+ ...(config.transition ? { transition: config.transition } : {}), ...(key ? { idempotencyKey: key } : {}),
18
+ };
9
19
  task.comments = [...(task.comments ?? []), comment];
10
20
  return comment;
11
21
  }
@@ -7,10 +7,27 @@ export function taskExecutionContext(job, taskId, executionId) {
7
7
  throw new Error(`Task ${taskId} not found`);
8
8
  if (task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state))
9
9
  throw new Error(`Stale Task execution ${taskId}/${executionId}; current state ${task.state}`);
10
+ const relevant = task.agentName === 'troubleshooter'
11
+ ? job.tasks.filter(item => item.id === task.parentTaskId)
12
+ : task.agentName === 'reviewer' ? task.dependencies.map(id => job.tasks.find(item => item.id === id)).filter((item) => Boolean(item)) : [];
10
13
  return { job: { id: job.id, repository: job.repository, title: job.title, objective: job.objective,
11
- projectPath: job.projectPath, state: job.state }, task, comments: task.comments ?? [], dependencies: task.dependencies.map(id => job.tasks.find(item => item.id === id) ?? { id, missing: true }),
14
+ projectPath: job.projectPath, state: job.state }, task, currentState: task.state, resolution: task.resolution ?? null, escalation: task.escalation ?? null,
15
+ comments: task.comments ?? [], relevantTasks: relevant.map(item => ({ id: item.id, title: item.title, state: item.state, attempt: item.attempt, executionId: item.executionId, resolution: item.resolution ?? null, escalation: item.escalation ?? null, comments: item.comments ?? [], result: item.result, error: item.error, failureReason: item.failureReason })),
16
+ dependencies: task.dependencies.map(id => job.tasks.find(item => item.id === id) ?? { id, missing: true }),
12
17
  plan: job.plans?.find(plan => plan.id === task.planningDecisionId) ?? null };
13
18
  }
19
+ /** Runtime-only task history; durable task.input remains the stable objective. */
20
+ export function taskAgentHistoryContext(job, task) {
21
+ if (task.agentName !== 'troubleshooter' && task.agentName !== 'reviewer')
22
+ return '';
23
+ const related = task.agentName === 'troubleshooter'
24
+ ? job.tasks.filter(item => item.id === task.parentTaskId)
25
+ : task.dependencies.map(id => job.tasks.find(item => item.id === id)).filter((item) => Boolean(item));
26
+ return `\n\n[Authoritative task state and append-only comments]\n${JSON.stringify({
27
+ currentTask: { id: task.id, state: task.state, attempt: task.attempt, executionId: task.executionId, resolution: task.resolution ?? null, escalation: task.escalation ?? null, comments: task.comments ?? [] },
28
+ relatedTasks: related.map(item => ({ id: item.id, title: item.title, state: item.state, attempt: item.attempt, executionId: item.executionId, resolution: item.resolution ?? null, escalation: item.escalation ?? null, comments: item.comments ?? [], result: item.result, error: item.error, failureReason: item.failureReason })),
29
+ }, null, 2)}\nTreat comments as supplementary history. Re-evaluate historical conditions using evidence from the original failing execution; a human response or prior condition alone does not prove recovery.`;
30
+ }
14
31
  /** Browser new-turn envelope: the local Contract and durable Task Context are authoritative. */
15
32
  export function taskReferencePrompt(repository, jobId, taskId, executionId, managedInput) {
16
33
  if (managedInput !== undefined && !managedInput.includes('[Execution event contract — mandatory]'))
@@ -1,4 +1,4 @@
1
- import { randomUUID } from 'node:crypto';
1
+ import { createHash, randomUUID } from 'node:crypto';
2
2
  import { eventFor } from './events.js';
3
3
  import { tryTransitionTaskEvent } from './scheduler.js';
4
4
  import { dispatchTaskResolutionSync, materializeResolution } from './workflow/task-resolution.js';
@@ -20,6 +20,11 @@ function validateRequest(request) {
20
20
  if (!request.context.trim())
21
21
  throw new Error('Escalation requires diagnostic context');
22
22
  }
23
+ function escalationFingerprint(request, executionId) {
24
+ return createHash('sha256').update(JSON.stringify([
25
+ request.requiredAction.trim(), request.resolutionCondition.trim(), request.context.trim(), executionId ?? null,
26
+ ])).digest('hex');
27
+ }
23
28
  /** Relay-owned escalation: the recovery Task, never the failed original, becomes BLOCKED. */
24
29
  export async function requestTroubleshooterEscalation(store, jobId, recoveryTaskId, request, now = new Date().toISOString()) {
25
30
  validateRequest(request);
@@ -35,6 +40,11 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
35
40
  const originalPlan = recovery.recoveryPlanId ? job.plans?.find(plan => plan.id === recovery.recoveryPlanId) : undefined;
36
41
  if ((!original || original.state !== 'FAILED') && (!originalPlan || originalPlan.status !== 'FAILED'))
37
42
  throw new Error(`Troubleshooter ${recoveryTaskId} has no current FAILED original Task or Plan`);
43
+ const fingerprint = escalationFingerprint(request, original?.executionId ?? originalPlan?.executionId);
44
+ if (recovery.escalation?.fingerprint === fingerprint) {
45
+ result = { task: structuredClone(recovery), escalation: structuredClone(recovery.escalation), duplicate: true };
46
+ return job;
47
+ }
38
48
  if (recovery.escalation?.status === 'pending' && recovery.state === 'BLOCKED') {
39
49
  result = { task: structuredClone(recovery), escalation: structuredClone(recovery.escalation), duplicate: true };
40
50
  return job;
@@ -56,6 +66,7 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
56
66
  throw new Error(`Troubleshooter ${recoveryTaskId} cannot be blocked`);
57
67
  const escalation = {
58
68
  id: randomUUID(), originalTaskId: original?.id, originalPlanId: originalPlan?.id,
69
+ fingerprint,
59
70
  originalExecutionId: original?.executionId ?? undefined,
60
71
  recoveryExecutionId: recovery.executionId ?? undefined, requiredAction: request.requiredAction.trim(),
61
72
  resolutionCondition: request.resolutionCondition.trim(), context: request.context.trim(), status: 'pending',
@@ -84,6 +95,8 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
84
95
  await store.saveTask(task); await store.saveJob(current); return current; });
85
96
  if (!result)
86
97
  throw new Error('Escalation did not produce a durable result');
98
+ result.task = structuredClone(job.tasks.find(task => task.id === recoveryTaskId) ?? result.task);
99
+ result.escalation = structuredClone(result.task.escalation ?? result.escalation);
87
100
  return { job, ...result };
88
101
  }
89
102
  /** Authorized human response; only a blocked Troubleshooter may be resumed. */
@@ -102,7 +115,6 @@ export async function resumeTroubleshooter(store, jobId, recoveryTaskId, respons
102
115
  if (!tryTransitionTaskEvent(task, 'task.unblocked', { autonomousJob: false }).accepted)
103
116
  throw new Error(`Troubleshooter ${recoveryTaskId} cannot be resumed`);
104
117
  task.escalation = { ...task.escalation, status: 'responded', response: response.trim(), respondedAt: now };
105
- task.input += `\n\n## Human response (${now})\n${response.trim()}\n\nRe-validate the original failure against this response. Required resolution condition: ${task.escalation.resolutionCondition}`;
106
118
  task.result = null;
107
119
  task.error = null;
108
120
  task.failureReason = undefined;
@@ -116,6 +128,7 @@ export async function resumeTroubleshooter(store, jobId, recoveryTaskId, respons
116
128
  await store.saveTask(task); await store.saveJob(current); return current; });
117
129
  if (!updated)
118
130
  throw new Error('Troubleshooter was not resumed');
131
+ updated = structuredClone(job.tasks.find(item => item.id === recoveryTaskId) ?? updated);
119
132
  return { job, task: updated };
120
133
  }
121
134
  export function hasPendingTroubleshooterEscalation(job, original) {
@@ -26,7 +26,7 @@ Retrieve this task's lifecycle event history before classifying the root cause o
26
26
  GET http://127.0.0.1:8787/api/events?job=${encodeURIComponent(job.id)}&task=${encodeURIComponent(task.id)}&limit=200
27
27
  Inspect event order, type/status, source, data/phase, duplicate events, missing expected lifecycle events, late events, and execution identifiers. If the event stream points to delivery/runtime failure, inspect the corresponding Relay/worker/tool logs too. Do not treat the final task.error string as the complete root cause.
28
28
 
29
- Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. The final task.error string alone is never sufficient. Classify the evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Select exactly one graph-enabled resolution path: exact_original_retry, same_job_dependency, blocker_job_dependency (with merge or verified mode), escalation_gate, or cancel_supersede. For confirmed adapter/model/provider failure, include --failure-kind, --reason, --evidence and --reroute in task recover; use --exclude-current-route only if the exact failed route should not be selected again. For semantic task failure use --failure-kind task_failure without rerouting. For uncertain failures use --failure-kind unknown and preserve routing unless evidence justifies rerouting. The recovery transaction stores diagnosis, clears old routing only on request, and next execution automatically reroutes with the evidence and exclusions. Troubleshooter must then reactivate/retry this exact original task and verify progress. Use the dedicated agents-relay task recover path with --task-id ${task.id} and --recovery-task-id ${recoveryId}; never use ordinary task retry from a Troubleshooter because manual retry intentionally supersedes active recovery. Do not manually choose a provider/model or create a replacement task; no replacement_task action exists. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. If diagnosis suggests a genuine human/external blocker, do not publish task.blocked directly and do not implement the result branch procedurally. Call agents-relay escalation with the original intent, blocker, and relevant context. Pass any caller-defined notification plan through --on-needs-user/--on-needs-user-file; do not hard-code a channel. The canonical escalation graph invokes Neo escalation and owns the resolved/failed/needs_user branch; follow its selected outcome. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
29
+ Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state, append-only comments, task-scoped event history, and relevant runtime logs before retry. The final task.error string alone is never sufficient. Independently verify every human response against evidence from the exact original failing execution context; a response or historical resolution condition alone never proves recovery. Classify evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Select one graph-enabled resolution path: exact_original_retry, same_job_dependency, blocker_job_dependency, escalation_gate, or cancel_supersede. For confirmed adapter/model/provider failure, include --failure-kind, --reason, --evidence and --reroute in task recover; use --exclude-current-route only if the exact failed route should not be selected again. For semantic task failure use --failure-kind task_failure without rerouting. For uncertain failures use --failure-kind unknown and preserve routing unless evidence justifies rerouting. Request the existing agents-relay task recover or agents-relay escalation operation; submission alone is not recovery. Reload authoritative SQLite state and verify the requested resolution and exact-original retry postconditions before reporting success. Bound self-healing retries to the durable Task retry budget. For a shared worker/runtime/tool defect, use its owning repository's P0/self-job workflow and verify the durable fix before requesting recovery. Never use ordinary task retry from a Troubleshooter because manual retry intentionally supersedes active recovery. Do not manually choose a provider/model or create a replacement task. If diagnosis suggests a genuine human/external blocker, invoke the escalation command; do not publish task.blocked or implement a workflow branch procedurally. The canonical escalation graph invokes Neo escalation and owns the resolved/failed/needs_user branch; Relay applies the selected outcome. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
30
30
  }
31
31
  function belongsToCurrentRun(job, task) {
32
32
  const slot = job.schedule?.currentRunSlot;
@@ -92,7 +92,7 @@ export function pendingTroubleshooterTasks(job) {
92
92
  recoveries.push(createTaskEntity(job, {
93
93
  jobId: job.id,
94
94
  id,
95
- title: `Diagnose and recover failed task ${task.id}`,
95
+ title: `Recover task ${task.id}`,
96
96
  recoveryForAttempt: task.attempt,
97
97
  kind: 'work',
98
98
  priority: 'P0',
@@ -43,8 +43,20 @@ export const jobLifecycleV2 = load('job-lifecycle', JOB_STATES, {
43
43
  implementationComplete: boolGuard('implementationComplete'),
44
44
  headAvailable: boolGuard('headAvailable'),
45
45
  mergeCapable: boolGuard('mergeCapable'),
46
- localCompletionAllowed: boolGuard('localCompletionAllowed'),
47
- localMergeReady: boolGuard('localMergeReady'),
46
+ localCompletionAllowed: (facts) => {
47
+ const values = facts.durableFacts;
48
+ return values.jobActive === true && values.autoMergeEnabled === true && values.schedulePresent !== true
49
+ && values.implementationComplete === true && values.completionGateSatisfied === true
50
+ && values.reviewSatisfied === true
51
+ && (values.localMergeRequired !== true || values.localMergeSucceeded === true);
52
+ },
53
+ localMergeReady: (facts) => {
54
+ const values = facts.durableFacts;
55
+ return values.jobActive === true && values.autoMergeEnabled === true && values.schedulePresent !== true
56
+ && values.implementationComplete === true && values.completionGateSatisfied === true
57
+ && values.reviewSatisfied === true && values.localMergePresent === true
58
+ && values.reviewHeadCurrent === true && values.mergeAttemptReady === true;
59
+ },
48
60
  });
49
61
  function topologyTargets(graph, from) {
50
62
  const source = [...graph.nodes.values()].find(node => node.kind === 'state' && node.semanticId === from);
@@ -25,7 +25,7 @@ function parseLabel(raw, line) {
25
25
  export function parseGraphV2(source) {
26
26
  const lines = source.split(/\r?\n/);
27
27
  // Mermaid YAML frontmatter is metadata only; never execute it as graph policy.
28
- // Preserve original source line positions for diagnostics and coverage reports.
28
+ // Keep source line positions stable even when graph metadata grows.
29
29
  let graphStart = 0;
30
30
  if (lines[0]?.trim() === '---') {
31
31
  let closing = -1;
@@ -34,8 +34,6 @@ export function parseGraphV2(source) {
34
34
  closing = i;
35
35
  break;
36
36
  }
37
- if (!/^(title|description):\s+[^\r\n]+$/.test(lines[i]))
38
- fail(i + 1, 'unsupported graph frontmatter field');
39
37
  }
40
38
  if (closing < 0)
41
39
  fail(1, 'unterminated graph frontmatter');