agents-relay 2.0.67 → 2.0.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +56 -20
  2. package/dist/adapters.js +39 -3
  3. package/dist/blockers.js +17 -4
  4. package/dist/cli-blocker.js +12 -30
  5. package/dist/cli-help.js +22 -17
  6. package/dist/cli-job-create.js +77 -0
  7. package/dist/cli-local-merge-config.js +11 -0
  8. package/dist/cli-projection.js +23 -55
  9. package/dist/cli-review.js +7 -24
  10. package/dist/cli-support.js +81 -8
  11. package/dist/cli-worker-task.js +63 -0
  12. package/dist/cli.js +223 -185
  13. package/dist/dashboard-html.js +15 -12
  14. package/dist/dashboard.js +27 -2
  15. package/dist/e2e-injection.js +76 -0
  16. package/dist/github-auth.js +4 -3
  17. package/dist/github-binding-order.js +15 -0
  18. package/dist/github-client.js +7 -2
  19. package/dist/github-pr-archive.js +22 -0
  20. package/dist/github-webhook.js +43 -10
  21. package/dist/job-actions.js +29 -0
  22. package/dist/local-authority-store.js +288 -0
  23. package/dist/local-commit-events.js +2 -0
  24. package/dist/local-git-merge.js +124 -0
  25. package/dist/local-job-path.js +7 -0
  26. package/dist/local-job-registry.js +29 -0
  27. package/dist/local-managed-store.js +21 -0
  28. package/dist/local-merge-delivery.js +53 -0
  29. package/dist/local-merge-lock.js +103 -0
  30. package/dist/local-merge-projection.js +60 -0
  31. package/dist/managed-worker-input.js +13 -6
  32. package/dist/manual-plan.js +29 -7
  33. package/dist/no-github.js +11 -0
  34. package/dist/planned-review-head.js +7 -8
  35. package/dist/planner-runtime.js +182 -41
  36. package/dist/planner.js +35 -2
  37. package/dist/pool.js +89 -136
  38. package/dist/projected-store.js +20 -3
  39. package/dist/projection-outbox.js +3 -1
  40. package/dist/publication-store.js +37 -0
  41. package/dist/reconciler-cancellation.js +21 -0
  42. package/dist/reconciler-review.js +99 -29
  43. package/dist/reconciler-runtime.js +72 -35
  44. package/dist/reconciler.js +124 -85
  45. package/dist/relay-store.js +108 -6
  46. package/dist/relayd-control.js +184 -0
  47. package/dist/relayd-dashboard-actions.js +108 -0
  48. package/dist/relayd-utilities.js +107 -0
  49. package/dist/relayd.js +304 -299
  50. package/dist/review-generation-fence.js +12 -0
  51. package/dist/runtime-status-events.js +21 -0
  52. package/dist/runtime-status.js +21 -7
  53. package/dist/scheduled-jobs.js +43 -36
  54. package/dist/service-external-actions.js +27 -0
  55. package/dist/service-publication.js +29 -0
  56. package/dist/store-github.js +17 -6
  57. package/dist/task-actions.js +59 -1
  58. package/dist/task-update.js +1 -1
  59. package/dist/troubleshooter.js +21 -2
  60. package/dist/unblock.js +47 -15
  61. package/dist/worker-rejection.js +11 -0
  62. package/dist/workflow/contracts.js +10 -1
  63. package/dist/workflow/definitions-v2.js +2 -0
  64. package/dist/workflow/job-reconcile-v2.js +9 -60
  65. package/dist/workflow/runtime.js +18 -3
  66. package/dist/workflow-effect-failures.js +30 -0
  67. package/dist/workspace.js +95 -6
  68. package/package.json +6 -3
  69. package/skills/agents-relay/agents/planner.agent.md +3 -0
  70. package/skills/agents-relay/contracts/orchestrator-workflows.md +4 -7
  71. package/workflow/job-lifecycle.mmd +19 -60
  72. package/workflow/review-lifecycle.mmd +5 -8
@@ -0,0 +1,12 @@
1
+ /** Called inside SQLite mutateJob against the latest durable Job snapshot. */
2
+ export function reviewGenerationMatches(current, original, task, headSha) {
3
+ const fresh = current.tasks.find(item => item.id === task.id);
4
+ if (!fresh || fresh.reviewHeadSha !== headSha || current.state !== 'ACTIVE'
5
+ || (current.headSha && current.headSha !== headSha)
6
+ || fresh.executionId !== task.executionId)
7
+ return false;
8
+ if (current.review)
9
+ return current.review.taskId === task.id && current.review.headSha === headSha
10
+ && current.review.outcome === (original.review?.outcome ?? 'pending');
11
+ return Boolean(task.planningDecisionId && !original.review?.taskId);
12
+ }
@@ -0,0 +1,21 @@
1
+ import { deriveJobRuntimeStatus } from './runtime-status.js';
2
+ import { eventFor } from './events.js';
3
+ /** Process-local change detector. No extra durable status field or polling required. */
4
+ export function createRuntimeStatusPublisher(bus) {
5
+ const last = new Map();
6
+ return (job) => {
7
+ const key = `${job.repository}\0${job.id}`;
8
+ const to = deriveJobRuntimeStatus(job);
9
+ const from = last.get(key);
10
+ if (from === to)
11
+ return;
12
+ last.set(key, to);
13
+ // First observation establishes a baseline rather than fabricating a transition.
14
+ if (from === undefined)
15
+ return;
16
+ const event = eventFor(job.id, '', null, 'job.runtime_status.changed', 'info', `${from} → ${to}`, 'orchestrator', { from, to, repository: job.repository, jobState: job.state, updatedAt: job.updatedAt }, { agent: 'relay', component: 'runtime-status' });
17
+ void bus.publish(event).catch(error => {
18
+ process.stderr.write(`runtime status change publish failed ${job.id}: ${String(error)}\n`);
19
+ });
20
+ };
21
+ }
@@ -12,13 +12,19 @@ export function implementationComplete(job) {
12
12
  const tasks = implementationTasks(job);
13
13
  return tasks.length > 0 && tasks.every(task => task.state === 'SUCCEEDED');
14
14
  }
15
- export function deriveJobRuntimeStatus(job, prState = 'OPEN') {
16
- if (prState === 'MERGED')
17
- return 'MERGED';
18
- if (prState === 'CLOSED' || job.state === 'CANCELLED')
15
+ export function deriveJobRuntimeStatus(job, _prState = 'OPEN') {
16
+ // External PR state is display metadata, never a Job runtime state input.
17
+ if (job.state === 'CANCELLED')
19
18
  return 'CANCELLED';
19
+ if (job.state === 'COMPLETED')
20
+ return 'COMPLETED';
20
21
  if (job.state === 'DRAFT')
21
22
  return 'IDLE';
23
+ const currentPlan = job.plans?.at(-1);
24
+ // An unresolved planner failure is a Job failure, not an idle Job.
25
+ // Active task execution still takes precedence below.
26
+ const plannerFailed = currentPlan?.status === 'FAILED' && !job.tasks.some(task => ['QUEUED', 'RUNNING'].includes(task.state));
27
+ const plannerRunning = currentPlan?.status === 'PLANNING';
22
28
  if (job.schedule) {
23
29
  if (job.tasks.some(task => task.state === 'RUNNING'))
24
30
  return 'RUNNING';
@@ -30,6 +36,10 @@ export function deriveJobRuntimeStatus(job, prState = 'OPEN') {
30
36
  return 'FAILED';
31
37
  if (job.tasks.some(task => task.state === 'DRAFT'))
32
38
  return 'WAITING';
39
+ if (plannerFailed)
40
+ return 'FAILED';
41
+ if (plannerRunning)
42
+ return 'RUNNING';
33
43
  return 'WAITING';
34
44
  }
35
45
  // Draft or queued reviewers must not mask implementation that is still active.
@@ -39,7 +49,7 @@ export function deriveJobRuntimeStatus(job, prState = 'OPEN') {
39
49
  if (implementation.some(task => task.state === 'QUEUED'))
40
50
  return 'QUEUED';
41
51
  const reviewRunning = job.tasks.some(task => task.agentName === 'reviewer' && task.state === 'RUNNING');
42
- if (job.review?.outcome === 'approved' && prState === 'OPEN')
52
+ if (job.review?.outcome === 'approved')
43
53
  return 'MERGE_PENDING';
44
54
  if (reviewRunning)
45
55
  return 'REVIEWING';
@@ -53,9 +63,13 @@ export function deriveJobRuntimeStatus(job, prState = 'OPEN') {
53
63
  return 'FAILED';
54
64
  if (job.tasks.some(task => task.state === 'DRAFT'))
55
65
  return 'WAITING';
56
- if (job.executionMode === 'autonomous' && job.autoMerge === false && job.reviewEnabled === false && implementationComplete(job) && prState === 'OPEN')
66
+ if (plannerFailed)
67
+ return 'FAILED';
68
+ if (plannerRunning)
69
+ return 'RUNNING';
70
+ if (job.executionMode === 'autonomous' && job.autoMerge === false && job.reviewEnabled === false && implementationComplete(job))
57
71
  return 'WAITING';
58
- if (job.reviewEnabled === false && implementationComplete(job) && prState === 'OPEN')
72
+ if (job.reviewEnabled === false && implementationComplete(job))
59
73
  return 'MERGE_PENDING';
60
74
  return 'IDLE';
61
75
  }
@@ -1,19 +1,31 @@
1
- import { GitHubStore } from './store.js';
2
1
  import { nextScheduleRun, scheduleBucket } from './scheduler.js';
3
2
  import { dispatchScheduledRun } from './workflow/scheduled-run-v2.js';
4
- async function refresh(client, repository, trustedAuthors, ref, storeFactory) {
5
- const store = storeFactory ? storeFactory() : new GitHubStore(client, repository, ref.prNumber, trustedAuthors);
6
- return await store.load(ref.jobId);
7
- }
8
- export async function runScheduledBucket(client, repository, trustedAuthors, ref, engine, now = new Date(), storeFactory) {
9
- let job = await refresh(client, repository, trustedAuthors, ref, storeFactory);
10
- const schedule = job.schedule;
11
- if (!schedule)
3
+ /** Local-first schedule entrypoint: immutable Job ID selects the SQLite store. */
4
+ export async function runScheduledBucketFromStore(store, ref, engine, now = new Date()) {
5
+ // SQLite authority provides an atomic claim. The fallback retains the
6
+ // existing in-memory test adapter, but cannot be used for production claims.
7
+ if (store.claimScheduledSlot) {
8
+ if (!store.finishScheduledSlot)
9
+ throw new Error('Atomic scheduled completion is required');
10
+ const claimed = await store.claimScheduledSlot(ref.jobId, ref.bucket, now);
11
+ if (claimed.status !== 'completed')
12
+ return claimed;
13
+ const slot = claimed.job.schedule?.currentRunSlot;
14
+ if (!slot)
15
+ throw new Error('Claimed schedule slot missing durable currentRunSlot');
16
+ await engine.reconcile(ref.jobId);
17
+ await engine.idle();
18
+ const job = await store.finishScheduledSlot(ref.jobId, slot, now);
19
+ return { job, status: 'completed' };
20
+ }
21
+ let job = await store.load(ref.jobId);
22
+ if (!job.schedule)
12
23
  throw new Error('Scheduled job ' + job.id + ' has no schedule metadata');
24
+ const schedule = job.schedule;
13
25
  const slot = new Date(schedule.nextRunAt);
14
26
  const active = job.tasks.some(task => ['QUEUED', 'RUNNING'].includes(task.state));
15
- const schedulerStore = storeFactory ? storeFactory() : new GitHubStore(client, repository, ref.prNumber, trustedAuthors);
16
- let actionStatus = null;
27
+ let claimed = false;
28
+ let status = 'skipped-idempotent';
17
29
  const decision = await dispatchScheduledRun({
18
30
  slotDue: scheduleBucket(slot) === ref.bucket && slot.getTime() <= now.getTime(),
19
31
  slotUnclaimed: !schedule.lastScheduledSlot || new Date(schedule.lastScheduledSlot).getTime() < slot.getTime(),
@@ -24,37 +36,32 @@ export async function runScheduledBucket(client, repository, trustedAuthors, ref
24
36
  if (actionId === 'schedule.skipActiveAdvance') {
25
37
  schedule.lastScheduledSlot = slot.toISOString();
26
38
  schedule.nextRunAt = nextScheduleRun(schedule, now).toISOString();
27
- job.updatedAt = now.toISOString();
28
- await schedulerStore.saveJob(job);
29
- actionStatus = 'skipped-active';
30
- return {};
39
+ status = 'skipped-active';
31
40
  }
32
- if (actionId === 'schedule.claimFromSlot' || actionId === 'schedule.claimFromNow') {
41
+ else if (actionId === 'schedule.claimFromSlot' || actionId === 'schedule.claimFromNow') {
33
42
  schedule.lastScheduledSlot = slot.toISOString();
34
43
  schedule.currentRunSlot = slot.toISOString();
35
44
  schedule.nextRunAt = nextScheduleRun(schedule, actionId === 'schedule.claimFromSlot' ? slot : now).toISOString();
36
- job.updatedAt = now.toISOString();
37
- await schedulerStore.saveJob(job);
38
- job = await schedulerStore.load(job.id);
39
- return {};
45
+ claimed = true;
46
+ status = 'completed';
40
47
  }
41
- throw new Error('Unsupported schedule graph action ' + actionId);
48
+ else
49
+ throw new Error('Unknown local schedule action: ' + actionId);
50
+ job.updatedAt = now.toISOString();
51
+ await store.saveJob(job);
52
+ return {};
42
53
  });
43
- if (!decision.accepted || decision.target === 'skip_idempotent')
44
- return { job, status: 'skipped-idempotent' };
45
- if (actionStatus === 'skipped-active' || decision.target === 'skip_active')
46
- return { job, status: 'skipped-active' };
47
- if (decision.target !== 'claimed')
48
- return { job, status: 'skipped-idempotent' };
49
- if (!engine)
50
- throw new Error('Scheduled jobs require the local relayd scheduler engine');
54
+ if (!decision.accepted || !claimed)
55
+ return { job, status };
51
56
  await engine.reconcile(job.id);
52
57
  await engine.idle();
53
- const finished = await schedulerStore.load(job.id);
54
- if (!finished.schedule)
55
- throw new Error('Scheduled job ' + job.id + ' lost its schedule metadata');
56
- finished.schedule.lastRunAt = now.toISOString();
57
- finished.updatedAt = now.toISOString();
58
- await schedulerStore.saveJob(finished);
59
- return { job: finished, status: 'completed' };
58
+ job = await store.load(job.id);
59
+ if (!job.schedule)
60
+ throw new Error('Scheduled job lost schedule metadata');
61
+ job.schedule.lastRunAt = now.toISOString();
62
+ // A claim and its completion can share the same wall-clock timestamp.
63
+ // Advance the snapshot revision to satisfy SQLite's stale-write guard.
64
+ job.updatedAt = new Date(Math.max(now.getTime(), Date.parse(job.updatedAt) + 1)).toISOString();
65
+ await store.saveJob(job);
66
+ return { job, status: 'completed' };
60
67
  }
@@ -0,0 +1,27 @@
1
+ import { LocalAuthorityDurableStore } from './local-authority-store.js';
2
+ import { errorMessage, observability } from './observability.js';
3
+ export async function flushServiceExternalActions(store, reconciler, jobId) {
4
+ if (!(store instanceof LocalAuthorityDurableStore) || store.noGitHub)
5
+ return 0;
6
+ const result = await store.projection.flushOnce();
7
+ if (result.projected > 0)
8
+ await reconciler.reconcile(jobId);
9
+ return result.projected;
10
+ }
11
+ export function startServiceExternalActionFlusher(store, reconciler, jobId, intervalMs) {
12
+ if (!(store instanceof LocalAuthorityDurableStore) || store.noGitHub)
13
+ return () => { };
14
+ const tick = () => {
15
+ void flushServiceExternalActions(store, reconciler, jobId).catch(error => observability().record({
16
+ stage: 'reconcile',
17
+ event: 'service.external_action_flush_failed',
18
+ level: 'warn',
19
+ outcome: 'failed',
20
+ message: errorMessage(error),
21
+ }));
22
+ };
23
+ tick();
24
+ const timer = setInterval(tick, intervalMs);
25
+ timer.unref();
26
+ return () => clearInterval(timer);
27
+ }
@@ -0,0 +1,29 @@
1
+ import { homedir } from 'node:os';
2
+ import { join } from 'node:path';
3
+ import { eventFor } from './events.js';
4
+ import { SQLiteRelayStore, relayStorePath } from './relay-store.js';
5
+ /** Single-Job service uses the same durable publish facts as the repository daemon. */
6
+ export async function handleServicePublication(job, wake, bus, directory = join(homedir(), '.agents-relay', 'stores')) {
7
+ if (!wake.publication && !wake.publishFailure)
8
+ return;
9
+ const sqlite = new SQLiteRelayStore(relayStorePath(directory, job.repository, job.prNumber));
10
+ try {
11
+ await sqlite.recordPublication(job.id, wake.deliveryId, wake.publication, wake.publishFailure ?? null, wake.publicationObservedAt);
12
+ }
13
+ finally {
14
+ sqlite.close();
15
+ }
16
+ if (bus) {
17
+ if (wake.publication) {
18
+ const event = eventFor(job.id, 'release', null, 'package.published', 'succeeded', 'Published ' + wake.publication.package + '@' + wake.publication.version, 'orchestrator', { ...wake.publication, deliveryId: wake.deliveryId, repository: job.repository, prNumber: job.prNumber });
19
+ event.event_id = 'github:' + job.id + ':' + wake.deliveryId + ':package.published';
20
+ await bus.publish(event);
21
+ }
22
+ if (wake.publishFailure) {
23
+ const event = eventFor(job.id, 'release', null, 'package.publish_failed', 'failed', 'Publish failed: ' + wake.publishFailure.stage, 'orchestrator', { ...wake.publishFailure, deliveryId: wake.deliveryId, repository: job.repository, prNumber: job.prNumber });
24
+ event.event_id = 'github:' + job.id + ':' + wake.deliveryId + ':package.publish_failed';
25
+ await bus.publish(event);
26
+ }
27
+ }
28
+ // Do not use a publication event to forge a Job state change.
29
+ }
@@ -153,12 +153,23 @@ export class GitHubStore {
153
153
  const payload = { merge_method: 'squash' };
154
154
  if (expectedHeadSha)
155
155
  payload.sha = expectedHeadSha;
156
- const result = await this.client.request('repos/' + this.repository + '/pulls/' + this.prNumber + '/merge', {
157
- method: 'PUT',
158
- body: JSON.stringify(payload),
159
- });
160
- if (result.merged !== true)
161
- throw new Error(typeof result.message === 'string' ? result.message : 'Pull request #' + this.prNumber + ' was not merged');
156
+ try {
157
+ const result = await this.client.request('repos/' + this.repository + '/pulls/' + this.prNumber + '/merge', {
158
+ method: 'PUT',
159
+ body: JSON.stringify(payload),
160
+ });
161
+ if (result.merged !== true)
162
+ throw new Error(typeof result.message === 'string' ? result.message : 'Pull request #' + this.prNumber + ' was not merged');
163
+ }
164
+ catch (error) {
165
+ // Crash recovery can replay a merge after GitHub committed but before
166
+ // its local acknowledgement. Treat replay as success ONLY after verifying
167
+ // the actual external PR is already MERGED; never assume success offline.
168
+ this.prNodeCache = null;
169
+ const state = await this.pullRequestState().catch(() => null);
170
+ if (state !== 'MERGED')
171
+ throw error;
172
+ }
162
173
  this.prNodeCache = null;
163
174
  }
164
175
  async closePullRequest() {
@@ -1,6 +1,7 @@
1
1
  import { extendRetryBudget, transitionTaskEvent, tryTransitionTaskEvent } from './scheduler.js';
2
2
  import { selectRelatedTaskMutations } from './workflow/task-action-policy.js';
3
3
  export const TASK_RETRY_ACTION = 'task.retry';
4
+ export const TASK_RECOVER_ACTION = 'task.recover';
4
5
  export const TASK_UNBLOCK_RETRY_ACTION = 'task.unblock.retry';
5
6
  export const TASK_CANCEL_ACTION = 'task.cancel';
6
7
  export const TASK_RELEASE_ACTION = 'task.release';
@@ -28,6 +29,24 @@ function clearExecution(task) {
28
29
  task.leaseExpiresAt = null;
29
30
  task.executionId = null;
30
31
  }
32
+ function applyTaskRetrySnapshot(task, policy = {}) {
33
+ if (task.kind === 'planner')
34
+ throw new Error(`Legacy planner Task ${task.id} is read-only; retry the Plan instead`);
35
+ const decision = tryTransitionTaskEvent(task, 'retry');
36
+ if (!decision.accepted)
37
+ throw lifecycleRejection(task, 'restartable');
38
+ if (policy.extendBudget)
39
+ task.maxAttempts = extendRetryBudget(task);
40
+ clearExecution(task);
41
+ }
42
+ /** Fixed-job authoritative unblock continuation, applied inside one Job snapshot mutation. */
43
+ export function applyTaskUnblockRetrySnapshot(task) {
44
+ if (!tryTransitionTaskEvent(task, 'retry').accepted)
45
+ throw lifecycleRejection(task, 'retryable after unblock');
46
+ task.maxAttempts = extendRetryBudget(task);
47
+ clearExecution(task);
48
+ delete task.blockedBy;
49
+ }
31
50
  /** Shared task.retry mutation. The canonical lifecycle graph is the sole legality authority. */
32
51
  export async function applyTaskRetryAction(task, policy = {}) {
33
52
  if (task.kind === 'planner')
@@ -130,13 +149,48 @@ async function applySelectedRelatedTaskMutations(job, mutations) {
130
149
  }
131
150
  /** Compatibility bridge: domain mutation only; adapters own wake/reconcile/output behavior. */
132
151
  export async function dispatchTaskAction(store, request) {
152
+ if (store.mutateJob && [TASK_RETRY_ACTION, TASK_RECOVER_ACTION, TASK_CANCEL_ACTION].includes(request.id)) {
153
+ let updatedTask;
154
+ const updated = await store.mutateJob(request.jobId, current => {
155
+ const task = current.tasks.find(item => item.id === request.taskId);
156
+ if (!task)
157
+ throw new Error(`Task ${request.taskId} not found`);
158
+ if (request.id === TASK_RECOVER_ACTION) {
159
+ const recovery = request.recoveryTaskId ? current.tasks.find(item => item.id === request.recoveryTaskId) : undefined;
160
+ if (!recovery || recovery.agentName !== 'troubleshooter' || recovery.parentTaskId !== task.id || !['QUEUED', 'RUNNING'].includes(recovery.state))
161
+ throw new Error(`task recover requires an active Troubleshooter recovery for ${task.id}`);
162
+ }
163
+ const relatedMutations = request.id === TASK_RECOVER_ACTION ? [] : selectRelatedTaskMutations(current, task, request.id);
164
+ if (request.id === TASK_RETRY_ACTION || request.id === TASK_RECOVER_ACTION)
165
+ applyTaskRetrySnapshot(task, { extendBudget: true });
166
+ else if (request.id === TASK_CANCEL_ACTION)
167
+ cancelTaskSnapshot(task);
168
+ for (const mutation of relatedMutations) {
169
+ const related = current.tasks.find(item => item.id === mutation.taskId);
170
+ if (!related)
171
+ throw new Error(`Related task ${mutation.taskId} not found`);
172
+ cancelTaskSnapshot(related, mutation.reason);
173
+ }
174
+ updatedTask = structuredClone(task);
175
+ return current;
176
+ });
177
+ if (!updatedTask)
178
+ throw new Error(`Task ${request.taskId} was not updated`);
179
+ return { job: updated, task: updatedTask };
180
+ }
133
181
  const job = await store.load(request.jobId);
134
182
  const task = job.tasks.find(item => item.id === request.taskId);
135
183
  if (!task)
136
184
  throw new Error(`Task ${request.taskId} not found`);
137
- const relatedMutations = selectRelatedTaskMutations(job, task, request.id);
185
+ const relatedMutations = request.id === TASK_RECOVER_ACTION ? [] : selectRelatedTaskMutations(job, task, request.id);
138
186
  if (request.id === TASK_RETRY_ACTION)
139
187
  await applyTaskRetryAction(task, { extendBudget: true });
188
+ else if (request.id === TASK_RECOVER_ACTION) {
189
+ const recovery = request.recoveryTaskId ? job.tasks.find(item => item.id === request.recoveryTaskId) : undefined;
190
+ if (!recovery || recovery.agentName !== 'troubleshooter' || recovery.parentTaskId !== task.id || !['QUEUED', 'RUNNING'].includes(recovery.state))
191
+ throw new Error(`task recover requires an active Troubleshooter recovery for ${task.id}`);
192
+ await applyTaskRetryAction(task, { extendBudget: true });
193
+ }
140
194
  else if (request.id === TASK_UNBLOCK_RETRY_ACTION)
141
195
  await applyTaskUnblockRetryAction(task);
142
196
  else if (request.id === TASK_CANCEL_ACTION)
@@ -157,3 +211,7 @@ export async function dispatchTaskAction(store, request) {
157
211
  await store.saveTask(related);
158
212
  return { job, task };
159
213
  }
214
+ function cancelTaskSnapshot(task, reason = 'Cancelled by user') {
215
+ if (!tryApplyTaskCancelAction(task, reason))
216
+ throw lifecycleRejection(task, 'cancellable');
217
+ }
@@ -44,7 +44,7 @@ export async function applyManagedTaskUpdate(store, jobId, taskId, proposed, exp
44
44
  }
45
45
  export function managedTaskUpdateHandler(locate, wake) {
46
46
  return async (jobId, taskId, task, expectedUpdatedAt) => {
47
- const target = locate(jobId);
47
+ const target = await locate(jobId);
48
48
  const updated = await applyManagedTaskUpdate(target.store, jobId, taskId, task, expectedUpdatedAt);
49
49
  await wake(target.repository, target.prNumber, jobId);
50
50
  return updated;
@@ -26,7 +26,7 @@ Retrieve this task's lifecycle event history before classifying the root cause o
26
26
  GET http://127.0.0.1:8787/api/events?job=${encodeURIComponent(job.id)}&task=${encodeURIComponent(task.id)}&limit=200
27
27
  Inspect event order, type/status, source, data/phase, duplicate events, missing expected lifecycle events, late events, and execution identifiers. If the event stream points to delivery/runtime failure, inspect the corresponding Relay/worker/tool logs too. Do not treat the final task.error string as the complete root cause.
28
28
 
29
- Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. The final task.error string alone is never sufficient. Classify the evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Only model/provider/runtime evidence may call the narrow model re-decision action; that action gathers fresh usage, reloads current policy, updates this exact Task's routing, and does not retry it. Troubleshooter must then reactivate/retry this exact original task and verify progress. Do not manually choose a provider/model or create a replacement task. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. If diagnosis suggests a genuine human/external blocker, do not publish task.blocked directly and do not implement the result branch procedurally. Call agents-relay escalation with the original intent, blocker, and relevant context. Pass any caller-defined notification plan through --on-needs-user/--on-needs-user-file; do not hard-code a channel. The canonical escalation graph invokes Neo escalation and owns the resolved/failed/needs_user branch; follow its selected outcome. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
29
+ Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. The final task.error string alone is never sufficient. Classify the evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Only model/provider/runtime evidence may call the narrow model re-decision action; that action gathers fresh usage, reloads current policy, updates this exact Task's routing, and does not retry it. Troubleshooter must then reactivate/retry this exact original task and verify progress. Use the dedicated agents-relay task recover path with --task-id ${task.id} and --recovery-task-id ${troubleshooterTaskId(task)}; never use ordinary task retry from a Troubleshooter because manual retry intentionally supersedes active recovery. Do not manually choose a provider/model or create a replacement task. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. If diagnosis suggests a genuine human/external blocker, do not publish task.blocked directly and do not implement the result branch procedurally. Call agents-relay escalation with the original intent, blocker, and relevant context. Pass any caller-defined notification plan through --on-needs-user/--on-needs-user-file; do not hard-code a channel. The canonical escalation graph invokes Neo escalation and owns the resolved/failed/needs_user branch; follow its selected outcome. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
30
30
  }
31
31
  function belongsToCurrentRun(job, task) {
32
32
  const slot = job.schedule?.currentRunSlot;
@@ -79,7 +79,7 @@ export function pendingTroubleshooterTasks(job) {
79
79
  continuation: null,
80
80
  continuationDeliveredAt: null,
81
81
  attempt: 0,
82
- maxAttempts: 1,
82
+ maxAttempts: 3,
83
83
  leaseOwner: null,
84
84
  leaseExpiresAt: null,
85
85
  executionId: null,
@@ -94,3 +94,22 @@ export function pendingTroubleshooterTasks(job) {
94
94
  }
95
95
  return recoveries;
96
96
  }
97
+ export function exhaustedTroubleshooterRecoveries(job) {
98
+ const results = [];
99
+ for (const recovery of job.tasks) {
100
+ if (recovery.agentName !== 'troubleshooter' || recovery.state !== 'FAILED' || recovery.attempt < recovery.maxAttempts || !recovery.parentTaskId)
101
+ continue;
102
+ const original = job.tasks.find(task => task.id === recovery.parentTaskId);
103
+ if (!original || !['FAILED', 'BLOCKED'].includes(original.state))
104
+ continue;
105
+ const alreadyRecorded = job.recoveryFailures?.some(item => item.taskId === original.id && item.recoveryTaskId === recovery.id);
106
+ if (alreadyRecorded)
107
+ continue;
108
+ results.push({
109
+ original,
110
+ recovery,
111
+ reason: recovery.error ?? recovery.failureReason ?? `Troubleshooter ${recovery.id} ${recovery.state.toLowerCase()}`,
112
+ });
113
+ }
114
+ return results;
115
+ }
package/dist/unblock.js CHANGED
@@ -1,5 +1,5 @@
1
- import { transitionTaskEvent } from './scheduler.js';
2
- import { applyTaskUnblockRetryAction } from './task-actions.js';
1
+ import { applyTaskUnblockRetrySnapshot } from './task-actions.js';
2
+ import { tryTransitionTaskEvent } from './scheduler.js';
3
3
  import { dispatchAuthoritativeUnblock } from './workflow/authoritative-unblock-v2.js';
4
4
  export function isEffectivelyBlocked(task) { return task.kind !== 'planner' && task.state === 'BLOCKED'; }
5
5
  function baselineFor(task) {
@@ -53,14 +53,29 @@ export async function unblockAutonomousTasks(options) {
53
53
  const now = options.now ?? new Date().toISOString();
54
54
  const currentHeadSha = await store.headSha?.() ?? job.headSha ?? null;
55
55
  const facts = unblockFacts(job, tasks, currentHeadSha, trigger.reason);
56
- for (const task of tasks) {
57
- const baseline = baselineFor(task);
58
- task.blockedBaseline = baseline;
59
- task.unblock = { at: now, source: trigger.source, reason: trigger.reason, baseline, currentHeadSha: currentHeadSha ?? undefined, facts, blockerJobId: trigger.blockerJobId, resolution: trigger.resolution };
60
- await transitionTaskEvent(task, 'task.unblocked');
61
- task.updatedAt = now;
62
- delete task.blockedBy;
63
- await store.saveTask(task);
56
+ const apply = (current) => {
57
+ for (const requested of tasks) {
58
+ const task = current.tasks.find(item => item.id === requested.id);
59
+ if (!task || !isEffectivelyBlocked(task))
60
+ throw new Error(`Task ${requested.id} changed before authoritative unblock`);
61
+ const baseline = baselineFor(task);
62
+ if (!tryTransitionTaskEvent(task, 'task.unblocked').accepted)
63
+ throw new Error(`Task ${task.id} cannot be authoritatively unblocked`);
64
+ task.blockedBaseline = baseline;
65
+ task.unblock = { at: now, source: trigger.source, reason: trigger.reason, baseline, currentHeadSha: currentHeadSha ?? undefined, facts, blockerJobId: trigger.blockerJobId, resolution: trigger.resolution };
66
+ task.updatedAt = now;
67
+ delete task.blockedBy;
68
+ }
69
+ current.updatedAt = now;
70
+ };
71
+ if (store.mutateJob) {
72
+ const committed = await store.mutateJob(job.id, current => { apply(current); return current; });
73
+ Object.assign(job, structuredClone(committed));
74
+ }
75
+ else {
76
+ apply(job);
77
+ for (const task of tasks)
78
+ await store.saveTask(task);
64
79
  }
65
80
  const planTrigger = unconsumedUnblockTrigger(job) ?? {
66
81
  source: 'unblock', taskIds: tasks.map(task => task.id), reason: trigger.reason,
@@ -84,11 +99,28 @@ export async function unblockTask(options) {
84
99
  options.task.blockedBaseline = baseline;
85
100
  const facts = unblockFacts(options.job, [options.task], currentHeadSha, options.trigger.reason);
86
101
  options.task.unblock = { at: now, source: options.trigger.source, reason: options.trigger.reason, baseline, currentHeadSha: currentHeadSha ?? undefined, facts, blockerJobId: options.trigger.blockerJobId, resolution: options.trigger.resolution };
87
- await transitionTaskEvent(options.task, 'task.unblocked');
88
- await options.store.saveTask(options.task);
89
- await applyTaskUnblockRetryAction(options.task);
90
- options.task.updatedAt = now;
91
- await options.store.saveTask(options.task);
102
+ const apply = (current) => {
103
+ const task = current.tasks.find(item => item.id === options.task.id);
104
+ if (!task || !isEffectivelyBlocked(task))
105
+ throw new Error(`Task ${options.task.id} changed before authoritative unblock`);
106
+ if (!tryTransitionTaskEvent(task, 'task.unblocked').accepted)
107
+ throw new Error(`Task ${task.id} cannot be authoritatively unblocked`);
108
+ const baseline = baselineFor(task);
109
+ task.blockedBaseline = baseline;
110
+ task.unblock = { at: now, source: options.trigger.source, reason: options.trigger.reason, baseline, currentHeadSha: currentHeadSha ?? undefined, facts, blockerJobId: options.trigger.blockerJobId, resolution: options.trigger.resolution };
111
+ applyTaskUnblockRetrySnapshot(task);
112
+ task.updatedAt = now;
113
+ delete task.blockedBy;
114
+ current.updatedAt = now;
115
+ };
116
+ if (options.store.mutateJob) {
117
+ const committed = await options.store.mutateJob(options.job.id, current => { apply(current); return current; });
118
+ Object.assign(options.job, structuredClone(committed));
119
+ }
120
+ else {
121
+ apply(options.job);
122
+ await options.store.saveTask(options.task);
123
+ }
92
124
  result = { replanning: false };
93
125
  return {};
94
126
  }
@@ -0,0 +1,11 @@
1
+ /** Classify genuine worker-process rejection text, regardless of E2E mode. */
2
+ export function classifyWorkerRejection(message) {
3
+ if (/rate[_ .-]?limit[_ .-]?exceeded|usage limit reached|quota[_ .-]?exceeded|insufficient[_ .-]?quota/i.test(message))
4
+ return 'rate_limit_exceeded';
5
+ if (/model[_ .-]?not[_ .-]?found|model (?:is )?(?:unavailable|not found|unsupported)|unsupported model/i.test(message))
6
+ return 'model_not_found';
7
+ if (/provider[_ .-]?unavailable|provider (?:is )?(?:unavailable|refused)|service unavailable/i.test(message))
8
+ return 'provider_unavailable';
9
+ return 'worker_process_failed';
10
+ }
11
+ export const reroutableWorkerRejections = new Set(['rate_limit_exceeded', 'model_not_found', 'provider_unavailable']);
@@ -12,6 +12,9 @@ export const WORKFLOW_EVENT_NAMES = [
12
12
  'hold',
13
13
  'lease.expired',
14
14
  'implementation.completed',
15
+ 'local.completed',
16
+ 'local.merge.requested',
17
+ 'effect.deliverLocalMerge.failed',
15
18
  'pr.closed',
16
19
  'pr.merged',
17
20
  'queue',
@@ -48,7 +51,6 @@ export const WORKFLOW_FACT_NAMES = [
48
51
  'scheduledRun',
49
52
  'hasUnblock',
50
53
  'manualPlan',
51
- 'prOpen',
52
54
  'reviewDisabled',
53
55
  'implementationComplete',
54
56
  'headAvailable',
@@ -67,6 +69,7 @@ export const WORKFLOW_EFFECT_NAMES = [
67
69
  'effect.launchTask',
68
70
  'effect.cancelActiveTasks',
69
71
  'effect.mergePullRequest',
72
+ 'effect.deliverLocalMerge',
70
73
  'effect.cleanupJobWorktree',
71
74
  ];
72
75
  export const WORKFLOW_EFFECT_POLICIES = {
@@ -88,6 +91,12 @@ export const WORKFLOW_EFFECT_POLICIES = {
88
91
  retry: 'workflow',
89
92
  observability: 'workflow.effect',
90
93
  },
94
+ 'effect.deliverLocalMerge': {
95
+ idempotency: 'job-local-source-sha',
96
+ failureEvent: 'effect.deliverLocalMerge.failed',
97
+ retry: 'workflow',
98
+ observability: 'workflow.effect',
99
+ },
91
100
  'effect.mergePullRequest': {
92
101
  idempotency: 'job-head-sha',
93
102
  failureEvent: 'effect.mergePullRequest.failed',
@@ -35,6 +35,8 @@ export const jobLifecycleV2 = load('job-lifecycle', JOB_STATES, {
35
35
  implementationComplete: boolGuard('implementationComplete'),
36
36
  headAvailable: boolGuard('headAvailable'),
37
37
  mergeCapable: boolGuard('mergeCapable'),
38
+ localCompletionAllowed: boolGuard('localCompletionAllowed'),
39
+ localMergeReady: boolGuard('localMergeReady'),
38
40
  });
39
41
  function topologyTargets(graph, from) {
40
42
  const source = [...graph.nodes.values()].find(node => node.kind === 'state' && node.semanticId === from);