agents-relay 2.0.67 → 2.0.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -20
- package/dist/adapters.js +39 -3
- package/dist/blockers.js +17 -4
- package/dist/cli-blocker.js +12 -30
- package/dist/cli-help.js +22 -17
- package/dist/cli-job-create.js +77 -0
- package/dist/cli-local-merge-config.js +11 -0
- package/dist/cli-projection.js +23 -55
- package/dist/cli-review.js +7 -24
- package/dist/cli-support.js +81 -8
- package/dist/cli-worker-task.js +63 -0
- package/dist/cli.js +223 -185
- package/dist/dashboard-html.js +15 -12
- package/dist/dashboard.js +27 -2
- package/dist/e2e-injection.js +76 -0
- package/dist/github-auth.js +4 -3
- package/dist/github-binding-order.js +15 -0
- package/dist/github-client.js +7 -2
- package/dist/github-pr-archive.js +22 -0
- package/dist/github-webhook.js +43 -10
- package/dist/job-actions.js +29 -0
- package/dist/local-authority-store.js +288 -0
- package/dist/local-commit-events.js +2 -0
- package/dist/local-git-merge.js +124 -0
- package/dist/local-job-path.js +7 -0
- package/dist/local-job-registry.js +29 -0
- package/dist/local-managed-store.js +21 -0
- package/dist/local-merge-delivery.js +53 -0
- package/dist/local-merge-lock.js +103 -0
- package/dist/local-merge-projection.js +60 -0
- package/dist/managed-worker-input.js +13 -6
- package/dist/manual-plan.js +29 -7
- package/dist/no-github.js +11 -0
- package/dist/planned-review-head.js +7 -8
- package/dist/planner-runtime.js +182 -41
- package/dist/planner.js +35 -2
- package/dist/pool.js +89 -136
- package/dist/projected-store.js +20 -3
- package/dist/projection-outbox.js +3 -1
- package/dist/publication-store.js +37 -0
- package/dist/reconciler-cancellation.js +21 -0
- package/dist/reconciler-review.js +99 -29
- package/dist/reconciler-runtime.js +72 -35
- package/dist/reconciler.js +124 -85
- package/dist/relay-store.js +108 -6
- package/dist/relayd-control.js +184 -0
- package/dist/relayd-dashboard-actions.js +108 -0
- package/dist/relayd-utilities.js +107 -0
- package/dist/relayd.js +304 -299
- package/dist/review-generation-fence.js +12 -0
- package/dist/runtime-status-events.js +21 -0
- package/dist/runtime-status.js +21 -7
- package/dist/scheduled-jobs.js +43 -36
- package/dist/service-external-actions.js +27 -0
- package/dist/service-publication.js +29 -0
- package/dist/store-github.js +17 -6
- package/dist/task-actions.js +59 -1
- package/dist/task-update.js +1 -1
- package/dist/troubleshooter.js +21 -2
- package/dist/unblock.js +47 -15
- package/dist/worker-rejection.js +11 -0
- package/dist/workflow/contracts.js +10 -1
- package/dist/workflow/definitions-v2.js +2 -0
- package/dist/workflow/job-reconcile-v2.js +9 -60
- package/dist/workflow/runtime.js +18 -3
- package/dist/workflow-effect-failures.js +30 -0
- package/dist/workspace.js +95 -6
- package/package.json +6 -3
- package/skills/agents-relay/agents/planner.agent.md +3 -0
- package/skills/agents-relay/contracts/orchestrator-workflows.md +4 -7
- package/workflow/job-lifecycle.mmd +19 -60
- package/workflow/review-lifecycle.mmd +5 -8
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Called inside SQLite mutateJob against the latest durable Job snapshot. */
|
|
2
|
+
export function reviewGenerationMatches(current, original, task, headSha) {
|
|
3
|
+
const fresh = current.tasks.find(item => item.id === task.id);
|
|
4
|
+
if (!fresh || fresh.reviewHeadSha !== headSha || current.state !== 'ACTIVE'
|
|
5
|
+
|| (current.headSha && current.headSha !== headSha)
|
|
6
|
+
|| fresh.executionId !== task.executionId)
|
|
7
|
+
return false;
|
|
8
|
+
if (current.review)
|
|
9
|
+
return current.review.taskId === task.id && current.review.headSha === headSha
|
|
10
|
+
&& current.review.outcome === (original.review?.outcome ?? 'pending');
|
|
11
|
+
return Boolean(task.planningDecisionId && !original.review?.taskId);
|
|
12
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { deriveJobRuntimeStatus } from './runtime-status.js';
|
|
2
|
+
import { eventFor } from './events.js';
|
|
3
|
+
/** Process-local change detector. No extra durable status field or polling required. */
|
|
4
|
+
export function createRuntimeStatusPublisher(bus) {
|
|
5
|
+
const last = new Map();
|
|
6
|
+
return (job) => {
|
|
7
|
+
const key = `${job.repository}\0${job.id}`;
|
|
8
|
+
const to = deriveJobRuntimeStatus(job);
|
|
9
|
+
const from = last.get(key);
|
|
10
|
+
if (from === to)
|
|
11
|
+
return;
|
|
12
|
+
last.set(key, to);
|
|
13
|
+
// First observation establishes a baseline rather than fabricating a transition.
|
|
14
|
+
if (from === undefined)
|
|
15
|
+
return;
|
|
16
|
+
const event = eventFor(job.id, '', null, 'job.runtime_status.changed', 'info', `${from} → ${to}`, 'orchestrator', { from, to, repository: job.repository, jobState: job.state, updatedAt: job.updatedAt }, { agent: 'relay', component: 'runtime-status' });
|
|
17
|
+
void bus.publish(event).catch(error => {
|
|
18
|
+
process.stderr.write(`runtime status change publish failed ${job.id}: ${String(error)}\n`);
|
|
19
|
+
});
|
|
20
|
+
};
|
|
21
|
+
}
|
package/dist/runtime-status.js
CHANGED
|
@@ -12,13 +12,19 @@ export function implementationComplete(job) {
|
|
|
12
12
|
const tasks = implementationTasks(job);
|
|
13
13
|
return tasks.length > 0 && tasks.every(task => task.state === 'SUCCEEDED');
|
|
14
14
|
}
|
|
15
|
-
export function deriveJobRuntimeStatus(job,
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
if (prState === 'CLOSED' || job.state === 'CANCELLED')
|
|
15
|
+
export function deriveJobRuntimeStatus(job, _prState = 'OPEN') {
|
|
16
|
+
// External PR state is display metadata, never a Job runtime state input.
|
|
17
|
+
if (job.state === 'CANCELLED')
|
|
19
18
|
return 'CANCELLED';
|
|
19
|
+
if (job.state === 'COMPLETED')
|
|
20
|
+
return 'COMPLETED';
|
|
20
21
|
if (job.state === 'DRAFT')
|
|
21
22
|
return 'IDLE';
|
|
23
|
+
const currentPlan = job.plans?.at(-1);
|
|
24
|
+
// An unresolved planner failure is a Job failure, not an idle Job.
|
|
25
|
+
// Active task execution still takes precedence below.
|
|
26
|
+
const plannerFailed = currentPlan?.status === 'FAILED' && !job.tasks.some(task => ['QUEUED', 'RUNNING'].includes(task.state));
|
|
27
|
+
const plannerRunning = currentPlan?.status === 'PLANNING';
|
|
22
28
|
if (job.schedule) {
|
|
23
29
|
if (job.tasks.some(task => task.state === 'RUNNING'))
|
|
24
30
|
return 'RUNNING';
|
|
@@ -30,6 +36,10 @@ export function deriveJobRuntimeStatus(job, prState = 'OPEN') {
|
|
|
30
36
|
return 'FAILED';
|
|
31
37
|
if (job.tasks.some(task => task.state === 'DRAFT'))
|
|
32
38
|
return 'WAITING';
|
|
39
|
+
if (plannerFailed)
|
|
40
|
+
return 'FAILED';
|
|
41
|
+
if (plannerRunning)
|
|
42
|
+
return 'RUNNING';
|
|
33
43
|
return 'WAITING';
|
|
34
44
|
}
|
|
35
45
|
// Draft or queued reviewers must not mask implementation that is still active.
|
|
@@ -39,7 +49,7 @@ export function deriveJobRuntimeStatus(job, prState = 'OPEN') {
|
|
|
39
49
|
if (implementation.some(task => task.state === 'QUEUED'))
|
|
40
50
|
return 'QUEUED';
|
|
41
51
|
const reviewRunning = job.tasks.some(task => task.agentName === 'reviewer' && task.state === 'RUNNING');
|
|
42
|
-
if (job.review?.outcome === 'approved'
|
|
52
|
+
if (job.review?.outcome === 'approved')
|
|
43
53
|
return 'MERGE_PENDING';
|
|
44
54
|
if (reviewRunning)
|
|
45
55
|
return 'REVIEWING';
|
|
@@ -53,9 +63,13 @@ export function deriveJobRuntimeStatus(job, prState = 'OPEN') {
|
|
|
53
63
|
return 'FAILED';
|
|
54
64
|
if (job.tasks.some(task => task.state === 'DRAFT'))
|
|
55
65
|
return 'WAITING';
|
|
56
|
-
if (
|
|
66
|
+
if (plannerFailed)
|
|
67
|
+
return 'FAILED';
|
|
68
|
+
if (plannerRunning)
|
|
69
|
+
return 'RUNNING';
|
|
70
|
+
if (job.executionMode === 'autonomous' && job.autoMerge === false && job.reviewEnabled === false && implementationComplete(job))
|
|
57
71
|
return 'WAITING';
|
|
58
|
-
if (job.reviewEnabled === false && implementationComplete(job)
|
|
72
|
+
if (job.reviewEnabled === false && implementationComplete(job))
|
|
59
73
|
return 'MERGE_PENDING';
|
|
60
74
|
return 'IDLE';
|
|
61
75
|
}
|
package/dist/scheduled-jobs.js
CHANGED
|
@@ -1,19 +1,31 @@
|
|
|
1
|
-
import { GitHubStore } from './store.js';
|
|
2
1
|
import { nextScheduleRun, scheduleBucket } from './scheduler.js';
|
|
3
2
|
import { dispatchScheduledRun } from './workflow/scheduled-run-v2.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
3
|
+
/** Local-first schedule entrypoint: immutable Job ID selects the SQLite store. */
|
|
4
|
+
export async function runScheduledBucketFromStore(store, ref, engine, now = new Date()) {
|
|
5
|
+
// SQLite authority provides an atomic claim. The fallback retains the
|
|
6
|
+
// existing in-memory test adapter, but cannot be used for production claims.
|
|
7
|
+
if (store.claimScheduledSlot) {
|
|
8
|
+
if (!store.finishScheduledSlot)
|
|
9
|
+
throw new Error('Atomic scheduled completion is required');
|
|
10
|
+
const claimed = await store.claimScheduledSlot(ref.jobId, ref.bucket, now);
|
|
11
|
+
if (claimed.status !== 'completed')
|
|
12
|
+
return claimed;
|
|
13
|
+
const slot = claimed.job.schedule?.currentRunSlot;
|
|
14
|
+
if (!slot)
|
|
15
|
+
throw new Error('Claimed schedule slot missing durable currentRunSlot');
|
|
16
|
+
await engine.reconcile(ref.jobId);
|
|
17
|
+
await engine.idle();
|
|
18
|
+
const job = await store.finishScheduledSlot(ref.jobId, slot, now);
|
|
19
|
+
return { job, status: 'completed' };
|
|
20
|
+
}
|
|
21
|
+
let job = await store.load(ref.jobId);
|
|
22
|
+
if (!job.schedule)
|
|
12
23
|
throw new Error('Scheduled job ' + job.id + ' has no schedule metadata');
|
|
24
|
+
const schedule = job.schedule;
|
|
13
25
|
const slot = new Date(schedule.nextRunAt);
|
|
14
26
|
const active = job.tasks.some(task => ['QUEUED', 'RUNNING'].includes(task.state));
|
|
15
|
-
|
|
16
|
-
let
|
|
27
|
+
let claimed = false;
|
|
28
|
+
let status = 'skipped-idempotent';
|
|
17
29
|
const decision = await dispatchScheduledRun({
|
|
18
30
|
slotDue: scheduleBucket(slot) === ref.bucket && slot.getTime() <= now.getTime(),
|
|
19
31
|
slotUnclaimed: !schedule.lastScheduledSlot || new Date(schedule.lastScheduledSlot).getTime() < slot.getTime(),
|
|
@@ -24,37 +36,32 @@ export async function runScheduledBucket(client, repository, trustedAuthors, ref
|
|
|
24
36
|
if (actionId === 'schedule.skipActiveAdvance') {
|
|
25
37
|
schedule.lastScheduledSlot = slot.toISOString();
|
|
26
38
|
schedule.nextRunAt = nextScheduleRun(schedule, now).toISOString();
|
|
27
|
-
|
|
28
|
-
await schedulerStore.saveJob(job);
|
|
29
|
-
actionStatus = 'skipped-active';
|
|
30
|
-
return {};
|
|
39
|
+
status = 'skipped-active';
|
|
31
40
|
}
|
|
32
|
-
if (actionId === 'schedule.claimFromSlot' || actionId === 'schedule.claimFromNow') {
|
|
41
|
+
else if (actionId === 'schedule.claimFromSlot' || actionId === 'schedule.claimFromNow') {
|
|
33
42
|
schedule.lastScheduledSlot = slot.toISOString();
|
|
34
43
|
schedule.currentRunSlot = slot.toISOString();
|
|
35
44
|
schedule.nextRunAt = nextScheduleRun(schedule, actionId === 'schedule.claimFromSlot' ? slot : now).toISOString();
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
job = await schedulerStore.load(job.id);
|
|
39
|
-
return {};
|
|
45
|
+
claimed = true;
|
|
46
|
+
status = 'completed';
|
|
40
47
|
}
|
|
41
|
-
|
|
48
|
+
else
|
|
49
|
+
throw new Error('Unknown local schedule action: ' + actionId);
|
|
50
|
+
job.updatedAt = now.toISOString();
|
|
51
|
+
await store.saveJob(job);
|
|
52
|
+
return {};
|
|
42
53
|
});
|
|
43
|
-
if (!decision.accepted ||
|
|
44
|
-
return { job, status
|
|
45
|
-
if (actionStatus === 'skipped-active' || decision.target === 'skip_active')
|
|
46
|
-
return { job, status: 'skipped-active' };
|
|
47
|
-
if (decision.target !== 'claimed')
|
|
48
|
-
return { job, status: 'skipped-idempotent' };
|
|
49
|
-
if (!engine)
|
|
50
|
-
throw new Error('Scheduled jobs require the local relayd scheduler engine');
|
|
54
|
+
if (!decision.accepted || !claimed)
|
|
55
|
+
return { job, status };
|
|
51
56
|
await engine.reconcile(job.id);
|
|
52
57
|
await engine.idle();
|
|
53
|
-
|
|
54
|
-
if (!
|
|
55
|
-
throw new Error('Scheduled job
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
job = await store.load(job.id);
|
|
59
|
+
if (!job.schedule)
|
|
60
|
+
throw new Error('Scheduled job lost schedule metadata');
|
|
61
|
+
job.schedule.lastRunAt = now.toISOString();
|
|
62
|
+
// A claim and its completion can share the same wall-clock timestamp.
|
|
63
|
+
// Advance the snapshot revision to satisfy SQLite's stale-write guard.
|
|
64
|
+
job.updatedAt = new Date(Math.max(now.getTime(), Date.parse(job.updatedAt) + 1)).toISOString();
|
|
65
|
+
await store.saveJob(job);
|
|
66
|
+
return { job, status: 'completed' };
|
|
60
67
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { LocalAuthorityDurableStore } from './local-authority-store.js';
|
|
2
|
+
import { errorMessage, observability } from './observability.js';
|
|
3
|
+
export async function flushServiceExternalActions(store, reconciler, jobId) {
|
|
4
|
+
if (!(store instanceof LocalAuthorityDurableStore) || store.noGitHub)
|
|
5
|
+
return 0;
|
|
6
|
+
const result = await store.projection.flushOnce();
|
|
7
|
+
if (result.projected > 0)
|
|
8
|
+
await reconciler.reconcile(jobId);
|
|
9
|
+
return result.projected;
|
|
10
|
+
}
|
|
11
|
+
export function startServiceExternalActionFlusher(store, reconciler, jobId, intervalMs) {
|
|
12
|
+
if (!(store instanceof LocalAuthorityDurableStore) || store.noGitHub)
|
|
13
|
+
return () => { };
|
|
14
|
+
const tick = () => {
|
|
15
|
+
void flushServiceExternalActions(store, reconciler, jobId).catch(error => observability().record({
|
|
16
|
+
stage: 'reconcile',
|
|
17
|
+
event: 'service.external_action_flush_failed',
|
|
18
|
+
level: 'warn',
|
|
19
|
+
outcome: 'failed',
|
|
20
|
+
message: errorMessage(error),
|
|
21
|
+
}));
|
|
22
|
+
};
|
|
23
|
+
tick();
|
|
24
|
+
const timer = setInterval(tick, intervalMs);
|
|
25
|
+
timer.unref();
|
|
26
|
+
return () => clearInterval(timer);
|
|
27
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { homedir } from 'node:os';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { eventFor } from './events.js';
|
|
4
|
+
import { SQLiteRelayStore, relayStorePath } from './relay-store.js';
|
|
5
|
+
/** Single-Job service uses the same durable publish facts as the repository daemon. */
|
|
6
|
+
export async function handleServicePublication(job, wake, bus, directory = join(homedir(), '.agents-relay', 'stores')) {
|
|
7
|
+
if (!wake.publication && !wake.publishFailure)
|
|
8
|
+
return;
|
|
9
|
+
const sqlite = new SQLiteRelayStore(relayStorePath(directory, job.repository, job.prNumber));
|
|
10
|
+
try {
|
|
11
|
+
await sqlite.recordPublication(job.id, wake.deliveryId, wake.publication, wake.publishFailure ?? null, wake.publicationObservedAt);
|
|
12
|
+
}
|
|
13
|
+
finally {
|
|
14
|
+
sqlite.close();
|
|
15
|
+
}
|
|
16
|
+
if (bus) {
|
|
17
|
+
if (wake.publication) {
|
|
18
|
+
const event = eventFor(job.id, 'release', null, 'package.published', 'succeeded', 'Published ' + wake.publication.package + '@' + wake.publication.version, 'orchestrator', { ...wake.publication, deliveryId: wake.deliveryId, repository: job.repository, prNumber: job.prNumber });
|
|
19
|
+
event.event_id = 'github:' + job.id + ':' + wake.deliveryId + ':package.published';
|
|
20
|
+
await bus.publish(event);
|
|
21
|
+
}
|
|
22
|
+
if (wake.publishFailure) {
|
|
23
|
+
const event = eventFor(job.id, 'release', null, 'package.publish_failed', 'failed', 'Publish failed: ' + wake.publishFailure.stage, 'orchestrator', { ...wake.publishFailure, deliveryId: wake.deliveryId, repository: job.repository, prNumber: job.prNumber });
|
|
24
|
+
event.event_id = 'github:' + job.id + ':' + wake.deliveryId + ':package.publish_failed';
|
|
25
|
+
await bus.publish(event);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
// Do not use a publication event to forge a Job state change.
|
|
29
|
+
}
|
package/dist/store-github.js
CHANGED
|
@@ -153,12 +153,23 @@ export class GitHubStore {
|
|
|
153
153
|
const payload = { merge_method: 'squash' };
|
|
154
154
|
if (expectedHeadSha)
|
|
155
155
|
payload.sha = expectedHeadSha;
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
156
|
+
try {
|
|
157
|
+
const result = await this.client.request('repos/' + this.repository + '/pulls/' + this.prNumber + '/merge', {
|
|
158
|
+
method: 'PUT',
|
|
159
|
+
body: JSON.stringify(payload),
|
|
160
|
+
});
|
|
161
|
+
if (result.merged !== true)
|
|
162
|
+
throw new Error(typeof result.message === 'string' ? result.message : 'Pull request #' + this.prNumber + ' was not merged');
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
// Crash recovery can replay a merge after GitHub committed but before
|
|
166
|
+
// its local acknowledgement. Treat replay as success ONLY after verifying
|
|
167
|
+
// the actual external PR is already MERGED; never assume success offline.
|
|
168
|
+
this.prNodeCache = null;
|
|
169
|
+
const state = await this.pullRequestState().catch(() => null);
|
|
170
|
+
if (state !== 'MERGED')
|
|
171
|
+
throw error;
|
|
172
|
+
}
|
|
162
173
|
this.prNodeCache = null;
|
|
163
174
|
}
|
|
164
175
|
async closePullRequest() {
|
package/dist/task-actions.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { extendRetryBudget, transitionTaskEvent, tryTransitionTaskEvent } from './scheduler.js';
|
|
2
2
|
import { selectRelatedTaskMutations } from './workflow/task-action-policy.js';
|
|
3
3
|
export const TASK_RETRY_ACTION = 'task.retry';
|
|
4
|
+
export const TASK_RECOVER_ACTION = 'task.recover';
|
|
4
5
|
export const TASK_UNBLOCK_RETRY_ACTION = 'task.unblock.retry';
|
|
5
6
|
export const TASK_CANCEL_ACTION = 'task.cancel';
|
|
6
7
|
export const TASK_RELEASE_ACTION = 'task.release';
|
|
@@ -28,6 +29,24 @@ function clearExecution(task) {
|
|
|
28
29
|
task.leaseExpiresAt = null;
|
|
29
30
|
task.executionId = null;
|
|
30
31
|
}
|
|
32
|
+
function applyTaskRetrySnapshot(task, policy = {}) {
|
|
33
|
+
if (task.kind === 'planner')
|
|
34
|
+
throw new Error(`Legacy planner Task ${task.id} is read-only; retry the Plan instead`);
|
|
35
|
+
const decision = tryTransitionTaskEvent(task, 'retry');
|
|
36
|
+
if (!decision.accepted)
|
|
37
|
+
throw lifecycleRejection(task, 'restartable');
|
|
38
|
+
if (policy.extendBudget)
|
|
39
|
+
task.maxAttempts = extendRetryBudget(task);
|
|
40
|
+
clearExecution(task);
|
|
41
|
+
}
|
|
42
|
+
/** Fixed-job authoritative unblock continuation, applied inside one Job snapshot mutation. */
|
|
43
|
+
export function applyTaskUnblockRetrySnapshot(task) {
|
|
44
|
+
if (!tryTransitionTaskEvent(task, 'retry').accepted)
|
|
45
|
+
throw lifecycleRejection(task, 'retryable after unblock');
|
|
46
|
+
task.maxAttempts = extendRetryBudget(task);
|
|
47
|
+
clearExecution(task);
|
|
48
|
+
delete task.blockedBy;
|
|
49
|
+
}
|
|
31
50
|
/** Shared task.retry mutation. The canonical lifecycle graph is the sole legality authority. */
|
|
32
51
|
export async function applyTaskRetryAction(task, policy = {}) {
|
|
33
52
|
if (task.kind === 'planner')
|
|
@@ -130,13 +149,48 @@ async function applySelectedRelatedTaskMutations(job, mutations) {
|
|
|
130
149
|
}
|
|
131
150
|
/** Compatibility bridge: domain mutation only; adapters own wake/reconcile/output behavior. */
|
|
132
151
|
export async function dispatchTaskAction(store, request) {
|
|
152
|
+
if (store.mutateJob && [TASK_RETRY_ACTION, TASK_RECOVER_ACTION, TASK_CANCEL_ACTION].includes(request.id)) {
|
|
153
|
+
let updatedTask;
|
|
154
|
+
const updated = await store.mutateJob(request.jobId, current => {
|
|
155
|
+
const task = current.tasks.find(item => item.id === request.taskId);
|
|
156
|
+
if (!task)
|
|
157
|
+
throw new Error(`Task ${request.taskId} not found`);
|
|
158
|
+
if (request.id === TASK_RECOVER_ACTION) {
|
|
159
|
+
const recovery = request.recoveryTaskId ? current.tasks.find(item => item.id === request.recoveryTaskId) : undefined;
|
|
160
|
+
if (!recovery || recovery.agentName !== 'troubleshooter' || recovery.parentTaskId !== task.id || !['QUEUED', 'RUNNING'].includes(recovery.state))
|
|
161
|
+
throw new Error(`task recover requires an active Troubleshooter recovery for ${task.id}`);
|
|
162
|
+
}
|
|
163
|
+
const relatedMutations = request.id === TASK_RECOVER_ACTION ? [] : selectRelatedTaskMutations(current, task, request.id);
|
|
164
|
+
if (request.id === TASK_RETRY_ACTION || request.id === TASK_RECOVER_ACTION)
|
|
165
|
+
applyTaskRetrySnapshot(task, { extendBudget: true });
|
|
166
|
+
else if (request.id === TASK_CANCEL_ACTION)
|
|
167
|
+
cancelTaskSnapshot(task);
|
|
168
|
+
for (const mutation of relatedMutations) {
|
|
169
|
+
const related = current.tasks.find(item => item.id === mutation.taskId);
|
|
170
|
+
if (!related)
|
|
171
|
+
throw new Error(`Related task ${mutation.taskId} not found`);
|
|
172
|
+
cancelTaskSnapshot(related, mutation.reason);
|
|
173
|
+
}
|
|
174
|
+
updatedTask = structuredClone(task);
|
|
175
|
+
return current;
|
|
176
|
+
});
|
|
177
|
+
if (!updatedTask)
|
|
178
|
+
throw new Error(`Task ${request.taskId} was not updated`);
|
|
179
|
+
return { job: updated, task: updatedTask };
|
|
180
|
+
}
|
|
133
181
|
const job = await store.load(request.jobId);
|
|
134
182
|
const task = job.tasks.find(item => item.id === request.taskId);
|
|
135
183
|
if (!task)
|
|
136
184
|
throw new Error(`Task ${request.taskId} not found`);
|
|
137
|
-
const relatedMutations = selectRelatedTaskMutations(job, task, request.id);
|
|
185
|
+
const relatedMutations = request.id === TASK_RECOVER_ACTION ? [] : selectRelatedTaskMutations(job, task, request.id);
|
|
138
186
|
if (request.id === TASK_RETRY_ACTION)
|
|
139
187
|
await applyTaskRetryAction(task, { extendBudget: true });
|
|
188
|
+
else if (request.id === TASK_RECOVER_ACTION) {
|
|
189
|
+
const recovery = request.recoveryTaskId ? job.tasks.find(item => item.id === request.recoveryTaskId) : undefined;
|
|
190
|
+
if (!recovery || recovery.agentName !== 'troubleshooter' || recovery.parentTaskId !== task.id || !['QUEUED', 'RUNNING'].includes(recovery.state))
|
|
191
|
+
throw new Error(`task recover requires an active Troubleshooter recovery for ${task.id}`);
|
|
192
|
+
await applyTaskRetryAction(task, { extendBudget: true });
|
|
193
|
+
}
|
|
140
194
|
else if (request.id === TASK_UNBLOCK_RETRY_ACTION)
|
|
141
195
|
await applyTaskUnblockRetryAction(task);
|
|
142
196
|
else if (request.id === TASK_CANCEL_ACTION)
|
|
@@ -157,3 +211,7 @@ export async function dispatchTaskAction(store, request) {
|
|
|
157
211
|
await store.saveTask(related);
|
|
158
212
|
return { job, task };
|
|
159
213
|
}
|
|
214
|
+
function cancelTaskSnapshot(task, reason = 'Cancelled by user') {
|
|
215
|
+
if (!tryApplyTaskCancelAction(task, reason))
|
|
216
|
+
throw lifecycleRejection(task, 'cancellable');
|
|
217
|
+
}
|
package/dist/task-update.js
CHANGED
|
@@ -44,7 +44,7 @@ export async function applyManagedTaskUpdate(store, jobId, taskId, proposed, exp
|
|
|
44
44
|
}
|
|
45
45
|
export function managedTaskUpdateHandler(locate, wake) {
|
|
46
46
|
return async (jobId, taskId, task, expectedUpdatedAt) => {
|
|
47
|
-
const target = locate(jobId);
|
|
47
|
+
const target = await locate(jobId);
|
|
48
48
|
const updated = await applyManagedTaskUpdate(target.store, jobId, taskId, task, expectedUpdatedAt);
|
|
49
49
|
await wake(target.repository, target.prNumber, jobId);
|
|
50
50
|
return updated;
|
package/dist/troubleshooter.js
CHANGED
|
@@ -26,7 +26,7 @@ Retrieve this task's lifecycle event history before classifying the root cause o
|
|
|
26
26
|
GET http://127.0.0.1:8787/api/events?job=${encodeURIComponent(job.id)}&task=${encodeURIComponent(task.id)}&limit=200
|
|
27
27
|
Inspect event order, type/status, source, data/phase, duplicate events, missing expected lifecycle events, late events, and execution identifiers. If the event stream points to delivery/runtime failure, inspect the corresponding Relay/worker/tool logs too. Do not treat the final task.error string as the complete root cause.
|
|
28
28
|
|
|
29
|
-
Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. The final task.error string alone is never sufficient. Classify the evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Only model/provider/runtime evidence may call the narrow model re-decision action; that action gathers fresh usage, reloads current policy, updates this exact Task's routing, and does not retry it. Troubleshooter must then reactivate/retry this exact original task and verify progress. Do not manually choose a provider/model or create a replacement task. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. If diagnosis suggests a genuine human/external blocker, do not publish task.blocked directly and do not implement the result branch procedurally. Call agents-relay escalation with the original intent, blocker, and relevant context. Pass any caller-defined notification plan through --on-needs-user/--on-needs-user-file; do not hard-code a channel. The canonical escalation graph invokes Neo escalation and owns the resolved/failed/needs_user branch; follow its selected outcome. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
|
|
29
|
+
Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. The final task.error string alone is never sufficient. Classify the evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Only model/provider/runtime evidence may call the narrow model re-decision action; that action gathers fresh usage, reloads current policy, updates this exact Task's routing, and does not retry it. Troubleshooter must then reactivate/retry this exact original task and verify progress. Use the dedicated agents-relay task recover path with --task-id ${task.id} and --recovery-task-id ${troubleshooterTaskId(task)}; never use ordinary task retry from a Troubleshooter because manual retry intentionally supersedes active recovery. Do not manually choose a provider/model or create a replacement task. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. If diagnosis suggests a genuine human/external blocker, do not publish task.blocked directly and do not implement the result branch procedurally. Call agents-relay escalation with the original intent, blocker, and relevant context. Pass any caller-defined notification plan through --on-needs-user/--on-needs-user-file; do not hard-code a channel. The canonical escalation graph invokes Neo escalation and owns the resolved/failed/needs_user branch; follow its selected outcome. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
|
|
30
30
|
}
|
|
31
31
|
function belongsToCurrentRun(job, task) {
|
|
32
32
|
const slot = job.schedule?.currentRunSlot;
|
|
@@ -79,7 +79,7 @@ export function pendingTroubleshooterTasks(job) {
|
|
|
79
79
|
continuation: null,
|
|
80
80
|
continuationDeliveredAt: null,
|
|
81
81
|
attempt: 0,
|
|
82
|
-
maxAttempts:
|
|
82
|
+
maxAttempts: 3,
|
|
83
83
|
leaseOwner: null,
|
|
84
84
|
leaseExpiresAt: null,
|
|
85
85
|
executionId: null,
|
|
@@ -94,3 +94,22 @@ export function pendingTroubleshooterTasks(job) {
|
|
|
94
94
|
}
|
|
95
95
|
return recoveries;
|
|
96
96
|
}
|
|
97
|
+
export function exhaustedTroubleshooterRecoveries(job) {
|
|
98
|
+
const results = [];
|
|
99
|
+
for (const recovery of job.tasks) {
|
|
100
|
+
if (recovery.agentName !== 'troubleshooter' || recovery.state !== 'FAILED' || recovery.attempt < recovery.maxAttempts || !recovery.parentTaskId)
|
|
101
|
+
continue;
|
|
102
|
+
const original = job.tasks.find(task => task.id === recovery.parentTaskId);
|
|
103
|
+
if (!original || !['FAILED', 'BLOCKED'].includes(original.state))
|
|
104
|
+
continue;
|
|
105
|
+
const alreadyRecorded = job.recoveryFailures?.some(item => item.taskId === original.id && item.recoveryTaskId === recovery.id);
|
|
106
|
+
if (alreadyRecorded)
|
|
107
|
+
continue;
|
|
108
|
+
results.push({
|
|
109
|
+
original,
|
|
110
|
+
recovery,
|
|
111
|
+
reason: recovery.error ?? recovery.failureReason ?? `Troubleshooter ${recovery.id} ${recovery.state.toLowerCase()}`,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
return results;
|
|
115
|
+
}
|
package/dist/unblock.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { applyTaskUnblockRetrySnapshot } from './task-actions.js';
|
|
2
|
+
import { tryTransitionTaskEvent } from './scheduler.js';
|
|
3
3
|
import { dispatchAuthoritativeUnblock } from './workflow/authoritative-unblock-v2.js';
|
|
4
4
|
export function isEffectivelyBlocked(task) { return task.kind !== 'planner' && task.state === 'BLOCKED'; }
|
|
5
5
|
function baselineFor(task) {
|
|
@@ -53,14 +53,29 @@ export async function unblockAutonomousTasks(options) {
|
|
|
53
53
|
const now = options.now ?? new Date().toISOString();
|
|
54
54
|
const currentHeadSha = await store.headSha?.() ?? job.headSha ?? null;
|
|
55
55
|
const facts = unblockFacts(job, tasks, currentHeadSha, trigger.reason);
|
|
56
|
-
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
56
|
+
const apply = (current) => {
|
|
57
|
+
for (const requested of tasks) {
|
|
58
|
+
const task = current.tasks.find(item => item.id === requested.id);
|
|
59
|
+
if (!task || !isEffectivelyBlocked(task))
|
|
60
|
+
throw new Error(`Task ${requested.id} changed before authoritative unblock`);
|
|
61
|
+
const baseline = baselineFor(task);
|
|
62
|
+
if (!tryTransitionTaskEvent(task, 'task.unblocked').accepted)
|
|
63
|
+
throw new Error(`Task ${task.id} cannot be authoritatively unblocked`);
|
|
64
|
+
task.blockedBaseline = baseline;
|
|
65
|
+
task.unblock = { at: now, source: trigger.source, reason: trigger.reason, baseline, currentHeadSha: currentHeadSha ?? undefined, facts, blockerJobId: trigger.blockerJobId, resolution: trigger.resolution };
|
|
66
|
+
task.updatedAt = now;
|
|
67
|
+
delete task.blockedBy;
|
|
68
|
+
}
|
|
69
|
+
current.updatedAt = now;
|
|
70
|
+
};
|
|
71
|
+
if (store.mutateJob) {
|
|
72
|
+
const committed = await store.mutateJob(job.id, current => { apply(current); return current; });
|
|
73
|
+
Object.assign(job, structuredClone(committed));
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
apply(job);
|
|
77
|
+
for (const task of tasks)
|
|
78
|
+
await store.saveTask(task);
|
|
64
79
|
}
|
|
65
80
|
const planTrigger = unconsumedUnblockTrigger(job) ?? {
|
|
66
81
|
source: 'unblock', taskIds: tasks.map(task => task.id), reason: trigger.reason,
|
|
@@ -84,11 +99,28 @@ export async function unblockTask(options) {
|
|
|
84
99
|
options.task.blockedBaseline = baseline;
|
|
85
100
|
const facts = unblockFacts(options.job, [options.task], currentHeadSha, options.trigger.reason);
|
|
86
101
|
options.task.unblock = { at: now, source: options.trigger.source, reason: options.trigger.reason, baseline, currentHeadSha: currentHeadSha ?? undefined, facts, blockerJobId: options.trigger.blockerJobId, resolution: options.trigger.resolution };
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
102
|
+
const apply = (current) => {
|
|
103
|
+
const task = current.tasks.find(item => item.id === options.task.id);
|
|
104
|
+
if (!task || !isEffectivelyBlocked(task))
|
|
105
|
+
throw new Error(`Task ${options.task.id} changed before authoritative unblock`);
|
|
106
|
+
if (!tryTransitionTaskEvent(task, 'task.unblocked').accepted)
|
|
107
|
+
throw new Error(`Task ${task.id} cannot be authoritatively unblocked`);
|
|
108
|
+
const baseline = baselineFor(task);
|
|
109
|
+
task.blockedBaseline = baseline;
|
|
110
|
+
task.unblock = { at: now, source: options.trigger.source, reason: options.trigger.reason, baseline, currentHeadSha: currentHeadSha ?? undefined, facts, blockerJobId: options.trigger.blockerJobId, resolution: options.trigger.resolution };
|
|
111
|
+
applyTaskUnblockRetrySnapshot(task);
|
|
112
|
+
task.updatedAt = now;
|
|
113
|
+
delete task.blockedBy;
|
|
114
|
+
current.updatedAt = now;
|
|
115
|
+
};
|
|
116
|
+
if (options.store.mutateJob) {
|
|
117
|
+
const committed = await options.store.mutateJob(options.job.id, current => { apply(current); return current; });
|
|
118
|
+
Object.assign(options.job, structuredClone(committed));
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
apply(options.job);
|
|
122
|
+
await options.store.saveTask(options.task);
|
|
123
|
+
}
|
|
92
124
|
result = { replanning: false };
|
|
93
125
|
return {};
|
|
94
126
|
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Classify genuine worker-process rejection text, regardless of E2E mode. */
|
|
2
|
+
export function classifyWorkerRejection(message) {
|
|
3
|
+
if (/rate[_ .-]?limit[_ .-]?exceeded|usage limit reached|quota[_ .-]?exceeded|insufficient[_ .-]?quota/i.test(message))
|
|
4
|
+
return 'rate_limit_exceeded';
|
|
5
|
+
if (/model[_ .-]?not[_ .-]?found|model (?:is )?(?:unavailable|not found|unsupported)|unsupported model/i.test(message))
|
|
6
|
+
return 'model_not_found';
|
|
7
|
+
if (/provider[_ .-]?unavailable|provider (?:is )?(?:unavailable|refused)|service unavailable/i.test(message))
|
|
8
|
+
return 'provider_unavailable';
|
|
9
|
+
return 'worker_process_failed';
|
|
10
|
+
}
|
|
11
|
+
export const reroutableWorkerRejections = new Set(['rate_limit_exceeded', 'model_not_found', 'provider_unavailable']);
|
|
@@ -12,6 +12,9 @@ export const WORKFLOW_EVENT_NAMES = [
|
|
|
12
12
|
'hold',
|
|
13
13
|
'lease.expired',
|
|
14
14
|
'implementation.completed',
|
|
15
|
+
'local.completed',
|
|
16
|
+
'local.merge.requested',
|
|
17
|
+
'effect.deliverLocalMerge.failed',
|
|
15
18
|
'pr.closed',
|
|
16
19
|
'pr.merged',
|
|
17
20
|
'queue',
|
|
@@ -48,7 +51,6 @@ export const WORKFLOW_FACT_NAMES = [
|
|
|
48
51
|
'scheduledRun',
|
|
49
52
|
'hasUnblock',
|
|
50
53
|
'manualPlan',
|
|
51
|
-
'prOpen',
|
|
52
54
|
'reviewDisabled',
|
|
53
55
|
'implementationComplete',
|
|
54
56
|
'headAvailable',
|
|
@@ -67,6 +69,7 @@ export const WORKFLOW_EFFECT_NAMES = [
|
|
|
67
69
|
'effect.launchTask',
|
|
68
70
|
'effect.cancelActiveTasks',
|
|
69
71
|
'effect.mergePullRequest',
|
|
72
|
+
'effect.deliverLocalMerge',
|
|
70
73
|
'effect.cleanupJobWorktree',
|
|
71
74
|
];
|
|
72
75
|
export const WORKFLOW_EFFECT_POLICIES = {
|
|
@@ -88,6 +91,12 @@ export const WORKFLOW_EFFECT_POLICIES = {
|
|
|
88
91
|
retry: 'workflow',
|
|
89
92
|
observability: 'workflow.effect',
|
|
90
93
|
},
|
|
94
|
+
'effect.deliverLocalMerge': {
|
|
95
|
+
idempotency: 'job-local-source-sha',
|
|
96
|
+
failureEvent: 'effect.deliverLocalMerge.failed',
|
|
97
|
+
retry: 'workflow',
|
|
98
|
+
observability: 'workflow.effect',
|
|
99
|
+
},
|
|
91
100
|
'effect.mergePullRequest': {
|
|
92
101
|
idempotency: 'job-head-sha',
|
|
93
102
|
failureEvent: 'effect.mergePullRequest.failed',
|
|
@@ -35,6 +35,8 @@ export const jobLifecycleV2 = load('job-lifecycle', JOB_STATES, {
|
|
|
35
35
|
implementationComplete: boolGuard('implementationComplete'),
|
|
36
36
|
headAvailable: boolGuard('headAvailable'),
|
|
37
37
|
mergeCapable: boolGuard('mergeCapable'),
|
|
38
|
+
localCompletionAllowed: boolGuard('localCompletionAllowed'),
|
|
39
|
+
localMergeReady: boolGuard('localMergeReady'),
|
|
38
40
|
});
|
|
39
41
|
function topologyTargets(graph, from) {
|
|
40
42
|
const source = [...graph.nodes.values()].find(node => node.kind === 'state' && node.semanticId === from);
|