agents-relay 2.0.58 → 2.0.60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,170 @@
1
+ import { spawn, spawnSync, } from 'node:child_process';
2
+ import { randomUUID } from 'node:crypto';
3
+ import { basename } from 'node:path';
4
+ import { observability } from './observability.js';
5
+ const owned = new Map();
6
+ const byChild = new WeakMap();
7
+ let exitHookInstalled = false;
8
+ function audit(record, event, outcome, data) {
9
+ observability().record({
10
+ stage: 'daemon',
11
+ event,
12
+ outcome,
13
+ level: outcome === 'failed' ? 'warn' : 'info',
14
+ message: `${record.invocation} process ${event.replace('process.', '')}`,
15
+ correlation: record.correlation,
16
+ data: {
17
+ invocationId: record.invocationId,
18
+ invocation: record.invocation,
19
+ executable: basename(record.command),
20
+ pid: record.pid,
21
+ processGroup: record.processGroup,
22
+ elapsedMs: Date.now() - record.startedAt,
23
+ ...data,
24
+ },
25
+ });
26
+ }
27
+ function signalOwned(record, signal) {
28
+ try {
29
+ if (record.processGroup !== null)
30
+ process.kill(-record.processGroup, signal);
31
+ else if (!record.child.kill(signal))
32
+ return 'gone';
33
+ return 'sent';
34
+ }
35
+ catch (error) {
36
+ if (error.code === 'ESRCH')
37
+ return 'gone';
38
+ audit(record, 'process.signal_failed', 'failed', { signal, error: error instanceof Error ? error.message : String(error) });
39
+ return 'error';
40
+ }
41
+ }
42
+ function forceOwned(record, reason) {
43
+ if (!owned.has(record.pid))
44
+ return;
45
+ if (process.platform === 'win32') {
46
+ const result = spawnSync('taskkill', ['/pid', String(record.pid), '/t', '/f'], { stdio: 'ignore', windowsHide: true });
47
+ audit(record, 'process.signal', result.status === 0 ? 'succeeded' : 'failed', { reason, signal: 'taskkill-/t-/f', exitStatus: result.status });
48
+ return;
49
+ }
50
+ const outcome = signalOwned(record, 'SIGKILL');
51
+ audit(record, 'process.signal', outcome === 'error' ? 'failed' : 'succeeded', { reason, signal: 'SIGKILL', delivered: outcome === 'sent', alreadyGone: outcome === 'gone' });
52
+ }
53
+ function installExitHook() {
54
+ if (exitHookInstalled)
55
+ return;
56
+ exitHookInstalled = true;
57
+ process.once('exit', () => {
58
+ for (const record of owned.values())
59
+ forceOwned(record, 'parent-shutdown');
60
+ });
61
+ }
62
+ function launchParentDeathSupervisor(pid, processGroup, graceMs) {
63
+ const posix = process.platform !== 'win32';
64
+ const script = posix ? [
65
+ "const pgid=Number(process.argv[1]),grace=Number(process.argv[2]);",
66
+ "const alive=()=>{try{process.kill(-pgid,0);return true}catch(e){return e&&e.code==='EPERM'}};",
67
+ "const stop=()=>{try{process.kill(-pgid,'SIGTERM')}catch{}setTimeout(()=>{try{process.kill(-pgid,'SIGKILL')}catch{}process.exit(0)},grace).unref()};",
68
+ "const timer=setInterval(()=>{if(!alive()){clearInterval(timer);process.exit(0);return}if(process.ppid===1){clearInterval(timer);stop()}},250);",
69
+ ].join('') : [
70
+ "const {spawnSync}=require('node:child_process'),owner=Number(process.argv[1]),pid=Number(process.argv[2]);",
71
+ "const alive=p=>{try{process.kill(p,0);return true}catch{return false}};",
72
+ "const timer=setInterval(()=>{if(!alive(pid)){clearInterval(timer);process.exit(0);return}if(!alive(owner)){clearInterval(timer);spawnSync('taskkill',['/pid',String(pid),'/t','/f'],{stdio:'ignore'});process.exit(0)}},250);",
73
+ ].join('');
74
+ const supervisorArgs = posix ? [String(processGroup), String(graceMs)] : [String(process.pid), String(pid)];
75
+ const supervisor = spawn(process.execPath, ['-e', script, ...supervisorArgs], {
76
+ detached: true,
77
+ stdio: 'ignore',
78
+ windowsHide: true,
79
+ });
80
+ supervisor.unref();
81
+ return supervisor;
82
+ }
83
+ /** Spawn one disposable Relay-owned process group. Arguments are intentionally excluded from audit logs. */
84
+ export function spawnOwnedProcess(command, args, options) {
85
+ installExitHook();
86
+ if (options.signal?.aborted)
87
+ throw new Error(`${options.invocation} aborted before launch`);
88
+ const detached = process.platform !== 'win32';
89
+ const child = spawn(command, [...args], { ...options.spawn, detached });
90
+ if (!child.pid)
91
+ return child;
92
+ const record = {
93
+ child,
94
+ invocationId: randomUUID(),
95
+ invocation: options.invocation,
96
+ command,
97
+ pid: child.pid,
98
+ processGroup: detached ? child.pid : null,
99
+ startedAt: Date.now(),
100
+ graceMs: options.graceMs ?? 750,
101
+ correlation: options.correlation,
102
+ terminating: false,
103
+ };
104
+ owned.set(record.pid, record);
105
+ byChild.set(child, record);
106
+ record.supervisor = launchParentDeathSupervisor(record.pid, record.processGroup, record.graceMs);
107
+ audit(record, 'process.launched', 'started', {});
108
+ const abort = () => { void terminateOwnedProcess(child, 'abort'); };
109
+ record.abort = abort;
110
+ options.signal?.addEventListener('abort', abort, { once: true });
111
+ if (options.timeoutMs !== undefined) {
112
+ record.timeoutTimer = setTimeout(() => { void terminateOwnedProcess(child, 'timeout'); }, options.timeoutMs);
113
+ record.timeoutTimer.unref();
114
+ }
115
+ child.once('exit', (code, signal) => {
116
+ audit(record, 'process.exited', code === 0 ? 'succeeded' : 'failed', { exitStatus: code, signal });
117
+ // `close` waits for inherited stdio. End any descendants left behind by a
118
+ // wrapper before they can keep the caller alive indefinitely.
119
+ if (!record.terminating)
120
+ void terminateOwnedProcess(child, 'leader-exit');
121
+ });
122
+ child.once('close', () => {
123
+ if (record.timeoutTimer)
124
+ clearTimeout(record.timeoutTimer);
125
+ options.signal?.removeEventListener('abort', abort);
126
+ // The leader closing is not proof that its process group is gone.
127
+ // In particular, unref'd grandchildren need not inherit stdio.
128
+ // Preserve ownership and the force-kill timer through the grace window.
129
+ if (!record.terminating)
130
+ void terminateOwnedProcess(child, 'leader-exit');
131
+ setTimeout(() => {
132
+ if (record.forceTimer)
133
+ clearTimeout(record.forceTimer);
134
+ record.supervisor?.kill('SIGTERM');
135
+ owned.delete(record.pid);
136
+ }, record.graceMs + 100).unref();
137
+ });
138
+ return child;
139
+ }
140
+ export async function terminateOwnedProcess(child, reason, graceMs) {
141
+ const record = byChild.get(child);
142
+ if (!record || !owned.has(record.pid))
143
+ return;
144
+ if (!record.terminating) {
145
+ record.terminating = true;
146
+ const outcome = signalOwned(record, 'SIGTERM');
147
+ audit(record, 'process.signal', outcome === 'error' ? 'failed' : 'succeeded', { reason, signal: 'SIGTERM', delivered: outcome === 'sent', alreadyGone: outcome === 'gone' });
148
+ record.forceTimer = setTimeout(() => forceOwned(record, reason), graceMs ?? record.graceMs);
149
+ record.forceTimer.unref();
150
+ }
151
+ if (child.exitCode !== null || child.signalCode !== null)
152
+ return;
153
+ await new Promise(resolve => {
154
+ const fallback = setTimeout(resolve, (graceMs ?? record.graceMs) + 1000);
155
+ fallback.unref();
156
+ child.once('close', () => { clearTimeout(fallback); resolve(); });
157
+ });
158
+ }
159
+ export async function shutdownOwnedProcesses(reason = 'parent-shutdown') {
160
+ await Promise.allSettled([...owned.values()].map(record => terminateOwnedProcess(record.child, reason)));
161
+ }
162
+ export function ownedProcessSnapshot() {
163
+ return [...owned.values()].map(record => ({
164
+ invocationId: record.invocationId,
165
+ invocation: record.invocation,
166
+ pid: record.pid,
167
+ processGroup: record.processGroup,
168
+ elapsedMs: Date.now() - record.startedAt,
169
+ }));
170
+ }
@@ -58,7 +58,6 @@ async function finishPlanner(ports, jobId, planId, executionId, error, raw) {
58
58
  plan.plannerResult = result;
59
59
  plan.assessment = result.assessment;
60
60
  plan.generatedTaskIds = result.next_tasks.map(item => item.id);
61
- plan.headSha = plan.trigger.currentHeadSha ?? await ports.store.headSha?.() ?? job.headSha;
62
61
  plan.error = undefined;
63
62
  plan.retryFeedback = undefined;
64
63
  plan.leaseOwner = null;
@@ -101,6 +100,9 @@ export async function launchPlanExecution(ports, job, plan) {
101
100
  const live = ports.live.get(fresh.id);
102
101
  if (fresh.executionId && fresh.leaseOwner === ports.options.owner && live?.executionId === fresh.executionId)
103
102
  return null;
103
+ fresh.headSha = await ports.store.headSha?.() ?? fresh.headSha;
104
+ if (fresh.headSha)
105
+ fresh.trigger = { ...fresh.trigger, currentHeadSha: fresh.headSha };
104
106
  const startedAt = (ports.options.now ?? new Date()).getTime();
105
107
  fresh.attempt = (fresh.attempt ?? 0) + 1;
106
108
  fresh.maxAttempts ??= fresh.manualPlanning ? 1 : 3;
package/dist/planner.js CHANGED
@@ -1,4 +1,3 @@
1
- import { spawn } from 'node:child_process';
2
1
  import { readFile } from 'node:fs/promises';
3
2
  import { fileURLToPath } from 'node:url';
4
3
  import { dirname, join } from 'node:path';
@@ -7,6 +6,7 @@ import { executionContextPrompt } from './execution-context.js';
7
6
  import { parseAgentGraph, validateAgentSelection } from './agent-graph.js';
8
7
  import { dispatchPlannerTrigger } from './workflow/planner-trigger-v2.js';
9
8
  import { createTaskEntity } from './workflow/entity-creation.js';
9
+ import { spawnOwnedProcess } from './owned-process.js';
10
10
  export function semanticTaskTree(tasks) {
11
11
  const work = tasks.filter(task => task.kind !== 'planner');
12
12
  const ids = new Set(work.map(task => task.id));
@@ -99,7 +99,9 @@ export function materializePlannerChild(job, planner, spec, executionContext) {
99
99
  const decisionId = 'round' in planner ? planner.id : planner.id;
100
100
  const source = 'round' in planner ? planner.source : planner.planningSource ?? 'model-planner';
101
101
  const round = 'round' in planner ? planner.round : planner.planningRound ?? 1;
102
- return createTaskEntity(job, { jobId: job.id, id: spec.id, kind: 'work', priority: spec.priority ?? job.priority, storySize: spec.storySize ?? 'M', projectName: spec.projectName, agentName: spec.agentName, agentGraph: spec.agentGraph, review: spec.review, parentTaskId: spec.parentTaskId ?? null, dependencies: spec.dependencies ?? [], capabilities: spec.capabilities ?? [], workerRequirements: spec.workerRequirements, adapter: 'codex', input: spec.input, output: spec.output, continuation: spec.continuation ?? null, continuationDeliveredAt: null, attempt: 0, maxAttempts: spec.maxAttempts ?? 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: spec.timeoutMs ?? taskOverallTimeoutMs(spec.storySize ?? 'M'), createdAt: now, updatedAt: now, planningDecisionId: decisionId, planningSource: source, planningRound: round });
102
+ const plannerHeadSha = 'round' in planner ? planner.headSha : undefined;
103
+ const reviewHeadSha = spec.agentName === 'reviewer' ? plannerHeadSha ?? job.headSha : undefined;
104
+ return createTaskEntity(job, { jobId: job.id, id: spec.id, kind: 'work', priority: spec.priority ?? job.priority, storySize: spec.storySize ?? 'M', projectName: spec.projectName, agentName: spec.agentName, agentGraph: spec.agentGraph, review: spec.review, parentTaskId: spec.parentTaskId ?? null, dependencies: spec.dependencies ?? [], capabilities: spec.capabilities ?? [], workerRequirements: spec.workerRequirements, adapter: 'codex', input: spec.input, output: spec.output, continuation: spec.continuation ?? null, continuationDeliveredAt: null, attempt: 0, maxAttempts: spec.maxAttempts ?? 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: spec.timeoutMs ?? taskOverallTimeoutMs(spec.storySize ?? 'M'), createdAt: now, updatedAt: now, planningDecisionId: decisionId, planningSource: source, planningRound: round, reviewHeadSha });
103
105
  }
104
106
  export function planningSpec(task) {
105
107
  if (!task.output && task.adapter !== 'orchestrator')
@@ -384,11 +386,11 @@ export class CommandObjectivePlanner {
384
386
  }
385
387
  async plan(context, signal) {
386
388
  return await new Promise((resolve, reject) => {
387
- const child = spawn(this.command, [], { stdio: ['pipe', 'pipe', 'pipe'] });
389
+ const child = spawnOwnedProcess(this.command, [], { invocation: 'planner.command', signal, correlation: { jobId: context.job.id }, spawn: { stdio: ['pipe', 'pipe', 'pipe'] } });
388
390
  let stdout = '';
389
391
  let stderr = '';
390
- child.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
391
- child.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
392
+ child.stdout?.on('data', (chunk) => { stdout += chunk.toString(); });
393
+ child.stderr?.on('data', (chunk) => { stderr += chunk.toString(); });
392
394
  child.on('error', reject);
393
395
  child.on('close', code => code === 0 ? (() => { try {
394
396
  resolve(parsePlannerOutput(stdout));
@@ -396,8 +398,7 @@ export class CommandObjectivePlanner {
396
398
  catch (error) {
397
399
  reject(error);
398
400
  } })() : reject(new Error(stderr.trim() || `Planner exited ${code}`)));
399
- signal.addEventListener('abort', () => child.kill('SIGTERM'), { once: true });
400
- child.stdin.end(plannerInput(context));
401
+ child.stdin?.end(plannerInput(context));
401
402
  });
402
403
  }
403
404
  }
@@ -430,7 +431,7 @@ export class AgentObjectivePlanner {
430
431
  const resolvedContext = context.executionContext ? executionContextPrompt(context.executionContext) + '\n\n' : '';
431
432
  const prompt = instructions.trim() + '\n\n## Shared Task / Job Contract\n\n' + contract.trim() + '\n\n' + resolvedContext + '## Durable planner input\n\n' + plannerInput(context);
432
433
  return await new Promise((resolve, reject) => {
433
- const child = spawn(this.command, ['exec', '--json', '--', prompt], { cwd: context.executionContext?.project.path, stdio: ['ignore', 'pipe', 'pipe'] });
434
+ const child = spawnOwnedProcess(this.command, ['exec', '--json', '--', prompt], { invocation: 'planner.agent', signal, correlation: { jobId: context.job.id }, spawn: { cwd: context.executionContext?.project.path, stdio: ['ignore', 'pipe', 'pipe'] } });
434
435
  let stdoutBuffer = '';
435
436
  let finalMessage = '';
436
437
  let stderrTail = '';
@@ -439,14 +440,14 @@ export class AgentObjectivePlanner {
439
440
  if (message)
440
441
  finalMessage = message;
441
442
  };
442
- child.stdout.on('data', (chunk) => {
443
+ child.stdout?.on('data', (chunk) => {
443
444
  stdoutBuffer += chunk.toString();
444
445
  const lines = stdoutBuffer.split(/\r?\n/);
445
446
  stdoutBuffer = lines.pop() ?? '';
446
447
  for (const line of lines)
447
448
  consume(line);
448
449
  });
449
- child.stderr.on('data', (chunk) => { stderrTail = (stderrTail + chunk.toString()).slice(-16384); });
450
+ child.stderr?.on('data', (chunk) => { stderrTail = (stderrTail + chunk.toString()).slice(-16384); });
450
451
  child.on('error', reject);
451
452
  child.on('close', code => {
452
453
  if (stdoutBuffer)
@@ -466,7 +467,6 @@ export class AgentObjectivePlanner {
466
467
  reject(error);
467
468
  }
468
469
  });
469
- signal.addEventListener('abort', () => child.kill('SIGTERM'), { once: true });
470
470
  });
471
471
  }
472
472
  }
@@ -0,0 +1,83 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { hostname } from 'node:os';
3
+ import { promisify } from 'node:util';
4
+ const execFileAsync = promisify(execFile);
5
+ /** ps lstart and PGID together guard against PID reuse after daemon restart. */
6
+ export async function observeProcess(pid) {
7
+ if (!Number.isSafeInteger(pid) || pid < 2)
8
+ return null;
9
+ try {
10
+ const { stdout } = await execFileAsync('ps', ['-p', String(pid), '-o', 'pgid=', '-o', 'lstart='], { timeout: 1500, maxBuffer: 2048 });
11
+ const match = stdout.trim().match(/^(\d+)\s+(.+)$/);
12
+ return match && Number(match[1]) > 0 ? { pgid: Number(match[1]), startToken: match[2].trim() } : null;
13
+ }
14
+ catch {
15
+ return null;
16
+ }
17
+ }
18
+ export async function captureProcessRecord(jobId, taskId, executionId, leaseOwner, pid) {
19
+ const snapshot = await observeProcess(pid);
20
+ return snapshot && snapshot.pgid === pid ? {
21
+ jobId, taskId, executionId, leaseOwner, host: hostname(), pid,
22
+ pgid: snapshot.pgid, startToken: snapshot.startToken, observedAt: new Date().toISOString(),
23
+ } : null;
24
+ }
25
+ export async function probeProcessRecord(record) {
26
+ if (record.host !== hostname() || !record.startToken || record.pid !== record.pgid)
27
+ return 'unknown';
28
+ const current = await observeProcess(record.pid);
29
+ if (current)
30
+ return current.pgid === record.pgid && current.startToken === record.startToken ? 'alive' : 'reused';
31
+ try {
32
+ process.kill(record.pid, 0);
33
+ return 'unknown';
34
+ }
35
+ catch (error) {
36
+ return error.code === 'ESRCH' ? 'gone' : 'unknown';
37
+ }
38
+ }
39
+ /** A separate still-running relay daemon must retain its live lease. */
40
+ export async function activeRelayLeaseOwner(owner) {
41
+ const match = owner.match(/^relayd-(\d+)$/);
42
+ if (!match || Number(match[1]) <= 1)
43
+ return false;
44
+ try {
45
+ const { stdout } = await execFileAsync('ps', ['-p', match[1], '-o', 'command='], { timeout: 1500, maxBuffer: 4096 });
46
+ return /\bagents-relayd\b/.test(stdout) || /(?:^|\/)relayd(?:-main)?\.js(?:\s|$)/.test(stdout);
47
+ }
48
+ catch {
49
+ return false;
50
+ }
51
+ }
52
+ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
53
+ /** Never signal another PID generation, remote host, or non-leader process group. */
54
+ export async function stopOrphanedProcessGroup(record, graceMs = 750) {
55
+ let state = await probeProcessRecord(record);
56
+ if (state === 'gone' || state === 'reused')
57
+ return true;
58
+ if (state !== 'alive' || process.platform === 'win32')
59
+ return false;
60
+ try {
61
+ process.kill(-record.pgid, 'SIGTERM');
62
+ }
63
+ catch (error) {
64
+ if (error.code !== 'ESRCH')
65
+ return false;
66
+ }
67
+ await sleep(graceMs);
68
+ state = await probeProcessRecord(record);
69
+ if (state === 'gone' || state === 'reused')
70
+ return true;
71
+ if (state !== 'alive')
72
+ return false;
73
+ try {
74
+ process.kill(-record.pgid, 'SIGKILL');
75
+ }
76
+ catch (error) {
77
+ if (error.code !== 'ESRCH')
78
+ return false;
79
+ }
80
+ await sleep(100);
81
+ state = await probeProcessRecord(record);
82
+ return state === 'gone' || state === 'reused';
83
+ }
@@ -192,6 +192,14 @@ export class ProjectedDurableStore {
192
192
  if (this.seededFacts)
193
193
  void this.relay.savePullRequestFacts(this.seededFacts).catch(() => { });
194
194
  }
195
+ // Process identities are operational SQLite records: never enqueue GitHub projection.
196
+ async recordExecutionProcess(record) { await this.relay.recordExecutionProcess(record); }
197
+ async loadExecutionProcess(jobId, taskId, executionId) {
198
+ return this.relay.loadExecutionProcess(jobId, taskId, executionId);
199
+ }
200
+ async finishExecutionProcess(jobId, taskId, executionId) {
201
+ await this.relay.finishExecutionProcess(jobId, taskId, executionId);
202
+ }
195
203
  /** Ingest trusted GitHub records without echoing them back through the projection outbox. */
196
204
  async hydrate(job) {
197
205
  await this.relay.ingestJobSnapshot(structuredClone(job));
@@ -25,6 +25,44 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
25
25
  noApprovedCurrentReview: !reviewTasks.some(task => task.state === 'SUCCEEDED' && job.review?.headSha === headSha && job.review.outcome === 'approved'),
26
26
  };
27
27
  }
28
+ static reviewRecord(value) {
29
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
30
+ }
31
+ static reviewHeadSha(event) {
32
+ const data = ReconcilerReviewBase.reviewRecord(event.data);
33
+ const outcome = ReconcilerReviewBase.reviewRecord(data.outcome);
34
+ const review = ReconcilerReviewBase.reviewRecord(data.review);
35
+ return [outcome.head_sha, outcome.headSha, review.head_sha, review.headSha, data.head_sha, data.headSha]
36
+ .find((value) => typeof value === 'string' && value.length > 0);
37
+ }
38
+ static reviewIssues(event) {
39
+ const data = ReconcilerReviewBase.reviewRecord(event.data);
40
+ const outcome = ReconcilerReviewBase.reviewRecord(data.outcome);
41
+ const review = ReconcilerReviewBase.reviewRecord(data.review);
42
+ const rawIssues = outcome.blocking_findings !== undefined ? outcome.blocking_findings
43
+ : outcome.blockingFindings !== undefined ? outcome.blockingFindings
44
+ : review.issues !== undefined ? review.issues
45
+ : data.issues;
46
+ if (!Array.isArray(rawIssues))
47
+ return [];
48
+ return rawIssues.map(item => {
49
+ if (typeof item === 'string')
50
+ return item.trim();
51
+ if (!item || typeof item !== 'object' || Array.isArray(item))
52
+ return '';
53
+ const value = item;
54
+ return (typeof value.summary === 'string' ? value.summary
55
+ : typeof value.message === 'string' ? value.message
56
+ : typeof value.finding === 'string' ? value.finding
57
+ : typeof value.title === 'string' ? value.title : '').trim();
58
+ }).filter(Boolean);
59
+ }
60
+ static normalizeReviewEvidence(event, task) {
61
+ const headSha = ReconcilerReviewBase.reviewHeadSha(event);
62
+ if (!headSha)
63
+ return null;
64
+ return { headSha, issues: ReconcilerReviewBase.reviewIssues(event) };
65
+ }
28
66
  async cancelStaleReviewTask(job) {
29
67
  const staleTask = job.review?.taskId ? job.tasks.find(task => task.id === job.review?.taskId) : undefined;
30
68
  if (!staleTask || !['DRAFT', 'QUEUED', 'RUNNING'].includes(staleTask.state))
@@ -179,7 +217,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
179
217
  id: task.id, kind: task.kind ?? 'work', storySize: task.storySize ?? 'M',
180
218
  priority: task.priority ?? null, agentName: task.agentName ?? null,
181
219
  parentTaskId: task.parentTaskId, dependencies: task.dependencies,
182
- capabilities: task.capabilities, input: task.input,
220
+ capabilities: task.capabilities, taskCapabilities: task.taskCapabilities, workerCapabilities: task.workerCapabilities, input: task.input,
183
221
  attachments: task.attachments ?? [], output: task.output ?? null,
184
222
  state: task.state, attempt: task.attempt, maxAttempts: task.maxAttempts,
185
223
  },
@@ -218,10 +256,14 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
218
256
  async applyReviewEvent(event) {
219
257
  const job = await this.store.load(event.job_id);
220
258
  const task = job.tasks.find(item => item.id === event.task_id && item.agentName === 'reviewer');
221
- const reviewData = event.data?.review && typeof event.data.review === 'object' ? event.data.review : undefined;
222
- const headSha = typeof reviewData?.headSha === 'string' ? reviewData.headSha : typeof event.data?.headSha === 'string' ? event.data.headSha : task?.reviewHeadSha;
223
- if (!task || !headSha || task.reviewHeadSha !== headSha)
259
+ if (!task)
260
+ return { applied: false, reconcile: false, reason: event.type + '.ignored' };
261
+ const normalizedEvidence = ReconcilerReviewBase.normalizeReviewEvidence(event, task);
262
+ if (!normalizedEvidence)
224
263
  return { applied: false, reconcile: false, reason: event.type + '.ignored' };
264
+ const headSha = normalizedEvidence.headSha;
265
+ if (task.reviewHeadSha !== headSha)
266
+ return { applied: false, reconcile: false, reason: event.type + '.stale' };
225
267
  const currentHeadSha = await this.store.headSha?.();
226
268
  if ((currentHeadSha && currentHeadSha !== headSha) || (job.headSha && job.headSha !== headSha))
227
269
  return { applied: false, reconcile: false, reason: event.type + '.stale' };
@@ -239,16 +281,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
239
281
  const reviewDecision = await dispatchReviewLifecycle(job.review?.outcome, event.type, { autoMerge: job.autoMerge !== false }, async (actionId) => {
240
282
  if (actionId === 'review.createFix') {
241
283
  needsReconcile = true;
242
- const rawIssues = Array.isArray(reviewData?.issues) ? reviewData.issues : [];
243
- const issues = rawIssues.map(item => {
244
- if (typeof item === 'string')
245
- return item.trim();
246
- if (item && typeof item === 'object') {
247
- const value = item;
248
- return (typeof value.summary === 'string' ? value.summary : typeof value.message === 'string' ? value.message : typeof value.title === 'string' ? value.title : '').trim();
249
- }
250
- return '';
251
- }).filter(Boolean);
284
+ const issues = normalizedEvidence.issues;
252
285
  if (issues.length === 0)
253
286
  issues.push(event.message || 'Address the reviewer-requested changes');
254
287
  const now = event.timestamp || new Date().toISOString();
@@ -275,7 +308,11 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
275
308
  const outcome = reviewOutcome(reviewDecision.target);
276
309
  if (!outcome)
277
310
  return { applied: false, reconcile: false, reason: event.type + '.ignored' };
278
- const evidence = { summary: event.message || 'Review ' + outcome, data: { ...(event.data ?? {}), review: reviewData ?? { headSha }, headSha, outcome } };
311
+ const reviewData = event.data?.review && typeof event.data.review === 'object' && !Array.isArray(event.data.review)
312
+ ? event.data.review : undefined;
313
+ const evidence = { summary: event.message || 'Review ' + outcome, data: {
314
+ ...(event.data ?? {}), review: { ...(reviewData ?? {}), headSha }, headSha, outcome, source: event.source,
315
+ } };
279
316
  job.review = { outcome, headSha, taskId: task.id, evidence, updatedAt: event.timestamp || new Date().toISOString() };
280
317
  job.updatedAt = job.review.updatedAt;
281
318
  await this.store.saveJob(job);
@@ -6,6 +6,8 @@ import { WorkflowEffectExecutor } from './workflow/effects.js';
6
6
  import { workflowEffectFailureEvent } from './workflow/contracts.js';
7
7
  import { ReconcileCoordinator } from './reconcile-coordinator.js';
8
8
  import { acceptedReportedTag, eventDataWithoutTag } from './task-tag.js';
9
+ import { rearmTerminalEventDeadline } from './terminal-event-watchdog.js';
10
+ import { monitorExecutionProcess } from './execution-process-tracker.js';
9
11
  export function executionEventSource(jobId, taskId, executionId) { return `job/${jobId}/task/${taskId}/execution/${executionId}`; }
10
12
  function eventSourceId(event) { return typeof event.source?.id === 'string' ? event.source.id : undefined; }
11
13
  function executionIdFromSource(sourceId, jobId, taskId) {
@@ -54,6 +56,7 @@ export class ReconcilerRuntimeBase {
54
56
  continuationLive = new Set();
55
57
  launchFailureClaims = new Set();
56
58
  terminalWaiters = new Map();
59
+ startAckCleanups = new Set();
57
60
  taskTransitions;
58
61
  eventCoordinator;
59
62
  detachedReconciles = new Set();
@@ -155,6 +158,7 @@ export class ReconcilerRuntimeBase {
155
158
  await this.eventCoordinator.settle();
156
159
  if (this.detachedReconciles.size > 0)
157
160
  await Promise.allSettled([...this.detachedReconciles]);
161
+ await Promise.allSettled([...this.startAckCleanups].map(stop => stop()));
158
162
  for (const live of this.live.values()) {
159
163
  const sourceId = executionEventSource(live.jobId, live.taskId, live.executionId);
160
164
  if (activeExecutionOwners.get(sourceId) === this)
@@ -204,13 +208,27 @@ export class ReconcilerRuntimeBase {
204
208
  rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_timeout: no worker task.started start-ack`));
205
209
  }, timeoutMs);
206
210
  let stopped = false;
207
- const stop = async () => { if (stopped)
208
- return; stopped = true; if (timer)
209
- clearTimeout(timer); await unsubscribe(); };
211
+ const stop = async () => {
212
+ if (stopped)
213
+ return;
214
+ stopped = true;
215
+ if (timer)
216
+ clearTimeout(timer);
217
+ if (!settled) {
218
+ settled = true;
219
+ rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_cancelled: start-ack owner closed`));
220
+ }
221
+ this.startAckCleanups.delete(stop);
222
+ await unsubscribe();
223
+ };
224
+ this.startAckCleanups.add(stop);
210
225
  return { ack, stop };
211
226
  }
212
227
  registerExecutionOwner(jobId, taskId, executionId) {
213
228
  activeExecutionOwners.set(executionEventSource(jobId, taskId, executionId), this);
229
+ const live = this.live.get(taskId);
230
+ if (live)
231
+ this.detach(monitorExecutionProcess(this.store, jobId, taskId, executionId, this.options.owner, live.execution), jobId, taskId, 'execution PID correlation');
214
232
  }
215
233
  cancelExecutionGeneration(jobId, taskId, executionId) {
216
234
  if (!executionId)
@@ -323,9 +341,10 @@ export class ReconcilerRuntimeBase {
323
341
  const task = job.tasks.find(item => item.id === taskId);
324
342
  if (!task || (task.executionId !== executionId && !terminalExecutionClaimed(jobId, taskId, executionId)))
325
343
  return;
326
- if (task.adapter !== 'chatgpt')
327
- this.clearTerminalTimer(taskId, executionId);
328
- await this.emit(eventFor(jobId, taskId, task.parentTaskId, task.adapter === 'chatgpt' ? 'task.process.async_exited' : 'task.process.exited', error ? 'failed' : 'succeeded', error ?? result?.summary ?? 'Worker runtime exited', 'orchestrator', { authoritative: false, waitingForTerminalEvent: true, processError: error }, { agent: 'relay', component: 'agents-relay' }));
344
+ if (task.adapter !== 'chatgpt' && task.executionId === executionId && live?.executionId === executionId)
345
+ rearmTerminalEventDeadline(this.terminalTimers, taskId, task.timeoutMs - (Date.now() - live.launchBarrier), this.options.postExitTerminalGraceMs ?? 15_000, () => this.detach(this.timeoutTask(jobId, taskId, executionId), jobId, taskId, 'post-exit terminal event timeout'));
346
+ await this.emit(eventFor(jobId, taskId, task.parentTaskId, task.adapter === 'chatgpt' ? 'task.process.async_exited' : 'task.process.exited', live?.execution.processExit ? (live.execution.processExit.exitCode === 0 ? 'succeeded' : 'failed') : (error ? 'failed' : 'succeeded'), result?.summary ?? (error ? 'Worker runtime exited with error' : 'Worker runtime exited'), 'orchestrator', { authoritative: false, waitingForTerminalEvent: true, exitCode: live?.execution.processExit?.exitCode ?? null,
347
+ signal: live?.execution.processExit?.signal ?? null, stderr: live?.execution.processExit?.stderr ?? '', processError: error, executionId, pid: live?.execution.pid ?? null }, { id: executionEventSource(jobId, taskId, executionId), agent: 'relay', component: 'agents-relay' }));
329
348
  }
330
349
  async timeoutTask(jobId, taskId, executionId) {
331
350
  const live = this.live.get(taskId);
@@ -470,7 +470,7 @@ export class Reconciler extends ReconcilerReviewBase {
470
470
  await this.failLaunch(job.id, task.id, task.executionId, error instanceof Error ? error.message : String(error));
471
471
  return;
472
472
  }
473
- void (execution.launched ?? Promise.resolve()).then(() => this.emit(eventFor(job.id, task.id, task.parentTaskId, 'task.process.launched', 'running', `Task ${task.id} process launched`, 'user', { adapter: adapter.id, model: task.routing?.model }, { agent: 'relay', component: 'agents-relay' }))).catch(error => this.failLaunch(job.id, task.id, durableExecutionId, error instanceof Error ? error.message : String(error), 'launch'));
473
+ void (execution.launched ?? Promise.resolve()).then(() => this.emit(eventFor(job.id, task.id, task.parentTaskId, 'task.process.launched', 'running', `Task ${task.id} process launched`, 'user', { adapter: adapter.id, model: task.routing?.model, pid: execution.pid ?? null, executionId: durableExecutionId }, { agent: 'relay', component: 'agents-relay' }))).catch(error => this.failLaunch(job.id, task.id, durableExecutionId, error instanceof Error ? error.message : String(error), 'launch'));
474
474
  if (startAckSetup)
475
475
  void startAckSetup.ack.finally(() => { void startAckSetup.stop(); }).catch(() => { });
476
476
  this.live.set(task.id, { jobId: job.id, taskId: task.id, executionId: durableExecutionId, execution, controller, launchBarrier: startedAt, workerStarted: false, processSettled: false });
@@ -74,6 +74,19 @@ export class SQLiteRelayStore {
74
74
  sequence INTEGER,
75
75
  received_at TEXT NOT NULL
76
76
  );
77
+ CREATE TABLE IF NOT EXISTS execution_process (
78
+ job_id TEXT NOT NULL,
79
+ task_id TEXT NOT NULL,
80
+ execution_id TEXT NOT NULL,
81
+ lease_owner TEXT NOT NULL,
82
+ host TEXT NOT NULL,
83
+ pid INTEGER NOT NULL,
84
+ pgid INTEGER NOT NULL,
85
+ start_token TEXT NOT NULL,
86
+ observed_at TEXT NOT NULL,
87
+ ended_at TEXT,
88
+ PRIMARY KEY(job_id,task_id,execution_id)
89
+ );
77
90
  ${ProjectionOutbox.createTableSql()}
78
91
  ${ProjectionOutbox.createIndexSql()}
79
92
  `);
@@ -201,6 +214,26 @@ export class SQLiteRelayStore {
201
214
  ON CONFLICT(id) DO UPDATE SET state=excluded.state,head_ref=excluded.head_ref,head_sha=excluded.head_sha,observed_at=excluded.observed_at
202
215
  `).run(facts.state, facts.head?.ref ?? null, facts.head?.sha ?? null, facts.observedAt);
203
216
  }
217
+ async recordExecutionProcess(record) {
218
+ const existing = await this.loadExecutionProcess(record.jobId, record.taskId, record.executionId);
219
+ if (existing && (existing.pid !== record.pid || existing.pgid !== record.pgid || existing.host !== record.host || existing.startToken !== record.startToken)) {
220
+ throw new Error(`Conflicting process identity for ${record.jobId}/${record.taskId}/${record.executionId}`);
221
+ }
222
+ this.db.prepare(`INSERT OR IGNORE INTO execution_process
223
+ (job_id,task_id,execution_id,lease_owner,host,pid,pgid,start_token,observed_at,ended_at)
224
+ VALUES(?,?,?,?,?,?,?,?,?,?)`).run(record.jobId, record.taskId, record.executionId, record.leaseOwner, record.host, record.pid, record.pgid, record.startToken, record.observedAt, record.endedAt ?? null);
225
+ }
226
+ async loadExecutionProcess(jobId, taskId, executionId) {
227
+ const row = this.db.prepare(`SELECT lease_owner,host,pid,pgid,start_token,observed_at,ended_at
228
+ FROM execution_process WHERE job_id=? AND task_id=? AND execution_id=?`).get(jobId, taskId, executionId);
229
+ return row ? { jobId, taskId, executionId, leaseOwner: row.lease_owner,
230
+ host: row.host, pid: row.pid, pgid: row.pgid, startToken: row.start_token,
231
+ observedAt: row.observed_at, endedAt: row.ended_at } : null;
232
+ }
233
+ async finishExecutionProcess(jobId, taskId, executionId, at = new Date().toISOString()) {
234
+ this.db.prepare('UPDATE execution_process SET ended_at=COALESCE(ended_at,?) WHERE job_id=? AND task_id=? AND execution_id=?')
235
+ .run(at, jobId, taskId, executionId);
236
+ }
204
237
  async markEventProcessed(eventId, stream = null, sequence = null) {
205
238
  const result = this.db.prepare(`
206
239
  INSERT OR IGNORE INTO event_receipt(event_id,stream,sequence,received_at) VALUES(?,?,?,?)