agents-relay 2.0.58 → 2.0.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters.js +15 -34
- package/dist/cli-escalation.js +4 -4
- package/dist/cli-shutdown.js +16 -0
- package/dist/cli-support.js +14 -1
- package/dist/cli.js +45 -18
- package/dist/continuation.js +3 -3
- package/dist/dashboard.js +2 -0
- package/dist/decision.js +53 -5
- package/dist/execution-process-tracker.js +35 -0
- package/dist/execution-recovery.js +7 -2
- package/dist/execution-routing.js +40 -8
- package/dist/github-client.js +8 -8
- package/dist/model-redecision.js +1 -1
- package/dist/owned-process.js +170 -0
- package/dist/planner-runtime.js +3 -1
- package/dist/planner.js +11 -11
- package/dist/process-identity.js +83 -0
- package/dist/projected-store.js +8 -0
- package/dist/reconciler-review.js +52 -15
- package/dist/reconciler-runtime.js +25 -6
- package/dist/reconciler.js +1 -1
- package/dist/relay-store.js +33 -0
- package/dist/relayd.js +9 -5
- package/dist/task-update.js +52 -0
- package/dist/terminal-event-watchdog.js +11 -0
- package/dist/usage.js +16 -10
- package/dist/workspace.js +14 -5
- package/package.json +1 -1
- package/workflow/task-lifecycle.mmd +1 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import { spawn, spawnSync, } from 'node:child_process';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { basename } from 'node:path';
|
|
4
|
+
import { observability } from './observability.js';
|
|
5
|
+
const owned = new Map();
|
|
6
|
+
const byChild = new WeakMap();
|
|
7
|
+
let exitHookInstalled = false;
|
|
8
|
+
function audit(record, event, outcome, data) {
|
|
9
|
+
observability().record({
|
|
10
|
+
stage: 'daemon',
|
|
11
|
+
event,
|
|
12
|
+
outcome,
|
|
13
|
+
level: outcome === 'failed' ? 'warn' : 'info',
|
|
14
|
+
message: `${record.invocation} process ${event.replace('process.', '')}`,
|
|
15
|
+
correlation: record.correlation,
|
|
16
|
+
data: {
|
|
17
|
+
invocationId: record.invocationId,
|
|
18
|
+
invocation: record.invocation,
|
|
19
|
+
executable: basename(record.command),
|
|
20
|
+
pid: record.pid,
|
|
21
|
+
processGroup: record.processGroup,
|
|
22
|
+
elapsedMs: Date.now() - record.startedAt,
|
|
23
|
+
...data,
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
function signalOwned(record, signal) {
|
|
28
|
+
try {
|
|
29
|
+
if (record.processGroup !== null)
|
|
30
|
+
process.kill(-record.processGroup, signal);
|
|
31
|
+
else if (!record.child.kill(signal))
|
|
32
|
+
return 'gone';
|
|
33
|
+
return 'sent';
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
if (error.code === 'ESRCH')
|
|
37
|
+
return 'gone';
|
|
38
|
+
audit(record, 'process.signal_failed', 'failed', { signal, error: error instanceof Error ? error.message : String(error) });
|
|
39
|
+
return 'error';
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function forceOwned(record, reason) {
|
|
43
|
+
if (!owned.has(record.pid))
|
|
44
|
+
return;
|
|
45
|
+
if (process.platform === 'win32') {
|
|
46
|
+
const result = spawnSync('taskkill', ['/pid', String(record.pid), '/t', '/f'], { stdio: 'ignore', windowsHide: true });
|
|
47
|
+
audit(record, 'process.signal', result.status === 0 ? 'succeeded' : 'failed', { reason, signal: 'taskkill-/t-/f', exitStatus: result.status });
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
const outcome = signalOwned(record, 'SIGKILL');
|
|
51
|
+
audit(record, 'process.signal', outcome === 'error' ? 'failed' : 'succeeded', { reason, signal: 'SIGKILL', delivered: outcome === 'sent', alreadyGone: outcome === 'gone' });
|
|
52
|
+
}
|
|
53
|
+
function installExitHook() {
|
|
54
|
+
if (exitHookInstalled)
|
|
55
|
+
return;
|
|
56
|
+
exitHookInstalled = true;
|
|
57
|
+
process.once('exit', () => {
|
|
58
|
+
for (const record of owned.values())
|
|
59
|
+
forceOwned(record, 'parent-shutdown');
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
function launchParentDeathSupervisor(pid, processGroup, graceMs) {
|
|
63
|
+
const posix = process.platform !== 'win32';
|
|
64
|
+
const script = posix ? [
|
|
65
|
+
"const pgid=Number(process.argv[1]),grace=Number(process.argv[2]);",
|
|
66
|
+
"const alive=()=>{try{process.kill(-pgid,0);return true}catch(e){return e&&e.code==='EPERM'}};",
|
|
67
|
+
"const stop=()=>{try{process.kill(-pgid,'SIGTERM')}catch{}setTimeout(()=>{try{process.kill(-pgid,'SIGKILL')}catch{}process.exit(0)},grace).unref()};",
|
|
68
|
+
"const timer=setInterval(()=>{if(!alive()){clearInterval(timer);process.exit(0);return}if(process.ppid===1){clearInterval(timer);stop()}},250);",
|
|
69
|
+
].join('') : [
|
|
70
|
+
"const {spawnSync}=require('node:child_process'),owner=Number(process.argv[1]),pid=Number(process.argv[2]);",
|
|
71
|
+
"const alive=p=>{try{process.kill(p,0);return true}catch{return false}};",
|
|
72
|
+
"const timer=setInterval(()=>{if(!alive(pid)){clearInterval(timer);process.exit(0);return}if(!alive(owner)){clearInterval(timer);spawnSync('taskkill',['/pid',String(pid),'/t','/f'],{stdio:'ignore'});process.exit(0)}},250);",
|
|
73
|
+
].join('');
|
|
74
|
+
const supervisorArgs = posix ? [String(processGroup), String(graceMs)] : [String(process.pid), String(pid)];
|
|
75
|
+
const supervisor = spawn(process.execPath, ['-e', script, ...supervisorArgs], {
|
|
76
|
+
detached: true,
|
|
77
|
+
stdio: 'ignore',
|
|
78
|
+
windowsHide: true,
|
|
79
|
+
});
|
|
80
|
+
supervisor.unref();
|
|
81
|
+
return supervisor;
|
|
82
|
+
}
|
|
83
|
+
/** Spawn one disposable Relay-owned process group. Arguments are intentionally excluded from audit logs. */
|
|
84
|
+
export function spawnOwnedProcess(command, args, options) {
|
|
85
|
+
installExitHook();
|
|
86
|
+
if (options.signal?.aborted)
|
|
87
|
+
throw new Error(`${options.invocation} aborted before launch`);
|
|
88
|
+
const detached = process.platform !== 'win32';
|
|
89
|
+
const child = spawn(command, [...args], { ...options.spawn, detached });
|
|
90
|
+
if (!child.pid)
|
|
91
|
+
return child;
|
|
92
|
+
const record = {
|
|
93
|
+
child,
|
|
94
|
+
invocationId: randomUUID(),
|
|
95
|
+
invocation: options.invocation,
|
|
96
|
+
command,
|
|
97
|
+
pid: child.pid,
|
|
98
|
+
processGroup: detached ? child.pid : null,
|
|
99
|
+
startedAt: Date.now(),
|
|
100
|
+
graceMs: options.graceMs ?? 750,
|
|
101
|
+
correlation: options.correlation,
|
|
102
|
+
terminating: false,
|
|
103
|
+
};
|
|
104
|
+
owned.set(record.pid, record);
|
|
105
|
+
byChild.set(child, record);
|
|
106
|
+
record.supervisor = launchParentDeathSupervisor(record.pid, record.processGroup, record.graceMs);
|
|
107
|
+
audit(record, 'process.launched', 'started', {});
|
|
108
|
+
const abort = () => { void terminateOwnedProcess(child, 'abort'); };
|
|
109
|
+
record.abort = abort;
|
|
110
|
+
options.signal?.addEventListener('abort', abort, { once: true });
|
|
111
|
+
if (options.timeoutMs !== undefined) {
|
|
112
|
+
record.timeoutTimer = setTimeout(() => { void terminateOwnedProcess(child, 'timeout'); }, options.timeoutMs);
|
|
113
|
+
record.timeoutTimer.unref();
|
|
114
|
+
}
|
|
115
|
+
child.once('exit', (code, signal) => {
|
|
116
|
+
audit(record, 'process.exited', code === 0 ? 'succeeded' : 'failed', { exitStatus: code, signal });
|
|
117
|
+
// `close` waits for inherited stdio. End any descendants left behind by a
|
|
118
|
+
// wrapper before they can keep the caller alive indefinitely.
|
|
119
|
+
if (!record.terminating)
|
|
120
|
+
void terminateOwnedProcess(child, 'leader-exit');
|
|
121
|
+
});
|
|
122
|
+
child.once('close', () => {
|
|
123
|
+
if (record.timeoutTimer)
|
|
124
|
+
clearTimeout(record.timeoutTimer);
|
|
125
|
+
options.signal?.removeEventListener('abort', abort);
|
|
126
|
+
// The leader closing is not proof that its process group is gone.
|
|
127
|
+
// In particular, unref'd grandchildren need not inherit stdio.
|
|
128
|
+
// Preserve ownership and the force-kill timer through the grace window.
|
|
129
|
+
if (!record.terminating)
|
|
130
|
+
void terminateOwnedProcess(child, 'leader-exit');
|
|
131
|
+
setTimeout(() => {
|
|
132
|
+
if (record.forceTimer)
|
|
133
|
+
clearTimeout(record.forceTimer);
|
|
134
|
+
record.supervisor?.kill('SIGTERM');
|
|
135
|
+
owned.delete(record.pid);
|
|
136
|
+
}, record.graceMs + 100).unref();
|
|
137
|
+
});
|
|
138
|
+
return child;
|
|
139
|
+
}
|
|
140
|
+
export async function terminateOwnedProcess(child, reason, graceMs) {
|
|
141
|
+
const record = byChild.get(child);
|
|
142
|
+
if (!record || !owned.has(record.pid))
|
|
143
|
+
return;
|
|
144
|
+
if (!record.terminating) {
|
|
145
|
+
record.terminating = true;
|
|
146
|
+
const outcome = signalOwned(record, 'SIGTERM');
|
|
147
|
+
audit(record, 'process.signal', outcome === 'error' ? 'failed' : 'succeeded', { reason, signal: 'SIGTERM', delivered: outcome === 'sent', alreadyGone: outcome === 'gone' });
|
|
148
|
+
record.forceTimer = setTimeout(() => forceOwned(record, reason), graceMs ?? record.graceMs);
|
|
149
|
+
record.forceTimer.unref();
|
|
150
|
+
}
|
|
151
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
152
|
+
return;
|
|
153
|
+
await new Promise(resolve => {
|
|
154
|
+
const fallback = setTimeout(resolve, (graceMs ?? record.graceMs) + 1000);
|
|
155
|
+
fallback.unref();
|
|
156
|
+
child.once('close', () => { clearTimeout(fallback); resolve(); });
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
export async function shutdownOwnedProcesses(reason = 'parent-shutdown') {
|
|
160
|
+
await Promise.allSettled([...owned.values()].map(record => terminateOwnedProcess(record.child, reason)));
|
|
161
|
+
}
|
|
162
|
+
export function ownedProcessSnapshot() {
|
|
163
|
+
return [...owned.values()].map(record => ({
|
|
164
|
+
invocationId: record.invocationId,
|
|
165
|
+
invocation: record.invocation,
|
|
166
|
+
pid: record.pid,
|
|
167
|
+
processGroup: record.processGroup,
|
|
168
|
+
elapsedMs: Date.now() - record.startedAt,
|
|
169
|
+
}));
|
|
170
|
+
}
|
package/dist/planner-runtime.js
CHANGED
|
@@ -58,7 +58,6 @@ async function finishPlanner(ports, jobId, planId, executionId, error, raw) {
|
|
|
58
58
|
plan.plannerResult = result;
|
|
59
59
|
plan.assessment = result.assessment;
|
|
60
60
|
plan.generatedTaskIds = result.next_tasks.map(item => item.id);
|
|
61
|
-
plan.headSha = plan.trigger.currentHeadSha ?? await ports.store.headSha?.() ?? job.headSha;
|
|
62
61
|
plan.error = undefined;
|
|
63
62
|
plan.retryFeedback = undefined;
|
|
64
63
|
plan.leaseOwner = null;
|
|
@@ -101,6 +100,9 @@ export async function launchPlanExecution(ports, job, plan) {
|
|
|
101
100
|
const live = ports.live.get(fresh.id);
|
|
102
101
|
if (fresh.executionId && fresh.leaseOwner === ports.options.owner && live?.executionId === fresh.executionId)
|
|
103
102
|
return null;
|
|
103
|
+
fresh.headSha = await ports.store.headSha?.() ?? fresh.headSha;
|
|
104
|
+
if (fresh.headSha)
|
|
105
|
+
fresh.trigger = { ...fresh.trigger, currentHeadSha: fresh.headSha };
|
|
104
106
|
const startedAt = (ports.options.now ?? new Date()).getTime();
|
|
105
107
|
fresh.attempt = (fresh.attempt ?? 0) + 1;
|
|
106
108
|
fresh.maxAttempts ??= fresh.manualPlanning ? 1 : 3;
|
package/dist/planner.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { spawn } from 'node:child_process';
|
|
2
1
|
import { readFile } from 'node:fs/promises';
|
|
3
2
|
import { fileURLToPath } from 'node:url';
|
|
4
3
|
import { dirname, join } from 'node:path';
|
|
@@ -7,6 +6,7 @@ import { executionContextPrompt } from './execution-context.js';
|
|
|
7
6
|
import { parseAgentGraph, validateAgentSelection } from './agent-graph.js';
|
|
8
7
|
import { dispatchPlannerTrigger } from './workflow/planner-trigger-v2.js';
|
|
9
8
|
import { createTaskEntity } from './workflow/entity-creation.js';
|
|
9
|
+
import { spawnOwnedProcess } from './owned-process.js';
|
|
10
10
|
export function semanticTaskTree(tasks) {
|
|
11
11
|
const work = tasks.filter(task => task.kind !== 'planner');
|
|
12
12
|
const ids = new Set(work.map(task => task.id));
|
|
@@ -99,7 +99,9 @@ export function materializePlannerChild(job, planner, spec, executionContext) {
|
|
|
99
99
|
const decisionId = 'round' in planner ? planner.id : planner.id;
|
|
100
100
|
const source = 'round' in planner ? planner.source : planner.planningSource ?? 'model-planner';
|
|
101
101
|
const round = 'round' in planner ? planner.round : planner.planningRound ?? 1;
|
|
102
|
-
|
|
102
|
+
const plannerHeadSha = 'round' in planner ? planner.headSha : undefined;
|
|
103
|
+
const reviewHeadSha = spec.agentName === 'reviewer' ? plannerHeadSha ?? job.headSha : undefined;
|
|
104
|
+
return createTaskEntity(job, { jobId: job.id, id: spec.id, kind: 'work', priority: spec.priority ?? job.priority, storySize: spec.storySize ?? 'M', projectName: spec.projectName, agentName: spec.agentName, agentGraph: spec.agentGraph, review: spec.review, parentTaskId: spec.parentTaskId ?? null, dependencies: spec.dependencies ?? [], capabilities: spec.capabilities ?? [], workerRequirements: spec.workerRequirements, adapter: 'codex', input: spec.input, output: spec.output, continuation: spec.continuation ?? null, continuationDeliveredAt: null, attempt: 0, maxAttempts: spec.maxAttempts ?? 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: spec.timeoutMs ?? taskOverallTimeoutMs(spec.storySize ?? 'M'), createdAt: now, updatedAt: now, planningDecisionId: decisionId, planningSource: source, planningRound: round, reviewHeadSha });
|
|
103
105
|
}
|
|
104
106
|
export function planningSpec(task) {
|
|
105
107
|
if (!task.output && task.adapter !== 'orchestrator')
|
|
@@ -384,11 +386,11 @@ export class CommandObjectivePlanner {
|
|
|
384
386
|
}
|
|
385
387
|
async plan(context, signal) {
|
|
386
388
|
return await new Promise((resolve, reject) => {
|
|
387
|
-
const child =
|
|
389
|
+
const child = spawnOwnedProcess(this.command, [], { invocation: 'planner.command', signal, correlation: { jobId: context.job.id }, spawn: { stdio: ['pipe', 'pipe', 'pipe'] } });
|
|
388
390
|
let stdout = '';
|
|
389
391
|
let stderr = '';
|
|
390
|
-
child.stdout
|
|
391
|
-
child.stderr
|
|
392
|
+
child.stdout?.on('data', (chunk) => { stdout += chunk.toString(); });
|
|
393
|
+
child.stderr?.on('data', (chunk) => { stderr += chunk.toString(); });
|
|
392
394
|
child.on('error', reject);
|
|
393
395
|
child.on('close', code => code === 0 ? (() => { try {
|
|
394
396
|
resolve(parsePlannerOutput(stdout));
|
|
@@ -396,8 +398,7 @@ export class CommandObjectivePlanner {
|
|
|
396
398
|
catch (error) {
|
|
397
399
|
reject(error);
|
|
398
400
|
} })() : reject(new Error(stderr.trim() || `Planner exited ${code}`)));
|
|
399
|
-
|
|
400
|
-
child.stdin.end(plannerInput(context));
|
|
401
|
+
child.stdin?.end(plannerInput(context));
|
|
401
402
|
});
|
|
402
403
|
}
|
|
403
404
|
}
|
|
@@ -430,7 +431,7 @@ export class AgentObjectivePlanner {
|
|
|
430
431
|
const resolvedContext = context.executionContext ? executionContextPrompt(context.executionContext) + '\n\n' : '';
|
|
431
432
|
const prompt = instructions.trim() + '\n\n## Shared Task / Job Contract\n\n' + contract.trim() + '\n\n' + resolvedContext + '## Durable planner input\n\n' + plannerInput(context);
|
|
432
433
|
return await new Promise((resolve, reject) => {
|
|
433
|
-
const child =
|
|
434
|
+
const child = spawnOwnedProcess(this.command, ['exec', '--json', '--', prompt], { invocation: 'planner.agent', signal, correlation: { jobId: context.job.id }, spawn: { cwd: context.executionContext?.project.path, stdio: ['ignore', 'pipe', 'pipe'] } });
|
|
434
435
|
let stdoutBuffer = '';
|
|
435
436
|
let finalMessage = '';
|
|
436
437
|
let stderrTail = '';
|
|
@@ -439,14 +440,14 @@ export class AgentObjectivePlanner {
|
|
|
439
440
|
if (message)
|
|
440
441
|
finalMessage = message;
|
|
441
442
|
};
|
|
442
|
-
child.stdout
|
|
443
|
+
child.stdout?.on('data', (chunk) => {
|
|
443
444
|
stdoutBuffer += chunk.toString();
|
|
444
445
|
const lines = stdoutBuffer.split(/\r?\n/);
|
|
445
446
|
stdoutBuffer = lines.pop() ?? '';
|
|
446
447
|
for (const line of lines)
|
|
447
448
|
consume(line);
|
|
448
449
|
});
|
|
449
|
-
child.stderr
|
|
450
|
+
child.stderr?.on('data', (chunk) => { stderrTail = (stderrTail + chunk.toString()).slice(-16384); });
|
|
450
451
|
child.on('error', reject);
|
|
451
452
|
child.on('close', code => {
|
|
452
453
|
if (stdoutBuffer)
|
|
@@ -466,7 +467,6 @@ export class AgentObjectivePlanner {
|
|
|
466
467
|
reject(error);
|
|
467
468
|
}
|
|
468
469
|
});
|
|
469
|
-
signal.addEventListener('abort', () => child.kill('SIGTERM'), { once: true });
|
|
470
470
|
});
|
|
471
471
|
}
|
|
472
472
|
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { hostname } from 'node:os';
|
|
3
|
+
import { promisify } from 'node:util';
|
|
4
|
+
const execFileAsync = promisify(execFile);
|
|
5
|
+
/** ps lstart and PGID together guard against PID reuse after daemon restart. */
|
|
6
|
+
export async function observeProcess(pid) {
|
|
7
|
+
if (!Number.isSafeInteger(pid) || pid < 2)
|
|
8
|
+
return null;
|
|
9
|
+
try {
|
|
10
|
+
const { stdout } = await execFileAsync('ps', ['-p', String(pid), '-o', 'pgid=', '-o', 'lstart='], { timeout: 1500, maxBuffer: 2048 });
|
|
11
|
+
const match = stdout.trim().match(/^(\d+)\s+(.+)$/);
|
|
12
|
+
return match && Number(match[1]) > 0 ? { pgid: Number(match[1]), startToken: match[2].trim() } : null;
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export async function captureProcessRecord(jobId, taskId, executionId, leaseOwner, pid) {
|
|
19
|
+
const snapshot = await observeProcess(pid);
|
|
20
|
+
return snapshot && snapshot.pgid === pid ? {
|
|
21
|
+
jobId, taskId, executionId, leaseOwner, host: hostname(), pid,
|
|
22
|
+
pgid: snapshot.pgid, startToken: snapshot.startToken, observedAt: new Date().toISOString(),
|
|
23
|
+
} : null;
|
|
24
|
+
}
|
|
25
|
+
export async function probeProcessRecord(record) {
|
|
26
|
+
if (record.host !== hostname() || !record.startToken || record.pid !== record.pgid)
|
|
27
|
+
return 'unknown';
|
|
28
|
+
const current = await observeProcess(record.pid);
|
|
29
|
+
if (current)
|
|
30
|
+
return current.pgid === record.pgid && current.startToken === record.startToken ? 'alive' : 'reused';
|
|
31
|
+
try {
|
|
32
|
+
process.kill(record.pid, 0);
|
|
33
|
+
return 'unknown';
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
return error.code === 'ESRCH' ? 'gone' : 'unknown';
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** A separate still-running relay daemon must retain its live lease. */
|
|
40
|
+
export async function activeRelayLeaseOwner(owner) {
|
|
41
|
+
const match = owner.match(/^relayd-(\d+)$/);
|
|
42
|
+
if (!match || Number(match[1]) <= 1)
|
|
43
|
+
return false;
|
|
44
|
+
try {
|
|
45
|
+
const { stdout } = await execFileAsync('ps', ['-p', match[1], '-o', 'command='], { timeout: 1500, maxBuffer: 4096 });
|
|
46
|
+
return /\bagents-relayd\b/.test(stdout) || /(?:^|\/)relayd(?:-main)?\.js(?:\s|$)/.test(stdout);
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
53
|
+
/** Never signal another PID generation, remote host, or non-leader process group. */
|
|
54
|
+
export async function stopOrphanedProcessGroup(record, graceMs = 750) {
|
|
55
|
+
let state = await probeProcessRecord(record);
|
|
56
|
+
if (state === 'gone' || state === 'reused')
|
|
57
|
+
return true;
|
|
58
|
+
if (state !== 'alive' || process.platform === 'win32')
|
|
59
|
+
return false;
|
|
60
|
+
try {
|
|
61
|
+
process.kill(-record.pgid, 'SIGTERM');
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
if (error.code !== 'ESRCH')
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
await sleep(graceMs);
|
|
68
|
+
state = await probeProcessRecord(record);
|
|
69
|
+
if (state === 'gone' || state === 'reused')
|
|
70
|
+
return true;
|
|
71
|
+
if (state !== 'alive')
|
|
72
|
+
return false;
|
|
73
|
+
try {
|
|
74
|
+
process.kill(-record.pgid, 'SIGKILL');
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
if (error.code !== 'ESRCH')
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
await sleep(100);
|
|
81
|
+
state = await probeProcessRecord(record);
|
|
82
|
+
return state === 'gone' || state === 'reused';
|
|
83
|
+
}
|
package/dist/projected-store.js
CHANGED
|
@@ -192,6 +192,14 @@ export class ProjectedDurableStore {
|
|
|
192
192
|
if (this.seededFacts)
|
|
193
193
|
void this.relay.savePullRequestFacts(this.seededFacts).catch(() => { });
|
|
194
194
|
}
|
|
195
|
+
// Process identities are operational SQLite records: never enqueue GitHub projection.
|
|
196
|
+
async recordExecutionProcess(record) { await this.relay.recordExecutionProcess(record); }
|
|
197
|
+
async loadExecutionProcess(jobId, taskId, executionId) {
|
|
198
|
+
return this.relay.loadExecutionProcess(jobId, taskId, executionId);
|
|
199
|
+
}
|
|
200
|
+
async finishExecutionProcess(jobId, taskId, executionId) {
|
|
201
|
+
await this.relay.finishExecutionProcess(jobId, taskId, executionId);
|
|
202
|
+
}
|
|
195
203
|
/** Ingest trusted GitHub records without echoing them back through the projection outbox. */
|
|
196
204
|
async hydrate(job) {
|
|
197
205
|
await this.relay.ingestJobSnapshot(structuredClone(job));
|
|
@@ -25,6 +25,44 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
25
25
|
noApprovedCurrentReview: !reviewTasks.some(task => task.state === 'SUCCEEDED' && job.review?.headSha === headSha && job.review.outcome === 'approved'),
|
|
26
26
|
};
|
|
27
27
|
}
|
|
28
|
+
static reviewRecord(value) {
|
|
29
|
+
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
30
|
+
}
|
|
31
|
+
static reviewHeadSha(event) {
|
|
32
|
+
const data = ReconcilerReviewBase.reviewRecord(event.data);
|
|
33
|
+
const outcome = ReconcilerReviewBase.reviewRecord(data.outcome);
|
|
34
|
+
const review = ReconcilerReviewBase.reviewRecord(data.review);
|
|
35
|
+
return [outcome.head_sha, outcome.headSha, review.head_sha, review.headSha, data.head_sha, data.headSha]
|
|
36
|
+
.find((value) => typeof value === 'string' && value.length > 0);
|
|
37
|
+
}
|
|
38
|
+
static reviewIssues(event) {
|
|
39
|
+
const data = ReconcilerReviewBase.reviewRecord(event.data);
|
|
40
|
+
const outcome = ReconcilerReviewBase.reviewRecord(data.outcome);
|
|
41
|
+
const review = ReconcilerReviewBase.reviewRecord(data.review);
|
|
42
|
+
const rawIssues = outcome.blocking_findings !== undefined ? outcome.blocking_findings
|
|
43
|
+
: outcome.blockingFindings !== undefined ? outcome.blockingFindings
|
|
44
|
+
: review.issues !== undefined ? review.issues
|
|
45
|
+
: data.issues;
|
|
46
|
+
if (!Array.isArray(rawIssues))
|
|
47
|
+
return [];
|
|
48
|
+
return rawIssues.map(item => {
|
|
49
|
+
if (typeof item === 'string')
|
|
50
|
+
return item.trim();
|
|
51
|
+
if (!item || typeof item !== 'object' || Array.isArray(item))
|
|
52
|
+
return '';
|
|
53
|
+
const value = item;
|
|
54
|
+
return (typeof value.summary === 'string' ? value.summary
|
|
55
|
+
: typeof value.message === 'string' ? value.message
|
|
56
|
+
: typeof value.finding === 'string' ? value.finding
|
|
57
|
+
: typeof value.title === 'string' ? value.title : '').trim();
|
|
58
|
+
}).filter(Boolean);
|
|
59
|
+
}
|
|
60
|
+
static normalizeReviewEvidence(event, task) {
|
|
61
|
+
const headSha = ReconcilerReviewBase.reviewHeadSha(event);
|
|
62
|
+
if (!headSha)
|
|
63
|
+
return null;
|
|
64
|
+
return { headSha, issues: ReconcilerReviewBase.reviewIssues(event) };
|
|
65
|
+
}
|
|
28
66
|
async cancelStaleReviewTask(job) {
|
|
29
67
|
const staleTask = job.review?.taskId ? job.tasks.find(task => task.id === job.review?.taskId) : undefined;
|
|
30
68
|
if (!staleTask || !['DRAFT', 'QUEUED', 'RUNNING'].includes(staleTask.state))
|
|
@@ -179,7 +217,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
179
217
|
id: task.id, kind: task.kind ?? 'work', storySize: task.storySize ?? 'M',
|
|
180
218
|
priority: task.priority ?? null, agentName: task.agentName ?? null,
|
|
181
219
|
parentTaskId: task.parentTaskId, dependencies: task.dependencies,
|
|
182
|
-
capabilities: task.capabilities, input: task.input,
|
|
220
|
+
capabilities: task.capabilities, taskCapabilities: task.taskCapabilities, workerCapabilities: task.workerCapabilities, input: task.input,
|
|
183
221
|
attachments: task.attachments ?? [], output: task.output ?? null,
|
|
184
222
|
state: task.state, attempt: task.attempt, maxAttempts: task.maxAttempts,
|
|
185
223
|
},
|
|
@@ -218,10 +256,14 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
218
256
|
async applyReviewEvent(event) {
|
|
219
257
|
const job = await this.store.load(event.job_id);
|
|
220
258
|
const task = job.tasks.find(item => item.id === event.task_id && item.agentName === 'reviewer');
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
259
|
+
if (!task)
|
|
260
|
+
return { applied: false, reconcile: false, reason: event.type + '.ignored' };
|
|
261
|
+
const normalizedEvidence = ReconcilerReviewBase.normalizeReviewEvidence(event, task);
|
|
262
|
+
if (!normalizedEvidence)
|
|
224
263
|
return { applied: false, reconcile: false, reason: event.type + '.ignored' };
|
|
264
|
+
const headSha = normalizedEvidence.headSha;
|
|
265
|
+
if (task.reviewHeadSha !== headSha)
|
|
266
|
+
return { applied: false, reconcile: false, reason: event.type + '.stale' };
|
|
225
267
|
const currentHeadSha = await this.store.headSha?.();
|
|
226
268
|
if ((currentHeadSha && currentHeadSha !== headSha) || (job.headSha && job.headSha !== headSha))
|
|
227
269
|
return { applied: false, reconcile: false, reason: event.type + '.stale' };
|
|
@@ -239,16 +281,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
239
281
|
const reviewDecision = await dispatchReviewLifecycle(job.review?.outcome, event.type, { autoMerge: job.autoMerge !== false }, async (actionId) => {
|
|
240
282
|
if (actionId === 'review.createFix') {
|
|
241
283
|
needsReconcile = true;
|
|
242
|
-
const
|
|
243
|
-
const issues = rawIssues.map(item => {
|
|
244
|
-
if (typeof item === 'string')
|
|
245
|
-
return item.trim();
|
|
246
|
-
if (item && typeof item === 'object') {
|
|
247
|
-
const value = item;
|
|
248
|
-
return (typeof value.summary === 'string' ? value.summary : typeof value.message === 'string' ? value.message : typeof value.title === 'string' ? value.title : '').trim();
|
|
249
|
-
}
|
|
250
|
-
return '';
|
|
251
|
-
}).filter(Boolean);
|
|
284
|
+
const issues = normalizedEvidence.issues;
|
|
252
285
|
if (issues.length === 0)
|
|
253
286
|
issues.push(event.message || 'Address the reviewer-requested changes');
|
|
254
287
|
const now = event.timestamp || new Date().toISOString();
|
|
@@ -275,7 +308,11 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
275
308
|
const outcome = reviewOutcome(reviewDecision.target);
|
|
276
309
|
if (!outcome)
|
|
277
310
|
return { applied: false, reconcile: false, reason: event.type + '.ignored' };
|
|
278
|
-
const
|
|
311
|
+
const reviewData = event.data?.review && typeof event.data.review === 'object' && !Array.isArray(event.data.review)
|
|
312
|
+
? event.data.review : undefined;
|
|
313
|
+
const evidence = { summary: event.message || 'Review ' + outcome, data: {
|
|
314
|
+
...(event.data ?? {}), review: { ...(reviewData ?? {}), headSha }, headSha, outcome, source: event.source,
|
|
315
|
+
} };
|
|
279
316
|
job.review = { outcome, headSha, taskId: task.id, evidence, updatedAt: event.timestamp || new Date().toISOString() };
|
|
280
317
|
job.updatedAt = job.review.updatedAt;
|
|
281
318
|
await this.store.saveJob(job);
|
|
@@ -6,6 +6,8 @@ import { WorkflowEffectExecutor } from './workflow/effects.js';
|
|
|
6
6
|
import { workflowEffectFailureEvent } from './workflow/contracts.js';
|
|
7
7
|
import { ReconcileCoordinator } from './reconcile-coordinator.js';
|
|
8
8
|
import { acceptedReportedTag, eventDataWithoutTag } from './task-tag.js';
|
|
9
|
+
import { rearmTerminalEventDeadline } from './terminal-event-watchdog.js';
|
|
10
|
+
import { monitorExecutionProcess } from './execution-process-tracker.js';
|
|
9
11
|
export function executionEventSource(jobId, taskId, executionId) { return `job/${jobId}/task/${taskId}/execution/${executionId}`; }
|
|
10
12
|
function eventSourceId(event) { return typeof event.source?.id === 'string' ? event.source.id : undefined; }
|
|
11
13
|
function executionIdFromSource(sourceId, jobId, taskId) {
|
|
@@ -54,6 +56,7 @@ export class ReconcilerRuntimeBase {
|
|
|
54
56
|
continuationLive = new Set();
|
|
55
57
|
launchFailureClaims = new Set();
|
|
56
58
|
terminalWaiters = new Map();
|
|
59
|
+
startAckCleanups = new Set();
|
|
57
60
|
taskTransitions;
|
|
58
61
|
eventCoordinator;
|
|
59
62
|
detachedReconciles = new Set();
|
|
@@ -155,6 +158,7 @@ export class ReconcilerRuntimeBase {
|
|
|
155
158
|
await this.eventCoordinator.settle();
|
|
156
159
|
if (this.detachedReconciles.size > 0)
|
|
157
160
|
await Promise.allSettled([...this.detachedReconciles]);
|
|
161
|
+
await Promise.allSettled([...this.startAckCleanups].map(stop => stop()));
|
|
158
162
|
for (const live of this.live.values()) {
|
|
159
163
|
const sourceId = executionEventSource(live.jobId, live.taskId, live.executionId);
|
|
160
164
|
if (activeExecutionOwners.get(sourceId) === this)
|
|
@@ -204,13 +208,27 @@ export class ReconcilerRuntimeBase {
|
|
|
204
208
|
rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_timeout: no worker task.started start-ack`));
|
|
205
209
|
}, timeoutMs);
|
|
206
210
|
let stopped = false;
|
|
207
|
-
const stop = async () => {
|
|
208
|
-
|
|
209
|
-
|
|
211
|
+
const stop = async () => {
|
|
212
|
+
if (stopped)
|
|
213
|
+
return;
|
|
214
|
+
stopped = true;
|
|
215
|
+
if (timer)
|
|
216
|
+
clearTimeout(timer);
|
|
217
|
+
if (!settled) {
|
|
218
|
+
settled = true;
|
|
219
|
+
rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_cancelled: start-ack owner closed`));
|
|
220
|
+
}
|
|
221
|
+
this.startAckCleanups.delete(stop);
|
|
222
|
+
await unsubscribe();
|
|
223
|
+
};
|
|
224
|
+
this.startAckCleanups.add(stop);
|
|
210
225
|
return { ack, stop };
|
|
211
226
|
}
|
|
212
227
|
registerExecutionOwner(jobId, taskId, executionId) {
|
|
213
228
|
activeExecutionOwners.set(executionEventSource(jobId, taskId, executionId), this);
|
|
229
|
+
const live = this.live.get(taskId);
|
|
230
|
+
if (live)
|
|
231
|
+
this.detach(monitorExecutionProcess(this.store, jobId, taskId, executionId, this.options.owner, live.execution), jobId, taskId, 'execution PID correlation');
|
|
214
232
|
}
|
|
215
233
|
cancelExecutionGeneration(jobId, taskId, executionId) {
|
|
216
234
|
if (!executionId)
|
|
@@ -323,9 +341,10 @@ export class ReconcilerRuntimeBase {
|
|
|
323
341
|
const task = job.tasks.find(item => item.id === taskId);
|
|
324
342
|
if (!task || (task.executionId !== executionId && !terminalExecutionClaimed(jobId, taskId, executionId)))
|
|
325
343
|
return;
|
|
326
|
-
if (task.adapter !== 'chatgpt')
|
|
327
|
-
this.
|
|
328
|
-
await this.emit(eventFor(jobId, taskId, task.parentTaskId, task.adapter === 'chatgpt' ? 'task.process.async_exited' : 'task.process.exited', error ? 'failed' : 'succeeded',
|
|
344
|
+
if (task.adapter !== 'chatgpt' && task.executionId === executionId && live?.executionId === executionId)
|
|
345
|
+
rearmTerminalEventDeadline(this.terminalTimers, taskId, task.timeoutMs - (Date.now() - live.launchBarrier), this.options.postExitTerminalGraceMs ?? 15_000, () => this.detach(this.timeoutTask(jobId, taskId, executionId), jobId, taskId, 'post-exit terminal event timeout'));
|
|
346
|
+
await this.emit(eventFor(jobId, taskId, task.parentTaskId, task.adapter === 'chatgpt' ? 'task.process.async_exited' : 'task.process.exited', live?.execution.processExit ? (live.execution.processExit.exitCode === 0 ? 'succeeded' : 'failed') : (error ? 'failed' : 'succeeded'), result?.summary ?? (error ? 'Worker runtime exited with error' : 'Worker runtime exited'), 'orchestrator', { authoritative: false, waitingForTerminalEvent: true, exitCode: live?.execution.processExit?.exitCode ?? null,
|
|
347
|
+
signal: live?.execution.processExit?.signal ?? null, stderr: live?.execution.processExit?.stderr ?? '', processError: error, executionId, pid: live?.execution.pid ?? null }, { id: executionEventSource(jobId, taskId, executionId), agent: 'relay', component: 'agents-relay' }));
|
|
329
348
|
}
|
|
330
349
|
async timeoutTask(jobId, taskId, executionId) {
|
|
331
350
|
const live = this.live.get(taskId);
|
package/dist/reconciler.js
CHANGED
|
@@ -470,7 +470,7 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
470
470
|
await this.failLaunch(job.id, task.id, task.executionId, error instanceof Error ? error.message : String(error));
|
|
471
471
|
return;
|
|
472
472
|
}
|
|
473
|
-
void (execution.launched ?? Promise.resolve()).then(() => this.emit(eventFor(job.id, task.id, task.parentTaskId, 'task.process.launched', 'running', `Task ${task.id} process launched`, 'user', { adapter: adapter.id, model: task.routing?.model }, { agent: 'relay', component: 'agents-relay' }))).catch(error => this.failLaunch(job.id, task.id, durableExecutionId, error instanceof Error ? error.message : String(error), 'launch'));
|
|
473
|
+
void (execution.launched ?? Promise.resolve()).then(() => this.emit(eventFor(job.id, task.id, task.parentTaskId, 'task.process.launched', 'running', `Task ${task.id} process launched`, 'user', { adapter: adapter.id, model: task.routing?.model, pid: execution.pid ?? null, executionId: durableExecutionId }, { agent: 'relay', component: 'agents-relay' }))).catch(error => this.failLaunch(job.id, task.id, durableExecutionId, error instanceof Error ? error.message : String(error), 'launch'));
|
|
474
474
|
if (startAckSetup)
|
|
475
475
|
void startAckSetup.ack.finally(() => { void startAckSetup.stop(); }).catch(() => { });
|
|
476
476
|
this.live.set(task.id, { jobId: job.id, taskId: task.id, executionId: durableExecutionId, execution, controller, launchBarrier: startedAt, workerStarted: false, processSettled: false });
|
package/dist/relay-store.js
CHANGED
|
@@ -74,6 +74,19 @@ export class SQLiteRelayStore {
|
|
|
74
74
|
sequence INTEGER,
|
|
75
75
|
received_at TEXT NOT NULL
|
|
76
76
|
);
|
|
77
|
+
CREATE TABLE IF NOT EXISTS execution_process (
|
|
78
|
+
job_id TEXT NOT NULL,
|
|
79
|
+
task_id TEXT NOT NULL,
|
|
80
|
+
execution_id TEXT NOT NULL,
|
|
81
|
+
lease_owner TEXT NOT NULL,
|
|
82
|
+
host TEXT NOT NULL,
|
|
83
|
+
pid INTEGER NOT NULL,
|
|
84
|
+
pgid INTEGER NOT NULL,
|
|
85
|
+
start_token TEXT NOT NULL,
|
|
86
|
+
observed_at TEXT NOT NULL,
|
|
87
|
+
ended_at TEXT,
|
|
88
|
+
PRIMARY KEY(job_id,task_id,execution_id)
|
|
89
|
+
);
|
|
77
90
|
${ProjectionOutbox.createTableSql()}
|
|
78
91
|
${ProjectionOutbox.createIndexSql()}
|
|
79
92
|
`);
|
|
@@ -201,6 +214,26 @@ export class SQLiteRelayStore {
|
|
|
201
214
|
ON CONFLICT(id) DO UPDATE SET state=excluded.state,head_ref=excluded.head_ref,head_sha=excluded.head_sha,observed_at=excluded.observed_at
|
|
202
215
|
`).run(facts.state, facts.head?.ref ?? null, facts.head?.sha ?? null, facts.observedAt);
|
|
203
216
|
}
|
|
217
|
+
async recordExecutionProcess(record) {
|
|
218
|
+
const existing = await this.loadExecutionProcess(record.jobId, record.taskId, record.executionId);
|
|
219
|
+
if (existing && (existing.pid !== record.pid || existing.pgid !== record.pgid || existing.host !== record.host || existing.startToken !== record.startToken)) {
|
|
220
|
+
throw new Error(`Conflicting process identity for ${record.jobId}/${record.taskId}/${record.executionId}`);
|
|
221
|
+
}
|
|
222
|
+
this.db.prepare(`INSERT OR IGNORE INTO execution_process
|
|
223
|
+
(job_id,task_id,execution_id,lease_owner,host,pid,pgid,start_token,observed_at,ended_at)
|
|
224
|
+
VALUES(?,?,?,?,?,?,?,?,?,?)`).run(record.jobId, record.taskId, record.executionId, record.leaseOwner, record.host, record.pid, record.pgid, record.startToken, record.observedAt, record.endedAt ?? null);
|
|
225
|
+
}
|
|
226
|
+
async loadExecutionProcess(jobId, taskId, executionId) {
|
|
227
|
+
const row = this.db.prepare(`SELECT lease_owner,host,pid,pgid,start_token,observed_at,ended_at
|
|
228
|
+
FROM execution_process WHERE job_id=? AND task_id=? AND execution_id=?`).get(jobId, taskId, executionId);
|
|
229
|
+
return row ? { jobId, taskId, executionId, leaseOwner: row.lease_owner,
|
|
230
|
+
host: row.host, pid: row.pid, pgid: row.pgid, startToken: row.start_token,
|
|
231
|
+
observedAt: row.observed_at, endedAt: row.ended_at } : null;
|
|
232
|
+
}
|
|
233
|
+
async finishExecutionProcess(jobId, taskId, executionId, at = new Date().toISOString()) {
|
|
234
|
+
this.db.prepare('UPDATE execution_process SET ended_at=COALESCE(ended_at,?) WHERE job_id=? AND task_id=? AND execution_id=?')
|
|
235
|
+
.run(at, jobId, taskId, executionId);
|
|
236
|
+
}
|
|
204
237
|
async markEventProcessed(eventId, stream = null, sequence = null) {
|
|
205
238
|
const result = this.db.prepare(`
|
|
206
239
|
INSERT OR IGNORE INTO event_receipt(event_id,stream,sequence,received_at) VALUES(?,?,?,?)
|