agents-relay 2.0.190 → 2.0.192
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { GraphRuntime } from './workflow/graph-v2.js';
|
|
3
|
+
import { dispatchWorkflowGraphSync } from './workflow/dispatcher.js';
|
|
4
|
+
const actions = ['assign', 'reject', 'enqueue', 'dequeue', 'bindSession', 'bindThread', 'touch', 'release', 'quarantine', 'releaseQuarantine', 'keepQuarantine', 'recoverTurn', 'keepMonitoring', 'extendTimeout'];
|
|
5
|
+
const events = [
|
|
6
|
+
'pool.claim', 'pool.enqueue', 'pool.expired', 'pool.cancel', 'pool.unavailable', 'pool.sessionBound',
|
|
7
|
+
'pool.threadBound', 'pool.activity', 'pool.terminalVerified', 'pool.outcomeUnknown', 'pool.stopVerified', 'pool.poll', 'pool.progressObserved',
|
|
8
|
+
];
|
|
9
|
+
export const browserPoolGraph = new GraphRuntime(readFileSync(new URL('../workflow/browser-worker-pool.mmd', import.meta.url), 'utf8'), {
|
|
10
|
+
states: new Set(['AVAILABLE', 'WAITING', 'LEASED', 'QUARANTINED']),
|
|
11
|
+
guards: { providerHealthy: facts => facts.durableFacts.providerHealthy === true, recoveryDue: facts => facts.durableFacts.recoveryDue === true, freshOutput: facts => facts.durableFacts.freshOutput === true },
|
|
12
|
+
actions: Object.fromEntries(actions.map(name => [`pool.${name}`, { id: `pool.${name}` }])),
|
|
13
|
+
events: Object.fromEntries(events.map(name => [name, { id: name }])),
|
|
14
|
+
});
|
|
15
|
+
/** Graph owns permission and the effect selected; SQLite executes the effect atomically. */
|
|
16
|
+
export function requirePoolDecision(state, event, expectedAction, providerHealthy = true) {
|
|
17
|
+
const decision = dispatchWorkflowGraphSync(browserPoolGraph, { initialState: state, input: { transport: 'event', id: event, payload: null }, durableFacts: { providerHealthy }, graphName: 'browser-worker-pool' });
|
|
18
|
+
if (!decision.accepted || decision.actions.length !== 1 || decision.actions[0].id !== `pool.${expectedAction}`)
|
|
19
|
+
throw new Error(`Browser Pool Graph rejected ${state}/${event}: expected pool.${expectedAction}`);
|
|
20
|
+
return decision;
|
|
21
|
+
}
|
|
22
|
+
/** The executable Graph selects the next monitoring action; no browser heartbeat counts as task progress. */
|
|
23
|
+
export function monitorPoolDecision(recoveryDue) {
|
|
24
|
+
const decision = dispatchWorkflowGraphSync(browserPoolGraph, { initialState: 'LEASED', input: { transport: 'event', id: 'pool.poll', payload: null }, durableFacts: { recoveryDue }, graphName: 'browser-worker-pool' });
|
|
25
|
+
if (!decision.accepted || decision.target !== 'LEASED' || decision.actions.length !== 1)
|
|
26
|
+
throw Error('Invalid Pool monitoring graph');
|
|
27
|
+
const id = decision.actions[0].id;
|
|
28
|
+
if (id === 'pool.recoverTurn')
|
|
29
|
+
return 'recoverTurn';
|
|
30
|
+
if (id === 'pool.keepMonitoring')
|
|
31
|
+
return 'keepMonitoring';
|
|
32
|
+
throw Error(`Unknown Pool monitoring action ${id}`);
|
|
33
|
+
}
|
|
34
|
+
/** Only verified new assistant output can extend Relay's inactivity deadline. */
|
|
35
|
+
export function progressPoolDecision(freshOutput) {
|
|
36
|
+
const decision = dispatchWorkflowGraphSync(browserPoolGraph, { initialState: 'LEASED', input: { transport: 'event', id: 'pool.progressObserved', payload: null }, durableFacts: { freshOutput }, graphName: 'browser-worker-pool' });
|
|
37
|
+
if (!decision.accepted || decision.target !== 'LEASED' || decision.actions.length !== 1)
|
|
38
|
+
throw Error('Invalid Pool progress graph');
|
|
39
|
+
const id = decision.actions[0].id;
|
|
40
|
+
if (id === 'pool.extendTimeout')
|
|
41
|
+
return 'extendTimeout';
|
|
42
|
+
if (id === 'pool.keepMonitoring')
|
|
43
|
+
return 'keepMonitoring';
|
|
44
|
+
throw Error(`Unknown Pool progress action ${id}`);
|
|
45
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import { requirePoolDecision } from './browser-worker-pool-graph.js';
|
|
2
3
|
import { mkdirSync } from 'node:fs';
|
|
3
4
|
import { dirname, join } from 'node:path';
|
|
4
5
|
import { homedir } from 'node:os';
|
|
@@ -77,8 +78,11 @@ export class BrowserWorkerPoolManager {
|
|
|
77
78
|
try {
|
|
78
79
|
const now = this.clock();
|
|
79
80
|
const health = this.db.prepare('SELECT unavailable_until FROM browser_provider_health WHERE provider=?').get(provider);
|
|
80
|
-
if (health.unavailable_until > now)
|
|
81
|
+
if (health.unavailable_until > now) {
|
|
82
|
+
const waiting = this.db.prepare('SELECT 1 FROM browser_queue WHERE execution_id=? AND provider=?').get(executionId, provider);
|
|
83
|
+
requirePoolDecision(waiting ? 'WAITING' : 'AVAILABLE', 'pool.claim', 'reject', false);
|
|
81
84
|
throw new BrowserPoolAdmissionError('PROVIDER_UNAVAILABLE', `${provider} circuit open until ${health.unavailable_until}`);
|
|
85
|
+
}
|
|
82
86
|
const existing = this.db.prepare('SELECT * FROM browser_slots WHERE execution_id=? AND provider=?').get(executionId, provider);
|
|
83
87
|
if (existing)
|
|
84
88
|
throw new Error(`Execution ${executionId} already owns ${provider}-${existing.slot}: refuse duplicate browser submission`);
|
|
@@ -87,6 +91,7 @@ export class BrowserWorkerPoolManager {
|
|
|
87
91
|
if (!row || row.execution_id === executionId) {
|
|
88
92
|
const idle = this.db.prepare("SELECT * FROM browser_slots WHERE provider=? AND state='IDLE' ORDER BY slot LIMIT 1").get(provider);
|
|
89
93
|
if (idle) {
|
|
94
|
+
requirePoolDecision(row ? 'WAITING' : 'AVAILABLE', 'pool.claim', 'assign', true);
|
|
90
95
|
const generation = idle.generation + 1;
|
|
91
96
|
this.db.prepare("UPDATE browser_slots SET state='LEASED',execution_id=?,generation=?,session_id=NULL,thread_url=NULL,touched_at=? WHERE provider=? AND slot=? AND state='IDLE'")
|
|
92
97
|
.run(executionId, generation, now, provider, idle.slot);
|
|
@@ -100,7 +105,11 @@ export class BrowserWorkerPoolManager {
|
|
|
100
105
|
this.db.exec('COMMIT');
|
|
101
106
|
}
|
|
102
107
|
}
|
|
103
|
-
dequeue(executionId) {
|
|
108
|
+
dequeue(executionId, event) {
|
|
109
|
+
const queued = this.db.prepare('SELECT 1 FROM browser_queue WHERE execution_id=?').get(executionId);
|
|
110
|
+
if (!queued)
|
|
111
|
+
return;
|
|
112
|
+
requirePoolDecision('WAITING', event, 'dequeue');
|
|
104
113
|
this.db.prepare('DELETE FROM browser_queue WHERE execution_id=?').run(executionId);
|
|
105
114
|
}
|
|
106
115
|
async acquire(provider, executionId, signal) {
|
|
@@ -109,10 +118,23 @@ export class BrowserWorkerPoolManager {
|
|
|
109
118
|
const immediate = this.tryClaim(provider, executionId);
|
|
110
119
|
if (immediate)
|
|
111
120
|
return immediate;
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
121
|
+
// Queue capacity is a cross-process resource constraint; hold SQLite's writer lock
|
|
122
|
+
// across the queue count and insertion (not only during slot claiming).
|
|
123
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
124
|
+
try {
|
|
125
|
+
const count = this.db.prepare('SELECT count(*) AS n FROM browser_queue WHERE provider=?').get(provider).n;
|
|
126
|
+
const present = this.db.prepare('SELECT 1 FROM browser_queue WHERE execution_id=?').get(executionId);
|
|
127
|
+
if (count >= this.maxQueue && !present)
|
|
128
|
+
throw new BrowserPoolAdmissionError('POOL_CAPACITY_FULL', `${provider} queue is full`);
|
|
129
|
+
if (!present) {
|
|
130
|
+
requirePoolDecision('AVAILABLE', 'pool.enqueue', 'enqueue');
|
|
131
|
+
this.db.prepare('INSERT INTO browser_queue(execution_id,provider,queued_at) VALUES(?,?,?)').run(executionId, provider, this.clock());
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
this.db.exec('COMMIT');
|
|
136
|
+
}
|
|
137
|
+
let dequeueEvent = 'pool.cancel';
|
|
116
138
|
try {
|
|
117
139
|
for (;;) {
|
|
118
140
|
if (signal?.aborted)
|
|
@@ -120,24 +142,32 @@ export class BrowserWorkerPoolManager {
|
|
|
120
142
|
const result = this.tryClaim(provider, executionId);
|
|
121
143
|
if (result)
|
|
122
144
|
return result;
|
|
123
|
-
if (this.clock() >= deadline)
|
|
145
|
+
if (this.clock() >= deadline) {
|
|
146
|
+
dequeueEvent = 'pool.expired';
|
|
124
147
|
throw new BrowserPoolAdmissionError('POOL_WAIT_EXPIRED', `${provider} queue wait expired without browser submission`);
|
|
148
|
+
}
|
|
125
149
|
await sleep(this.pollMs);
|
|
126
150
|
}
|
|
127
151
|
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
if (error instanceof BrowserPoolAdmissionError && error.code === 'PROVIDER_UNAVAILABLE')
|
|
154
|
+
dequeueEvent = 'pool.unavailable';
|
|
155
|
+
throw error;
|
|
156
|
+
}
|
|
128
157
|
finally {
|
|
129
|
-
this.dequeue(executionId);
|
|
158
|
+
this.dequeue(executionId, dequeueEvent);
|
|
130
159
|
}
|
|
131
160
|
}
|
|
132
|
-
update(lease, state, sessionId, threadUrl) {
|
|
133
|
-
|
|
161
|
+
update(lease, state, sessionId, threadUrl, event, action) {
|
|
162
|
+
requirePoolDecision('LEASED', event, action);
|
|
163
|
+
const result = this.db.prepare("UPDATE browser_slots SET state=?,session_id=?,thread_url=?,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=? AND state='LEASED'")
|
|
134
164
|
.run(state, sessionId, threadUrl, this.clock(), lease.provider, lease.slot, lease.executionId, lease.generation);
|
|
135
165
|
if (Number(result.changes) !== 1)
|
|
136
166
|
throw Error('Stale browser pool lease generation; refusing mutation');
|
|
137
167
|
}
|
|
138
168
|
bindSession(lease, sessionId) {
|
|
139
169
|
lease.sessionId = sessionId;
|
|
140
|
-
this.update(lease, 'LEASED', sessionId, lease.threadUrl);
|
|
170
|
+
this.update(lease, 'LEASED', sessionId, lease.threadUrl, 'pool.sessionBound', 'bindSession');
|
|
141
171
|
}
|
|
142
172
|
bindThread(lease, url) {
|
|
143
173
|
const expected = lease.provider === 'z' ? 'chat.z.ai' : 'chatgpt.com';
|
|
@@ -145,20 +175,30 @@ export class BrowserWorkerPoolManager {
|
|
|
145
175
|
if (parsed.protocol !== 'https:' || parsed.hostname !== expected || !/^\/c\/[\w-]+\/?$/.test(parsed.pathname))
|
|
146
176
|
throw Error('Unsafe browser thread URL');
|
|
147
177
|
lease.threadUrl = parsed.origin + parsed.pathname;
|
|
148
|
-
this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl);
|
|
178
|
+
this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl, 'pool.threadBound', 'bindThread');
|
|
149
179
|
}
|
|
150
|
-
touch(lease) { this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl); }
|
|
180
|
+
touch(lease) { this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl, 'pool.activity', 'touch'); }
|
|
151
181
|
release(lease) {
|
|
182
|
+
requirePoolDecision('LEASED', 'pool.terminalVerified', 'release');
|
|
152
183
|
const result = this.db.prepare("UPDATE browser_slots SET state='IDLE',execution_id=NULL,session_id=NULL,thread_url=NULL,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=? AND state='LEASED'")
|
|
153
184
|
.run(this.clock(), lease.provider, lease.slot, lease.executionId, lease.generation);
|
|
154
185
|
if (Number(result.changes) !== 1)
|
|
155
186
|
throw Error('Stale browser pool lease release');
|
|
156
187
|
}
|
|
157
|
-
quarantine(lease) {
|
|
188
|
+
quarantine(lease) {
|
|
189
|
+
const row = this.db.prepare('SELECT state FROM browser_slots WHERE provider=? AND slot=? AND execution_id=? AND generation=?')
|
|
190
|
+
.get(lease.provider, lease.slot, lease.executionId, lease.generation);
|
|
191
|
+
if (row?.state === 'QUARANTINED') {
|
|
192
|
+
requirePoolDecision('QUARANTINED', 'pool.outcomeUnknown', 'keepQuarantine');
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
this.update(lease, 'QUARANTINED', lease.sessionId, lease.threadUrl, 'pool.outcomeUnknown', 'quarantine');
|
|
196
|
+
}
|
|
158
197
|
/** Recovery can only release quarantined slots after verifying the old worker is no longer active. */
|
|
159
198
|
releaseQuarantine(lease, verifiedStopped) {
|
|
160
199
|
if (!verifiedStopped)
|
|
161
200
|
throw Error('Cannot release quarantined slot without independent stop verification');
|
|
201
|
+
requirePoolDecision('QUARANTINED', 'pool.stopVerified', 'releaseQuarantine');
|
|
162
202
|
const result = this.db.prepare("UPDATE browser_slots SET state='IDLE',execution_id=NULL,session_id=NULL,thread_url=NULL,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=? AND state='QUARANTINED'")
|
|
163
203
|
.run(this.clock(), lease.provider, lease.slot, lease.executionId, lease.generation);
|
|
164
204
|
if (Number(result.changes) !== 1)
|
|
@@ -4,6 +4,7 @@ import { join } from 'node:path';
|
|
|
4
4
|
import { homedir } from 'node:os';
|
|
5
5
|
import { spawnOwnedProcess } from './owned-process.js';
|
|
6
6
|
import { BrowserWorkerPoolManager, BrowserPoolAdmissionError } from './browser-worker-pool.js';
|
|
7
|
+
import { monitorPoolDecision, progressPoolDecision } from './browser-worker-pool-graph.js';
|
|
7
8
|
const WORKSPACE = 'Neo';
|
|
8
9
|
const OUTPUT_LIMIT = 2_000_000;
|
|
9
10
|
let manager;
|
|
@@ -13,6 +14,7 @@ function cliCommand(args, input, signal) {
|
|
|
13
14
|
return new Promise((resolve, reject) => {
|
|
14
15
|
const child = spawnOwnedProcess(executable, args, {
|
|
15
16
|
invocation: 'browser-worker-pool:webhelm', signal,
|
|
17
|
+
timeoutMs: args[0] === 'session' && args[1] === 'exec' ? 25_000 : 150_000,
|
|
16
18
|
spawn: { shell: false, stdio: [input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'] },
|
|
17
19
|
});
|
|
18
20
|
let out = '', err = '';
|
|
@@ -76,36 +78,41 @@ const state=await js('(()=>{const nodes=[...document.querySelectorAll(\"[data-me
|
|
|
76
78
|
console.log(JSON.stringify({url,state}));`;
|
|
77
79
|
function fingerprint(data) { return `${data.state?.text || ''}|${data.state?.streamLength || 0}`; }
|
|
78
80
|
export function pooledWebHelmRun(request, signal) {
|
|
81
|
+
const pool = request.pool ?? browserWorkerPool();
|
|
79
82
|
let lease;
|
|
80
83
|
let sessionId;
|
|
81
84
|
let released = false;
|
|
82
85
|
let stopped = false;
|
|
83
86
|
let monitor;
|
|
87
|
+
let completionRecoveryTimer;
|
|
84
88
|
let browserKeepAliveAt = 0, observedAt = Date.now(), lastFingerprint = '', sampled = false;
|
|
85
89
|
let recoveries = 0, monitoring = false, threadId;
|
|
86
90
|
let submissionAttempted = false, verifiedSubmitted = false, uncertain = false, recoveryOutcomeUncertain = false;
|
|
91
|
+
let providerPreflightFailed = false;
|
|
87
92
|
const stop = async () => {
|
|
88
93
|
if (stopped)
|
|
89
94
|
return;
|
|
90
95
|
stopped = true;
|
|
91
96
|
if (monitor)
|
|
92
97
|
clearInterval(monitor);
|
|
98
|
+
if (completionRecoveryTimer)
|
|
99
|
+
clearTimeout(completionRecoveryTimer);
|
|
93
100
|
if (!lease)
|
|
94
101
|
return;
|
|
95
102
|
if (uncertain) {
|
|
96
|
-
|
|
103
|
+
pool.quarantine(lease);
|
|
97
104
|
return;
|
|
98
105
|
}
|
|
99
106
|
if (!sessionId) {
|
|
100
|
-
|
|
107
|
+
pool.release(lease);
|
|
101
108
|
return;
|
|
102
109
|
}
|
|
103
110
|
try {
|
|
104
111
|
await cliCommand(['session', 'stop', sessionId]);
|
|
105
|
-
|
|
112
|
+
pool.release(lease);
|
|
106
113
|
}
|
|
107
114
|
catch {
|
|
108
|
-
|
|
115
|
+
pool.quarantine(lease);
|
|
109
116
|
}
|
|
110
117
|
};
|
|
111
118
|
const release = async () => { released = true; if (lease)
|
|
@@ -113,10 +120,30 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
113
120
|
const cancel = () => { released = true; if (!lease)
|
|
114
121
|
return; if (verifiedSubmitted || submissionAttempted) {
|
|
115
122
|
uncertain = true;
|
|
116
|
-
|
|
123
|
+
pool.quarantine(lease);
|
|
117
124
|
}
|
|
118
125
|
else
|
|
119
126
|
void stop().catch(() => { }); };
|
|
127
|
+
const sendRecovery = async () => {
|
|
128
|
+
if (stopped || released || uncertain || recoveryOutcomeUncertain || !sessionId || !threadId || !request.recover || recoveries >= 2)
|
|
129
|
+
return;
|
|
130
|
+
recoveries++;
|
|
131
|
+
const cfg = await request.recover(threadId);
|
|
132
|
+
try {
|
|
133
|
+
const answer = await configRun(request.provider, 'new-turn', sessionId, cfg);
|
|
134
|
+
if (!['submitted', 'completed'].includes(String(answer.status)) || !answer.verified_by)
|
|
135
|
+
throw Error('Recovery turn not independently verified');
|
|
136
|
+
await request.onRecovery?.(threadId, String(answer.verified_by));
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
// An ambiguous recovery can already have been delivered. Preserve the original session.
|
|
140
|
+
recoveryOutcomeUncertain = true;
|
|
141
|
+
uncertain = true;
|
|
142
|
+
if (lease)
|
|
143
|
+
pool.quarantine(lease);
|
|
144
|
+
throw error;
|
|
145
|
+
}
|
|
146
|
+
};
|
|
120
147
|
const probe = async () => {
|
|
121
148
|
if (stopped || monitoring || !sessionId || !lease?.threadUrl)
|
|
122
149
|
return;
|
|
@@ -137,29 +164,20 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
137
164
|
lastFingerprint = current;
|
|
138
165
|
return;
|
|
139
166
|
}
|
|
140
|
-
|
|
167
|
+
const fresh = current !== lastFingerprint && current !== '|0';
|
|
168
|
+
if (progressPoolDecision(fresh) === 'extendTimeout') {
|
|
141
169
|
lastFingerprint = current;
|
|
142
170
|
observedAt = now;
|
|
143
|
-
|
|
171
|
+
pool.touch(lease);
|
|
172
|
+
// Relay correlates task.progress with the original execution and re-arms its inactivity watchdog.
|
|
144
173
|
await request.onActivity?.('Browser Worker produced new assistant output');
|
|
145
174
|
}
|
|
146
|
-
|
|
147
|
-
!observation.state?.generating && !recoveryOutcomeUncertain && recoveries < 2 && request.recover && threadId
|
|
175
|
+
const due = now - observedAt >= Math.max(request.recoveryMinMs ?? 60_000, Math.round(request.timeoutMs * .6)) &&
|
|
176
|
+
!observation.state?.generating && !recoveryOutcomeUncertain && recoveries < 2 && !!request.recover && !!threadId;
|
|
177
|
+
if (monitorPoolDecision(due) === 'recoverTurn' && request.recover && threadId) {
|
|
148
178
|
// Count before the send. An ambiguous response must never cause automatic retransmission.
|
|
149
|
-
recoveries++;
|
|
150
179
|
observedAt = now;
|
|
151
|
-
|
|
152
|
-
try {
|
|
153
|
-
const answer = await configRun(request.provider, 'new-turn', sessionId, cfg);
|
|
154
|
-
if (answer.status !== 'submitted' || !answer.verified_by)
|
|
155
|
-
throw Error('Recovery turn not independently verified');
|
|
156
|
-
await request.onActivity?.('Verified same-thread recovery turn submitted');
|
|
157
|
-
}
|
|
158
|
-
catch (error) {
|
|
159
|
-
// The recovery may have reached the remote site: never submit another automatically.
|
|
160
|
-
recoveryOutcomeUncertain = true;
|
|
161
|
-
throw error;
|
|
162
|
-
}
|
|
180
|
+
await sendRecovery(); // A recovery receipt cannot extend Relay's inactivity deadline.
|
|
163
181
|
}
|
|
164
182
|
}
|
|
165
183
|
catch {
|
|
@@ -172,18 +190,32 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
172
190
|
};
|
|
173
191
|
const promise = (async () => {
|
|
174
192
|
try {
|
|
175
|
-
lease = await
|
|
193
|
+
lease = await pool.acquire(request.provider, request.executionId, signal);
|
|
176
194
|
if (released || signal.aborted)
|
|
177
195
|
throw Error('Browser worker cancelled before submission');
|
|
178
196
|
// Provider code must be present before opening any site or submitting work.
|
|
179
|
-
|
|
197
|
+
let profile;
|
|
198
|
+
try {
|
|
199
|
+
profile = parseEnvelope(await cliCommand(['platform', 'profile', request.provider], undefined, signal));
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
providerPreflightFailed = true;
|
|
203
|
+
throw error;
|
|
204
|
+
}
|
|
180
205
|
if (profile.platform !== request.provider)
|
|
181
206
|
throw Error(`WebHelm platform ${request.provider} missing`);
|
|
182
|
-
|
|
207
|
+
let session;
|
|
208
|
+
try {
|
|
209
|
+
session = parseEnvelope(await cliCommand(['session', 'start', '--workspace', WORKSPACE], undefined, signal));
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
providerPreflightFailed = true;
|
|
213
|
+
throw error;
|
|
214
|
+
}
|
|
183
215
|
if (typeof session.session_id !== 'string')
|
|
184
216
|
throw Error('No WebHelm session ID');
|
|
185
217
|
sessionId = session.session_id;
|
|
186
|
-
|
|
218
|
+
pool.bindSession(lease, sessionId);
|
|
187
219
|
// Preserve WebHelm's existing extension lifecycle. Pool capacity is six tabs total.
|
|
188
220
|
await cliCommand(['session', 'exec', sessionId], `await workspace_resize(${JSON.stringify(WORKSPACE)},6);`);
|
|
189
221
|
if (released || signal.aborted)
|
|
@@ -191,18 +223,24 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
191
223
|
const cfg = await request.build();
|
|
192
224
|
submissionAttempted = true;
|
|
193
225
|
const receipt = await configRun(request.provider, cfg.action, sessionId, cfg.config, signal);
|
|
194
|
-
if (receipt.status
|
|
226
|
+
if (!['submitted', 'completed'].includes(String(receipt.status)) || !receipt.verified_by)
|
|
195
227
|
throw Error('Submission outcome unknown: no verified receipt');
|
|
196
228
|
verifiedSubmitted = true;
|
|
197
229
|
const url = typeof receipt.thread_url === 'string' ? receipt.thread_url :
|
|
198
230
|
typeof receipt.thread_id === 'string' ? `https://${request.provider === 'z' ? 'chat.z.ai' : 'chatgpt.com'}/c/${receipt.thread_id}` : '';
|
|
199
|
-
|
|
231
|
+
pool.bindThread(lease, url);
|
|
200
232
|
threadId = String(receipt.thread_id || new URL(url).pathname.split('/').at(-1));
|
|
201
|
-
|
|
233
|
+
pool.reportHealthy(request.provider);
|
|
202
234
|
if (released) {
|
|
203
235
|
await stop();
|
|
204
236
|
return JSON.stringify(receipt);
|
|
205
237
|
}
|
|
238
|
+
// Completion proof is independent of CDP polling. Give the worker time to emit
|
|
239
|
+
// its correlated terminal event; the terminal event closes this lease before recovery.
|
|
240
|
+
if (receipt.status === 'completed' && request.recover) {
|
|
241
|
+
completionRecoveryTimer = setTimeout(() => { void sendRecovery().catch(() => { }); }, request.completionRecoveryGraceMs ?? 12_000);
|
|
242
|
+
completionRecoveryTimer.unref();
|
|
243
|
+
}
|
|
206
244
|
monitor = setInterval(() => { void probe(); }, request.monitorIntervalMs ?? 30_000);
|
|
207
245
|
monitor.unref();
|
|
208
246
|
return JSON.stringify(receipt);
|
|
@@ -216,12 +254,13 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
216
254
|
const rejection = safe && !cancelled && !(error instanceof BrowserPoolAdmissionError)
|
|
217
255
|
? new BrowserPoolAdmissionError('PROVIDER_UNAVAILABLE', `${request.provider} pre-submission failure: ${reason}`)
|
|
218
256
|
: error;
|
|
219
|
-
|
|
220
|
-
|
|
257
|
+
// A local prompt/configuration error is not evidence that the remote provider is down.
|
|
258
|
+
if (safe && !cancelled && providerPreflightFailed && lease && !(error instanceof BrowserPoolAdmissionError))
|
|
259
|
+
pool.reportPreSubmissionOutage(request.provider);
|
|
221
260
|
if (lease && !stopped) {
|
|
222
261
|
if (!safe) {
|
|
223
262
|
uncertain = true;
|
|
224
|
-
|
|
263
|
+
pool.quarantine(lease);
|
|
225
264
|
}
|
|
226
265
|
else
|
|
227
266
|
await stop();
|
package/dist/dashboard-html.js
CHANGED
|
@@ -42,7 +42,7 @@ ${dashboardStyles}
|
|
|
42
42
|
</header>
|
|
43
43
|
<main>
|
|
44
44
|
<section id="jobs-view" role="tabpanel" aria-labelledby="jobs-tab" class="panel jobs-panel">
|
|
45
|
-
<div class="panel-head"><h2>Jobs</h2><div class="panel-actions"><div class="job-filters"><input id="job-search" class="job-search" type="search" placeholder="Search
|
|
45
|
+
<div class="panel-head"><h2>Jobs</h2><div class="panel-actions"><div class="job-filters"><input id="job-search" class="job-search" type="search" placeholder="Search" aria-label="Search jobs by title or PR number"><select id="status-filter" class="job-filter" aria-label="Filter jobs by status (default excludes cancelled)"><option value="">No cancelled</option><option value="DRAFT">DRAFT</option><option value="ACTIVE">ACTIVE</option><option value="COMPLETED">COMPLETED</option><option value="CANCELLED">CANCELLED</option></select><select id="repo-filter" class="job-filter" aria-label="Filter jobs by repository"><option value="">All repos</option></select></div><div class="jobs-actions"><button id="jobs-refresh" class="refresh-button" type="button">Refresh</button></div><span id="jobs-feedback" class="jobs-feedback subtle" role="status" aria-live="polite"></span></div></div>
|
|
46
46
|
<div class="panel-body" aria-live="polite"><ul id="job-list" class="job-list"><li class="skeleton" aria-hidden="true"></li></ul></div>
|
|
47
47
|
</section>
|
|
48
48
|
<div class="workspace" id="jobs-workspace">
|
|
@@ -142,7 +142,6 @@ const shortcutsView=document.querySelector('#shortcuts-view');
|
|
|
142
142
|
const jobsWorkspace=document.querySelector('#jobs-workspace');
|
|
143
143
|
const jobList=document.querySelector('#job-list');
|
|
144
144
|
const jobsRefresh=document.querySelector('#jobs-refresh');
|
|
145
|
-
const jobsReset=document.querySelector('#jobs-reset');
|
|
146
145
|
const jobSearch=document.querySelector('#job-search');
|
|
147
146
|
const repoFilter=document.querySelector('#repo-filter');
|
|
148
147
|
const statusFilter=document.querySelector('#status-filter');
|
|
@@ -223,7 +222,7 @@ function matchesSearch(job){const query=searchQuery.trim().toLowerCase();if(!que
|
|
|
223
222
|
function filteredJobs(){return jobs.filter(job=>(!selectedRepo||selectedRepo===job.repository)&&matchesQuickFilter(job)&&matchesSearch(job)).slice(0,20);}
|
|
224
223
|
function syncFilters(){
|
|
225
224
|
const repos=jobRepositories.length?jobRepositories:[...new Set(jobs.map(job=>job.repository).filter(Boolean))].sort();
|
|
226
|
-
const repoOptions=[new Option('All
|
|
225
|
+
const repoOptions=[new Option('All repos',''),...repos.map(repo=>new Option(repo,repo))];
|
|
227
226
|
if(repoFilter.options.length!==repoOptions.length||[...repoFilter.options].some((option,index)=>option.value!==repoOptions[index].value))repoFilter.replaceChildren(...repoOptions);
|
|
228
227
|
if(selectedRepo&&!repos.includes(selectedRepo))selectedRepo='';
|
|
229
228
|
if(!['','DRAFT','ACTIVE','COMPLETED','CANCELLED'].includes(selectedStatus))selectedStatus='';
|
|
@@ -337,7 +336,6 @@ jobsTab.addEventListener('click',()=>{location.hash=selectedJob?'#/jobs/'+encode
|
|
|
337
336
|
jobsRefresh.addEventListener('click',()=>void refresh(true));
|
|
338
337
|
eventsAllTasks.addEventListener('click',()=>{eventTaskId=null;clearEvents();renderEventScope();void loadEventHistory();});
|
|
339
338
|
function applyLocalJobFilters(){rememberJobFilters();renderJobList();}
|
|
340
|
-
jobsReset.addEventListener('click',()=>{searchQuery='';selectedStatus='';selectedRepo='';syncFilters();applyLocalJobFilters();});
|
|
341
339
|
repoFilter.addEventListener('change',()=>{selectedRepo=repoFilter.value;applyLocalJobFilters();});
|
|
342
340
|
jobSearch.addEventListener('input',()=>{searchQuery=jobSearch.value;applyLocalJobFilters();});
|
|
343
341
|
statusFilter.addEventListener('change',()=>{selectedStatus=statusFilter.value;applyLocalJobFilters();});
|
package/dist/dashboard-style.js
CHANGED
|
@@ -158,15 +158,14 @@ export const dashboardStyles = ` <style>
|
|
|
158
158
|
body[data-mobile-jobs-view="detail"] #jobs-view { display:none; }
|
|
159
159
|
.mobile-only { display:inline-flex; }
|
|
160
160
|
#jobs-view .panel-head { display:grid; align-items:start; }
|
|
161
|
-
#jobs-view .panel-actions { display:grid; grid-template-columns:minmax(0,1fr) auto; width:100%; gap:.
|
|
161
|
+
#jobs-view .panel-actions { display:grid; grid-template-columns:minmax(0,1fr) auto; align-items:center; width:100%; gap:.25rem; }
|
|
162
162
|
.runtime-summary { display:none; }
|
|
163
|
-
#jobs-view .job-filters {
|
|
164
|
-
#jobs-view .job-
|
|
165
|
-
#jobs-view .job-
|
|
166
|
-
#jobs-view .
|
|
167
|
-
#jobs-view .jobs-
|
|
168
|
-
#jobs-view
|
|
169
|
-
#jobs-view .refresh-button { min-height:2.5rem; }
|
|
163
|
+
#jobs-view .job-filters { display:grid; grid-template-columns:minmax(0,1fr) 5.8rem 4.6rem; align-items:center; gap:.25rem; min-width:0; }
|
|
164
|
+
#jobs-view .job-search { width:100%; min-width:0; max-width:none; min-height:2.25rem; }
|
|
165
|
+
#jobs-view .job-filter { width:100%; max-width:none; min-width:0; text-transform:none; font-size:.69rem; min-height:2.25rem; padding:.2rem .15rem; }
|
|
166
|
+
#jobs-view .jobs-actions { justify-content:flex-end; }
|
|
167
|
+
#jobs-view .jobs-feedback { grid-column:1 / -1; text-align:left; }
|
|
168
|
+
#jobs-view #jobs-refresh { min-height:2.25rem; padding:.28rem .35rem; }
|
|
170
169
|
#jobs-view .panel-body { max-height:none; }
|
|
171
170
|
#jobs-workspace { gap:.65rem; }
|
|
172
171
|
#jobs-workspace .panel-body { max-height:none; }
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agents-relay",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.192",
|
|
4
4
|
"description": "Durable async agent jobs coordinated through GitHub pull requests",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
"test:publish": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 --test-force-exit test/*.test.js",
|
|
20
20
|
"test:fast": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=2 test/workflow-lifecycle.test.js test/workflow-v2-lifecycle-parity.test.js test/workflow-effects.test.js test/relay-store.test.js test/local-authority.test.js test/local-jobid-routing.test.js test/local-sqlite-jobid.test.js test/no-github.test.js test/reconcile-coordinator.test.js test/service-publication.test.js test/github-webhook.test.js",
|
|
21
21
|
"test:full": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 test/*.test.js && npm run test:verification:l3",
|
|
22
|
-
"test:verification:l1": "npm run build && node --test test/verification/harness.test.mjs test/verification/contract.test.mjs test/verification/graph-inventory.test.mjs test/verification/dashboard-drilldown.test.mjs test/verification/evidence-matrix.test.mjs test/verification/task-resolution-classification.test.mjs test/verification/planner-contract.test.mjs test/verification/force-restate-validation.test.mjs test/planner-child-recovery.test.js",
|
|
22
|
+
"test:verification:l1": "npm run build && node --test test/verification/harness.test.mjs test/verification/contract.test.mjs test/browser-worker-pool-graph.test.js test/verification/graph-inventory.test.mjs test/verification/dashboard-drilldown.test.mjs test/verification/evidence-matrix.test.mjs test/verification/task-resolution-classification.test.mjs test/verification/planner-contract.test.mjs test/verification/force-restate-validation.test.mjs test/planner-child-recovery.test.js",
|
|
23
23
|
"test:verification:l3": "npm run build && node --test --test-concurrency=1 test/verification/harness.test.mjs test/local-sqlite-jobid.test.js test/verification/l3-sqlite.test.mjs test/verification/l3-ingress.test.mjs test/verification/l3-replay.test.mjs test/verification/objective-authority.test.mjs test/verification/l3-convergence.test.mjs test/verification/l3-review-convergence.test.mjs test/verification/l3-human-escalation.test.mjs test/verification/l3-escalation-notifier.test.mjs test/verification/l3-unknown-effect.test.mjs test/verification/l3-cli-actions.test.mjs test/verification/l3-planner-round.test.mjs test/verification/l3-planner-cli.test.mjs test/verification/l3-task-resolution.test.mjs test/verification/l3-plan-apply-convergence.test.mjs test/verification/full-recovery-gates.test.mjs test/verification/production-wiring.test.mjs test/verification/l3-full-path.test.mjs test/verification/l3-projection-recovery.test.mjs test/verification/l3-daemon-startup-recovery.test.mjs",
|
|
24
24
|
"test:verification": "npm run test:verification:l1 && npm run test:verification:l2 && npm run test:verification:l3 && npm run test:verification:cross-a && node --test test/workflow-production-outbox.test.js && npm run test:verification:matrix",
|
|
25
25
|
"test:verification:cross-a": "npm run build && RELAY_A_DIST=$PWD/dist node --test test/verification/action-ledger-cross-branch.test.mjs test/verification/outbox-cross-a.test.mjs",
|
|
@@ -28,7 +28,8 @@
|
|
|
28
28
|
"test:verification:l2:live": "node test/verification/l2/live-provider.mjs",
|
|
29
29
|
"test:verification:l2:live-codex": "npm run build && node test/verification/l2/live-codex-agents.mjs",
|
|
30
30
|
"test:verification:l2:live-planner": "npm run build && RELAY_L2_LIVE=1 node --test test/verification/l2/live-planner-convergence.test.mjs",
|
|
31
|
-
"test:l2:agents": "npm run build && node test/l2-agent-prompts/evaluate.mjs"
|
|
31
|
+
"test:l2:agents": "npm run build && node test/l2-agent-prompts/evaluate.mjs",
|
|
32
|
+
"test:verification:browser-pool:live": "npm run build && node test/live/browser-worker-pool.e2e.mjs && node test/live/browser-worker-pool-recovery.e2e.mjs"
|
|
32
33
|
},
|
|
33
34
|
"devDependencies": {
|
|
34
35
|
"@types/node": "^22.10.2",
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Browser Worker Pool Lifecycle
|
|
3
|
+
description: Resource admission, FIFO queue, provider-health rejection, exclusive lease, terminal release and uncertain-outcome quarantine; Relay Task Graph remains authoritative for Task completion.
|
|
4
|
+
---
|
|
5
|
+
flowchart LR
|
|
6
|
+
%% relay: initial available
|
|
7
|
+
available["state: AVAILABLE"]
|
|
8
|
+
waiting["state: WAITING"]
|
|
9
|
+
leased["state: LEASED"]
|
|
10
|
+
quarantined["state: QUARANTINED"]
|
|
11
|
+
healthyDirect["guard: providerHealthy"]
|
|
12
|
+
healthyQueued["guard: providerHealthy"]
|
|
13
|
+
assignDirect["action: pool.assign"]
|
|
14
|
+
assignQueued["action: pool.assign"]
|
|
15
|
+
rejectDirect["action: pool.reject"]
|
|
16
|
+
rejectQueued["action: pool.reject"]
|
|
17
|
+
enqueue["action: pool.enqueue"]
|
|
18
|
+
expire["action: pool.dequeue"]
|
|
19
|
+
cancel["action: pool.dequeue"]
|
|
20
|
+
circuit["action: pool.dequeue"]
|
|
21
|
+
bindSession["action: pool.bindSession"]
|
|
22
|
+
bindThread["action: pool.bindThread"]
|
|
23
|
+
activity["action: pool.touch"]
|
|
24
|
+
release["action: pool.release"]
|
|
25
|
+
quarantine["action: pool.quarantine"]
|
|
26
|
+
verifiedRelease["action: pool.releaseQuarantine"]
|
|
27
|
+
keepQuarantine["action: pool.keepQuarantine"]
|
|
28
|
+
recoveryAllowed["guard: recoveryDue"]
|
|
29
|
+
recoverTurn["action: pool.recoverTurn"]
|
|
30
|
+
keepMonitoring["action: pool.keepMonitoring"]
|
|
31
|
+
verifiedProgress["guard: freshOutput"]
|
|
32
|
+
extendTimeout["action: pool.extendTimeout"]
|
|
33
|
+
available -->|on: pool.claim| healthyDirect
|
|
34
|
+
healthyDirect -->|true| assignDirect
|
|
35
|
+
assignDirect --> leased
|
|
36
|
+
healthyDirect -->|false| rejectDirect
|
|
37
|
+
rejectDirect --> available
|
|
38
|
+
available -->|on: pool.enqueue| enqueue
|
|
39
|
+
enqueue --> waiting
|
|
40
|
+
waiting -->|on: pool.claim| healthyQueued
|
|
41
|
+
healthyQueued -->|true| assignQueued
|
|
42
|
+
assignQueued --> leased
|
|
43
|
+
healthyQueued -->|false| rejectQueued
|
|
44
|
+
rejectQueued --> waiting
|
|
45
|
+
waiting -->|on: pool.expired| expire
|
|
46
|
+
expire --> available
|
|
47
|
+
waiting -->|on: pool.cancel| cancel
|
|
48
|
+
cancel --> available
|
|
49
|
+
waiting -->|on: pool.unavailable| circuit
|
|
50
|
+
circuit --> available
|
|
51
|
+
leased -->|on: pool.sessionBound| bindSession
|
|
52
|
+
bindSession --> leased
|
|
53
|
+
leased -->|on: pool.threadBound| bindThread
|
|
54
|
+
bindThread --> leased
|
|
55
|
+
leased -->|on: pool.activity| activity
|
|
56
|
+
activity --> leased
|
|
57
|
+
leased -->|on: pool.terminalVerified| release
|
|
58
|
+
release --> available
|
|
59
|
+
leased -->|on: pool.outcomeUnknown| quarantine
|
|
60
|
+
quarantine --> quarantined
|
|
61
|
+
quarantined -->|on: pool.stopVerified| verifiedRelease
|
|
62
|
+
verifiedRelease --> available
|
|
63
|
+
quarantined -->|on: pool.outcomeUnknown| keepQuarantine
|
|
64
|
+
keepQuarantine --> quarantined
|
|
65
|
+
leased -->|on: pool.poll| recoveryAllowed
|
|
66
|
+
recoveryAllowed -->|true| recoverTurn
|
|
67
|
+
recoverTurn --> leased
|
|
68
|
+
recoveryAllowed -->|false| keepMonitoring
|
|
69
|
+
keepMonitoring --> leased
|
|
70
|
+
leased -->|on: pool.progressObserved| verifiedProgress
|
|
71
|
+
verifiedProgress -->|true| extendTimeout
|
|
72
|
+
extendTimeout --> leased
|
|
73
|
+
verifiedProgress -->|false| keepMonitoring
|