agents-relay 2.0.191 → 2.0.193
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { GraphRuntime } from './workflow/graph-v2.js';
|
|
3
|
+
import { dispatchWorkflowGraphSync } from './workflow/dispatcher.js';
|
|
4
|
+
const actions = ['assign', 'reject', 'enqueue', 'dequeue', 'bindSession', 'bindThread', 'touch', 'release', 'quarantine', 'releaseQuarantine', 'keepQuarantine', 'recoverTurn', 'keepMonitoring', 'extendTimeout'];
|
|
5
|
+
const events = [
|
|
6
|
+
'pool.claim', 'pool.enqueue', 'pool.expired', 'pool.cancel', 'pool.unavailable', 'pool.sessionBound',
|
|
7
|
+
'pool.threadBound', 'pool.activity', 'pool.terminalVerified', 'pool.outcomeUnknown', 'pool.stopVerified', 'pool.poll', 'pool.progressObserved',
|
|
8
|
+
];
|
|
9
|
+
export const browserPoolGraph = new GraphRuntime(readFileSync(new URL('../workflow/browser-worker-pool.mmd', import.meta.url), 'utf8'), {
|
|
10
|
+
states: new Set(['AVAILABLE', 'WAITING', 'LEASED', 'QUARANTINED']),
|
|
11
|
+
guards: { providerHealthy: facts => facts.durableFacts.providerHealthy === true, recoveryDue: facts => facts.durableFacts.recoveryDue === true, freshOutput: facts => facts.durableFacts.freshOutput === true },
|
|
12
|
+
actions: Object.fromEntries(actions.map(name => [`pool.${name}`, { id: `pool.${name}` }])),
|
|
13
|
+
events: Object.fromEntries(events.map(name => [name, { id: name }])),
|
|
14
|
+
});
|
|
15
|
+
/** Graph owns permission and the effect selected; SQLite executes the effect atomically. */
|
|
16
|
+
export function requirePoolDecision(state, event, expectedAction, providerHealthy = true) {
|
|
17
|
+
const decision = dispatchWorkflowGraphSync(browserPoolGraph, { initialState: state, input: { transport: 'event', id: event, payload: null }, durableFacts: { providerHealthy }, graphName: 'browser-worker-pool' });
|
|
18
|
+
if (!decision.accepted || decision.actions.length !== 1 || decision.actions[0].id !== `pool.${expectedAction}`)
|
|
19
|
+
throw new Error(`Browser Pool Graph rejected ${state}/${event}: expected pool.${expectedAction}`);
|
|
20
|
+
return decision;
|
|
21
|
+
}
|
|
22
|
+
/** The executable Graph selects the next monitoring action; no browser heartbeat counts as task progress. */
|
|
23
|
+
export function monitorPoolDecision(recoveryDue) {
|
|
24
|
+
const decision = dispatchWorkflowGraphSync(browserPoolGraph, { initialState: 'LEASED', input: { transport: 'event', id: 'pool.poll', payload: null }, durableFacts: { recoveryDue }, graphName: 'browser-worker-pool' });
|
|
25
|
+
if (!decision.accepted || decision.target !== 'LEASED' || decision.actions.length !== 1)
|
|
26
|
+
throw Error('Invalid Pool monitoring graph');
|
|
27
|
+
const id = decision.actions[0].id;
|
|
28
|
+
if (id === 'pool.recoverTurn')
|
|
29
|
+
return 'recoverTurn';
|
|
30
|
+
if (id === 'pool.keepMonitoring')
|
|
31
|
+
return 'keepMonitoring';
|
|
32
|
+
throw Error(`Unknown Pool monitoring action ${id}`);
|
|
33
|
+
}
|
|
34
|
+
/** Only verified new assistant output can extend Relay's inactivity deadline. */
|
|
35
|
+
export function progressPoolDecision(freshOutput) {
|
|
36
|
+
const decision = dispatchWorkflowGraphSync(browserPoolGraph, { initialState: 'LEASED', input: { transport: 'event', id: 'pool.progressObserved', payload: null }, durableFacts: { freshOutput }, graphName: 'browser-worker-pool' });
|
|
37
|
+
if (!decision.accepted || decision.target !== 'LEASED' || decision.actions.length !== 1)
|
|
38
|
+
throw Error('Invalid Pool progress graph');
|
|
39
|
+
const id = decision.actions[0].id;
|
|
40
|
+
if (id === 'pool.extendTimeout')
|
|
41
|
+
return 'extendTimeout';
|
|
42
|
+
if (id === 'pool.keepMonitoring')
|
|
43
|
+
return 'keepMonitoring';
|
|
44
|
+
throw Error(`Unknown Pool progress action ${id}`);
|
|
45
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import { requirePoolDecision } from './browser-worker-pool-graph.js';
|
|
2
3
|
import { mkdirSync } from 'node:fs';
|
|
3
4
|
import { dirname, join } from 'node:path';
|
|
4
5
|
import { homedir } from 'node:os';
|
|
@@ -77,8 +78,11 @@ export class BrowserWorkerPoolManager {
|
|
|
77
78
|
try {
|
|
78
79
|
const now = this.clock();
|
|
79
80
|
const health = this.db.prepare('SELECT unavailable_until FROM browser_provider_health WHERE provider=?').get(provider);
|
|
80
|
-
if (health.unavailable_until > now)
|
|
81
|
+
if (health.unavailable_until > now) {
|
|
82
|
+
const waiting = this.db.prepare('SELECT 1 FROM browser_queue WHERE execution_id=? AND provider=?').get(executionId, provider);
|
|
83
|
+
requirePoolDecision(waiting ? 'WAITING' : 'AVAILABLE', 'pool.claim', 'reject', false);
|
|
81
84
|
throw new BrowserPoolAdmissionError('PROVIDER_UNAVAILABLE', `${provider} circuit open until ${health.unavailable_until}`);
|
|
85
|
+
}
|
|
82
86
|
const existing = this.db.prepare('SELECT * FROM browser_slots WHERE execution_id=? AND provider=?').get(executionId, provider);
|
|
83
87
|
if (existing)
|
|
84
88
|
throw new Error(`Execution ${executionId} already owns ${provider}-${existing.slot}: refuse duplicate browser submission`);
|
|
@@ -87,6 +91,7 @@ export class BrowserWorkerPoolManager {
|
|
|
87
91
|
if (!row || row.execution_id === executionId) {
|
|
88
92
|
const idle = this.db.prepare("SELECT * FROM browser_slots WHERE provider=? AND state='IDLE' ORDER BY slot LIMIT 1").get(provider);
|
|
89
93
|
if (idle) {
|
|
94
|
+
requirePoolDecision(row ? 'WAITING' : 'AVAILABLE', 'pool.claim', 'assign', true);
|
|
90
95
|
const generation = idle.generation + 1;
|
|
91
96
|
this.db.prepare("UPDATE browser_slots SET state='LEASED',execution_id=?,generation=?,session_id=NULL,thread_url=NULL,touched_at=? WHERE provider=? AND slot=? AND state='IDLE'")
|
|
92
97
|
.run(executionId, generation, now, provider, idle.slot);
|
|
@@ -100,7 +105,11 @@ export class BrowserWorkerPoolManager {
|
|
|
100
105
|
this.db.exec('COMMIT');
|
|
101
106
|
}
|
|
102
107
|
}
|
|
103
|
-
dequeue(executionId) {
|
|
108
|
+
dequeue(executionId, event) {
|
|
109
|
+
const queued = this.db.prepare('SELECT 1 FROM browser_queue WHERE execution_id=?').get(executionId);
|
|
110
|
+
if (!queued)
|
|
111
|
+
return;
|
|
112
|
+
requirePoolDecision('WAITING', event, 'dequeue');
|
|
104
113
|
this.db.prepare('DELETE FROM browser_queue WHERE execution_id=?').run(executionId);
|
|
105
114
|
}
|
|
106
115
|
async acquire(provider, executionId, signal) {
|
|
@@ -109,10 +118,23 @@ export class BrowserWorkerPoolManager {
|
|
|
109
118
|
const immediate = this.tryClaim(provider, executionId);
|
|
110
119
|
if (immediate)
|
|
111
120
|
return immediate;
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
121
|
+
// Queue capacity is a cross-process resource constraint; hold SQLite's writer lock
|
|
122
|
+
// across the queue count and insertion (not only during slot claiming).
|
|
123
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
124
|
+
try {
|
|
125
|
+
const count = this.db.prepare('SELECT count(*) AS n FROM browser_queue WHERE provider=?').get(provider).n;
|
|
126
|
+
const present = this.db.prepare('SELECT 1 FROM browser_queue WHERE execution_id=?').get(executionId);
|
|
127
|
+
if (count >= this.maxQueue && !present)
|
|
128
|
+
throw new BrowserPoolAdmissionError('POOL_CAPACITY_FULL', `${provider} queue is full`);
|
|
129
|
+
if (!present) {
|
|
130
|
+
requirePoolDecision('AVAILABLE', 'pool.enqueue', 'enqueue');
|
|
131
|
+
this.db.prepare('INSERT INTO browser_queue(execution_id,provider,queued_at) VALUES(?,?,?)').run(executionId, provider, this.clock());
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
this.db.exec('COMMIT');
|
|
136
|
+
}
|
|
137
|
+
let dequeueEvent = 'pool.cancel';
|
|
116
138
|
try {
|
|
117
139
|
for (;;) {
|
|
118
140
|
if (signal?.aborted)
|
|
@@ -120,24 +142,32 @@ export class BrowserWorkerPoolManager {
|
|
|
120
142
|
const result = this.tryClaim(provider, executionId);
|
|
121
143
|
if (result)
|
|
122
144
|
return result;
|
|
123
|
-
if (this.clock() >= deadline)
|
|
145
|
+
if (this.clock() >= deadline) {
|
|
146
|
+
dequeueEvent = 'pool.expired';
|
|
124
147
|
throw new BrowserPoolAdmissionError('POOL_WAIT_EXPIRED', `${provider} queue wait expired without browser submission`);
|
|
148
|
+
}
|
|
125
149
|
await sleep(this.pollMs);
|
|
126
150
|
}
|
|
127
151
|
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
if (error instanceof BrowserPoolAdmissionError && error.code === 'PROVIDER_UNAVAILABLE')
|
|
154
|
+
dequeueEvent = 'pool.unavailable';
|
|
155
|
+
throw error;
|
|
156
|
+
}
|
|
128
157
|
finally {
|
|
129
|
-
this.dequeue(executionId);
|
|
158
|
+
this.dequeue(executionId, dequeueEvent);
|
|
130
159
|
}
|
|
131
160
|
}
|
|
132
|
-
update(lease, state, sessionId, threadUrl) {
|
|
133
|
-
|
|
161
|
+
update(lease, state, sessionId, threadUrl, event, action) {
|
|
162
|
+
requirePoolDecision('LEASED', event, action);
|
|
163
|
+
const result = this.db.prepare("UPDATE browser_slots SET state=?,session_id=?,thread_url=?,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=? AND state='LEASED'")
|
|
134
164
|
.run(state, sessionId, threadUrl, this.clock(), lease.provider, lease.slot, lease.executionId, lease.generation);
|
|
135
165
|
if (Number(result.changes) !== 1)
|
|
136
166
|
throw Error('Stale browser pool lease generation; refusing mutation');
|
|
137
167
|
}
|
|
138
168
|
bindSession(lease, sessionId) {
|
|
139
169
|
lease.sessionId = sessionId;
|
|
140
|
-
this.update(lease, 'LEASED', sessionId, lease.threadUrl);
|
|
170
|
+
this.update(lease, 'LEASED', sessionId, lease.threadUrl, 'pool.sessionBound', 'bindSession');
|
|
141
171
|
}
|
|
142
172
|
bindThread(lease, url) {
|
|
143
173
|
const expected = lease.provider === 'z' ? 'chat.z.ai' : 'chatgpt.com';
|
|
@@ -145,20 +175,30 @@ export class BrowserWorkerPoolManager {
|
|
|
145
175
|
if (parsed.protocol !== 'https:' || parsed.hostname !== expected || !/^\/c\/[\w-]+\/?$/.test(parsed.pathname))
|
|
146
176
|
throw Error('Unsafe browser thread URL');
|
|
147
177
|
lease.threadUrl = parsed.origin + parsed.pathname;
|
|
148
|
-
this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl);
|
|
178
|
+
this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl, 'pool.threadBound', 'bindThread');
|
|
149
179
|
}
|
|
150
|
-
touch(lease) { this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl); }
|
|
180
|
+
touch(lease) { this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl, 'pool.activity', 'touch'); }
|
|
151
181
|
release(lease) {
|
|
182
|
+
requirePoolDecision('LEASED', 'pool.terminalVerified', 'release');
|
|
152
183
|
const result = this.db.prepare("UPDATE browser_slots SET state='IDLE',execution_id=NULL,session_id=NULL,thread_url=NULL,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=? AND state='LEASED'")
|
|
153
184
|
.run(this.clock(), lease.provider, lease.slot, lease.executionId, lease.generation);
|
|
154
185
|
if (Number(result.changes) !== 1)
|
|
155
186
|
throw Error('Stale browser pool lease release');
|
|
156
187
|
}
|
|
157
|
-
quarantine(lease) {
|
|
188
|
+
quarantine(lease) {
|
|
189
|
+
const row = this.db.prepare('SELECT state FROM browser_slots WHERE provider=? AND slot=? AND execution_id=? AND generation=?')
|
|
190
|
+
.get(lease.provider, lease.slot, lease.executionId, lease.generation);
|
|
191
|
+
if (row?.state === 'QUARANTINED') {
|
|
192
|
+
requirePoolDecision('QUARANTINED', 'pool.outcomeUnknown', 'keepQuarantine');
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
this.update(lease, 'QUARANTINED', lease.sessionId, lease.threadUrl, 'pool.outcomeUnknown', 'quarantine');
|
|
196
|
+
}
|
|
158
197
|
/** Recovery can only release quarantined slots after verifying the old worker is no longer active. */
|
|
159
198
|
releaseQuarantine(lease, verifiedStopped) {
|
|
160
199
|
if (!verifiedStopped)
|
|
161
200
|
throw Error('Cannot release quarantined slot without independent stop verification');
|
|
201
|
+
requirePoolDecision('QUARANTINED', 'pool.stopVerified', 'releaseQuarantine');
|
|
162
202
|
const result = this.db.prepare("UPDATE browser_slots SET state='IDLE',execution_id=NULL,session_id=NULL,thread_url=NULL,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=? AND state='QUARANTINED'")
|
|
163
203
|
.run(this.clock(), lease.provider, lease.slot, lease.executionId, lease.generation);
|
|
164
204
|
if (Number(result.changes) !== 1)
|
|
@@ -4,6 +4,7 @@ import { join } from 'node:path';
|
|
|
4
4
|
import { homedir } from 'node:os';
|
|
5
5
|
import { spawnOwnedProcess } from './owned-process.js';
|
|
6
6
|
import { BrowserWorkerPoolManager, BrowserPoolAdmissionError } from './browser-worker-pool.js';
|
|
7
|
+
import { monitorPoolDecision, progressPoolDecision } from './browser-worker-pool-graph.js';
|
|
7
8
|
const WORKSPACE = 'Neo';
|
|
8
9
|
const OUTPUT_LIMIT = 2_000_000;
|
|
9
10
|
let manager;
|
|
@@ -13,6 +14,7 @@ function cliCommand(args, input, signal) {
|
|
|
13
14
|
return new Promise((resolve, reject) => {
|
|
14
15
|
const child = spawnOwnedProcess(executable, args, {
|
|
15
16
|
invocation: 'browser-worker-pool:webhelm', signal,
|
|
17
|
+
timeoutMs: args[0] === 'session' && args[1] === 'exec' ? 25_000 : 150_000,
|
|
16
18
|
spawn: { shell: false, stdio: [input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'] },
|
|
17
19
|
});
|
|
18
20
|
let out = '', err = '';
|
|
@@ -76,36 +78,41 @@ const state=await js('(()=>{const nodes=[...document.querySelectorAll(\"[data-me
|
|
|
76
78
|
console.log(JSON.stringify({url,state}));`;
|
|
77
79
|
function fingerprint(data) { return `${data.state?.text || ''}|${data.state?.streamLength || 0}`; }
|
|
78
80
|
export function pooledWebHelmRun(request, signal) {
|
|
81
|
+
const pool = request.pool ?? browserWorkerPool();
|
|
79
82
|
let lease;
|
|
80
83
|
let sessionId;
|
|
81
84
|
let released = false;
|
|
82
85
|
let stopped = false;
|
|
83
86
|
let monitor;
|
|
87
|
+
let completionRecoveryTimer;
|
|
84
88
|
let browserKeepAliveAt = 0, observedAt = Date.now(), lastFingerprint = '', sampled = false;
|
|
85
89
|
let recoveries = 0, monitoring = false, threadId;
|
|
86
90
|
let submissionAttempted = false, verifiedSubmitted = false, uncertain = false, recoveryOutcomeUncertain = false;
|
|
91
|
+
let providerPreflightFailed = false;
|
|
87
92
|
const stop = async () => {
|
|
88
93
|
if (stopped)
|
|
89
94
|
return;
|
|
90
95
|
stopped = true;
|
|
91
96
|
if (monitor)
|
|
92
97
|
clearInterval(monitor);
|
|
98
|
+
if (completionRecoveryTimer)
|
|
99
|
+
clearTimeout(completionRecoveryTimer);
|
|
93
100
|
if (!lease)
|
|
94
101
|
return;
|
|
95
102
|
if (uncertain) {
|
|
96
|
-
|
|
103
|
+
pool.quarantine(lease);
|
|
97
104
|
return;
|
|
98
105
|
}
|
|
99
106
|
if (!sessionId) {
|
|
100
|
-
|
|
107
|
+
pool.release(lease);
|
|
101
108
|
return;
|
|
102
109
|
}
|
|
103
110
|
try {
|
|
104
111
|
await cliCommand(['session', 'stop', sessionId]);
|
|
105
|
-
|
|
112
|
+
pool.release(lease);
|
|
106
113
|
}
|
|
107
114
|
catch {
|
|
108
|
-
|
|
115
|
+
pool.quarantine(lease);
|
|
109
116
|
}
|
|
110
117
|
};
|
|
111
118
|
const release = async () => { released = true; if (lease)
|
|
@@ -113,10 +120,30 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
113
120
|
const cancel = () => { released = true; if (!lease)
|
|
114
121
|
return; if (verifiedSubmitted || submissionAttempted) {
|
|
115
122
|
uncertain = true;
|
|
116
|
-
|
|
123
|
+
pool.quarantine(lease);
|
|
117
124
|
}
|
|
118
125
|
else
|
|
119
126
|
void stop().catch(() => { }); };
|
|
127
|
+
const sendRecovery = async () => {
|
|
128
|
+
if (stopped || released || uncertain || recoveryOutcomeUncertain || !sessionId || !threadId || !request.recover || recoveries >= 2)
|
|
129
|
+
return;
|
|
130
|
+
recoveries++;
|
|
131
|
+
const cfg = await request.recover(threadId);
|
|
132
|
+
try {
|
|
133
|
+
const answer = await configRun(request.provider, 'new-turn', sessionId, cfg);
|
|
134
|
+
if (!['submitted', 'completed'].includes(String(answer.status)) || !answer.verified_by)
|
|
135
|
+
throw Error('Recovery turn not independently verified');
|
|
136
|
+
await request.onRecovery?.(threadId, String(answer.verified_by));
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
// An ambiguous recovery can already have been delivered. Preserve the original session.
|
|
140
|
+
recoveryOutcomeUncertain = true;
|
|
141
|
+
uncertain = true;
|
|
142
|
+
if (lease)
|
|
143
|
+
pool.quarantine(lease);
|
|
144
|
+
throw error;
|
|
145
|
+
}
|
|
146
|
+
};
|
|
120
147
|
const probe = async () => {
|
|
121
148
|
if (stopped || monitoring || !sessionId || !lease?.threadUrl)
|
|
122
149
|
return;
|
|
@@ -137,29 +164,20 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
137
164
|
lastFingerprint = current;
|
|
138
165
|
return;
|
|
139
166
|
}
|
|
140
|
-
|
|
167
|
+
const fresh = current !== lastFingerprint && current !== '|0';
|
|
168
|
+
if (progressPoolDecision(fresh) === 'extendTimeout') {
|
|
141
169
|
lastFingerprint = current;
|
|
142
170
|
observedAt = now;
|
|
143
|
-
|
|
171
|
+
pool.touch(lease);
|
|
172
|
+
// Relay correlates task.progress with the original execution and re-arms its inactivity watchdog.
|
|
144
173
|
await request.onActivity?.('Browser Worker produced new assistant output');
|
|
145
174
|
}
|
|
146
|
-
|
|
147
|
-
!observation.state?.generating && !recoveryOutcomeUncertain && recoveries < 2 && request.recover && threadId
|
|
175
|
+
const due = now - observedAt >= Math.max(request.recoveryMinMs ?? 60_000, Math.round(request.timeoutMs * .6)) &&
|
|
176
|
+
!observation.state?.generating && !recoveryOutcomeUncertain && recoveries < 2 && !!request.recover && !!threadId;
|
|
177
|
+
if (monitorPoolDecision(due) === 'recoverTurn' && request.recover && threadId) {
|
|
148
178
|
// Count before the send. An ambiguous response must never cause automatic retransmission.
|
|
149
|
-
recoveries++;
|
|
150
179
|
observedAt = now;
|
|
151
|
-
|
|
152
|
-
try {
|
|
153
|
-
const answer = await configRun(request.provider, 'new-turn', sessionId, cfg);
|
|
154
|
-
if (answer.status !== 'submitted' || !answer.verified_by)
|
|
155
|
-
throw Error('Recovery turn not independently verified');
|
|
156
|
-
await request.onActivity?.('Verified same-thread recovery turn submitted');
|
|
157
|
-
}
|
|
158
|
-
catch (error) {
|
|
159
|
-
// The recovery may have reached the remote site: never submit another automatically.
|
|
160
|
-
recoveryOutcomeUncertain = true;
|
|
161
|
-
throw error;
|
|
162
|
-
}
|
|
180
|
+
await sendRecovery(); // A recovery receipt cannot extend Relay's inactivity deadline.
|
|
163
181
|
}
|
|
164
182
|
}
|
|
165
183
|
catch {
|
|
@@ -172,18 +190,32 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
172
190
|
};
|
|
173
191
|
const promise = (async () => {
|
|
174
192
|
try {
|
|
175
|
-
lease = await
|
|
193
|
+
lease = await pool.acquire(request.provider, request.executionId, signal);
|
|
176
194
|
if (released || signal.aborted)
|
|
177
195
|
throw Error('Browser worker cancelled before submission');
|
|
178
196
|
// Provider code must be present before opening any site or submitting work.
|
|
179
|
-
|
|
197
|
+
let profile;
|
|
198
|
+
try {
|
|
199
|
+
profile = parseEnvelope(await cliCommand(['platform', 'profile', request.provider], undefined, signal));
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
providerPreflightFailed = true;
|
|
203
|
+
throw error;
|
|
204
|
+
}
|
|
180
205
|
if (profile.platform !== request.provider)
|
|
181
206
|
throw Error(`WebHelm platform ${request.provider} missing`);
|
|
182
|
-
|
|
207
|
+
let session;
|
|
208
|
+
try {
|
|
209
|
+
session = parseEnvelope(await cliCommand(['session', 'start', '--workspace', WORKSPACE], undefined, signal));
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
providerPreflightFailed = true;
|
|
213
|
+
throw error;
|
|
214
|
+
}
|
|
183
215
|
if (typeof session.session_id !== 'string')
|
|
184
216
|
throw Error('No WebHelm session ID');
|
|
185
217
|
sessionId = session.session_id;
|
|
186
|
-
|
|
218
|
+
pool.bindSession(lease, sessionId);
|
|
187
219
|
// Preserve WebHelm's existing extension lifecycle. Pool capacity is six tabs total.
|
|
188
220
|
await cliCommand(['session', 'exec', sessionId], `await workspace_resize(${JSON.stringify(WORKSPACE)},6);`);
|
|
189
221
|
if (released || signal.aborted)
|
|
@@ -191,18 +223,24 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
191
223
|
const cfg = await request.build();
|
|
192
224
|
submissionAttempted = true;
|
|
193
225
|
const receipt = await configRun(request.provider, cfg.action, sessionId, cfg.config, signal);
|
|
194
|
-
if (receipt.status
|
|
226
|
+
if (!['submitted', 'completed'].includes(String(receipt.status)) || !receipt.verified_by)
|
|
195
227
|
throw Error('Submission outcome unknown: no verified receipt');
|
|
196
228
|
verifiedSubmitted = true;
|
|
197
229
|
const url = typeof receipt.thread_url === 'string' ? receipt.thread_url :
|
|
198
230
|
typeof receipt.thread_id === 'string' ? `https://${request.provider === 'z' ? 'chat.z.ai' : 'chatgpt.com'}/c/${receipt.thread_id}` : '';
|
|
199
|
-
|
|
231
|
+
pool.bindThread(lease, url);
|
|
200
232
|
threadId = String(receipt.thread_id || new URL(url).pathname.split('/').at(-1));
|
|
201
|
-
|
|
233
|
+
pool.reportHealthy(request.provider);
|
|
202
234
|
if (released) {
|
|
203
235
|
await stop();
|
|
204
236
|
return JSON.stringify(receipt);
|
|
205
237
|
}
|
|
238
|
+
// Completion proof is independent of CDP polling. Give the worker time to emit
|
|
239
|
+
// its correlated terminal event; the terminal event closes this lease before recovery.
|
|
240
|
+
if (receipt.status === 'completed' && request.recover) {
|
|
241
|
+
completionRecoveryTimer = setTimeout(() => { void sendRecovery().catch(() => { }); }, request.completionRecoveryGraceMs ?? 12_000);
|
|
242
|
+
completionRecoveryTimer.unref();
|
|
243
|
+
}
|
|
206
244
|
monitor = setInterval(() => { void probe(); }, request.monitorIntervalMs ?? 30_000);
|
|
207
245
|
monitor.unref();
|
|
208
246
|
return JSON.stringify(receipt);
|
|
@@ -216,12 +254,13 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
216
254
|
const rejection = safe && !cancelled && !(error instanceof BrowserPoolAdmissionError)
|
|
217
255
|
? new BrowserPoolAdmissionError('PROVIDER_UNAVAILABLE', `${request.provider} pre-submission failure: ${reason}`)
|
|
218
256
|
: error;
|
|
219
|
-
|
|
220
|
-
|
|
257
|
+
// A local prompt/configuration error is not evidence that the remote provider is down.
|
|
258
|
+
if (safe && !cancelled && providerPreflightFailed && lease && !(error instanceof BrowserPoolAdmissionError))
|
|
259
|
+
pool.reportPreSubmissionOutage(request.provider);
|
|
221
260
|
if (lease && !stopped) {
|
|
222
261
|
if (!safe) {
|
|
223
262
|
uncertain = true;
|
|
224
|
-
|
|
263
|
+
pool.quarantine(lease);
|
|
225
264
|
}
|
|
226
265
|
else
|
|
227
266
|
await stop();
|
package/dist/usage.js
CHANGED
|
@@ -9,7 +9,7 @@ export class BrowserWorkspaceUsageReader {
|
|
|
9
9
|
timeoutMs;
|
|
10
10
|
now;
|
|
11
11
|
staleAfterMs;
|
|
12
|
-
constructor(command = '
|
|
12
|
+
constructor(command = process.env.AGENTS_RELAY_WEBHELM_COMMAND || join(homedir(), '.local', 'bin', 'webhelm'), timeoutMs = 30_000, now = () => new Date(), staleAfterMs = 15 * 60_000) {
|
|
13
13
|
this.command = command;
|
|
14
14
|
this.timeoutMs = timeoutMs;
|
|
15
15
|
this.now = now;
|
|
@@ -18,7 +18,7 @@ export class BrowserWorkspaceUsageReader {
|
|
|
18
18
|
async read() {
|
|
19
19
|
const observedAt = this.now().toISOString();
|
|
20
20
|
return await new Promise(resolve => {
|
|
21
|
-
const child = spawnOwnedProcess(this.command,
|
|
21
|
+
const child = spawnOwnedProcess(this.command, ['capacity'], { invocation: 'usage.browser-workspace', timeoutMs: this.timeoutMs, spawn: { stdio: ['ignore', 'pipe', 'ignore'] } });
|
|
22
22
|
let output = '';
|
|
23
23
|
let settled = false;
|
|
24
24
|
const finish = (capacity) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agents-relay",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.193",
|
|
4
4
|
"description": "Durable async agent jobs coordinated through GitHub pull requests",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
"test:publish": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 --test-force-exit test/*.test.js",
|
|
20
20
|
"test:fast": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=2 test/workflow-lifecycle.test.js test/workflow-v2-lifecycle-parity.test.js test/workflow-effects.test.js test/relay-store.test.js test/local-authority.test.js test/local-jobid-routing.test.js test/local-sqlite-jobid.test.js test/no-github.test.js test/reconcile-coordinator.test.js test/service-publication.test.js test/github-webhook.test.js",
|
|
21
21
|
"test:full": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 test/*.test.js && npm run test:verification:l3",
|
|
22
|
-
"test:verification:l1": "npm run build && node --test test/verification/harness.test.mjs test/verification/contract.test.mjs test/verification/graph-inventory.test.mjs test/verification/dashboard-drilldown.test.mjs test/verification/evidence-matrix.test.mjs test/verification/task-resolution-classification.test.mjs test/verification/planner-contract.test.mjs test/verification/force-restate-validation.test.mjs test/planner-child-recovery.test.js",
|
|
22
|
+
"test:verification:l1": "npm run build && node --test test/verification/harness.test.mjs test/verification/contract.test.mjs test/browser-worker-pool-graph.test.js test/verification/graph-inventory.test.mjs test/verification/dashboard-drilldown.test.mjs test/verification/evidence-matrix.test.mjs test/verification/task-resolution-classification.test.mjs test/verification/planner-contract.test.mjs test/verification/force-restate-validation.test.mjs test/planner-child-recovery.test.js",
|
|
23
23
|
"test:verification:l3": "npm run build && node --test --test-concurrency=1 test/verification/harness.test.mjs test/local-sqlite-jobid.test.js test/verification/l3-sqlite.test.mjs test/verification/l3-ingress.test.mjs test/verification/l3-replay.test.mjs test/verification/objective-authority.test.mjs test/verification/l3-convergence.test.mjs test/verification/l3-review-convergence.test.mjs test/verification/l3-human-escalation.test.mjs test/verification/l3-escalation-notifier.test.mjs test/verification/l3-unknown-effect.test.mjs test/verification/l3-cli-actions.test.mjs test/verification/l3-planner-round.test.mjs test/verification/l3-planner-cli.test.mjs test/verification/l3-task-resolution.test.mjs test/verification/l3-plan-apply-convergence.test.mjs test/verification/full-recovery-gates.test.mjs test/verification/production-wiring.test.mjs test/verification/l3-full-path.test.mjs test/verification/l3-projection-recovery.test.mjs test/verification/l3-daemon-startup-recovery.test.mjs",
|
|
24
24
|
"test:verification": "npm run test:verification:l1 && npm run test:verification:l2 && npm run test:verification:l3 && npm run test:verification:cross-a && node --test test/workflow-production-outbox.test.js && npm run test:verification:matrix",
|
|
25
25
|
"test:verification:cross-a": "npm run build && RELAY_A_DIST=$PWD/dist node --test test/verification/action-ledger-cross-branch.test.mjs test/verification/outbox-cross-a.test.mjs",
|
|
@@ -28,7 +28,8 @@
|
|
|
28
28
|
"test:verification:l2:live": "node test/verification/l2/live-provider.mjs",
|
|
29
29
|
"test:verification:l2:live-codex": "npm run build && node test/verification/l2/live-codex-agents.mjs",
|
|
30
30
|
"test:verification:l2:live-planner": "npm run build && RELAY_L2_LIVE=1 node --test test/verification/l2/live-planner-convergence.test.mjs",
|
|
31
|
-
"test:l2:agents": "npm run build && node test/l2-agent-prompts/evaluate.mjs"
|
|
31
|
+
"test:l2:agents": "npm run build && node test/l2-agent-prompts/evaluate.mjs",
|
|
32
|
+
"test:verification:browser-pool:live": "npm run build && node test/live/browser-worker-pool.e2e.mjs && node test/live/browser-worker-pool-recovery.e2e.mjs"
|
|
32
33
|
},
|
|
33
34
|
"devDependencies": {
|
|
34
35
|
"@types/node": "^22.10.2",
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Browser Worker Pool Lifecycle
|
|
3
|
+
description: Resource admission, FIFO queue, provider-health rejection, exclusive lease, terminal release and uncertain-outcome quarantine; Relay Task Graph remains authoritative for Task completion.
|
|
4
|
+
---
|
|
5
|
+
flowchart LR
|
|
6
|
+
%% relay: initial available
|
|
7
|
+
available["state: AVAILABLE"]
|
|
8
|
+
waiting["state: WAITING"]
|
|
9
|
+
leased["state: LEASED"]
|
|
10
|
+
quarantined["state: QUARANTINED"]
|
|
11
|
+
healthyDirect["guard: providerHealthy"]
|
|
12
|
+
healthyQueued["guard: providerHealthy"]
|
|
13
|
+
assignDirect["action: pool.assign"]
|
|
14
|
+
assignQueued["action: pool.assign"]
|
|
15
|
+
rejectDirect["action: pool.reject"]
|
|
16
|
+
rejectQueued["action: pool.reject"]
|
|
17
|
+
enqueue["action: pool.enqueue"]
|
|
18
|
+
expire["action: pool.dequeue"]
|
|
19
|
+
cancel["action: pool.dequeue"]
|
|
20
|
+
circuit["action: pool.dequeue"]
|
|
21
|
+
bindSession["action: pool.bindSession"]
|
|
22
|
+
bindThread["action: pool.bindThread"]
|
|
23
|
+
activity["action: pool.touch"]
|
|
24
|
+
release["action: pool.release"]
|
|
25
|
+
quarantine["action: pool.quarantine"]
|
|
26
|
+
verifiedRelease["action: pool.releaseQuarantine"]
|
|
27
|
+
keepQuarantine["action: pool.keepQuarantine"]
|
|
28
|
+
recoveryAllowed["guard: recoveryDue"]
|
|
29
|
+
recoverTurn["action: pool.recoverTurn"]
|
|
30
|
+
keepMonitoring["action: pool.keepMonitoring"]
|
|
31
|
+
verifiedProgress["guard: freshOutput"]
|
|
32
|
+
extendTimeout["action: pool.extendTimeout"]
|
|
33
|
+
available -->|on: pool.claim| healthyDirect
|
|
34
|
+
healthyDirect -->|true| assignDirect
|
|
35
|
+
assignDirect --> leased
|
|
36
|
+
healthyDirect -->|false| rejectDirect
|
|
37
|
+
rejectDirect --> available
|
|
38
|
+
available -->|on: pool.enqueue| enqueue
|
|
39
|
+
enqueue --> waiting
|
|
40
|
+
waiting -->|on: pool.claim| healthyQueued
|
|
41
|
+
healthyQueued -->|true| assignQueued
|
|
42
|
+
assignQueued --> leased
|
|
43
|
+
healthyQueued -->|false| rejectQueued
|
|
44
|
+
rejectQueued --> waiting
|
|
45
|
+
waiting -->|on: pool.expired| expire
|
|
46
|
+
expire --> available
|
|
47
|
+
waiting -->|on: pool.cancel| cancel
|
|
48
|
+
cancel --> available
|
|
49
|
+
waiting -->|on: pool.unavailable| circuit
|
|
50
|
+
circuit --> available
|
|
51
|
+
leased -->|on: pool.sessionBound| bindSession
|
|
52
|
+
bindSession --> leased
|
|
53
|
+
leased -->|on: pool.threadBound| bindThread
|
|
54
|
+
bindThread --> leased
|
|
55
|
+
leased -->|on: pool.activity| activity
|
|
56
|
+
activity --> leased
|
|
57
|
+
leased -->|on: pool.terminalVerified| release
|
|
58
|
+
release --> available
|
|
59
|
+
leased -->|on: pool.outcomeUnknown| quarantine
|
|
60
|
+
quarantine --> quarantined
|
|
61
|
+
quarantined -->|on: pool.stopVerified| verifiedRelease
|
|
62
|
+
verifiedRelease --> available
|
|
63
|
+
quarantined -->|on: pool.outcomeUnknown| keepQuarantine
|
|
64
|
+
keepQuarantine --> quarantined
|
|
65
|
+
leased -->|on: pool.poll| recoveryAllowed
|
|
66
|
+
recoveryAllowed -->|true| recoverTurn
|
|
67
|
+
recoverTurn --> leased
|
|
68
|
+
recoveryAllowed -->|false| keepMonitoring
|
|
69
|
+
keepMonitoring --> leased
|
|
70
|
+
leased -->|on: pool.progressObserved| verifiedProgress
|
|
71
|
+
verifiedProgress -->|true| extendTimeout
|
|
72
|
+
extendTimeout --> leased
|
|
73
|
+
verifiedProgress -->|false| keepMonitoring
|