agents-relay 2.0.191 → 2.0.192

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,45 @@
1
+ import { readFileSync } from 'node:fs';
2
+ import { GraphRuntime } from './workflow/graph-v2.js';
3
+ import { dispatchWorkflowGraphSync } from './workflow/dispatcher.js';
4
+ const actions = ['assign', 'reject', 'enqueue', 'dequeue', 'bindSession', 'bindThread', 'touch', 'release', 'quarantine', 'releaseQuarantine', 'keepQuarantine', 'recoverTurn', 'keepMonitoring', 'extendTimeout'];
5
+ const events = [
6
+ 'pool.claim', 'pool.enqueue', 'pool.expired', 'pool.cancel', 'pool.unavailable', 'pool.sessionBound',
7
+ 'pool.threadBound', 'pool.activity', 'pool.terminalVerified', 'pool.outcomeUnknown', 'pool.stopVerified', 'pool.poll', 'pool.progressObserved',
8
+ ];
9
+ export const browserPoolGraph = new GraphRuntime(readFileSync(new URL('../workflow/browser-worker-pool.mmd', import.meta.url), 'utf8'), {
10
+ states: new Set(['AVAILABLE', 'WAITING', 'LEASED', 'QUARANTINED']),
11
+ guards: { providerHealthy: facts => facts.durableFacts.providerHealthy === true, recoveryDue: facts => facts.durableFacts.recoveryDue === true, freshOutput: facts => facts.durableFacts.freshOutput === true },
12
+ actions: Object.fromEntries(actions.map(name => [`pool.${name}`, { id: `pool.${name}` }])),
13
+ events: Object.fromEntries(events.map(name => [name, { id: name }])),
14
+ });
15
+ /** Graph owns permission and the effect selected; SQLite executes the effect atomically. */
16
+ export function requirePoolDecision(state, event, expectedAction, providerHealthy = true) {
17
+ const decision = dispatchWorkflowGraphSync(browserPoolGraph, { initialState: state, input: { transport: 'event', id: event, payload: null }, durableFacts: { providerHealthy }, graphName: 'browser-worker-pool' });
18
+ if (!decision.accepted || decision.actions.length !== 1 || decision.actions[0].id !== `pool.${expectedAction}`)
19
+ throw new Error(`Browser Pool Graph rejected ${state}/${event}: expected pool.${expectedAction}`);
20
+ return decision;
21
+ }
22
+ /** The executable Graph selects the next monitoring action; no browser heartbeat counts as task progress. */
23
+ export function monitorPoolDecision(recoveryDue) {
24
+ const decision = dispatchWorkflowGraphSync(browserPoolGraph, { initialState: 'LEASED', input: { transport: 'event', id: 'pool.poll', payload: null }, durableFacts: { recoveryDue }, graphName: 'browser-worker-pool' });
25
+ if (!decision.accepted || decision.target !== 'LEASED' || decision.actions.length !== 1)
26
+ throw Error('Invalid Pool monitoring graph');
27
+ const id = decision.actions[0].id;
28
+ if (id === 'pool.recoverTurn')
29
+ return 'recoverTurn';
30
+ if (id === 'pool.keepMonitoring')
31
+ return 'keepMonitoring';
32
+ throw Error(`Unknown Pool monitoring action ${id}`);
33
+ }
34
+ /** Only verified new assistant output can extend Relay's inactivity deadline. */
35
+ export function progressPoolDecision(freshOutput) {
36
+ const decision = dispatchWorkflowGraphSync(browserPoolGraph, { initialState: 'LEASED', input: { transport: 'event', id: 'pool.progressObserved', payload: null }, durableFacts: { freshOutput }, graphName: 'browser-worker-pool' });
37
+ if (!decision.accepted || decision.target !== 'LEASED' || decision.actions.length !== 1)
38
+ throw Error('Invalid Pool progress graph');
39
+ const id = decision.actions[0].id;
40
+ if (id === 'pool.extendTimeout')
41
+ return 'extendTimeout';
42
+ if (id === 'pool.keepMonitoring')
43
+ return 'keepMonitoring';
44
+ throw Error(`Unknown Pool progress action ${id}`);
45
+ }
@@ -1,4 +1,5 @@
1
1
  import { DatabaseSync } from 'node:sqlite';
2
+ import { requirePoolDecision } from './browser-worker-pool-graph.js';
2
3
  import { mkdirSync } from 'node:fs';
3
4
  import { dirname, join } from 'node:path';
4
5
  import { homedir } from 'node:os';
@@ -77,8 +78,11 @@ export class BrowserWorkerPoolManager {
77
78
  try {
78
79
  const now = this.clock();
79
80
  const health = this.db.prepare('SELECT unavailable_until FROM browser_provider_health WHERE provider=?').get(provider);
80
- if (health.unavailable_until > now)
81
+ if (health.unavailable_until > now) {
82
+ const waiting = this.db.prepare('SELECT 1 FROM browser_queue WHERE execution_id=? AND provider=?').get(executionId, provider);
83
+ requirePoolDecision(waiting ? 'WAITING' : 'AVAILABLE', 'pool.claim', 'reject', false);
81
84
  throw new BrowserPoolAdmissionError('PROVIDER_UNAVAILABLE', `${provider} circuit open until ${health.unavailable_until}`);
85
+ }
82
86
  const existing = this.db.prepare('SELECT * FROM browser_slots WHERE execution_id=? AND provider=?').get(executionId, provider);
83
87
  if (existing)
84
88
  throw new Error(`Execution ${executionId} already owns ${provider}-${existing.slot}: refuse duplicate browser submission`);
@@ -87,6 +91,7 @@ export class BrowserWorkerPoolManager {
87
91
  if (!row || row.execution_id === executionId) {
88
92
  const idle = this.db.prepare("SELECT * FROM browser_slots WHERE provider=? AND state='IDLE' ORDER BY slot LIMIT 1").get(provider);
89
93
  if (idle) {
94
+ requirePoolDecision(row ? 'WAITING' : 'AVAILABLE', 'pool.claim', 'assign', true);
90
95
  const generation = idle.generation + 1;
91
96
  this.db.prepare("UPDATE browser_slots SET state='LEASED',execution_id=?,generation=?,session_id=NULL,thread_url=NULL,touched_at=? WHERE provider=? AND slot=? AND state='IDLE'")
92
97
  .run(executionId, generation, now, provider, idle.slot);
@@ -100,7 +105,11 @@ export class BrowserWorkerPoolManager {
100
105
  this.db.exec('COMMIT');
101
106
  }
102
107
  }
103
- dequeue(executionId) {
108
+ dequeue(executionId, event) {
109
+ const queued = this.db.prepare('SELECT 1 FROM browser_queue WHERE execution_id=?').get(executionId);
110
+ if (!queued)
111
+ return;
112
+ requirePoolDecision('WAITING', event, 'dequeue');
104
113
  this.db.prepare('DELETE FROM browser_queue WHERE execution_id=?').run(executionId);
105
114
  }
106
115
  async acquire(provider, executionId, signal) {
@@ -109,10 +118,23 @@ export class BrowserWorkerPoolManager {
109
118
  const immediate = this.tryClaim(provider, executionId);
110
119
  if (immediate)
111
120
  return immediate;
112
- const count = this.db.prepare('SELECT count(*) AS n FROM browser_queue WHERE provider=?').get(provider).n;
113
- if (count >= this.maxQueue)
114
- throw new BrowserPoolAdmissionError('POOL_CAPACITY_FULL', `${provider} queue is full`);
115
- this.db.prepare('INSERT OR IGNORE INTO browser_queue(execution_id,provider,queued_at) VALUES(?,?,?)').run(executionId, provider, this.clock());
121
+ // Queue capacity is a cross-process resource constraint; hold SQLite's writer lock
122
+ // across the queue count and insertion (not only during slot claiming).
123
+ this.db.exec('BEGIN IMMEDIATE');
124
+ try {
125
+ const count = this.db.prepare('SELECT count(*) AS n FROM browser_queue WHERE provider=?').get(provider).n;
126
+ const present = this.db.prepare('SELECT 1 FROM browser_queue WHERE execution_id=?').get(executionId);
127
+ if (count >= this.maxQueue && !present)
128
+ throw new BrowserPoolAdmissionError('POOL_CAPACITY_FULL', `${provider} queue is full`);
129
+ if (!present) {
130
+ requirePoolDecision('AVAILABLE', 'pool.enqueue', 'enqueue');
131
+ this.db.prepare('INSERT INTO browser_queue(execution_id,provider,queued_at) VALUES(?,?,?)').run(executionId, provider, this.clock());
132
+ }
133
+ }
134
+ finally {
135
+ this.db.exec('COMMIT');
136
+ }
137
+ let dequeueEvent = 'pool.cancel';
116
138
  try {
117
139
  for (;;) {
118
140
  if (signal?.aborted)
@@ -120,24 +142,32 @@ export class BrowserWorkerPoolManager {
120
142
  const result = this.tryClaim(provider, executionId);
121
143
  if (result)
122
144
  return result;
123
- if (this.clock() >= deadline)
145
+ if (this.clock() >= deadline) {
146
+ dequeueEvent = 'pool.expired';
124
147
  throw new BrowserPoolAdmissionError('POOL_WAIT_EXPIRED', `${provider} queue wait expired without browser submission`);
148
+ }
125
149
  await sleep(this.pollMs);
126
150
  }
127
151
  }
152
+ catch (error) {
153
+ if (error instanceof BrowserPoolAdmissionError && error.code === 'PROVIDER_UNAVAILABLE')
154
+ dequeueEvent = 'pool.unavailable';
155
+ throw error;
156
+ }
128
157
  finally {
129
- this.dequeue(executionId);
158
+ this.dequeue(executionId, dequeueEvent);
130
159
  }
131
160
  }
132
- update(lease, state, sessionId, threadUrl) {
133
- const result = this.db.prepare('UPDATE browser_slots SET state=?,session_id=?,thread_url=?,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=?')
161
+ update(lease, state, sessionId, threadUrl, event, action) {
162
+ requirePoolDecision('LEASED', event, action);
163
+ const result = this.db.prepare("UPDATE browser_slots SET state=?,session_id=?,thread_url=?,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=? AND state='LEASED'")
134
164
  .run(state, sessionId, threadUrl, this.clock(), lease.provider, lease.slot, lease.executionId, lease.generation);
135
165
  if (Number(result.changes) !== 1)
136
166
  throw Error('Stale browser pool lease generation; refusing mutation');
137
167
  }
138
168
  bindSession(lease, sessionId) {
139
169
  lease.sessionId = sessionId;
140
- this.update(lease, 'LEASED', sessionId, lease.threadUrl);
170
+ this.update(lease, 'LEASED', sessionId, lease.threadUrl, 'pool.sessionBound', 'bindSession');
141
171
  }
142
172
  bindThread(lease, url) {
143
173
  const expected = lease.provider === 'z' ? 'chat.z.ai' : 'chatgpt.com';
@@ -145,20 +175,30 @@ export class BrowserWorkerPoolManager {
145
175
  if (parsed.protocol !== 'https:' || parsed.hostname !== expected || !/^\/c\/[\w-]+\/?$/.test(parsed.pathname))
146
176
  throw Error('Unsafe browser thread URL');
147
177
  lease.threadUrl = parsed.origin + parsed.pathname;
148
- this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl);
178
+ this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl, 'pool.threadBound', 'bindThread');
149
179
  }
150
- touch(lease) { this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl); }
180
+ touch(lease) { this.update(lease, 'LEASED', lease.sessionId, lease.threadUrl, 'pool.activity', 'touch'); }
151
181
  release(lease) {
182
+ requirePoolDecision('LEASED', 'pool.terminalVerified', 'release');
152
183
  const result = this.db.prepare("UPDATE browser_slots SET state='IDLE',execution_id=NULL,session_id=NULL,thread_url=NULL,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=? AND state='LEASED'")
153
184
  .run(this.clock(), lease.provider, lease.slot, lease.executionId, lease.generation);
154
185
  if (Number(result.changes) !== 1)
155
186
  throw Error('Stale browser pool lease release');
156
187
  }
157
- quarantine(lease) { this.update(lease, 'QUARANTINED', lease.sessionId, lease.threadUrl); }
188
+ quarantine(lease) {
189
+ const row = this.db.prepare('SELECT state FROM browser_slots WHERE provider=? AND slot=? AND execution_id=? AND generation=?')
190
+ .get(lease.provider, lease.slot, lease.executionId, lease.generation);
191
+ if (row?.state === 'QUARANTINED') {
192
+ requirePoolDecision('QUARANTINED', 'pool.outcomeUnknown', 'keepQuarantine');
193
+ return;
194
+ }
195
+ this.update(lease, 'QUARANTINED', lease.sessionId, lease.threadUrl, 'pool.outcomeUnknown', 'quarantine');
196
+ }
158
197
  /** Recovery can only release quarantined slots after verifying the old worker is no longer active. */
159
198
  releaseQuarantine(lease, verifiedStopped) {
160
199
  if (!verifiedStopped)
161
200
  throw Error('Cannot release quarantined slot without independent stop verification');
201
+ requirePoolDecision('QUARANTINED', 'pool.stopVerified', 'releaseQuarantine');
162
202
  const result = this.db.prepare("UPDATE browser_slots SET state='IDLE',execution_id=NULL,session_id=NULL,thread_url=NULL,touched_at=? WHERE provider=? AND slot=? AND execution_id=? AND generation=? AND state='QUARANTINED'")
163
203
  .run(this.clock(), lease.provider, lease.slot, lease.executionId, lease.generation);
164
204
  if (Number(result.changes) !== 1)
@@ -4,6 +4,7 @@ import { join } from 'node:path';
4
4
  import { homedir } from 'node:os';
5
5
  import { spawnOwnedProcess } from './owned-process.js';
6
6
  import { BrowserWorkerPoolManager, BrowserPoolAdmissionError } from './browser-worker-pool.js';
7
+ import { monitorPoolDecision, progressPoolDecision } from './browser-worker-pool-graph.js';
7
8
  const WORKSPACE = 'Neo';
8
9
  const OUTPUT_LIMIT = 2_000_000;
9
10
  let manager;
@@ -13,6 +14,7 @@ function cliCommand(args, input, signal) {
13
14
  return new Promise((resolve, reject) => {
14
15
  const child = spawnOwnedProcess(executable, args, {
15
16
  invocation: 'browser-worker-pool:webhelm', signal,
17
+ timeoutMs: args[0] === 'session' && args[1] === 'exec' ? 25_000 : 150_000,
16
18
  spawn: { shell: false, stdio: [input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'] },
17
19
  });
18
20
  let out = '', err = '';
@@ -76,36 +78,41 @@ const state=await js('(()=>{const nodes=[...document.querySelectorAll(\"[data-me
76
78
  console.log(JSON.stringify({url,state}));`;
77
79
  function fingerprint(data) { return `${data.state?.text || ''}|${data.state?.streamLength || 0}`; }
78
80
  export function pooledWebHelmRun(request, signal) {
81
+ const pool = request.pool ?? browserWorkerPool();
79
82
  let lease;
80
83
  let sessionId;
81
84
  let released = false;
82
85
  let stopped = false;
83
86
  let monitor;
87
+ let completionRecoveryTimer;
84
88
  let browserKeepAliveAt = 0, observedAt = Date.now(), lastFingerprint = '', sampled = false;
85
89
  let recoveries = 0, monitoring = false, threadId;
86
90
  let submissionAttempted = false, verifiedSubmitted = false, uncertain = false, recoveryOutcomeUncertain = false;
91
+ let providerPreflightFailed = false;
87
92
  const stop = async () => {
88
93
  if (stopped)
89
94
  return;
90
95
  stopped = true;
91
96
  if (monitor)
92
97
  clearInterval(monitor);
98
+ if (completionRecoveryTimer)
99
+ clearTimeout(completionRecoveryTimer);
93
100
  if (!lease)
94
101
  return;
95
102
  if (uncertain) {
96
- browserWorkerPool().quarantine(lease);
103
+ pool.quarantine(lease);
97
104
  return;
98
105
  }
99
106
  if (!sessionId) {
100
- browserWorkerPool().release(lease);
107
+ pool.release(lease);
101
108
  return;
102
109
  }
103
110
  try {
104
111
  await cliCommand(['session', 'stop', sessionId]);
105
- browserWorkerPool().release(lease);
112
+ pool.release(lease);
106
113
  }
107
114
  catch {
108
- browserWorkerPool().quarantine(lease);
115
+ pool.quarantine(lease);
109
116
  }
110
117
  };
111
118
  const release = async () => { released = true; if (lease)
@@ -113,10 +120,30 @@ export function pooledWebHelmRun(request, signal) {
113
120
  const cancel = () => { released = true; if (!lease)
114
121
  return; if (verifiedSubmitted || submissionAttempted) {
115
122
  uncertain = true;
116
- browserWorkerPool().quarantine(lease);
123
+ pool.quarantine(lease);
117
124
  }
118
125
  else
119
126
  void stop().catch(() => { }); };
127
+ const sendRecovery = async () => {
128
+ if (stopped || released || uncertain || recoveryOutcomeUncertain || !sessionId || !threadId || !request.recover || recoveries >= 2)
129
+ return;
130
+ recoveries++;
131
+ const cfg = await request.recover(threadId);
132
+ try {
133
+ const answer = await configRun(request.provider, 'new-turn', sessionId, cfg);
134
+ if (!['submitted', 'completed'].includes(String(answer.status)) || !answer.verified_by)
135
+ throw Error('Recovery turn not independently verified');
136
+ await request.onRecovery?.(threadId, String(answer.verified_by));
137
+ }
138
+ catch (error) {
139
+ // An ambiguous recovery can already have been delivered. Preserve the original session.
140
+ recoveryOutcomeUncertain = true;
141
+ uncertain = true;
142
+ if (lease)
143
+ pool.quarantine(lease);
144
+ throw error;
145
+ }
146
+ };
120
147
  const probe = async () => {
121
148
  if (stopped || monitoring || !sessionId || !lease?.threadUrl)
122
149
  return;
@@ -137,29 +164,20 @@ export function pooledWebHelmRun(request, signal) {
137
164
  lastFingerprint = current;
138
165
  return;
139
166
  }
140
- if (current !== lastFingerprint && current !== '|0') {
167
+ const fresh = current !== lastFingerprint && current !== '|0';
168
+ if (progressPoolDecision(fresh) === 'extendTimeout') {
141
169
  lastFingerprint = current;
142
170
  observedAt = now;
143
- browserWorkerPool().touch(lease);
171
+ pool.touch(lease);
172
+ // Relay correlates task.progress with the original execution and re-arms its inactivity watchdog.
144
173
  await request.onActivity?.('Browser Worker produced new assistant output');
145
174
  }
146
- if (now - observedAt >= Math.max(request.recoveryMinMs ?? 60_000, Math.round(request.timeoutMs * .6)) &&
147
- !observation.state?.generating && !recoveryOutcomeUncertain && recoveries < 2 && request.recover && threadId) {
175
+ const due = now - observedAt >= Math.max(request.recoveryMinMs ?? 60_000, Math.round(request.timeoutMs * .6)) &&
176
+ !observation.state?.generating && !recoveryOutcomeUncertain && recoveries < 2 && !!request.recover && !!threadId;
177
+ if (monitorPoolDecision(due) === 'recoverTurn' && request.recover && threadId) {
148
178
  // Count before the send. An ambiguous response must never cause automatic retransmission.
149
- recoveries++;
150
179
  observedAt = now;
151
- const cfg = await request.recover(threadId);
152
- try {
153
- const answer = await configRun(request.provider, 'new-turn', sessionId, cfg);
154
- if (answer.status !== 'submitted' || !answer.verified_by)
155
- throw Error('Recovery turn not independently verified');
156
- await request.onActivity?.('Verified same-thread recovery turn submitted');
157
- }
158
- catch (error) {
159
- // The recovery may have reached the remote site: never submit another automatically.
160
- recoveryOutcomeUncertain = true;
161
- throw error;
162
- }
180
+ await sendRecovery(); // A recovery receipt cannot extend Relay's inactivity deadline.
163
181
  }
164
182
  }
165
183
  catch {
@@ -172,18 +190,32 @@ export function pooledWebHelmRun(request, signal) {
172
190
  };
173
191
  const promise = (async () => {
174
192
  try {
175
- lease = await browserWorkerPool().acquire(request.provider, request.executionId, signal);
193
+ lease = await pool.acquire(request.provider, request.executionId, signal);
176
194
  if (released || signal.aborted)
177
195
  throw Error('Browser worker cancelled before submission');
178
196
  // Provider code must be present before opening any site or submitting work.
179
- const profile = parseEnvelope(await cliCommand(['platform', 'profile', request.provider], undefined, signal));
197
+ let profile;
198
+ try {
199
+ profile = parseEnvelope(await cliCommand(['platform', 'profile', request.provider], undefined, signal));
200
+ }
201
+ catch (error) {
202
+ providerPreflightFailed = true;
203
+ throw error;
204
+ }
180
205
  if (profile.platform !== request.provider)
181
206
  throw Error(`WebHelm platform ${request.provider} missing`);
182
- const session = parseEnvelope(await cliCommand(['session', 'start', '--workspace', WORKSPACE], undefined, signal));
207
+ let session;
208
+ try {
209
+ session = parseEnvelope(await cliCommand(['session', 'start', '--workspace', WORKSPACE], undefined, signal));
210
+ }
211
+ catch (error) {
212
+ providerPreflightFailed = true;
213
+ throw error;
214
+ }
183
215
  if (typeof session.session_id !== 'string')
184
216
  throw Error('No WebHelm session ID');
185
217
  sessionId = session.session_id;
186
- browserWorkerPool().bindSession(lease, sessionId);
218
+ pool.bindSession(lease, sessionId);
187
219
  // Preserve WebHelm's existing extension lifecycle. Pool capacity is six tabs total.
188
220
  await cliCommand(['session', 'exec', sessionId], `await workspace_resize(${JSON.stringify(WORKSPACE)},6);`);
189
221
  if (released || signal.aborted)
@@ -191,18 +223,24 @@ export function pooledWebHelmRun(request, signal) {
191
223
  const cfg = await request.build();
192
224
  submissionAttempted = true;
193
225
  const receipt = await configRun(request.provider, cfg.action, sessionId, cfg.config, signal);
194
- if (receipt.status !== 'submitted' || !receipt.verified_by)
226
+ if (!['submitted', 'completed'].includes(String(receipt.status)) || !receipt.verified_by)
195
227
  throw Error('Submission outcome unknown: no verified receipt');
196
228
  verifiedSubmitted = true;
197
229
  const url = typeof receipt.thread_url === 'string' ? receipt.thread_url :
198
230
  typeof receipt.thread_id === 'string' ? `https://${request.provider === 'z' ? 'chat.z.ai' : 'chatgpt.com'}/c/${receipt.thread_id}` : '';
199
- browserWorkerPool().bindThread(lease, url);
231
+ pool.bindThread(lease, url);
200
232
  threadId = String(receipt.thread_id || new URL(url).pathname.split('/').at(-1));
201
- browserWorkerPool().reportHealthy(request.provider);
233
+ pool.reportHealthy(request.provider);
202
234
  if (released) {
203
235
  await stop();
204
236
  return JSON.stringify(receipt);
205
237
  }
238
+ // Completion proof is independent of CDP polling. Give the worker time to emit
239
+ // its correlated terminal event; the terminal event closes this lease before recovery.
240
+ if (receipt.status === 'completed' && request.recover) {
241
+ completionRecoveryTimer = setTimeout(() => { void sendRecovery().catch(() => { }); }, request.completionRecoveryGraceMs ?? 12_000);
242
+ completionRecoveryTimer.unref();
243
+ }
206
244
  monitor = setInterval(() => { void probe(); }, request.monitorIntervalMs ?? 30_000);
207
245
  monitor.unref();
208
246
  return JSON.stringify(receipt);
@@ -216,12 +254,13 @@ export function pooledWebHelmRun(request, signal) {
216
254
  const rejection = safe && !cancelled && !(error instanceof BrowserPoolAdmissionError)
217
255
  ? new BrowserPoolAdmissionError('PROVIDER_UNAVAILABLE', `${request.provider} pre-submission failure: ${reason}`)
218
256
  : error;
219
- if (safe && !cancelled && lease && !(error instanceof BrowserPoolAdmissionError))
220
- browserWorkerPool().reportPreSubmissionOutage(request.provider);
257
+ // A local prompt/configuration error is not evidence that the remote provider is down.
258
+ if (safe && !cancelled && providerPreflightFailed && lease && !(error instanceof BrowserPoolAdmissionError))
259
+ pool.reportPreSubmissionOutage(request.provider);
221
260
  if (lease && !stopped) {
222
261
  if (!safe) {
223
262
  uncertain = true;
224
- browserWorkerPool().quarantine(lease);
263
+ pool.quarantine(lease);
225
264
  }
226
265
  else
227
266
  await stop();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agents-relay",
3
- "version": "2.0.191",
3
+ "version": "2.0.192",
4
4
  "description": "Durable async agent jobs coordinated through GitHub pull requests",
5
5
  "repository": {
6
6
  "type": "git",
@@ -19,7 +19,7 @@
19
19
  "test:publish": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 --test-force-exit test/*.test.js",
20
20
  "test:fast": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=2 test/workflow-lifecycle.test.js test/workflow-v2-lifecycle-parity.test.js test/workflow-effects.test.js test/relay-store.test.js test/local-authority.test.js test/local-jobid-routing.test.js test/local-sqlite-jobid.test.js test/no-github.test.js test/reconcile-coordinator.test.js test/service-publication.test.js test/github-webhook.test.js",
21
21
  "test:full": "npm run build && node --import ./test/test-store-env.mjs --test --test-concurrency=1 test/*.test.js && npm run test:verification:l3",
22
- "test:verification:l1": "npm run build && node --test test/verification/harness.test.mjs test/verification/contract.test.mjs test/verification/graph-inventory.test.mjs test/verification/dashboard-drilldown.test.mjs test/verification/evidence-matrix.test.mjs test/verification/task-resolution-classification.test.mjs test/verification/planner-contract.test.mjs test/verification/force-restate-validation.test.mjs test/planner-child-recovery.test.js",
22
+ "test:verification:l1": "npm run build && node --test test/verification/harness.test.mjs test/verification/contract.test.mjs test/browser-worker-pool-graph.test.js test/verification/graph-inventory.test.mjs test/verification/dashboard-drilldown.test.mjs test/verification/evidence-matrix.test.mjs test/verification/task-resolution-classification.test.mjs test/verification/planner-contract.test.mjs test/verification/force-restate-validation.test.mjs test/planner-child-recovery.test.js",
23
23
  "test:verification:l3": "npm run build && node --test --test-concurrency=1 test/verification/harness.test.mjs test/local-sqlite-jobid.test.js test/verification/l3-sqlite.test.mjs test/verification/l3-ingress.test.mjs test/verification/l3-replay.test.mjs test/verification/objective-authority.test.mjs test/verification/l3-convergence.test.mjs test/verification/l3-review-convergence.test.mjs test/verification/l3-human-escalation.test.mjs test/verification/l3-escalation-notifier.test.mjs test/verification/l3-unknown-effect.test.mjs test/verification/l3-cli-actions.test.mjs test/verification/l3-planner-round.test.mjs test/verification/l3-planner-cli.test.mjs test/verification/l3-task-resolution.test.mjs test/verification/l3-plan-apply-convergence.test.mjs test/verification/full-recovery-gates.test.mjs test/verification/production-wiring.test.mjs test/verification/l3-full-path.test.mjs test/verification/l3-projection-recovery.test.mjs test/verification/l3-daemon-startup-recovery.test.mjs",
24
24
  "test:verification": "npm run test:verification:l1 && npm run test:verification:l2 && npm run test:verification:l3 && npm run test:verification:cross-a && node --test test/workflow-production-outbox.test.js && npm run test:verification:matrix",
25
25
  "test:verification:cross-a": "npm run build && RELAY_A_DIST=$PWD/dist node --test test/verification/action-ledger-cross-branch.test.mjs test/verification/outbox-cross-a.test.mjs",
@@ -28,7 +28,8 @@
28
28
  "test:verification:l2:live": "node test/verification/l2/live-provider.mjs",
29
29
  "test:verification:l2:live-codex": "npm run build && node test/verification/l2/live-codex-agents.mjs",
30
30
  "test:verification:l2:live-planner": "npm run build && RELAY_L2_LIVE=1 node --test test/verification/l2/live-planner-convergence.test.mjs",
31
- "test:l2:agents": "npm run build && node test/l2-agent-prompts/evaluate.mjs"
31
+ "test:l2:agents": "npm run build && node test/l2-agent-prompts/evaluate.mjs",
32
+ "test:verification:browser-pool:live": "npm run build && node test/live/browser-worker-pool.e2e.mjs && node test/live/browser-worker-pool-recovery.e2e.mjs"
32
33
  },
33
34
  "devDependencies": {
34
35
  "@types/node": "^22.10.2",
@@ -0,0 +1,73 @@
1
+ ---
2
+ title: Browser Worker Pool Lifecycle
3
+ description: Resource admission, FIFO queue, provider-health rejection, exclusive lease, terminal release and uncertain-outcome quarantine; Relay Task Graph remains authoritative for Task completion.
4
+ ---
5
+ flowchart LR
6
+ %% relay: initial available
7
+ available["state: AVAILABLE"]
8
+ waiting["state: WAITING"]
9
+ leased["state: LEASED"]
10
+ quarantined["state: QUARANTINED"]
11
+ healthyDirect["guard: providerHealthy"]
12
+ healthyQueued["guard: providerHealthy"]
13
+ assignDirect["action: pool.assign"]
14
+ assignQueued["action: pool.assign"]
15
+ rejectDirect["action: pool.reject"]
16
+ rejectQueued["action: pool.reject"]
17
+ enqueue["action: pool.enqueue"]
18
+ expire["action: pool.dequeue"]
19
+ cancel["action: pool.dequeue"]
20
+ circuit["action: pool.dequeue"]
21
+ bindSession["action: pool.bindSession"]
22
+ bindThread["action: pool.bindThread"]
23
+ activity["action: pool.touch"]
24
+ release["action: pool.release"]
25
+ quarantine["action: pool.quarantine"]
26
+ verifiedRelease["action: pool.releaseQuarantine"]
27
+ keepQuarantine["action: pool.keepQuarantine"]
28
+ recoveryAllowed["guard: recoveryDue"]
29
+ recoverTurn["action: pool.recoverTurn"]
30
+ keepMonitoring["action: pool.keepMonitoring"]
31
+ verifiedProgress["guard: freshOutput"]
32
+ extendTimeout["action: pool.extendTimeout"]
33
+ available -->|on: pool.claim| healthyDirect
34
+ healthyDirect -->|true| assignDirect
35
+ assignDirect --> leased
36
+ healthyDirect -->|false| rejectDirect
37
+ rejectDirect --> available
38
+ available -->|on: pool.enqueue| enqueue
39
+ enqueue --> waiting
40
+ waiting -->|on: pool.claim| healthyQueued
41
+ healthyQueued -->|true| assignQueued
42
+ assignQueued --> leased
43
+ healthyQueued -->|false| rejectQueued
44
+ rejectQueued --> waiting
45
+ waiting -->|on: pool.expired| expire
46
+ expire --> available
47
+ waiting -->|on: pool.cancel| cancel
48
+ cancel --> available
49
+ waiting -->|on: pool.unavailable| circuit
50
+ circuit --> available
51
+ leased -->|on: pool.sessionBound| bindSession
52
+ bindSession --> leased
53
+ leased -->|on: pool.threadBound| bindThread
54
+ bindThread --> leased
55
+ leased -->|on: pool.activity| activity
56
+ activity --> leased
57
+ leased -->|on: pool.terminalVerified| release
58
+ release --> available
59
+ leased -->|on: pool.outcomeUnknown| quarantine
60
+ quarantine --> quarantined
61
+ quarantined -->|on: pool.stopVerified| verifiedRelease
62
+ verifiedRelease --> available
63
+ quarantined -->|on: pool.outcomeUnknown| keepQuarantine
64
+ keepQuarantine --> quarantined
65
+ leased -->|on: pool.poll| recoveryAllowed
66
+ recoveryAllowed -->|true| recoverTurn
67
+ recoverTurn --> leased
68
+ recoveryAllowed -->|false| keepMonitoring
69
+ keepMonitoring --> leased
70
+ leased -->|on: pool.progressObserved| verifiedProgress
71
+ verifiedProgress -->|true| extendTimeout
72
+ extendTimeout --> leased
73
+ verifiedProgress -->|false| keepMonitoring