@stratusagent/gateway 0.2.5 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/index.d.ts +113 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +645 -86
- package/dist/index.js.map +1 -1
- package/package.json +7 -6
- package/dist/tsconfig.test.tsbuildinfo +0 -1
package/dist/index.js
CHANGED
|
@@ -1,13 +1,25 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
2
|
import { chmodSync, mkdirSync } from 'node:fs';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
-
import { AgentRegistry, AgentRunner, EventBus, RunAbortedError, ToolRegistry, } from '@stratusagent/core';
|
|
5
|
+
import { AgentRegistry, AgentRunner, EventBus, RunAbortedError, ToolRegistry, readPendingApproval, } from '@stratusagent/core';
|
|
6
6
|
import { createDelegateTool, createRememberTool, } from '@stratusagent/agents';
|
|
7
7
|
import { createLocalCommandExecutor } from '@stratusagent/executor-local';
|
|
8
8
|
import { createDemoTool, createFileMemoryStore, createRuntimeProvider, DEFAULT_STRATUS_AGENT, loadRosterSouls, FALLBACK_ACTIVE_METADATA_KEY, loadSoulFile, memoryFilePath, migrateLegacyMemory, ConfigFileError, loadConfigFile, readNonEmptyString, resolveConfigLocation, resolveConfiguredSoul, resolveRuntimeConfig, stratusHomePath, withLegacyDefaultMemories, } from '@stratusagent/state';
|
|
9
9
|
const SESSIONS_DB_FILENAME = 'sessions.db';
|
|
10
10
|
const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
|
|
11
|
+
/**
|
|
12
|
+
* How long a parked call waits for a person. Long enough to survive a
|
|
13
|
+
* meeting, short enough that a request nobody saw does not hold a turn (and
|
|
14
|
+
* the Slack thread it is answering) open indefinitely.
|
|
15
|
+
*/
|
|
16
|
+
const DEFAULT_APPROVAL_TIMEOUT_MS = 900_000;
|
|
17
|
+
/**
|
|
18
|
+
* Node's largest `setTimeout` delay. Anything above it does not become a
|
|
19
|
+
* long wait — it silently becomes a 1ms one, so a config asking for a
|
|
20
|
+
* 30-day approval window would expire every request almost immediately.
|
|
21
|
+
*/
|
|
22
|
+
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
11
23
|
export class SqliteSessionStore {
|
|
12
24
|
db;
|
|
13
25
|
constructor(filePath, options = {}) {
|
|
@@ -84,6 +96,17 @@ export class SqliteSessionStore {
|
|
|
84
96
|
.prepare('INSERT OR REPLACE INTO sessions (id, agent_id, status, body, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)')
|
|
85
97
|
.run(updated.id, updated.agent.id, updated.status, JSON.stringify(updated), updated.createdAt, updated.updatedAt);
|
|
86
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* Session ids in a state, oldest first — the index a restarting daemon
|
|
101
|
+
* sweeps for turns parked on a human. The `status` column carries it, so
|
|
102
|
+
* no conversation body is deserialized to answer the question.
|
|
103
|
+
*/
|
|
104
|
+
async listIdsByStatus(status) {
|
|
105
|
+
const rows = this.db
|
|
106
|
+
.prepare('SELECT id FROM sessions WHERE status = ? ORDER BY updated_at ASC')
|
|
107
|
+
.all(status);
|
|
108
|
+
return rows.map((row) => row.id);
|
|
109
|
+
}
|
|
87
110
|
/** Newest-first session listing for one agent (or all agents). */
|
|
88
111
|
list(agentId) {
|
|
89
112
|
const rows = (agentId
|
|
@@ -101,6 +124,75 @@ export class SqliteSessionStore {
|
|
|
101
124
|
this.db.close();
|
|
102
125
|
}
|
|
103
126
|
}
|
|
127
|
+
/**
|
|
128
|
+
* Apply a soul's provider/model pins to a selection, demoting the
|
|
129
|
+
* daemon-wide defaults the pins outrank.
|
|
130
|
+
*
|
|
131
|
+
* Exported because this is not only dispatch logic: anything that wants to
|
|
132
|
+
* know what a served agent will actually resolve to — a startup billing
|
|
133
|
+
* check, a diagnostic — has to normalize the same way, and a second copy
|
|
134
|
+
* of these rules drifts from this one.
|
|
135
|
+
*/
|
|
136
|
+
export const applySoulPins = (pins, selection, env, context) => {
|
|
137
|
+
selection.presetSoul = pins;
|
|
138
|
+
if (!pins.provider && !pins.model) {
|
|
139
|
+
return { selection, env };
|
|
140
|
+
}
|
|
141
|
+
const processEnv = { ...(env.processEnv ?? process.env) };
|
|
142
|
+
// The daemon-wide default can come from the selection, the environment,
|
|
143
|
+
// or the config file. A config with no provider key predates the
|
|
144
|
+
// anthropic option and is openai-specific (the resolver treats it that
|
|
145
|
+
// way), so a real file without the key still names openai as the
|
|
146
|
+
// default. Env values normalize exactly as the resolver normalizes
|
|
147
|
+
// them: an empty or whitespace-padded STRATUS_PROVIDER is no default at
|
|
148
|
+
// all, not a mismatching one.
|
|
149
|
+
const defaultProvider = context.selectionProvider
|
|
150
|
+
?? readNonEmptyString(processEnv.STRATUS_PROVIDER)
|
|
151
|
+
?? readNonEmptyString(processEnv.STRATUSCLAW_PROVIDER)
|
|
152
|
+
?? context.configProvider
|
|
153
|
+
?? (context.configPresent ? 'openai' : undefined);
|
|
154
|
+
if (pins.provider) {
|
|
155
|
+
delete selection.provider;
|
|
156
|
+
delete processEnv.STRATUS_PROVIDER;
|
|
157
|
+
delete processEnv.STRATUSCLAW_PROVIDER;
|
|
158
|
+
if (defaultProvider !== undefined && defaultProvider !== 'demo' && pins.provider !== defaultProvider) {
|
|
159
|
+
// The default model, endpoint, and generic credentials were all
|
|
160
|
+
// chosen for the default provider — none may ride along to the
|
|
161
|
+
// soul's: a base URL would point the pinned provider at the wrong
|
|
162
|
+
// service, and a generic API key would be sent to it. With NO
|
|
163
|
+
// default selected anywhere — or the credential-less demo provider
|
|
164
|
+
// as the default — there is nothing those values could have been
|
|
165
|
+
// chosen for except whatever provider the soul selects — exactly
|
|
166
|
+
// the resolver's own reading of a generic credential — so they stay.
|
|
167
|
+
delete selection.model;
|
|
168
|
+
delete processEnv.STRATUS_MODEL;
|
|
169
|
+
delete processEnv.STRATUSCLAW_MODEL;
|
|
170
|
+
delete selection.baseUrl;
|
|
171
|
+
delete processEnv.STRATUS_BASE_URL;
|
|
172
|
+
delete processEnv.STRATUSCLAW_BASE_URL;
|
|
173
|
+
delete processEnv.STRATUS_API_KEY;
|
|
174
|
+
delete processEnv.STRATUSCLAW_API_KEY;
|
|
175
|
+
delete processEnv.STRATUS_API_KEY_ENV;
|
|
176
|
+
delete processEnv.STRATUSCLAW_API_KEY_ENV;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
if (pins.model) {
|
|
180
|
+
delete selection.model;
|
|
181
|
+
delete processEnv.STRATUS_MODEL;
|
|
182
|
+
delete processEnv.STRATUSCLAW_MODEL;
|
|
183
|
+
}
|
|
184
|
+
return { selection, env: { ...env, processEnv } };
|
|
185
|
+
};
|
|
186
|
+
/**
|
|
187
|
+
* Recorded on a turn the previous process was still running when it died.
|
|
188
|
+
*
|
|
189
|
+
* Exported because "distinguishable" is the whole point: a surface, a
|
|
190
|
+
* test, or an operator reading `lastError` has to be able to tell a turn
|
|
191
|
+
* nobody finished apart from one a provider or a tool actually failed, and
|
|
192
|
+
* matching on prose that lives in one place beats each caller inventing
|
|
193
|
+
* its own guess at the wording.
|
|
194
|
+
*/
|
|
195
|
+
export const ABANDONED_TURN_ERROR = 'stratusd stopped while this turn was still running; it was not resumed. Send the message again.';
|
|
104
196
|
/**
|
|
105
197
|
* The always-on Stratus process. One gateway holds the roster, the durable
|
|
106
198
|
* session store, the shared event bus, and a pool of runners keyed by
|
|
@@ -122,6 +214,253 @@ export const createGateway = (options = {}) => {
|
|
|
122
214
|
? new SqliteSessionStore(options.sessionDbPath)
|
|
123
215
|
: new SqliteSessionStore(path.join(stratusHomePath(env), SESSIONS_DB_FILENAME), { ownedDirectory: true });
|
|
124
216
|
const memory = withLegacyDefaultMemories(createFileMemoryStore(memoryFilePath(env)));
|
|
217
|
+
// ---- approval brokering -------------------------------------------------
|
|
218
|
+
const approvalTimeoutMs = options.approvalTimeoutMs ?? DEFAULT_APPROVAL_TIMEOUT_MS;
|
|
219
|
+
// Refused at construction, not absorbed: a negative or NaN timeout fails
|
|
220
|
+
// the `> 0` test below and quietly means "never expire" — the one
|
|
221
|
+
// behavior documented for an explicit 0, and the last one a caller who
|
|
222
|
+
// typed a bad number wants. Config-file values are already rejected with
|
|
223
|
+
// a better message; this catches the programmatic path.
|
|
224
|
+
if (!Number.isFinite(approvalTimeoutMs) || approvalTimeoutMs < 0) {
|
|
225
|
+
throw new Error(`approvalTimeoutMs must be a non-negative number of milliseconds (0 to wait indefinitely); received ${String(options.approvalTimeoutMs)}.`);
|
|
226
|
+
}
|
|
227
|
+
// Clamped, not trusted: a delay past Node's timer range is not a long
|
|
228
|
+
// wait, it is a 1ms one, so an over-large value would turn every approval
|
|
229
|
+
// into an instant expiry — the opposite of what it asked for. The CLI
|
|
230
|
+
// rejects these at config load with a better message; this is the
|
|
231
|
+
// backstop for a programmatic caller.
|
|
232
|
+
const effectiveApprovalTimeoutMs = Math.min(approvalTimeoutMs, MAX_TIMER_DELAY_MS);
|
|
233
|
+
if (approvalTimeoutMs > MAX_TIMER_DELAY_MS) {
|
|
234
|
+
warn(`approval timeout ${approvalTimeoutMs}ms is above Node's maximum timer delay; using ${MAX_TIMER_DELAY_MS}ms (~24.8 days)`);
|
|
235
|
+
}
|
|
236
|
+
const pendingApprovals = new Map();
|
|
237
|
+
/**
|
|
238
|
+
* In-flight `tool.approval-resolved` emissions. `EventBus.emit` awaits its
|
|
239
|
+
* subscribers in order, so one async handler ahead of a channel adapter
|
|
240
|
+
* suspends the emission before the adapter ever sees it. Nothing today is
|
|
241
|
+
* async in front of one — which is exactly the problem: the shutdown
|
|
242
|
+
* guarantee below would rest on that staying true, and the first async
|
|
243
|
+
* subscriber anyone adds would silently leave live-looking buttons in a
|
|
244
|
+
* workspace the daemon has left.
|
|
245
|
+
*/
|
|
246
|
+
const approvalEmissions = new Set();
|
|
247
|
+
/**
|
|
248
|
+
* Publishes one parked call and settles when somebody answers it, the
|
|
249
|
+
* request expires, or the turn is cancelled.
|
|
250
|
+
*
|
|
251
|
+
* All three endings run through the same `settle`, which removes the
|
|
252
|
+
* request from the registry BEFORE resolving. That ordering is the whole
|
|
253
|
+
* point: a request that is no longer pending cannot be resolved twice,
|
|
254
|
+
* so a click racing the timeout — or arriving after an abort — is refused
|
|
255
|
+
* rather than executing a tool for a turn that has already moved on.
|
|
256
|
+
*/
|
|
257
|
+
const requestApproval = (request) => new Promise((resolve) => {
|
|
258
|
+
// A shutdown denies what is parked once, at the top of stop(). Turns
|
|
259
|
+
// already running keep going through the drain, though, and one of them
|
|
260
|
+
// can reach a gated tool AFTER that snapshot — finishing a provider
|
|
261
|
+
// call, or moving to the next call in the same response. Parking it
|
|
262
|
+
// would deadlock the drain against a question nobody is left to answer:
|
|
263
|
+
// stop() waits for the turn, the turn waits for the approval, and the
|
|
264
|
+
// approval waits out its timeout — forever when there is none.
|
|
265
|
+
if (stopping) {
|
|
266
|
+
resolve('deny');
|
|
267
|
+
void bus.emit({
|
|
268
|
+
type: 'tool.approval-resolved',
|
|
269
|
+
sessionId: request.session.id,
|
|
270
|
+
requestId: randomUUID(),
|
|
271
|
+
answer: 'deny',
|
|
272
|
+
reason: 'cancelled',
|
|
273
|
+
});
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const requestId = randomUUID();
|
|
277
|
+
let timer;
|
|
278
|
+
const onAbort = () => settle('deny', 'cancelled');
|
|
279
|
+
/**
|
|
280
|
+
* Settles once the request has finished being announced.
|
|
281
|
+
*
|
|
282
|
+
* Both emissions go through the same subscriber list, in order, and
|
|
283
|
+
* `EventBus.emit` awaits each one — so an async subscriber ahead of a
|
|
284
|
+
* channel can hold the announcement while a timeout or abort fires
|
|
285
|
+
* behind it, and the channel would hear that a request it has never
|
|
286
|
+
* heard of was resolved. It drops that, then renders the announcement
|
|
287
|
+
* when it finally arrives, leaving buttons for a settled request that
|
|
288
|
+
* nothing will ever retract.
|
|
289
|
+
*
|
|
290
|
+
* Chaining is the fix rather than remembering orphaned resolutions in
|
|
291
|
+
* the channel: a resolution that arrives before its request is not
|
|
292
|
+
* something a consumer can act on, so it should not be able to happen.
|
|
293
|
+
* Starts resolved, because a turn already aborted never announces at
|
|
294
|
+
* all and its denial should not wait on an emission that never runs.
|
|
295
|
+
*/
|
|
296
|
+
let announced = Promise.resolve();
|
|
297
|
+
const settle = (answer, reason, actor) => {
|
|
298
|
+
// Identity, not presence: a request that already settled is gone from
|
|
299
|
+
// the registry, and comparing against this exact function is what
|
|
300
|
+
// makes every ending — click, timeout, abort, shutdown — idempotent.
|
|
301
|
+
if (pendingApprovals.get(requestId) !== settle) {
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
pendingApprovals.delete(requestId);
|
|
305
|
+
if (timer) {
|
|
306
|
+
clearTimeout(timer);
|
|
307
|
+
}
|
|
308
|
+
request.signal?.removeEventListener('abort', onAbort);
|
|
309
|
+
resolve(answer);
|
|
310
|
+
const emitted = announced.then(() => bus.emit({
|
|
311
|
+
type: 'tool.approval-resolved',
|
|
312
|
+
sessionId: request.session.id,
|
|
313
|
+
requestId,
|
|
314
|
+
answer,
|
|
315
|
+
reason,
|
|
316
|
+
...(actor ? { actor } : {}),
|
|
317
|
+
}));
|
|
318
|
+
approvalEmissions.add(emitted);
|
|
319
|
+
void emitted.finally(() => approvalEmissions.delete(emitted));
|
|
320
|
+
};
|
|
321
|
+
pendingApprovals.set(requestId, settle);
|
|
322
|
+
// What is left of the window, not a fresh one: a re-asked request
|
|
323
|
+
// carries when it originally parked, and the elapsed time counts.
|
|
324
|
+
const alreadyWaitedMs = request.parkedAt ? Date.now() - Date.parse(request.parkedAt) : 0;
|
|
325
|
+
const remainingTimeoutMs = effectiveApprovalTimeoutMs > 0
|
|
326
|
+
? effectiveApprovalTimeoutMs - (Number.isFinite(alreadyWaitedMs) ? Math.max(0, alreadyWaitedMs) : 0)
|
|
327
|
+
: 0;
|
|
328
|
+
if (effectiveApprovalTimeoutMs > 0 && remainingTimeoutMs <= 0) {
|
|
329
|
+
// Nothing left to wait with. Settled here rather than armed with a
|
|
330
|
+
// zero timer, which `setTimeout` would still run a tick later — long
|
|
331
|
+
// enough to announce a request that is already over.
|
|
332
|
+
settle('deny', 'timeout');
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
if (remainingTimeoutMs > 0) {
|
|
336
|
+
// Deliberately NOT unref'd. A parked turn is real outstanding work,
|
|
337
|
+
// and its timer is the only thing that will ever finish it — letting
|
|
338
|
+
// the process exit out from under one would abandon the turn instead
|
|
339
|
+
// of denying it. Shutdown is handled where it belongs, in stop(),
|
|
340
|
+
// which settles every outstanding request before draining.
|
|
341
|
+
timer = setTimeout(() => settle('deny', 'timeout'), remainingTimeoutMs);
|
|
342
|
+
}
|
|
343
|
+
// The abort listener is attached before the request is announced, so a
|
|
344
|
+
// turn cancelled while the event is still being delivered still ends
|
|
345
|
+
// the request rather than leaving it pending forever.
|
|
346
|
+
if (request.signal) {
|
|
347
|
+
if (request.signal.aborted) {
|
|
348
|
+
settle('deny', 'cancelled');
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
request.signal.addEventListener('abort', onAbort, { once: true });
|
|
352
|
+
}
|
|
353
|
+
// `announced` is assigned BEFORE the emission starts, because emit runs
|
|
354
|
+
// its first subscribers synchronously — one of them settling the
|
|
355
|
+
// request inline would otherwise chain onto the already-resolved
|
|
356
|
+
// placeholder and race the very announcement it is answering.
|
|
357
|
+
let markAnnounced = () => { };
|
|
358
|
+
announced = new Promise((resolveAnnounced) => {
|
|
359
|
+
markAnnounced = resolveAnnounced;
|
|
360
|
+
});
|
|
361
|
+
const announcing = bus.emit({
|
|
362
|
+
type: 'tool.approval-requested',
|
|
363
|
+
sessionId: request.session.id,
|
|
364
|
+
agentId: request.session.agent.id,
|
|
365
|
+
requestId,
|
|
366
|
+
call: request.call,
|
|
367
|
+
risk: request.risk,
|
|
368
|
+
...(request.session.metadata ? { metadata: request.session.metadata } : {}),
|
|
369
|
+
...(remainingTimeoutMs > 0
|
|
370
|
+
? { expiresAt: new Date(Date.now() + remainingTimeoutMs).toISOString() }
|
|
371
|
+
: {}),
|
|
372
|
+
});
|
|
373
|
+
// Drained at shutdown alongside the resolutions: a channel that has not
|
|
374
|
+
// yet been told a request exists cannot be told it was denied either.
|
|
375
|
+
approvalEmissions.add(announcing);
|
|
376
|
+
void announcing.finally(() => {
|
|
377
|
+
approvalEmissions.delete(announcing);
|
|
378
|
+
markAnnounced();
|
|
379
|
+
});
|
|
380
|
+
});
|
|
381
|
+
const resolveApproval = (input) => {
|
|
382
|
+
const settle = pendingApprovals.get(input.requestId);
|
|
383
|
+
if (!settle) {
|
|
384
|
+
return false;
|
|
385
|
+
}
|
|
386
|
+
settle(input.answer, input.reason ?? 'decided', input.actor);
|
|
387
|
+
return true;
|
|
388
|
+
};
|
|
389
|
+
/**
|
|
390
|
+
* How a turn's watchdog stops its own clock, registered while it runs.
|
|
391
|
+
*
|
|
392
|
+
* Recording that a tool phase is open is not enough on its own: the
|
|
393
|
+
* timer is only re-evaluated when an event arrives, so a phase opened by
|
|
394
|
+
* something that emits nothing — an approval policy handed straight to
|
|
395
|
+
* the gateway — would leave an already-armed timer running to fire
|
|
396
|
+
* mid-wait.
|
|
397
|
+
*/
|
|
398
|
+
const suspendWatchdog = new Map();
|
|
399
|
+
/**
|
|
400
|
+
* Approval waits in progress, per session.
|
|
401
|
+
*
|
|
402
|
+
* Deliberately a balanced counter and not a set of call ids. An earlier
|
|
403
|
+
* shape had the approval wrapper *open* a phase that the watchdog's
|
|
404
|
+
* event observer was expected to close, and that handoff was wrong in
|
|
405
|
+
* every way it could be: the observer does not exist when no watchdog is
|
|
406
|
+
* armed, so nothing closed it; a policy that threw closed nothing; and a
|
|
407
|
+
* stale entry did not merely leak, it suppressed the watchdog for the
|
|
408
|
+
* next turn on that session. Four separate exits, each patched in turn.
|
|
409
|
+
*
|
|
410
|
+
* A counter incremented and decremented in one `finally` cannot have any
|
|
411
|
+
* of those bugs, because the thing that opens the phase is the thing
|
|
412
|
+
* that closes it, in the same scope, on every path out.
|
|
413
|
+
*/
|
|
414
|
+
const approvalWaits = new Map();
|
|
415
|
+
const holdApprovalWait = (sessionId) => {
|
|
416
|
+
approvalWaits.set(sessionId, (approvalWaits.get(sessionId) ?? 0) + 1);
|
|
417
|
+
// Stop any clock already running: this wait produces no events, so
|
|
418
|
+
// nothing else would.
|
|
419
|
+
suspendWatchdog.get(sessionId)?.();
|
|
420
|
+
let released = false;
|
|
421
|
+
return () => {
|
|
422
|
+
if (released) {
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
released = true;
|
|
426
|
+
const remaining = (approvalWaits.get(sessionId) ?? 1) - 1;
|
|
427
|
+
if (remaining > 0) {
|
|
428
|
+
approvalWaits.set(sessionId, remaining);
|
|
429
|
+
}
|
|
430
|
+
else {
|
|
431
|
+
approvalWaits.delete(sessionId);
|
|
432
|
+
}
|
|
433
|
+
};
|
|
434
|
+
};
|
|
435
|
+
const configuredApprovals = typeof options.approvals === 'function'
|
|
436
|
+
? options.approvals({ request: requestApproval })
|
|
437
|
+
: options.approvals;
|
|
438
|
+
/**
|
|
439
|
+
* Every approval opens a tool phase, whatever policy answers it.
|
|
440
|
+
*
|
|
441
|
+
* `tool.approval-requested` is the broker's event, so it exists only for
|
|
442
|
+
* the remote path; a policy handed straight to the gateway emits nothing
|
|
443
|
+
* at all, and `tool.called` comes only *after* the answer. Without this,
|
|
444
|
+
* a slow custom policy on a streaming provider that hosts its own loop
|
|
445
|
+
* would be indistinguishable from a stalled turn — the case this
|
|
446
|
+
* watchdog work exists to stop, arriving through a different door.
|
|
447
|
+
*/
|
|
448
|
+
const approvals = configuredApprovals
|
|
449
|
+
? {
|
|
450
|
+
async approve(context) {
|
|
451
|
+
const release = holdApprovalWait(context.session.id);
|
|
452
|
+
try {
|
|
453
|
+
return await configuredApprovals.approve(context);
|
|
454
|
+
}
|
|
455
|
+
finally {
|
|
456
|
+
// Every path out, including a policy that throws. What happens
|
|
457
|
+
// after the answer is the tool events' business, and they are
|
|
458
|
+
// observed by the watchdog itself.
|
|
459
|
+
release();
|
|
460
|
+
}
|
|
461
|
+
},
|
|
462
|
+
}
|
|
463
|
+
: undefined;
|
|
125
464
|
const registry = new AgentRegistry();
|
|
126
465
|
const sources = new Map();
|
|
127
466
|
// ---- roster -------------------------------------------------------------
|
|
@@ -190,15 +529,21 @@ export const createGateway = (options = {}) => {
|
|
|
190
529
|
// The built-in id is reserved BEFORE ordinary roster entries load: a
|
|
191
530
|
// roster file that happens to declare id "stratus" must not hijack
|
|
192
531
|
// the documented built-in fallback for agentId-less dispatches — the
|
|
193
|
-
//
|
|
532
|
+
// guard below skips it with a warning. Only the explicitly
|
|
194
533
|
// configured default soul may take the id over (defaultAgentId
|
|
195
534
|
// replaces a pathless source).
|
|
196
535
|
registerSource({ definition: { ...DEFAULT_STRATUS_AGENT } });
|
|
536
|
+
// Throws on two roster files claiming one id: that is a collision with
|
|
537
|
+
// no correct winner, and refusing to serve beats letting sort order
|
|
538
|
+
// decide whose sessions, memory, and credentials an agent inherits.
|
|
197
539
|
const entries = await loadRosterSouls(env, warn);
|
|
198
540
|
for (const entry of entries) {
|
|
199
541
|
if (sources.has(entry.soul.agent.id)) {
|
|
200
|
-
|
|
201
|
-
|
|
542
|
+
// Only reachable for the reserved built-in id now — roster-vs-
|
|
543
|
+
// roster duplicates never get this far. Reserved is not the same
|
|
544
|
+
// failure as duplicated: a soul may not take the fallback over,
|
|
545
|
+
// and must not be able to take the daemon down by trying.
|
|
546
|
+
warn(`agent id ${entry.soul.agent.id} is reserved for the built-in agent; ignoring ${entry.path}`);
|
|
202
547
|
continue;
|
|
203
548
|
}
|
|
204
549
|
registerSource({ definition: entry.soul.agent, soulPath: entry.path, soul: entry.soul });
|
|
@@ -298,7 +643,7 @@ export const createGateway = (options = {}) => {
|
|
|
298
643
|
provider,
|
|
299
644
|
tools,
|
|
300
645
|
executor: createLocalCommandExecutor(),
|
|
301
|
-
...(
|
|
646
|
+
...(approvals ? { approvals } : {}),
|
|
302
647
|
store,
|
|
303
648
|
bus,
|
|
304
649
|
agents: registry,
|
|
@@ -320,8 +665,16 @@ export const createGateway = (options = {}) => {
|
|
|
320
665
|
* that kills slow-but-healthy turns, so it stays off for them (their
|
|
321
666
|
* tool phases are covered by executor timeouts regardless).
|
|
322
667
|
*/
|
|
323
|
-
const fallbackStreamsDeltas = (fallback) =>
|
|
324
|
-
|
|
668
|
+
const fallbackStreamsDeltas = (fallback) =>
|
|
669
|
+
// The same two modes as a primary. A subscription fallback that is not
|
|
670
|
+
// recognised as streaming gets no watchdog at all once a session
|
|
671
|
+
// switches to it, so a stall there would run to the provider's own
|
|
672
|
+
// ten-minute timer instead of the idle timeout the operator set.
|
|
673
|
+
fallback.provider === 'anthropic' && Boolean(fallback.apiKey || fallback.authToken);
|
|
674
|
+
const streamsDeltas = (config) =>
|
|
675
|
+
// Both Anthropic modes stream now: an API key through the Messages
|
|
676
|
+
// API, a subscription token through the Agent SDK's partial messages.
|
|
677
|
+
config.provider === 'anthropic' && Boolean(config.apiKey || config.authToken);
|
|
325
678
|
/**
|
|
326
679
|
* Progress-based abort, armed only while the turn is actually awaiting a
|
|
327
680
|
* streaming provider — the one phase where silence means a stall. Tool
|
|
@@ -354,6 +707,18 @@ export const createGateway = (options = {}) => {
|
|
|
354
707
|
// the primary until a reset delta announces the sticky fallback switch.
|
|
355
708
|
let streamingActive = idleEnabled;
|
|
356
709
|
let pendingTools = 0;
|
|
710
|
+
/**
|
|
711
|
+
* Tool calls this turn has started and not finished.
|
|
712
|
+
*
|
|
713
|
+
* `pendingTools` above reads a count off `provider.response`, which
|
|
714
|
+
* only a provider that hands its calls back to the kernel loop emits.
|
|
715
|
+
* A provider hosting its own loop dispatches inside `generate`, so that
|
|
716
|
+
* count is zero for the whole of a hosted tool. These are the kernel's
|
|
717
|
+
* own tool events, which every path emits because every path runs its
|
|
718
|
+
* tools through the same executor — and they are turn-local, so an
|
|
719
|
+
* aborted turn takes them with it rather than leaving anything behind.
|
|
720
|
+
*/
|
|
721
|
+
const openToolCalls = new Set();
|
|
357
722
|
// Set at a text-only provider.response: the turn is wrapping up
|
|
358
723
|
// (saves, completion events); nothing left for the timer to guard.
|
|
359
724
|
let turnSettling = false;
|
|
@@ -391,9 +756,13 @@ export const createGateway = (options = {}) => {
|
|
|
391
756
|
}
|
|
392
757
|
switch (event.type) {
|
|
393
758
|
case 'provider.delta':
|
|
394
|
-
//
|
|
395
|
-
// the timer follows
|
|
396
|
-
|
|
759
|
+
// Only a *fallback* reset marks the mid-turn, session-sticky
|
|
760
|
+
// switch that the timer follows. A provider retrying its own
|
|
761
|
+
// attempt abandons its partial output too, but nothing about
|
|
762
|
+
// who is serving the turn changed — reading that as a switch
|
|
763
|
+
// would disarm the watchdog for the rest of a turn that is
|
|
764
|
+
// still running on the provider it started on.
|
|
765
|
+
if (event.delta.type === 'reset' && event.delta.reason === 'fallback') {
|
|
397
766
|
streamingActive = fallbackStreams;
|
|
398
767
|
}
|
|
399
768
|
break;
|
|
@@ -407,8 +776,23 @@ export const createGateway = (options = {}) => {
|
|
|
407
776
|
turnSettling = true;
|
|
408
777
|
}
|
|
409
778
|
break;
|
|
779
|
+
case 'tool.approval-requested':
|
|
780
|
+
// Redundant with the approval wrapper for a gateway-owned
|
|
781
|
+
// transport, and not redundant for a recovered turn, whose
|
|
782
|
+
// request is re-announced without going through a policy again.
|
|
783
|
+
openToolCalls.add(event.call.id);
|
|
784
|
+
break;
|
|
785
|
+
case 'tool.called':
|
|
786
|
+
openToolCalls.add(event.call.id);
|
|
787
|
+
break;
|
|
410
788
|
case 'tool.completed':
|
|
789
|
+
// Completion names the call through its result; denial carries
|
|
790
|
+
// the call itself. Both settle it.
|
|
791
|
+
openToolCalls.delete(event.result.callId);
|
|
792
|
+
pendingTools = Math.max(0, pendingTools - 1);
|
|
793
|
+
break;
|
|
411
794
|
case 'tool.denied':
|
|
795
|
+
openToolCalls.delete(event.call.id);
|
|
412
796
|
pendingTools = Math.max(0, pendingTools - 1);
|
|
413
797
|
break;
|
|
414
798
|
default:
|
|
@@ -420,10 +804,15 @@ export const createGateway = (options = {}) => {
|
|
|
420
804
|
if (!('sessionId' in event) || event.sessionId !== sessionId) {
|
|
421
805
|
return;
|
|
422
806
|
}
|
|
423
|
-
if (streamingActive
|
|
807
|
+
if (streamingActive
|
|
808
|
+
&& pendingTools === 0
|
|
809
|
+
&& openToolCalls.size === 0
|
|
810
|
+
&& (approvalWaits.get(sessionId) ?? 0) === 0
|
|
811
|
+
&& !turnSettling) {
|
|
424
812
|
armTimer();
|
|
425
813
|
}
|
|
426
814
|
});
|
|
815
|
+
suspendWatchdog.set(sessionId, suspendTimer);
|
|
427
816
|
if (streamingActive) {
|
|
428
817
|
armTimer();
|
|
429
818
|
}
|
|
@@ -440,6 +829,7 @@ export const createGateway = (options = {}) => {
|
|
|
440
829
|
suspendTimer();
|
|
441
830
|
unsubscribeObserver();
|
|
442
831
|
unsubscribeDrain();
|
|
832
|
+
suspendWatchdog.delete(sessionId);
|
|
443
833
|
external?.removeEventListener('abort', onExternalAbort);
|
|
444
834
|
}
|
|
445
835
|
};
|
|
@@ -450,23 +840,16 @@ export const createGateway = (options = {}) => {
|
|
|
450
840
|
const sessionChains = new Map();
|
|
451
841
|
const inflight = new Set();
|
|
452
842
|
let stopping = false;
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
// configured default soul has changed since — only a brand-new session
|
|
464
|
-
// takes the current default. (Loaded here, before the watchdog, also
|
|
465
|
-
// because whether this turn streams depends on durable session state.)
|
|
466
|
-
const existing = await store.get(input.sessionId);
|
|
467
|
-
const agentId = input.agentId ?? existing?.agent.id ?? await defaultAgentId();
|
|
468
|
-
const source = await refreshAgent(agentId);
|
|
469
|
-
const agent = source.definition;
|
|
843
|
+
/**
|
|
844
|
+
* The runtime one agent would run on right now — config snapshot, soul
|
|
845
|
+
* pins, degradation and all.
|
|
846
|
+
*
|
|
847
|
+
* Extracted so recovery resolves it exactly as a dispatch does. A second
|
|
848
|
+
* copy of this chain is the difference between a recovered turn finishing
|
|
849
|
+
* on its agent's provider and finishing on the daemon's default, and it
|
|
850
|
+
* would drift the first time either side gained a rule.
|
|
851
|
+
*/
|
|
852
|
+
const runtimeForAgent = async (source) => {
|
|
470
853
|
// Config re-resolves per dispatch, so a changed model, credential, or
|
|
471
854
|
// soul pin applies without a restart. The pool dedupes runners by the
|
|
472
855
|
// resolved configuration. The gateway-wide selection and the daemon's
|
|
@@ -509,55 +892,12 @@ export const createGateway = (options = {}) => {
|
|
|
509
892
|
// to resolveRuntimeConfig instead would read the file a second
|
|
510
893
|
// time, racing a concurrent replacement — this turn could keep one
|
|
511
894
|
// agent's definition while adopting another's provider pins.
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
// resolver treats it that way), so a real file without the key
|
|
519
|
-
// still names openai as the default.
|
|
520
|
-
// Env values normalize exactly as the resolver normalizes them:
|
|
521
|
-
// an empty or whitespace-padded STRATUS_PROVIDER is no default at
|
|
522
|
-
// all, not a mismatching one.
|
|
523
|
-
const defaultProvider = options.selection?.provider
|
|
524
|
-
?? readNonEmptyString(processEnv.STRATUS_PROVIDER)
|
|
525
|
-
?? readNonEmptyString(processEnv.STRATUSCLAW_PROVIDER)
|
|
526
|
-
?? configSnapshot.config.provider
|
|
527
|
-
?? (configSnapshot.path !== undefined ? 'openai' : undefined);
|
|
528
|
-
if (pins.provider) {
|
|
529
|
-
delete selection.provider;
|
|
530
|
-
delete processEnv.STRATUS_PROVIDER;
|
|
531
|
-
delete processEnv.STRATUSCLAW_PROVIDER;
|
|
532
|
-
if (defaultProvider !== undefined && defaultProvider !== 'demo' && pins.provider !== defaultProvider) {
|
|
533
|
-
// The default model, endpoint, and generic credentials were all
|
|
534
|
-
// chosen for the default provider — none may ride along to the
|
|
535
|
-
// soul's: a base URL would point the pinned provider at the
|
|
536
|
-
// wrong service, and a generic API key would be sent to it.
|
|
537
|
-
// With NO default selected anywhere — or the credential-less
|
|
538
|
-
// demo provider as the default — there is nothing those
|
|
539
|
-
// values could have been chosen for except whatever provider
|
|
540
|
-
// the soul selects — exactly the resolver's own reading of a
|
|
541
|
-
// generic credential — so they stay.
|
|
542
|
-
delete selection.model;
|
|
543
|
-
delete processEnv.STRATUS_MODEL;
|
|
544
|
-
delete processEnv.STRATUSCLAW_MODEL;
|
|
545
|
-
delete selection.baseUrl;
|
|
546
|
-
delete processEnv.STRATUS_BASE_URL;
|
|
547
|
-
delete processEnv.STRATUSCLAW_BASE_URL;
|
|
548
|
-
delete processEnv.STRATUS_API_KEY;
|
|
549
|
-
delete processEnv.STRATUSCLAW_API_KEY;
|
|
550
|
-
delete processEnv.STRATUS_API_KEY_ENV;
|
|
551
|
-
delete processEnv.STRATUSCLAW_API_KEY_ENV;
|
|
552
|
-
}
|
|
553
|
-
}
|
|
554
|
-
if (pins.model) {
|
|
555
|
-
delete selection.model;
|
|
556
|
-
delete processEnv.STRATUS_MODEL;
|
|
557
|
-
delete processEnv.STRATUSCLAW_MODEL;
|
|
558
|
-
}
|
|
559
|
-
resolveEnv = { ...env, processEnv };
|
|
560
|
-
}
|
|
895
|
+
const normalized = applySoulPins(pins, selection, env, {
|
|
896
|
+
...(options.selection?.provider !== undefined ? { selectionProvider: options.selection.provider } : {}),
|
|
897
|
+
...(configSnapshot.config.provider !== undefined ? { configProvider: configSnapshot.config.provider } : {}),
|
|
898
|
+
configPresent: configSnapshot.path !== undefined,
|
|
899
|
+
});
|
|
900
|
+
resolveEnv = normalized.env;
|
|
561
901
|
}
|
|
562
902
|
else {
|
|
563
903
|
// A soul-less agent (the built-in default) resolves with no soul at
|
|
@@ -568,7 +908,26 @@ export const createGateway = (options = {}) => {
|
|
|
568
908
|
}
|
|
569
909
|
// The preset snapshot means resolution reads no config file — every
|
|
570
910
|
// failure from here is a credential or provider problem and surfaces.
|
|
571
|
-
|
|
911
|
+
return resolveRuntimeConfig(selection, resolveEnv);
|
|
912
|
+
};
|
|
913
|
+
const dispatchInternal = async (input) => {
|
|
914
|
+
// A dispatch whose signal fired while it queued behind another turn
|
|
915
|
+
// must not touch durable state: without this check, the runner would
|
|
916
|
+
// load the session, append the cancelled user message, and save it as
|
|
917
|
+
// failed — polluting future model history with input never processed.
|
|
918
|
+
if (input.signal?.aborted) {
|
|
919
|
+
throw new RunAbortedError();
|
|
920
|
+
}
|
|
921
|
+
// A session pins its agent: when the caller names none, an existing
|
|
922
|
+
// conversation keeps the agent it was created with even if the
|
|
923
|
+
// configured default soul has changed since — only a brand-new session
|
|
924
|
+
// takes the current default. (Loaded here, before the watchdog, also
|
|
925
|
+
// because whether this turn streams depends on durable session state.)
|
|
926
|
+
const existing = await store.get(input.sessionId);
|
|
927
|
+
const agentId = input.agentId ?? existing?.agent.id ?? await defaultAgentId();
|
|
928
|
+
const source = await refreshAgent(agentId);
|
|
929
|
+
const agent = source.definition;
|
|
930
|
+
const config = await runtimeForAgent(source);
|
|
572
931
|
const runner = runnerFor(config);
|
|
573
932
|
const metadata = {
|
|
574
933
|
...(config.provider === 'demo'
|
|
@@ -610,23 +969,193 @@ export const createGateway = (options = {}) => {
|
|
|
610
969
|
});
|
|
611
970
|
});
|
|
612
971
|
};
|
|
613
|
-
|
|
972
|
+
/**
|
|
973
|
+
* Finishes the turns that were parked on a human when this daemon last
|
|
974
|
+
* went down.
|
|
975
|
+
*
|
|
976
|
+
* Run after the channels are up, deliberately: recovery re-asks, and a
|
|
977
|
+
* request emitted into a gateway with no live adapter would be declined
|
|
978
|
+
* for want of anywhere to put it — turning "your approval survived the
|
|
979
|
+
* restart" into "your approval was denied by the restart".
|
|
980
|
+
*
|
|
981
|
+
* Each session is recovered independently and failures are per-session:
|
|
982
|
+
* one unrecoverable turn must not stop the daemon from serving, or from
|
|
983
|
+
* recovering the others.
|
|
984
|
+
*/
|
|
985
|
+
const recoverParkedTurns = async () => {
|
|
986
|
+
if (!store.listIdsByStatus) {
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
const parked = await store.listIdsByStatus('pending_approval');
|
|
990
|
+
if (parked.length === 0) {
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
993
|
+
log(`recovering ${parked.length} turn(s) parked on approval`);
|
|
994
|
+
// Started together, not one after another. Each recovery re-asks and
|
|
995
|
+
// then waits on a human, so awaiting them in sequence would mean the
|
|
996
|
+
// second parked turn is not even *announced* until the first is
|
|
997
|
+
// answered — fifteen minutes later by default, and never at all with a
|
|
998
|
+
// zero timeout. They are independent turns; only same-session work is
|
|
999
|
+
// ordered, and onSessionChain already guarantees that.
|
|
1000
|
+
await Promise.allSettled(parked.map((sessionId) => recoverOne(sessionId)));
|
|
1001
|
+
};
|
|
1002
|
+
const recoverOne = (sessionId) => onSessionChain(sessionId, async () => {
|
|
614
1003
|
if (stopping) {
|
|
615
|
-
|
|
1004
|
+
return;
|
|
1005
|
+
}
|
|
1006
|
+
try {
|
|
1007
|
+
const session = await store.get(sessionId);
|
|
1008
|
+
if (!session) {
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
const record = readPendingApproval(session);
|
|
1012
|
+
// A wait that outlived its window while the daemon was down is
|
|
1013
|
+
// honoured, not restarted: the request really did go unanswered for
|
|
1014
|
+
// the whole time it was configured to wait, and downtime is not a
|
|
1015
|
+
// reason to extend a security decision. Measured from when the turn
|
|
1016
|
+
// parked against this daemon's timeout — the transport's original
|
|
1017
|
+
// deadline was chosen after the checkpoint was written and is gone
|
|
1018
|
+
// with the process that chose it. Denying goes through the same
|
|
1019
|
+
// recovery path, so the queue behind it still drains.
|
|
1020
|
+
const parkedAt = record ? Date.parse(record.parkedAt) : Number.NaN;
|
|
1021
|
+
const expired = effectiveApprovalTimeoutMs > 0
|
|
1022
|
+
&& Number.isFinite(parkedAt)
|
|
1023
|
+
&& Date.now() - parkedAt >= effectiveApprovalTimeoutMs;
|
|
1024
|
+
if (expired) {
|
|
1025
|
+
log(`${sessionId}: the approval for ${record?.call.toolName} outlived its window while the daemon was down; denying it`);
|
|
1026
|
+
}
|
|
1027
|
+
// Resolved the way a dispatch for this agent would resolve it, so a
|
|
1028
|
+
// recovered turn finishes on the same provider, model, and
|
|
1029
|
+
// credentials it was parked on.
|
|
1030
|
+
const source = await refreshAgent(session.agent.id);
|
|
1031
|
+
// And on the CURRENT definition, saved before recovery reads the
|
|
1032
|
+
// session back. An allowlist is a permission boundary: a soul that
|
|
1033
|
+
// dropped a tool while the daemon was down must not have that tool
|
|
1034
|
+
// executed by the turn that outlived the change, and
|
|
1035
|
+
// `allowedToolsFor` reads the definition frozen into the session
|
|
1036
|
+
// ahead of the registry. Dispatch refreshes it the same way before
|
|
1037
|
+
// resuming.
|
|
1038
|
+
session.agent = source.definition;
|
|
1039
|
+
await store.save(session);
|
|
1040
|
+
const runner = runnerFor(await runtimeForAgent(source));
|
|
1041
|
+
await runner.recoverPendingApproval(sessionId, { denyPending: expired });
|
|
1042
|
+
}
|
|
1043
|
+
catch (error) {
|
|
1044
|
+
warn(`could not recover parked session ${sessionId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1045
|
+
}
|
|
1046
|
+
});
|
|
1047
|
+
/**
|
|
1048
|
+
* Turns the last process was still running when it stopped being able to
|
|
1049
|
+
* finish them.
|
|
1050
|
+
*
|
|
1051
|
+
* `pending_approval` is the one window a turn can be picked up from, and
|
|
1052
|
+
* it has its own sweep above. Everything else a turn does is saved as
|
|
1053
|
+
* `running`: the provider call, the executor, and — on a provider that
|
|
1054
|
+
* hosts its own loop — the approval wait too, since a hosted call is
|
|
1055
|
+
* deliberately not checkpointed (`executeHostedToolCall` passes
|
|
1056
|
+
* `recoverable: false`, because the SDK's inner loop is not something a
|
|
1057
|
+
* restart can rebuild). None of those can be resumed, and nothing else
|
|
1058
|
+
* sweeps them, so the record goes on claiming the turn is running for as
|
|
1059
|
+
* long as the session exists.
|
|
1060
|
+
*
|
|
1061
|
+
* Note what this is *not* for. A graceful stop denies everything parked
|
|
1062
|
+
* and drains the turns those denials release, so an operator restarting
|
|
1063
|
+
* the daemon normally has nothing here. These are the turns a crash, a
|
|
1064
|
+
* SIGKILL, or a lost machine left behind.
|
|
1065
|
+
*/
|
|
1066
|
+
const listAbandonedTurns = async () => {
|
|
1067
|
+
if (!store.listIdsByStatus) {
|
|
1068
|
+
return [];
|
|
1069
|
+
}
|
|
1070
|
+
return store.listIdsByStatus('running');
|
|
1071
|
+
};
|
|
1072
|
+
/**
|
|
1073
|
+
* Marks each of them failed, with a reason that says what happened.
|
|
1074
|
+
*
|
|
1075
|
+
* Read BEFORE the channels start and applied after, which is the only
|
|
1076
|
+
* ordering that is both honest and safe. Reading first means the list is
|
|
1077
|
+
* exactly what the last process left, with no chance of a turn this
|
|
1078
|
+
* process started being in it. Applying after means the failure is
|
|
1079
|
+
* emitted into a gateway whose surfaces are listening, rather than into
|
|
1080
|
+
* one where nothing is.
|
|
1081
|
+
*
|
|
1082
|
+
* The gap between the two is real, though: an inbound message for one of
|
|
1083
|
+
* these sessions can arrive as soon as the channels are up. What makes
|
|
1084
|
+
* that safe is single-flight — every writer of a session goes through
|
|
1085
|
+
* `onSessionChain`, so this runs either wholly before that turn (it
|
|
1086
|
+
* fails, and `resume()` then clears `lastError` and starts fresh) or
|
|
1087
|
+
* wholly after it (the status has moved off `running`, and the re-read
|
|
1088
|
+
* below leaves it alone). There is no interleaving in which this
|
|
1089
|
+
* overwrites a live turn.
|
|
1090
|
+
*
|
|
1091
|
+
* What that invariant does NOT cover is a second daemon on the same
|
|
1092
|
+
* database, whose turns are running with no chain of ours in front of
|
|
1093
|
+
* them. Two processes sharing a session store is already unsupported —
|
|
1094
|
+
* the parked sweep above would re-ask that daemon's approvals for the
|
|
1095
|
+
* same reason — and no check here would make it supported.
|
|
1096
|
+
*/
|
|
1097
|
+
const failAbandonedTurns = async (abandoned) => {
|
|
1098
|
+
if (abandoned.length === 0) {
|
|
1099
|
+
return;
|
|
1100
|
+
}
|
|
1101
|
+
log(`failing ${abandoned.length} turn(s) left running by the last stratusd`);
|
|
1102
|
+
await Promise.allSettled(abandoned.map((id) => failAbandonedTurn(id)));
|
|
1103
|
+
};
|
|
1104
|
+
const failAbandonedTurn = (id) => onSessionChain(id, async () => {
|
|
1105
|
+
if (stopping) {
|
|
1106
|
+
return;
|
|
1107
|
+
}
|
|
1108
|
+
try {
|
|
1109
|
+
const session = await store.get(id);
|
|
1110
|
+
// Re-read on the chain, because the snapshot is from before the
|
|
1111
|
+
// channels came up: a message that arrived since has already taken
|
|
1112
|
+
// this session somewhere else, and that turn owns it now.
|
|
1113
|
+
if (!session || session.status !== 'running') {
|
|
1114
|
+
return;
|
|
1115
|
+
}
|
|
1116
|
+
session.status = 'failed';
|
|
1117
|
+
session.lastError = ABANDONED_TURN_ERROR;
|
|
1118
|
+
await store.save(session);
|
|
1119
|
+
await bus.emit({ type: 'session.updated', sessionId: id, status: 'failed' });
|
|
1120
|
+
await bus.emit({ type: 'session.failed', sessionId: id, error: ABANDONED_TURN_ERROR });
|
|
616
1121
|
}
|
|
617
|
-
|
|
618
|
-
|
|
1122
|
+
catch (error) {
|
|
1123
|
+
warn(`could not fail abandoned turn ${id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1124
|
+
}
|
|
1125
|
+
});
|
|
1126
|
+
/**
|
|
1127
|
+
* Runs `work` as the session's next turn: queued behind whatever that
|
|
1128
|
+
* session is already doing, and tracked so shutdown drains it.
|
|
1129
|
+
*
|
|
1130
|
+
* Recovery goes through here as well as dispatch, and must. Channels are
|
|
1131
|
+
* live before a sweep finishes, so an inbound message for a parked
|
|
1132
|
+
* session could otherwise `resume()` alongside `recoverPendingApproval()`
|
|
1133
|
+
* — one closing the parked call as interrupted while the other re-enters
|
|
1134
|
+
* it, both saving divergent copies of the same transcript, and the tool
|
|
1135
|
+
* possibly running twice. Single-flight per session is the invariant that
|
|
1136
|
+
* stops that, and it only holds if everything that writes a session goes
|
|
1137
|
+
* through it.
|
|
1138
|
+
*/
|
|
1139
|
+
const onSessionChain = (sessionId, work) => {
|
|
1140
|
+
const previous = sessionChains.get(sessionId) ?? Promise.resolve();
|
|
1141
|
+
const turn = previous.then(work, work);
|
|
619
1142
|
const settled = turn.catch(() => { });
|
|
620
|
-
sessionChains.set(
|
|
1143
|
+
sessionChains.set(sessionId, settled);
|
|
621
1144
|
inflight.add(settled);
|
|
622
1145
|
void settled.finally(() => {
|
|
623
1146
|
inflight.delete(settled);
|
|
624
|
-
if (sessionChains.get(
|
|
625
|
-
sessionChains.delete(
|
|
1147
|
+
if (sessionChains.get(sessionId) === settled) {
|
|
1148
|
+
sessionChains.delete(sessionId);
|
|
626
1149
|
}
|
|
627
1150
|
});
|
|
628
1151
|
return turn;
|
|
629
1152
|
};
|
|
1153
|
+
const dispatch = async (input) => {
|
|
1154
|
+
if (stopping) {
|
|
1155
|
+
throw new Error('The gateway is stopping and no longer accepts new work.');
|
|
1156
|
+
}
|
|
1157
|
+
return onSessionChain(input.sessionId, () => dispatchInternal(input));
|
|
1158
|
+
};
|
|
630
1159
|
const startedChannels = [];
|
|
631
1160
|
const gateway = {
|
|
632
1161
|
bus,
|
|
@@ -636,6 +1165,11 @@ export const createGateway = (options = {}) => {
|
|
|
636
1165
|
await loadRoster();
|
|
637
1166
|
const named = registry.list().map((agent) => agent.name).join(', ');
|
|
638
1167
|
log(`stratusd ready — ${registry.list().length} agent(s): ${named}`);
|
|
1168
|
+
// Read before anything can dispatch, so this is exactly what the
|
|
1169
|
+
// last process left running and not a turn of ours caught in flight.
|
|
1170
|
+
// Applied further down, once there is somewhere for the failure to
|
|
1171
|
+
// be heard.
|
|
1172
|
+
const abandoned = await listAbandonedTurns();
|
|
639
1173
|
// Channels come up after the roster so their first inbound message
|
|
640
1174
|
// already has agents to dispatch to. One failing adapter must not
|
|
641
1175
|
// keep the rest (or the gateway) down.
|
|
@@ -657,6 +1191,19 @@ export const createGateway = (options = {}) => {
|
|
|
657
1191
|
}
|
|
658
1192
|
}
|
|
659
1193
|
}
|
|
1194
|
+
// Last, and not awaited: recovery re-asks, so it needs the channels
|
|
1195
|
+
// above already listening — but a turn parked behind a slow approver
|
|
1196
|
+
// must not hold up start(), or a daemon with one outstanding request
|
|
1197
|
+
// would refuse to finish booting.
|
|
1198
|
+
void recoverParkedTurns().catch((error) => {
|
|
1199
|
+
warn(`recovering parked turns failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1200
|
+
});
|
|
1201
|
+
// Not awaited for the same reason, and independent of the sweep
|
|
1202
|
+
// above: these sessions are not parked, so the two never touch the
|
|
1203
|
+
// same one.
|
|
1204
|
+
void failAbandonedTurns(abandoned).catch((error) => {
|
|
1205
|
+
warn(`failing abandoned turns failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1206
|
+
});
|
|
660
1207
|
},
|
|
661
1208
|
// Drain: channels stop taking messages, in-flight turns finish, new
|
|
662
1209
|
// dispatches are refused, then the database closes. SIGTERM handling
|
|
@@ -666,6 +1213,17 @@ export const createGateway = (options = {}) => {
|
|
|
666
1213
|
// in-flight turns (already-started dispatches keep running), drain,
|
|
667
1214
|
// and close.
|
|
668
1215
|
stopping = true;
|
|
1216
|
+
// Deny what is parked before draining, or the drain waits out every
|
|
1217
|
+
// outstanding approval timeout — a shutdown would hang for as long as
|
|
1218
|
+
// the longest request had left. Each denial is a real decision the
|
|
1219
|
+
// turn continues from, so the drain below still finishes those turns.
|
|
1220
|
+
for (const settle of [...pendingApprovals.values()]) {
|
|
1221
|
+
settle('deny', 'cancelled');
|
|
1222
|
+
}
|
|
1223
|
+
// Let those resolutions reach their subscribers before the channels
|
|
1224
|
+
// go down, or a channel that renders approvals never learns to
|
|
1225
|
+
// retract the buttons it is still showing.
|
|
1226
|
+
await Promise.allSettled([...approvalEmissions]);
|
|
669
1227
|
await Promise.allSettled(startedChannels.map((adapter) => adapter.stop()));
|
|
670
1228
|
startedChannels.length = 0;
|
|
671
1229
|
await Promise.allSettled([...inflight]);
|
|
@@ -676,6 +1234,7 @@ export const createGateway = (options = {}) => {
|
|
|
676
1234
|
agents() {
|
|
677
1235
|
return registry.list();
|
|
678
1236
|
},
|
|
1237
|
+
resolveApproval,
|
|
679
1238
|
};
|
|
680
1239
|
return gateway;
|
|
681
1240
|
};
|