@stratusagent/gateway 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,13 +1,25 @@
1
- import { createHash } from 'node:crypto';
1
+ import { createHash, randomUUID } from 'node:crypto';
2
2
  import { chmodSync, mkdirSync } from 'node:fs';
3
3
  import path from 'node:path';
4
4
  import { DatabaseSync } from 'node:sqlite';
5
- import { AgentRegistry, AgentRunner, EventBus, RunAbortedError, ToolRegistry, } from '@stratusagent/core';
5
+ import { AgentRegistry, AgentRunner, EventBus, RunAbortedError, ToolRegistry, readPendingApproval, } from '@stratusagent/core';
6
6
  import { createDelegateTool, createRememberTool, } from '@stratusagent/agents';
7
7
  import { createLocalCommandExecutor } from '@stratusagent/executor-local';
8
8
  import { createDemoTool, createFileMemoryStore, createRuntimeProvider, DEFAULT_STRATUS_AGENT, loadRosterSouls, FALLBACK_ACTIVE_METADATA_KEY, loadSoulFile, memoryFilePath, migrateLegacyMemory, ConfigFileError, loadConfigFile, readNonEmptyString, resolveConfigLocation, resolveConfiguredSoul, resolveRuntimeConfig, stratusHomePath, withLegacyDefaultMemories, } from '@stratusagent/state';
9
9
  const SESSIONS_DB_FILENAME = 'sessions.db';
10
10
  const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
11
+ /**
12
+ * How long a parked call waits for a person. Long enough to survive a
13
+ * meeting, short enough that a request nobody saw does not hold a turn (and
14
+ * the Slack thread it is answering) open indefinitely.
15
+ */
16
+ const DEFAULT_APPROVAL_TIMEOUT_MS = 900_000;
17
+ /**
18
+ * Node's largest `setTimeout` delay. Anything above it does not become a
19
+ * long wait — it silently becomes a 1ms one, so a config asking for a
20
+ * 30-day approval window would expire every request almost immediately.
21
+ */
22
+ const MAX_TIMER_DELAY_MS = 2_147_483_647;
11
23
  export class SqliteSessionStore {
12
24
  db;
13
25
  constructor(filePath, options = {}) {
@@ -84,6 +96,17 @@ export class SqliteSessionStore {
84
96
  .prepare('INSERT OR REPLACE INTO sessions (id, agent_id, status, body, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)')
85
97
  .run(updated.id, updated.agent.id, updated.status, JSON.stringify(updated), updated.createdAt, updated.updatedAt);
86
98
  }
99
+ /**
100
+ * Session ids in a state, oldest first — the index a restarting daemon
101
+ * sweeps for turns parked on a human. The `status` column carries it, so
102
+ * no conversation body is deserialized to answer the question.
103
+ */
104
+ async listIdsByStatus(status) {
105
+ const rows = this.db
106
+ .prepare('SELECT id FROM sessions WHERE status = ? ORDER BY updated_at ASC')
107
+ .all(status);
108
+ return rows.map((row) => row.id);
109
+ }
87
110
  /** Newest-first session listing for one agent (or all agents). */
88
111
  list(agentId) {
89
112
  const rows = (agentId
@@ -160,6 +183,16 @@ export const applySoulPins = (pins, selection, env, context) => {
160
183
  }
161
184
  return { selection, env: { ...env, processEnv } };
162
185
  };
186
+ /**
187
+ * Recorded on a turn the previous process was still running when it died.
188
+ *
189
+ * Exported because "distinguishable" is the whole point: a surface, a
190
+ * test, or an operator reading `lastError` has to be able to tell a turn
191
+ * nobody finished apart from one a provider or a tool actually failed, and
192
+ * matching on prose that lives in one place beats each caller inventing
193
+ * its own guess at the wording.
194
+ */
195
+ export const ABANDONED_TURN_ERROR = 'stratusd stopped while this turn was still running; it was not resumed. Send the message again.';
163
196
  /**
164
197
  * The always-on Stratus process. One gateway holds the roster, the durable
165
198
  * session store, the shared event bus, and a pool of runners keyed by
@@ -181,6 +214,253 @@ export const createGateway = (options = {}) => {
181
214
  ? new SqliteSessionStore(options.sessionDbPath)
182
215
  : new SqliteSessionStore(path.join(stratusHomePath(env), SESSIONS_DB_FILENAME), { ownedDirectory: true });
183
216
  const memory = withLegacyDefaultMemories(createFileMemoryStore(memoryFilePath(env)));
217
+ // ---- approval brokering -------------------------------------------------
218
+ const approvalTimeoutMs = options.approvalTimeoutMs ?? DEFAULT_APPROVAL_TIMEOUT_MS;
219
+ // Refused at construction, not absorbed: a negative or NaN timeout fails
220
+ // the `> 0` test below and quietly means "never expire" — the one
221
+ // behavior documented for an explicit 0, and the last one a caller who
222
+ // typed a bad number wants. Config-file values are already rejected with
223
+ // a better message; this catches the programmatic path.
224
+ if (!Number.isFinite(approvalTimeoutMs) || approvalTimeoutMs < 0) {
225
+ throw new Error(`approvalTimeoutMs must be a non-negative number of milliseconds (0 to wait indefinitely); received ${String(options.approvalTimeoutMs)}.`);
226
+ }
227
+ // Clamped, not trusted: a delay past Node's timer range is not a long
228
+ // wait, it is a 1ms one, so an over-large value would turn every approval
229
+ // into an instant expiry — the opposite of what it asked for. The CLI
230
+ // rejects these at config load with a better message; this is the
231
+ // backstop for a programmatic caller.
232
+ const effectiveApprovalTimeoutMs = Math.min(approvalTimeoutMs, MAX_TIMER_DELAY_MS);
233
+ if (approvalTimeoutMs > MAX_TIMER_DELAY_MS) {
234
+ warn(`approval timeout ${approvalTimeoutMs}ms is above Node's maximum timer delay; using ${MAX_TIMER_DELAY_MS}ms (~24.8 days)`);
235
+ }
236
+ const pendingApprovals = new Map();
237
+ /**
238
+ * In-flight `tool.approval-resolved` emissions. `EventBus.emit` awaits its
239
+ * subscribers in order, so one async handler ahead of a channel adapter
240
+ * suspends the emission before the adapter ever sees it. Nothing today is
241
+ * async in front of one — which is exactly the problem: the shutdown
242
+ * guarantee below would rest on that staying true, and the first async
243
+ * subscriber anyone adds would silently leave live-looking buttons in a
244
+ * workspace the daemon has left.
245
+ */
246
+ const approvalEmissions = new Set();
247
+ /**
248
+ * Publishes one parked call and settles when somebody answers it, the
249
+ * request expires, or the turn is cancelled.
250
+ *
251
+ * All three endings run through the same `settle`, which removes the
252
+ * request from the registry BEFORE resolving. That ordering is the whole
253
+ * point: a request that is no longer pending cannot be resolved twice,
254
+ * so a click racing the timeout — or arriving after an abort — is refused
255
+ * rather than executing a tool for a turn that has already moved on.
256
+ */
257
+ const requestApproval = (request) => new Promise((resolve) => {
258
+ // A shutdown denies what is parked once, at the top of stop(). Turns
259
+ // already running keep going through the drain, though, and one of them
260
+ // can reach a gated tool AFTER that snapshot — finishing a provider
261
+ // call, or moving to the next call in the same response. Parking it
262
+ // would deadlock the drain against a question nobody is left to answer:
263
+ // stop() waits for the turn, the turn waits for the approval, and the
264
+ // approval waits out its timeout — forever when there is none.
265
+ if (stopping) {
266
+ resolve('deny');
267
+ void bus.emit({
268
+ type: 'tool.approval-resolved',
269
+ sessionId: request.session.id,
270
+ requestId: randomUUID(),
271
+ answer: 'deny',
272
+ reason: 'cancelled',
273
+ });
274
+ return;
275
+ }
276
+ const requestId = randomUUID();
277
+ let timer;
278
+ const onAbort = () => settle('deny', 'cancelled');
279
+ /**
280
+ * Settles once the request has finished being announced.
281
+ *
282
+ * Both emissions go through the same subscriber list, in order, and
283
+ * `EventBus.emit` awaits each one — so an async subscriber ahead of a
284
+ * channel can hold the announcement while a timeout or abort fires
285
+ * behind it, and the channel would hear that a request it has never
286
+ * heard of was resolved. It drops that, then renders the announcement
287
+ * when it finally arrives, leaving buttons for a settled request that
288
+ * nothing will ever retract.
289
+ *
290
+ * Chaining is the fix rather than remembering orphaned resolutions in
291
+ * the channel: a resolution that arrives before its request is not
292
+ * something a consumer can act on, so it should not be able to happen.
293
+ * Starts resolved, because a turn already aborted never announces at
294
+ * all and its denial should not wait on an emission that never runs.
295
+ */
296
+ let announced = Promise.resolve();
297
+ const settle = (answer, reason, actor) => {
298
+ // Identity, not presence: a request that already settled is gone from
299
+ // the registry, and comparing against this exact function is what
300
+ // makes every ending — click, timeout, abort, shutdown — idempotent.
301
+ if (pendingApprovals.get(requestId) !== settle) {
302
+ return;
303
+ }
304
+ pendingApprovals.delete(requestId);
305
+ if (timer) {
306
+ clearTimeout(timer);
307
+ }
308
+ request.signal?.removeEventListener('abort', onAbort);
309
+ resolve(answer);
310
+ const emitted = announced.then(() => bus.emit({
311
+ type: 'tool.approval-resolved',
312
+ sessionId: request.session.id,
313
+ requestId,
314
+ answer,
315
+ reason,
316
+ ...(actor ? { actor } : {}),
317
+ }));
318
+ approvalEmissions.add(emitted);
319
+ void emitted.finally(() => approvalEmissions.delete(emitted));
320
+ };
321
+ pendingApprovals.set(requestId, settle);
322
+ // What is left of the window, not a fresh one: a re-asked request
323
+ // carries when it originally parked, and the elapsed time counts.
324
+ const alreadyWaitedMs = request.parkedAt ? Date.now() - Date.parse(request.parkedAt) : 0;
325
+ const remainingTimeoutMs = effectiveApprovalTimeoutMs > 0
326
+ ? effectiveApprovalTimeoutMs - (Number.isFinite(alreadyWaitedMs) ? Math.max(0, alreadyWaitedMs) : 0)
327
+ : 0;
328
+ if (effectiveApprovalTimeoutMs > 0 && remainingTimeoutMs <= 0) {
329
+ // Nothing left to wait with. Settled here rather than armed with a
330
+ // zero timer, which `setTimeout` would still run a tick later — long
331
+ // enough to announce a request that is already over.
332
+ settle('deny', 'timeout');
333
+ return;
334
+ }
335
+ if (remainingTimeoutMs > 0) {
336
+ // Deliberately NOT unref'd. A parked turn is real outstanding work,
337
+ // and its timer is the only thing that will ever finish it — letting
338
+ // the process exit out from under one would abandon the turn instead
339
+ // of denying it. Shutdown is handled where it belongs, in stop(),
340
+ // which settles every outstanding request before draining.
341
+ timer = setTimeout(() => settle('deny', 'timeout'), remainingTimeoutMs);
342
+ }
343
+ // The abort listener is attached before the request is announced, so a
344
+ // turn cancelled while the event is still being delivered still ends
345
+ // the request rather than leaving it pending forever.
346
+ if (request.signal) {
347
+ if (request.signal.aborted) {
348
+ settle('deny', 'cancelled');
349
+ return;
350
+ }
351
+ request.signal.addEventListener('abort', onAbort, { once: true });
352
+ }
353
+ // `announced` is assigned BEFORE the emission starts, because emit runs
354
+ // its first subscribers synchronously — one of them settling the
355
+ // request inline would otherwise chain onto the already-resolved
356
+ // placeholder and race the very announcement it is answering.
357
+ let markAnnounced = () => { };
358
+ announced = new Promise((resolveAnnounced) => {
359
+ markAnnounced = resolveAnnounced;
360
+ });
361
+ const announcing = bus.emit({
362
+ type: 'tool.approval-requested',
363
+ sessionId: request.session.id,
364
+ agentId: request.session.agent.id,
365
+ requestId,
366
+ call: request.call,
367
+ risk: request.risk,
368
+ ...(request.session.metadata ? { metadata: request.session.metadata } : {}),
369
+ ...(remainingTimeoutMs > 0
370
+ ? { expiresAt: new Date(Date.now() + remainingTimeoutMs).toISOString() }
371
+ : {}),
372
+ });
373
+ // Drained at shutdown alongside the resolutions: a channel that has not
374
+ // yet been told a request exists cannot be told it was denied either.
375
+ approvalEmissions.add(announcing);
376
+ void announcing.finally(() => {
377
+ approvalEmissions.delete(announcing);
378
+ markAnnounced();
379
+ });
380
+ });
381
+ const resolveApproval = (input) => {
382
+ const settle = pendingApprovals.get(input.requestId);
383
+ if (!settle) {
384
+ return false;
385
+ }
386
+ settle(input.answer, input.reason ?? 'decided', input.actor);
387
+ return true;
388
+ };
389
+ /**
390
+ * How a turn's watchdog stops its own clock, registered while it runs.
391
+ *
392
+ * Recording that a tool phase is open is not enough on its own: the
393
+ * timer is only re-evaluated when an event arrives, so a phase opened by
394
+ * something that emits nothing — an approval policy handed straight to
395
+ * the gateway — would leave an already-armed timer running to fire
396
+ * mid-wait.
397
+ */
398
+ const suspendWatchdog = new Map();
399
+ /**
400
+ * Approval waits in progress, per session.
401
+ *
402
+ * Deliberately a balanced counter and not a set of call ids. An earlier
403
+ * shape had the approval wrapper *open* a phase that the watchdog's
404
+ * event observer was expected to close, and that handoff was wrong in
405
+ * every way it could be: the observer does not exist when no watchdog is
406
+ * armed, so nothing closed it; a policy that threw closed nothing; and a
407
+ * stale entry did not merely leak, it suppressed the watchdog for the
408
+ * next turn on that session. Four separate exits, each patched in turn.
409
+ *
410
+ * A counter incremented and decremented in one `finally` cannot have any
411
+ * of those bugs, because the thing that opens the phase is the thing
412
+ * that closes it, in the same scope, on every path out.
413
+ */
414
+ const approvalWaits = new Map();
415
+ const holdApprovalWait = (sessionId) => {
416
+ approvalWaits.set(sessionId, (approvalWaits.get(sessionId) ?? 0) + 1);
417
+ // Stop any clock already running: this wait produces no events, so
418
+ // nothing else would.
419
+ suspendWatchdog.get(sessionId)?.();
420
+ let released = false;
421
+ return () => {
422
+ if (released) {
423
+ return;
424
+ }
425
+ released = true;
426
+ const remaining = (approvalWaits.get(sessionId) ?? 1) - 1;
427
+ if (remaining > 0) {
428
+ approvalWaits.set(sessionId, remaining);
429
+ }
430
+ else {
431
+ approvalWaits.delete(sessionId);
432
+ }
433
+ };
434
+ };
435
+ const configuredApprovals = typeof options.approvals === 'function'
436
+ ? options.approvals({ request: requestApproval })
437
+ : options.approvals;
438
+ /**
439
+ * Every approval opens a tool phase, whatever policy answers it.
440
+ *
441
+ * `tool.approval-requested` is the broker's event, so it exists only for
442
+ * the remote path; a policy handed straight to the gateway emits nothing
443
+ * at all, and `tool.called` comes only *after* the answer. Without this,
444
+ * a slow custom policy on a streaming provider that hosts its own loop
445
+ * would be indistinguishable from a stalled turn — the case this
446
+ * watchdog work exists to stop, arriving through a different door.
447
+ */
448
+ const approvals = configuredApprovals
449
+ ? {
450
+ async approve(context) {
451
+ const release = holdApprovalWait(context.session.id);
452
+ try {
453
+ return await configuredApprovals.approve(context);
454
+ }
455
+ finally {
456
+ // Every path out, including a policy that throws. What happens
457
+ // after the answer is the tool events' business, and they are
458
+ // observed by the watchdog itself.
459
+ release();
460
+ }
461
+ },
462
+ }
463
+ : undefined;
184
464
  const registry = new AgentRegistry();
185
465
  const sources = new Map();
186
466
  // ---- roster -------------------------------------------------------------
@@ -249,15 +529,21 @@ export const createGateway = (options = {}) => {
249
529
  // The built-in id is reserved BEFORE ordinary roster entries load: a
250
530
  // roster file that happens to declare id "stratus" must not hijack
251
531
  // the documented built-in fallback for agentId-less dispatches — the
252
- // duplicate guard below skips it with a warning. Only the explicitly
532
+ // guard below skips it with a warning. Only the explicitly
253
533
  // configured default soul may take the id over (defaultAgentId
254
534
  // replaces a pathless source).
255
535
  registerSource({ definition: { ...DEFAULT_STRATUS_AGENT } });
536
+ // Throws on two roster files claiming one id: that is a collision with
537
+ // no correct winner, and refusing to serve beats letting sort order
538
+ // decide whose sessions, memory, and credentials an agent inherits.
256
539
  const entries = await loadRosterSouls(env, warn);
257
540
  for (const entry of entries) {
258
541
  if (sources.has(entry.soul.agent.id)) {
259
- const existing = sources.get(entry.soul.agent.id);
260
- warn(`duplicate agent id ${entry.soul.agent.id} (${existing?.soulPath ?? 'built-in'} vs ${entry.path}); keeping the first`);
542
+ // Only reachable for the reserved built-in id now — roster-vs-
543
+ // roster duplicates never get this far. Reserved is not the same
544
+ // failure as duplicated: a soul may not take the fallback over,
545
+ // and must not be able to take the daemon down by trying.
546
+ warn(`agent id ${entry.soul.agent.id} is reserved for the built-in agent; ignoring ${entry.path}`);
261
547
  continue;
262
548
  }
263
549
  registerSource({ definition: entry.soul.agent, soulPath: entry.path, soul: entry.soul });
@@ -357,7 +643,7 @@ export const createGateway = (options = {}) => {
357
643
  provider,
358
644
  tools,
359
645
  executor: createLocalCommandExecutor(),
360
- ...(options.approvals ? { approvals: options.approvals } : {}),
646
+ ...(approvals ? { approvals } : {}),
361
647
  store,
362
648
  bus,
363
649
  agents: registry,
@@ -379,8 +665,16 @@ export const createGateway = (options = {}) => {
379
665
  * that kills slow-but-healthy turns, so it stays off for them (their
380
666
  * tool phases are covered by executor timeouts regardless).
381
667
  */
382
- const fallbackStreamsDeltas = (fallback) => fallback.provider === 'anthropic' && Boolean(fallback.apiKey);
383
- const streamsDeltas = (config) => config.provider === 'anthropic' && Boolean(config.apiKey);
668
+ const fallbackStreamsDeltas = (fallback) =>
669
+ // The same two modes as a primary. A subscription fallback that is not
670
+ // recognised as streaming gets no watchdog at all once a session
671
+ // switches to it, so a stall there would run to the provider's own
672
+ // ten-minute timer instead of the idle timeout the operator set.
673
+ fallback.provider === 'anthropic' && Boolean(fallback.apiKey || fallback.authToken);
674
+ const streamsDeltas = (config) =>
675
+ // Both Anthropic modes stream now: an API key through the Messages
676
+ // API, a subscription token through the Agent SDK's partial messages.
677
+ config.provider === 'anthropic' && Boolean(config.apiKey || config.authToken);
384
678
  /**
385
679
  * Progress-based abort, armed only while the turn is actually awaiting a
386
680
  * streaming provider — the one phase where silence means a stall. Tool
@@ -413,6 +707,18 @@ export const createGateway = (options = {}) => {
413
707
  // the primary until a reset delta announces the sticky fallback switch.
414
708
  let streamingActive = idleEnabled;
415
709
  let pendingTools = 0;
710
+ /**
711
+ * Tool calls this turn has started and not finished.
712
+ *
713
+ * `pendingTools` above reads a count off `provider.response`, which
714
+ * only a provider that hands its calls back to the kernel loop emits.
715
+ * A provider hosting its own loop dispatches inside `generate`, so that
716
+ * count is zero for the whole of a hosted tool. These are the kernel's
717
+ * own tool events, which every path emits because every path runs its
718
+ * tools through the same executor — and they are turn-local, so an
719
+ * aborted turn takes them with it rather than leaving anything behind.
720
+ */
721
+ const openToolCalls = new Set();
416
722
  // Set at a text-only provider.response: the turn is wrapping up
417
723
  // (saves, completion events); nothing left for the timer to guard.
418
724
  let turnSettling = false;
@@ -450,9 +756,13 @@ export const createGateway = (options = {}) => {
450
756
  }
451
757
  switch (event.type) {
452
758
  case 'provider.delta':
453
- // A reset marks the mid-turn, session-sticky fallback switch:
454
- // the timer follows whether the now-active provider streams.
455
- if (event.delta.type === 'reset') {
759
+ // Only a *fallback* reset marks the mid-turn, session-sticky
760
+ // switch that the timer follows. A provider retrying its own
761
+ // attempt abandons its partial output too, but nothing about
762
+ // who is serving the turn changed — reading that as a switch
763
+ // would disarm the watchdog for the rest of a turn that is
764
+ // still running on the provider it started on.
765
+ if (event.delta.type === 'reset' && event.delta.reason === 'fallback') {
456
766
  streamingActive = fallbackStreams;
457
767
  }
458
768
  break;
@@ -466,8 +776,23 @@ export const createGateway = (options = {}) => {
466
776
  turnSettling = true;
467
777
  }
468
778
  break;
779
+ case 'tool.approval-requested':
780
+ // Redundant with the approval wrapper for a gateway-owned
781
+ // transport, and not redundant for a recovered turn, whose
782
+ // request is re-announced without going through a policy again.
783
+ openToolCalls.add(event.call.id);
784
+ break;
785
+ case 'tool.called':
786
+ openToolCalls.add(event.call.id);
787
+ break;
469
788
  case 'tool.completed':
789
+ // Completion names the call through its result; denial carries
790
+ // the call itself. Both settle it.
791
+ openToolCalls.delete(event.result.callId);
792
+ pendingTools = Math.max(0, pendingTools - 1);
793
+ break;
470
794
  case 'tool.denied':
795
+ openToolCalls.delete(event.call.id);
471
796
  pendingTools = Math.max(0, pendingTools - 1);
472
797
  break;
473
798
  default:
@@ -479,10 +804,15 @@ export const createGateway = (options = {}) => {
479
804
  if (!('sessionId' in event) || event.sessionId !== sessionId) {
480
805
  return;
481
806
  }
482
- if (streamingActive && pendingTools === 0 && !turnSettling) {
807
+ if (streamingActive
808
+ && pendingTools === 0
809
+ && openToolCalls.size === 0
810
+ && (approvalWaits.get(sessionId) ?? 0) === 0
811
+ && !turnSettling) {
483
812
  armTimer();
484
813
  }
485
814
  });
815
+ suspendWatchdog.set(sessionId, suspendTimer);
486
816
  if (streamingActive) {
487
817
  armTimer();
488
818
  }
@@ -499,6 +829,7 @@ export const createGateway = (options = {}) => {
499
829
  suspendTimer();
500
830
  unsubscribeObserver();
501
831
  unsubscribeDrain();
832
+ suspendWatchdog.delete(sessionId);
502
833
  external?.removeEventListener('abort', onExternalAbort);
503
834
  }
504
835
  };
@@ -509,23 +840,16 @@ export const createGateway = (options = {}) => {
509
840
  const sessionChains = new Map();
510
841
  const inflight = new Set();
511
842
  let stopping = false;
512
- const dispatchInternal = async (input) => {
513
- // A dispatch whose signal fired while it queued behind another turn
514
- // must not touch durable state: without this check, the runner would
515
- // load the session, append the cancelled user message, and save it as
516
- // failed — polluting future model history with input never processed.
517
- if (input.signal?.aborted) {
518
- throw new RunAbortedError();
519
- }
520
- // A session pins its agent: when the caller names none, an existing
521
- // conversation keeps the agent it was created with even if the
522
- // configured default soul has changed since — only a brand-new session
523
- // takes the current default. (Loaded here, before the watchdog, also
524
- // because whether this turn streams depends on durable session state.)
525
- const existing = await store.get(input.sessionId);
526
- const agentId = input.agentId ?? existing?.agent.id ?? await defaultAgentId();
527
- const source = await refreshAgent(agentId);
528
- const agent = source.definition;
843
+ /**
844
+ * The runtime one agent would run on right now — config snapshot, soul
845
+ * pins, degradation and all.
846
+ *
847
+ * Extracted so recovery resolves it exactly as a dispatch does. A second
848
+ * copy of this chain is the difference between a recovered turn finishing
849
+ * on its agent's provider and finishing on the daemon's default, and it
850
+ * would drift the first time either side gained a rule.
851
+ */
852
+ const runtimeForAgent = async (source) => {
529
853
  // Config re-resolves per dispatch, so a changed model, credential, or
530
854
  // soul pin applies without a restart. The pool dedupes runners by the
531
855
  // resolved configuration. The gateway-wide selection and the daemon's
@@ -584,7 +908,26 @@ export const createGateway = (options = {}) => {
584
908
  }
585
909
  // The preset snapshot means resolution reads no config file — every
586
910
  // failure from here is a credential or provider problem and surfaces.
587
- const config = await resolveRuntimeConfig(selection, resolveEnv);
911
+ return resolveRuntimeConfig(selection, resolveEnv);
912
+ };
913
+ const dispatchInternal = async (input) => {
914
+ // A dispatch whose signal fired while it queued behind another turn
915
+ // must not touch durable state: without this check, the runner would
916
+ // load the session, append the cancelled user message, and save it as
917
+ // failed — polluting future model history with input never processed.
918
+ if (input.signal?.aborted) {
919
+ throw new RunAbortedError();
920
+ }
921
+ // A session pins its agent: when the caller names none, an existing
922
+ // conversation keeps the agent it was created with even if the
923
+ // configured default soul has changed since — only a brand-new session
924
+ // takes the current default. (Loaded here, before the watchdog, also
925
+ // because whether this turn streams depends on durable session state.)
926
+ const existing = await store.get(input.sessionId);
927
+ const agentId = input.agentId ?? existing?.agent.id ?? await defaultAgentId();
928
+ const source = await refreshAgent(agentId);
929
+ const agent = source.definition;
930
+ const config = await runtimeForAgent(source);
588
931
  const runner = runnerFor(config);
589
932
  const metadata = {
590
933
  ...(config.provider === 'demo'
@@ -626,23 +969,193 @@ export const createGateway = (options = {}) => {
626
969
  });
627
970
  });
628
971
  };
629
- const dispatch = async (input) => {
972
+ /**
973
+ * Finishes the turns that were parked on a human when this daemon last
974
+ * went down.
975
+ *
976
+ * Run after the channels are up, deliberately: recovery re-asks, and a
977
+ * request emitted into a gateway with no live adapter would be declined
978
+ * for want of anywhere to put it — turning "your approval survived the
979
+ * restart" into "your approval was denied by the restart".
980
+ *
981
+ * Each session is recovered independently and failures are per-session:
982
+ * one unrecoverable turn must not stop the daemon from serving, or from
983
+ * recovering the others.
984
+ */
985
+ const recoverParkedTurns = async () => {
986
+ if (!store.listIdsByStatus) {
987
+ return;
988
+ }
989
+ const parked = await store.listIdsByStatus('pending_approval');
990
+ if (parked.length === 0) {
991
+ return;
992
+ }
993
+ log(`recovering ${parked.length} turn(s) parked on approval`);
994
+ // Started together, not one after another. Each recovery re-asks and
995
+ // then waits on a human, so awaiting them in sequence would mean the
996
+ // second parked turn is not even *announced* until the first is
997
+ // answered — fifteen minutes later by default, and never at all with a
998
+ // zero timeout. They are independent turns; only same-session work is
999
+ // ordered, and onSessionChain already guarantees that.
1000
+ await Promise.allSettled(parked.map((sessionId) => recoverOne(sessionId)));
1001
+ };
1002
+ const recoverOne = (sessionId) => onSessionChain(sessionId, async () => {
630
1003
  if (stopping) {
631
- throw new Error('The gateway is stopping and no longer accepts new work.');
1004
+ return;
1005
+ }
1006
+ try {
1007
+ const session = await store.get(sessionId);
1008
+ if (!session) {
1009
+ return;
1010
+ }
1011
+ const record = readPendingApproval(session);
1012
+ // A wait that outlived its window while the daemon was down is
1013
+ // honoured, not restarted: the request really did go unanswered for
1014
+ // the whole time it was configured to wait, and downtime is not a
1015
+ // reason to extend a security decision. Measured from when the turn
1016
+ // parked against this daemon's timeout — the transport's original
1017
+ // deadline was chosen after the checkpoint was written and is gone
1018
+ // with the process that chose it. Denying goes through the same
1019
+ // recovery path, so the queue behind it still drains.
1020
+ const parkedAt = record ? Date.parse(record.parkedAt) : Number.NaN;
1021
+ const expired = effectiveApprovalTimeoutMs > 0
1022
+ && Number.isFinite(parkedAt)
1023
+ && Date.now() - parkedAt >= effectiveApprovalTimeoutMs;
1024
+ if (expired) {
1025
+ log(`${sessionId}: the approval for ${record?.call.toolName} outlived its window while the daemon was down; denying it`);
1026
+ }
1027
+ // Resolved the way a dispatch for this agent would resolve it, so a
1028
+ // recovered turn finishes on the same provider, model, and
1029
+ // credentials it was parked on.
1030
+ const source = await refreshAgent(session.agent.id);
1031
+ // And on the CURRENT definition, saved before recovery reads the
1032
+ // session back. An allowlist is a permission boundary: a soul that
1033
+ // dropped a tool while the daemon was down must not have that tool
1034
+ // executed by the turn that outlived the change, and
1035
+ // `allowedToolsFor` reads the definition frozen into the session
1036
+ // ahead of the registry. Dispatch refreshes it the same way before
1037
+ // resuming.
1038
+ session.agent = source.definition;
1039
+ await store.save(session);
1040
+ const runner = runnerFor(await runtimeForAgent(source));
1041
+ await runner.recoverPendingApproval(sessionId, { denyPending: expired });
1042
+ }
1043
+ catch (error) {
1044
+ warn(`could not recover parked session ${sessionId}: ${error instanceof Error ? error.message : String(error)}`);
1045
+ }
1046
+ });
1047
+ /**
1048
+ * Turns the last process was still running when it stopped being able to
1049
+ * finish them.
1050
+ *
1051
+ * `pending_approval` is the one window a turn can be picked up from, and
1052
+ * it has its own sweep above. Everything else a turn does is saved as
1053
+ * `running`: the provider call, the executor, and — on a provider that
1054
+ * hosts its own loop — the approval wait too, since a hosted call is
1055
+ * deliberately not checkpointed (`executeHostedToolCall` passes
1056
+ * `recoverable: false`, because the SDK's inner loop is not something a
1057
+ * restart can rebuild). None of those can be resumed, and nothing else
1058
+ * sweeps them, so the record goes on claiming the turn is running for as
1059
+ * long as the session exists.
1060
+ *
1061
+ * Note what this is *not* for. A graceful stop denies everything parked
1062
+ * and drains the turns those denials release, so an operator restarting
1063
+ * the daemon normally has nothing here. These are the turns a crash, a
1064
+ * SIGKILL, or a lost machine left behind.
1065
+ */
1066
+ const listAbandonedTurns = async () => {
1067
+ if (!store.listIdsByStatus) {
1068
+ return [];
1069
+ }
1070
+ return store.listIdsByStatus('running');
1071
+ };
1072
+ /**
1073
+ * Marks each of them failed, with a reason that says what happened.
1074
+ *
1075
+ * Read BEFORE the channels start and applied after, which is the only
1076
+ * ordering that is both honest and safe. Reading first means the list is
1077
+ * exactly what the last process left, with no chance of a turn this
1078
+ * process started being in it. Applying after means the failure is
1079
+ * emitted into a gateway whose surfaces are listening, rather than into
1080
+ * one where nothing is.
1081
+ *
1082
+ * The gap between the two is real, though: an inbound message for one of
1083
+ * these sessions can arrive as soon as the channels are up. What makes
1084
+ * that safe is single-flight — every writer of a session goes through
1085
+ * `onSessionChain`, so this runs either wholly before that turn (it
1086
+ * fails, and `resume()` then clears `lastError` and starts fresh) or
1087
+ * wholly after it (the status has moved off `running`, and the re-read
1088
+ * below leaves it alone). There is no interleaving in which this
1089
+ * overwrites a live turn.
1090
+ *
1091
+ * What that invariant does NOT cover is a second daemon on the same
1092
+ * database, whose turns are running with no chain of ours in front of
1093
+ * them. Two processes sharing a session store is already unsupported —
1094
+ * the parked sweep above would re-ask that daemon's approvals for the
1095
+ * same reason — and no check here would make it supported.
1096
+ */
1097
+ const failAbandonedTurns = async (abandoned) => {
1098
+ if (abandoned.length === 0) {
1099
+ return;
1100
+ }
1101
+ log(`failing ${abandoned.length} turn(s) left running by the last stratusd`);
1102
+ await Promise.allSettled(abandoned.map((id) => failAbandonedTurn(id)));
1103
+ };
1104
+ const failAbandonedTurn = (id) => onSessionChain(id, async () => {
1105
+ if (stopping) {
1106
+ return;
632
1107
  }
633
- const previous = sessionChains.get(input.sessionId) ?? Promise.resolve();
634
- const turn = previous.then(() => dispatchInternal(input), () => dispatchInternal(input));
1108
+ try {
1109
+ const session = await store.get(id);
1110
+ // Re-read on the chain, because the snapshot is from before the
1111
+ // channels came up: a message that arrived since has already taken
1112
+ // this session somewhere else, and that turn owns it now.
1113
+ if (!session || session.status !== 'running') {
1114
+ return;
1115
+ }
1116
+ session.status = 'failed';
1117
+ session.lastError = ABANDONED_TURN_ERROR;
1118
+ await store.save(session);
1119
+ await bus.emit({ type: 'session.updated', sessionId: id, status: 'failed' });
1120
+ await bus.emit({ type: 'session.failed', sessionId: id, error: ABANDONED_TURN_ERROR });
1121
+ }
1122
+ catch (error) {
1123
+ warn(`could not fail abandoned turn ${id}: ${error instanceof Error ? error.message : String(error)}`);
1124
+ }
1125
+ });
1126
+ /**
1127
+ * Runs `work` as the session's next turn: queued behind whatever that
1128
+ * session is already doing, and tracked so shutdown drains it.
1129
+ *
1130
+ * Recovery goes through here as well as dispatch, and must. Channels are
1131
+ * live before a sweep finishes, so an inbound message for a parked
1132
+ * session could otherwise `resume()` alongside `recoverPendingApproval()`
1133
+ * — one closing the parked call as interrupted while the other re-enters
1134
+ * it, both saving divergent copies of the same transcript, and the tool
1135
+ * possibly running twice. Single-flight per session is the invariant that
1136
+ * stops that, and it only holds if everything that writes a session goes
1137
+ * through it.
1138
+ */
1139
+ const onSessionChain = (sessionId, work) => {
1140
+ const previous = sessionChains.get(sessionId) ?? Promise.resolve();
1141
+ const turn = previous.then(work, work);
635
1142
  const settled = turn.catch(() => { });
636
- sessionChains.set(input.sessionId, settled);
1143
+ sessionChains.set(sessionId, settled);
637
1144
  inflight.add(settled);
638
1145
  void settled.finally(() => {
639
1146
  inflight.delete(settled);
640
- if (sessionChains.get(input.sessionId) === settled) {
641
- sessionChains.delete(input.sessionId);
1147
+ if (sessionChains.get(sessionId) === settled) {
1148
+ sessionChains.delete(sessionId);
642
1149
  }
643
1150
  });
644
1151
  return turn;
645
1152
  };
1153
+ const dispatch = async (input) => {
1154
+ if (stopping) {
1155
+ throw new Error('The gateway is stopping and no longer accepts new work.');
1156
+ }
1157
+ return onSessionChain(input.sessionId, () => dispatchInternal(input));
1158
+ };
646
1159
  const startedChannels = [];
647
1160
  const gateway = {
648
1161
  bus,
@@ -652,6 +1165,11 @@ export const createGateway = (options = {}) => {
652
1165
  await loadRoster();
653
1166
  const named = registry.list().map((agent) => agent.name).join(', ');
654
1167
  log(`stratusd ready — ${registry.list().length} agent(s): ${named}`);
1168
+ // Read before anything can dispatch, so this is exactly what the
1169
+ // last process left running and not a turn of ours caught in flight.
1170
+ // Applied further down, once there is somewhere for the failure to
1171
+ // be heard.
1172
+ const abandoned = await listAbandonedTurns();
655
1173
  // Channels come up after the roster so their first inbound message
656
1174
  // already has agents to dispatch to. One failing adapter must not
657
1175
  // keep the rest (or the gateway) down.
@@ -673,6 +1191,19 @@ export const createGateway = (options = {}) => {
673
1191
  }
674
1192
  }
675
1193
  }
1194
+ // Last, and not awaited: recovery re-asks, so it needs the channels
1195
+ // above already listening — but a turn parked behind a slow approver
1196
+ // must not hold up start(), or a daemon with one outstanding request
1197
+ // would refuse to finish booting.
1198
+ void recoverParkedTurns().catch((error) => {
1199
+ warn(`recovering parked turns failed: ${error instanceof Error ? error.message : String(error)}`);
1200
+ });
1201
+ // Not awaited for the same reason, and independent of the sweep
1202
+ // above: these sessions are not parked, so the two never touch the
1203
+ // same one.
1204
+ void failAbandonedTurns(abandoned).catch((error) => {
1205
+ warn(`failing abandoned turns failed: ${error instanceof Error ? error.message : String(error)}`);
1206
+ });
676
1207
  },
677
1208
  // Drain: channels stop taking messages, in-flight turns finish, new
678
1209
  // dispatches are refused, then the database closes. SIGTERM handling
@@ -682,6 +1213,17 @@ export const createGateway = (options = {}) => {
682
1213
  // in-flight turns (already-started dispatches keep running), drain,
683
1214
  // and close.
684
1215
  stopping = true;
1216
+ // Deny what is parked before draining, or the drain waits out every
1217
+ // outstanding approval timeout — a shutdown would hang for as long as
1218
+ // the longest request had left. Each denial is a real decision the
1219
+ // turn continues from, so the drain below still finishes those turns.
1220
+ for (const settle of [...pendingApprovals.values()]) {
1221
+ settle('deny', 'cancelled');
1222
+ }
1223
+ // Let those resolutions reach their subscribers before the channels
1224
+ // go down, or a channel that renders approvals never learns to
1225
+ // retract the buttons it is still showing.
1226
+ await Promise.allSettled([...approvalEmissions]);
685
1227
  await Promise.allSettled(startedChannels.map((adapter) => adapter.stop()));
686
1228
  startedChannels.length = 0;
687
1229
  await Promise.allSettled([...inflight]);
@@ -692,6 +1234,7 @@ export const createGateway = (options = {}) => {
692
1234
  agents() {
693
1235
  return registry.list();
694
1236
  },
1237
+ resolveApproval,
695
1238
  };
696
1239
  return gateway;
697
1240
  };