@stratusagent/gateway 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +119 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +433 -67
- package/dist/index.js.map +1 -1
- package/dist/lock.d.ts +50 -0
- package/dist/lock.d.ts.map +1 -0
- package/dist/lock.js +128 -0
- package/dist/lock.js.map +1 -0
- package/dist/schedules.d.ts +13 -0
- package/dist/schedules.d.ts.map +1 -1
- package/dist/schedules.js +18 -2
- package/dist/schedules.js.map +1 -1
- package/package.json +7 -7
package/dist/index.js
CHANGED
|
@@ -10,13 +10,14 @@ import { createHash, randomUUID } from 'node:crypto';
|
|
|
10
10
|
import { chmodSync, mkdirSync } from 'node:fs';
|
|
11
11
|
import path from 'node:path';
|
|
12
12
|
import { DatabaseSync } from 'node:sqlite';
|
|
13
|
-
import { abortErrorFor, AgentRegistry, AgentRunner, EventBus, RunAbortedError, SkillRegistry, ToolRegistry, createSkillReadTool, missingSkillRequirements, PENDING_APPROVAL_METADATA_KEY, readPendingApproval, } from '@stratusagent/core';
|
|
13
|
+
import { abortErrorFor, AgentRegistry, AgentRunner, EventBus, RunAbortedError, SkillRegistry, ToolRegistry, createSkillReadTool, missingSkillRequirements, PENDING_APPROVAL_METADATA_KEY, latestTurnReply, readPendingApproval, } from '@stratusagent/core';
|
|
14
14
|
import { createDelegateTool, createForgetTool, createMessageSendTool, createRecallTool, createRememberTool, createScheduleTools, DELEGATED_BY_METADATA_KEY, DELEGATION_DEPTH_METADATA_KEY, delegatingSessionIdOf, isDelegatedSession, outstandingDelegationFor, ROOT_SESSION_ID_METADATA_KEY, SCHEDULE_ID_METADATA_KEY, SCHEDULED_TURN_METADATA_KEY, withoutDelegation, } from '@stratusagent/agents';
|
|
15
15
|
import { createSchedulerRuntime, isScheduleSessionId, SqliteScheduleStore, SCHEDULE_SESSION_ID_PREFIX, } from "./schedules.js";
|
|
16
|
+
export { claimHome, homeLockPath, HomeClaimedError, } from "./lock.js";
|
|
16
17
|
export { SqliteScheduleStore, createSchedulerRuntime, isScheduleSessionId, SCHEDULE_SESSION_ID_PREFIX, } from "./schedules.js";
|
|
17
18
|
import { createLocalCommandExecutor } from '@stratusagent/executor-local';
|
|
18
19
|
import { loadPlugins, } from '@stratusagent/plugins';
|
|
19
|
-
import { createDemoTool, createFileMemoryStore, createRuntimeProvider, DEFAULT_STRATUS_AGENT, loadOperatorSkills, loadRosterSouls, FALLBACK_ACTIVE_METADATA_KEY, loadSoulFile, memoryFilePath, migrateLegacyMemory, ConfigFileError, loadConfigFile, resolveConfigLocation, resolveConfiguredSoul, resolveRuntimeConfig, applySoulPins, stratusHomePath, workspacesDirPath, withLegacyDefaultMemories, } from '@stratusagent/state';
|
|
20
|
+
import { createDemoTool, createFileCredentialResolver, createFileMemoryStore, createRuntimeProvider, DEFAULT_STRATUS_AGENT, loadOperatorSkills, loadRosterSouls, FALLBACK_ACTIVE_METADATA_KEY, loadSoulFile, memoryFilePath, migrateLegacyMemory, ConfigFileError, loadConfigFile, resolveConfigLocation, resolveConfiguredSoul, resolveRuntimeConfig, applySoulPins, stratusHomePath, workspacesDirPath, withLegacyDefaultMemories, } from '@stratusagent/state';
|
|
20
21
|
const SESSIONS_DB_FILENAME = 'sessions.db';
|
|
21
22
|
/**
|
|
22
23
|
* Where a daemon with no explicit `sessionDbPath` keeps its database — the
|
|
@@ -221,6 +222,26 @@ export class SqliteSessionStore {
|
|
|
221
222
|
* question even when the rule answering it is not gateway code.
|
|
222
223
|
*/
|
|
223
224
|
export { applySoulPins } from '@stratusagent/state';
|
|
225
|
+
/** What a turn cut short by an announced restart records as its failure. */
|
|
226
|
+
export const RESTARTING_TURN_ERROR = 'Run aborted: stratusd is restarting';
|
|
227
|
+
/**
|
|
228
|
+
* How long an announced restart lets in-flight turns finish before aborting
|
|
229
|
+
* them. Matches the systemd unit's `TimeoutStopSec`, so the two windows an
|
|
230
|
+
* operator can hit agree.
|
|
231
|
+
*/
|
|
232
|
+
export const DEFAULT_RESTART_DRAIN_TIMEOUT_MS = 30_000;
|
|
233
|
+
/**
|
|
234
|
+
* `restart()` on a gateway whose host gave it no way back. Exported so the
|
|
235
|
+
* control API can answer it as "not supported here" rather than as a
|
|
236
|
+
* failure of the daemon.
|
|
237
|
+
*/
|
|
238
|
+
export class RestartUnsupportedError extends Error {
|
|
239
|
+
constructor() {
|
|
240
|
+
super('This daemon cannot restart itself: the host that started it did not say how. '
|
|
241
|
+
+ 'Restart it from the service manager (stratus service stop, then start) or from the terminal it runs in.');
|
|
242
|
+
this.name = 'RestartUnsupportedError';
|
|
243
|
+
}
|
|
244
|
+
}
|
|
224
245
|
/**
|
|
225
246
|
* Recorded on a turn the previous process was still running when it died.
|
|
226
247
|
*
|
|
@@ -459,6 +480,8 @@ export const createGateway = (options = {}) => {
|
|
|
459
480
|
agentId: request.session.agent.id,
|
|
460
481
|
call: request.call,
|
|
461
482
|
risk: request.risk,
|
|
483
|
+
...(request.origin !== undefined ? { origin: request.origin } : {}),
|
|
484
|
+
...(request.oneShot ? { oneShot: true } : {}),
|
|
462
485
|
parkedAt: request.parkedAt ?? new Date().toISOString(),
|
|
463
486
|
...(expiresAt ? { expiresAt } : {}),
|
|
464
487
|
...(request.session.metadata ? { metadata: request.session.metadata } : {}),
|
|
@@ -504,6 +527,8 @@ export const createGateway = (options = {}) => {
|
|
|
504
527
|
requestId,
|
|
505
528
|
call: request.call,
|
|
506
529
|
risk: request.risk,
|
|
530
|
+
...(request.origin !== undefined ? { origin: request.origin } : {}),
|
|
531
|
+
...(request.oneShot ? { oneShot: true } : {}),
|
|
507
532
|
...(request.session.metadata ? { metadata: request.session.metadata } : {}),
|
|
508
533
|
...(expiresAt ? { expiresAt } : {}),
|
|
509
534
|
});
|
|
@@ -731,22 +756,73 @@ export const createGateway = (options = {}) => {
|
|
|
731
756
|
}
|
|
732
757
|
};
|
|
733
758
|
/**
|
|
734
|
-
*
|
|
759
|
+
* Rebuild the skill catalog from what is on disk right now, into a fresh
|
|
760
|
+
* registry, and swap it in whole. See `Gateway.reloadSkills`.
|
|
761
|
+
*
|
|
762
|
+
* Strict on the operator directory: the previous set is still serving,
|
|
763
|
+
* so a file that will not load is a reason to keep it, not to serve the
|
|
764
|
+
* rest. The plugins' skills are re-registered from the load records, in
|
|
765
|
+
* load order, after the operator's — the order `start()` used, which is
|
|
766
|
+
* what makes an operator's bare id win its alias and a bare id two
|
|
767
|
+
* plugins want stay contested. Nothing is imported again: a plugin is
|
|
768
|
+
* code, and code gets a restart.
|
|
769
|
+
*/
|
|
770
|
+
const loadSkills = async () => {
|
|
771
|
+
const next = new SkillRegistry();
|
|
772
|
+
const operator = await loadOperatorSkills(env, next, warn, { strict: true });
|
|
773
|
+
for (const plugin of loadedPlugins) {
|
|
774
|
+
for (const record of plugin.skills) {
|
|
775
|
+
// The live catalog holds the very `Skill` the loader staged, under
|
|
776
|
+
// its canonical id; a rebuild re-registers that object rather than
|
|
777
|
+
// re-reading the plugin's file, so a plugin's skills are exactly
|
|
778
|
+
// what loaded with it until it is loaded again.
|
|
779
|
+
const skill = skillCatalog.resolve(record.id);
|
|
780
|
+
if (!skill) {
|
|
781
|
+
throw new Error(`Plugin ${plugin.package} lists skill ${record.id}, but the catalog no longer holds it. Restart the daemon.`);
|
|
782
|
+
}
|
|
783
|
+
next.register(skill);
|
|
784
|
+
next.registerAlias(record.bareId, skill.id);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
const before = new Set(operatorSkills.map((skill) => skill.id));
|
|
788
|
+
const after = new Set(operator.map((skill) => skill.id));
|
|
789
|
+
skillCatalog.replaceWith(next);
|
|
790
|
+
operatorSkills = operator;
|
|
791
|
+
const added = operator.filter((skill) => !before.has(skill.id)).map((skill) => skill.id);
|
|
792
|
+
const removed = [...before].filter((id) => !after.has(id));
|
|
793
|
+
const change = [
|
|
794
|
+
...(added.length > 0 ? [`added ${added.join(', ')}`] : []),
|
|
795
|
+
...(removed.length > 0 ? [`removed ${removed.join(', ')}`] : []),
|
|
796
|
+
];
|
|
797
|
+
log(`skills reloaded — ${skillCatalog.list().length} skill(s)${change.length > 0 ? `: ${change.join('; ')}` : ' (no change)'}`);
|
|
798
|
+
// The same advisory check the roster load runs: a removed skill an
|
|
799
|
+
// agent still lists, or a new one whose `requires:` the agent's tools
|
|
800
|
+
// do not cover, is worth a line now rather than at the next restart.
|
|
801
|
+
for (const agent of registry.list()) {
|
|
802
|
+
for (const { skill, missing } of missingSkillRequirements(agent, skillCatalog)) {
|
|
803
|
+
warn(`agent ${agent.id} enables skill ${skill.id}, which expects tools the agent is not allowed: ${missing.join(', ')}`);
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
};
|
|
807
|
+
/**
|
|
808
|
+
* Roster and skill rebuilds, one at a time.
|
|
735
809
|
*
|
|
736
810
|
* `loadRoster` prunes by comparing the registry against the ids *this pass*
|
|
737
811
|
* saw, so two overlapping passes destroy each other's work: the older one
|
|
738
812
|
* snapshots a roster without agent B, the newer one registers B and
|
|
739
813
|
* finishes, and then the older one reaches its prune and unregisters B —
|
|
740
814
|
* which two clients creating agents at once will do, each having been told
|
|
741
|
-
* 201 for an agent that is no longer dispatchable.
|
|
815
|
+
* 201 for an agent that is no longer dispatchable. Skill reloads share the
|
|
816
|
+
* chain: each is a whole-catalog swap, and the roster load reads the
|
|
817
|
+
* catalog for its advisory warnings.
|
|
742
818
|
*
|
|
743
819
|
* A chain rather than a lock: a reload is idempotent, so a caller arriving
|
|
744
820
|
* mid-rebuild wants the *next* complete one, and waiting for it is exactly
|
|
745
821
|
* what this gives them.
|
|
746
822
|
*/
|
|
747
823
|
let reloads = Promise.resolve();
|
|
748
|
-
const reloadsInOrder = async () => {
|
|
749
|
-
const next = reloads.then(
|
|
824
|
+
const reloadsInOrder = async (work) => {
|
|
825
|
+
const next = reloads.then(work, work);
|
|
750
826
|
// Swallowed for the chain only — the caller still sees the rejection, but
|
|
751
827
|
// a roster that failed to load must not block every reload after it.
|
|
752
828
|
reloads = next.catch(() => undefined);
|
|
@@ -936,9 +1012,25 @@ export const createGateway = (options = {}) => {
|
|
|
936
1012
|
const withWatchdog = async (sessionId, external, idleEnabled, fallbackStreams, run) => {
|
|
937
1013
|
const effectiveIdleMs = idleEnabled || fallbackStreams ? idleTimeoutMs : 0;
|
|
938
1014
|
if (effectiveIdleMs <= 0 && !external) {
|
|
939
|
-
|
|
1015
|
+
// No watchdog and nobody outside to cancel it — but still a turn an
|
|
1016
|
+
// announced restart must be able to cut short at the window.
|
|
1017
|
+
const idle = new AbortController();
|
|
1018
|
+
turnControllers.add(idle);
|
|
1019
|
+
if (abortingTurns) {
|
|
1020
|
+
idle.abort(new RunAbortedError(RESTARTING_TURN_ERROR));
|
|
1021
|
+
}
|
|
1022
|
+
try {
|
|
1023
|
+
return await run(idle.signal);
|
|
1024
|
+
}
|
|
1025
|
+
finally {
|
|
1026
|
+
turnControllers.delete(idle);
|
|
1027
|
+
}
|
|
940
1028
|
}
|
|
941
1029
|
const controller = new AbortController();
|
|
1030
|
+
turnControllers.add(controller);
|
|
1031
|
+
if (abortingTurns) {
|
|
1032
|
+
controller.abort(new RunAbortedError(RESTARTING_TURN_ERROR));
|
|
1033
|
+
}
|
|
942
1034
|
// The external reason travels: a caller that aborted with its own
|
|
943
1035
|
// `RunAbortedError` gets it recorded on the session, like the watchdog's.
|
|
944
1036
|
const onExternalAbort = () => controller.abort(external?.reason);
|
|
@@ -1075,6 +1167,7 @@ export const createGateway = (options = {}) => {
|
|
|
1075
1167
|
unsubscribeObserver();
|
|
1076
1168
|
unsubscribeDrain();
|
|
1077
1169
|
suspendWatchdog.delete(sessionId);
|
|
1170
|
+
turnControllers.delete(controller);
|
|
1078
1171
|
external?.removeEventListener('abort', onExternalAbort);
|
|
1079
1172
|
}
|
|
1080
1173
|
};
|
|
@@ -1085,6 +1178,37 @@ export const createGateway = (options = {}) => {
|
|
|
1085
1178
|
const sessionChains = new Map();
|
|
1086
1179
|
const inflight = new Set();
|
|
1087
1180
|
let stopping = false;
|
|
1181
|
+
/**
|
|
1182
|
+
* The abort handle of every turn running right now. A restart's drain
|
|
1183
|
+
* window ends by aborting these — the same signal the watchdog fires —
|
|
1184
|
+
* so a turn that outlives the window fails as aborted, its session
|
|
1185
|
+
* saved, instead of being cut off by a process exit and found as
|
|
1186
|
+
* abandoned by the next one.
|
|
1187
|
+
*/
|
|
1188
|
+
const turnControllers = new Set();
|
|
1189
|
+
/**
|
|
1190
|
+
* Set once a restart's window has closed and the turns in it were
|
|
1191
|
+
* aborted. A turn that starts after that — one queued behind the
|
|
1192
|
+
* aborted turn on the same session, accepted before the announcement —
|
|
1193
|
+
* is aborted the moment it gets its controller, with the same reason,
|
|
1194
|
+
* rather than running unseen until the shutdown closes the store under
|
|
1195
|
+
* it and leaves it recorded as nothing at all.
|
|
1196
|
+
*/
|
|
1197
|
+
let abortingTurns = false;
|
|
1198
|
+
/**
|
|
1199
|
+
* Set once `start()` has finished. A restart before that would stop the
|
|
1200
|
+
* channels started so far and close the stores while the rest of
|
|
1201
|
+
* startup is still running — reachable, since the control API can be
|
|
1202
|
+
* up and answering `POST /restart` while a later channel is still
|
|
1203
|
+
* connecting — and the startup then fails on a closed store instead of
|
|
1204
|
+
* the daemon coming back.
|
|
1205
|
+
*/
|
|
1206
|
+
let serving = false;
|
|
1207
|
+
/** Set once `restart()` announced; the refusal a caller reads says why. */
|
|
1208
|
+
let restartStatus;
|
|
1209
|
+
const refusal = () => new Error(restartStatus
|
|
1210
|
+
? 'stratusd is restarting and will accept new work once it is back up.'
|
|
1211
|
+
: 'The gateway is stopping and no longer accepts new work.');
|
|
1088
1212
|
/**
|
|
1089
1213
|
* The turn id running on each session, for callers that named one.
|
|
1090
1214
|
*
|
|
@@ -1369,7 +1493,31 @@ export const createGateway = (options = {}) => {
|
|
|
1369
1493
|
session.agent = source.definition;
|
|
1370
1494
|
await store.save(session);
|
|
1371
1495
|
const runner = runnerFor(await runtimeForAgent(source));
|
|
1372
|
-
|
|
1496
|
+
// Tracked like a dispatched turn's controller: a recovered turn
|
|
1497
|
+
// that runs on past its approval is a turn like any other, and a
|
|
1498
|
+
// restart's window has to be able to cut it short with the same
|
|
1499
|
+
// reason — or it runs unseen to the second window and the restart
|
|
1500
|
+
// reports undrained for a turn it could have finished cleanly.
|
|
1501
|
+
const controller = new AbortController();
|
|
1502
|
+
turnControllers.add(controller);
|
|
1503
|
+
if (abortingTurns) {
|
|
1504
|
+
controller.abort(new RunAbortedError(RESTARTING_TURN_ERROR));
|
|
1505
|
+
}
|
|
1506
|
+
try {
|
|
1507
|
+
await runner.recoverPendingApproval(sessionId, { denyPending: expired, signal: controller.signal });
|
|
1508
|
+
}
|
|
1509
|
+
finally {
|
|
1510
|
+
turnControllers.delete(controller);
|
|
1511
|
+
// A recovered firing's row outlived the process that would have
|
|
1512
|
+
// retired it, and the firing is over either way — a recovery that
|
|
1513
|
+
// failed has durably failed its session, which is as finished as
|
|
1514
|
+
// completing. The scheduler decides whether there is a row to
|
|
1515
|
+
// retire, and only this firing's own.
|
|
1516
|
+
const scheduleId = session.metadata?.[SCHEDULE_ID_METADATA_KEY];
|
|
1517
|
+
if (typeof scheduleId === 'string') {
|
|
1518
|
+
scheduler.retireSpentOneShot(scheduleId, sessionId);
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1373
1521
|
}
|
|
1374
1522
|
catch (error) {
|
|
1375
1523
|
warn(`could not recover parked session ${sessionId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -1507,13 +1655,13 @@ export const createGateway = (options = {}) => {
|
|
|
1507
1655
|
*/
|
|
1508
1656
|
const dispatchScheduledFiring = (input) => {
|
|
1509
1657
|
if (stopping) {
|
|
1510
|
-
throw
|
|
1658
|
+
throw refusal();
|
|
1511
1659
|
}
|
|
1512
1660
|
return onSessionChain(input.sessionId, () => dispatchInternal(input));
|
|
1513
1661
|
};
|
|
1514
1662
|
const dispatch = async (input) => {
|
|
1515
1663
|
if (stopping) {
|
|
1516
|
-
throw
|
|
1664
|
+
throw refusal();
|
|
1517
1665
|
}
|
|
1518
1666
|
// The reserved namespace is the scheduler's alone. A caller that could
|
|
1519
1667
|
// name a firing's session id — derivable from `GET /schedules` — would
|
|
@@ -1572,7 +1720,16 @@ export const createGateway = (options = {}) => {
|
|
|
1572
1720
|
tools,
|
|
1573
1721
|
skills: skillCatalog,
|
|
1574
1722
|
bus,
|
|
1723
|
+
// The file-backed resolver, so a plugin needing a key reaches the
|
|
1724
|
+
// *calling* agent's — its allowlist checked, its own entry before the
|
|
1725
|
+
// fleet's shared one.
|
|
1726
|
+
credentials: createFileCredentialResolver(env),
|
|
1575
1727
|
workspaceRoot: workspacesDirPath(env),
|
|
1728
|
+
// The structured log, so a plugin's lifecycle lines — an MCP server
|
|
1729
|
+
// that dropped, a reconnect that failed — are in `stratus logs` and
|
|
1730
|
+
// not only on a stderr the service manager owns.
|
|
1731
|
+
log,
|
|
1732
|
+
warn,
|
|
1576
1733
|
});
|
|
1577
1734
|
loadedPlugins = result.loaded;
|
|
1578
1735
|
pluginFailures = result.failures;
|
|
@@ -1626,7 +1783,10 @@ export const createGateway = (options = {}) => {
|
|
|
1626
1783
|
startedChannels.push(adapter);
|
|
1627
1784
|
}
|
|
1628
1785
|
catch (error) {
|
|
1629
|
-
|
|
1786
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
1787
|
+
if (!adapter.required) {
|
|
1788
|
+
warn(`channel ${adapter.name} failed to start: ${reason}`);
|
|
1789
|
+
}
|
|
1630
1790
|
// A start() that rejected may still hold sockets or listeners it
|
|
1631
1791
|
// acquired before failing; it never reaches startedChannels, so
|
|
1632
1792
|
// this is its only cleanup.
|
|
@@ -1636,6 +1796,17 @@ export const createGateway = (options = {}) => {
|
|
|
1636
1796
|
catch (stopError) {
|
|
1637
1797
|
warn(`channel ${adapter.name} cleanup after failed start also failed: ${stopError instanceof Error ? stopError.message : String(stopError)}`);
|
|
1638
1798
|
}
|
|
1799
|
+
if (adapter.required) {
|
|
1800
|
+
// Before the scheduler and the sweeps below, deliberately: a
|
|
1801
|
+
// daemon that is not going to serve must not fire a catch-up
|
|
1802
|
+
// slot or re-ask a parked approval on its way out. The channels
|
|
1803
|
+
// already up are left to the shutdown this throw becomes, which
|
|
1804
|
+
// denies what they parked and lets those denials reach them
|
|
1805
|
+
// BEFORE it stops them — stopped here, a channel that had parked
|
|
1806
|
+
// a gated turn in the window would never learn to retract its
|
|
1807
|
+
// buttons.
|
|
1808
|
+
throw new Error(`${adapter.name} could not start, and the gateway cannot serve without it: ${reason}`);
|
|
1809
|
+
}
|
|
1639
1810
|
}
|
|
1640
1811
|
}
|
|
1641
1812
|
// After the channels, so a catch-up firing has somewhere to report and
|
|
@@ -1657,100 +1828,295 @@ export const createGateway = (options = {}) => {
|
|
|
1657
1828
|
warn(`failing abandoned turns failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1658
1829
|
});
|
|
1659
1830
|
};
|
|
1660
|
-
/**
|
|
1831
|
+
/**
|
|
1832
|
+
* Both stores, once. Reached from `stop()` and from a `start()` that
|
|
1833
|
+
* failed, and a host may call the one after the other; node:sqlite
|
|
1834
|
+
* refuses a second close, so this remembers.
|
|
1835
|
+
*/
|
|
1836
|
+
let storesClosed = false;
|
|
1837
|
+
const closeStores = () => {
|
|
1838
|
+
if (storesClosed) {
|
|
1839
|
+
return;
|
|
1840
|
+
}
|
|
1841
|
+
storesClosed = true;
|
|
1842
|
+
scheduleStore.close();
|
|
1843
|
+
store.close();
|
|
1844
|
+
};
|
|
1845
|
+
/**
|
|
1846
|
+
* Release what the plugins acquired, reporting rather than throwing.
|
|
1847
|
+
* Resolves with whether every plugin let go: a dispose that rejected may
|
|
1848
|
+
* still hold the browser or socket it was closing, which a restart has
|
|
1849
|
+
* to know before it starts a daemon beside it.
|
|
1850
|
+
*/
|
|
1661
1851
|
const disposePlugins = async () => {
|
|
1852
|
+
let released = true;
|
|
1662
1853
|
await Promise.allSettled(loadedPlugins.map(async (plugin) => {
|
|
1663
1854
|
try {
|
|
1664
1855
|
await plugin.instance.dispose?.();
|
|
1665
1856
|
}
|
|
1666
1857
|
catch (error) {
|
|
1858
|
+
released = false;
|
|
1667
1859
|
warn(`plugin ${plugin.package} failed to shut down: ${error instanceof Error ? error.message : String(error)}`);
|
|
1668
1860
|
}
|
|
1669
1861
|
}));
|
|
1670
1862
|
loadedPlugins = [];
|
|
1863
|
+
return released;
|
|
1671
1864
|
};
|
|
1672
1865
|
const startedChannels = [];
|
|
1866
|
+
/** Resolves with `true` when `work` settles before `ms` elapse. */
|
|
1867
|
+
const settlesWithin = (work, ms) => new Promise((resolve) => {
|
|
1868
|
+
const timer = setTimeout(() => resolve(false), ms);
|
|
1869
|
+
void work.then(() => { clearTimeout(timer); resolve(true); }, () => { clearTimeout(timer); resolve(true); });
|
|
1870
|
+
});
|
|
1871
|
+
let stopped;
|
|
1872
|
+
/**
|
|
1873
|
+
* The one shutdown sequence — for a stop, for a restart, and for a
|
|
1874
|
+
* start() that failed, whose caller never reaches stop().
|
|
1875
|
+
*
|
|
1876
|
+
* `abortAfterMs` is the difference: a stop waits for every in-flight
|
|
1877
|
+
* turn however long it takes (the service manager's own timeout is the
|
|
1878
|
+
* bound there), while a restart waits that long and then aborts what is
|
|
1879
|
+
* still running, so the daemon can come back instead of waiting on one
|
|
1880
|
+
* stuck turn. Resolves with whether everything finished — after an abort,
|
|
1881
|
+
* a turn that still ignores its signal for a second window is shut down
|
|
1882
|
+
* around, and the host is told so it can exit rather than start a new
|
|
1883
|
+
* daemon beside it. Every wait after the window opens is bounded by it,
|
|
1884
|
+
* the scheduler's own drain included: that drain tracks the same firing
|
|
1885
|
+
* the turn drain gave up on, and an unbounded wait there would hold the
|
|
1886
|
+
* restart forever on exactly the turn the window exists for.
|
|
1887
|
+
*/
|
|
1888
|
+
const shutdown = async (abortAfterMs) => {
|
|
1889
|
+
// Refuse new work first; then let channels finish rendering their
|
|
1890
|
+
// in-flight turns (already-started dispatches keep running), drain,
|
|
1891
|
+
// and close.
|
|
1892
|
+
stopping = true;
|
|
1893
|
+
// No new firings from here: the tick stops before anything drains,
|
|
1894
|
+
// so the drain below is over a set that can only shrink. A tick
|
|
1895
|
+
// already past its `stopping` check loses to `dispatch`'s own
|
|
1896
|
+
// refusal — the second gate is why this needs no handshake.
|
|
1897
|
+
scheduler.stop();
|
|
1898
|
+
// Deny what is parked before draining, or the drain waits out every
|
|
1899
|
+
// outstanding approval timeout — a shutdown would hang for as long as
|
|
1900
|
+
// the longest request had left. Each denial is a real decision the
|
|
1901
|
+
// turn continues from, so the drain below still finishes those turns.
|
|
1902
|
+
for (const parked of [...pendingApprovals.values()]) {
|
|
1903
|
+
parked.settle('deny', 'cancelled');
|
|
1904
|
+
}
|
|
1905
|
+
// Everything a restart's window has to cover, in the order a stop
|
|
1906
|
+
// does it. The approval resolutions reach their subscribers before the
|
|
1907
|
+
// channels go down, or a channel that renders approvals never learns
|
|
1908
|
+
// to retract the buttons it is still showing; the channels' stop awaits
|
|
1909
|
+
// the rendering they have in flight, which is a running turn's output.
|
|
1910
|
+
// All three inside the window: a subscriber or a channel stalled on a
|
|
1911
|
+
// stuck turn would otherwise hold the restart outside it.
|
|
1912
|
+
// A channel whose stop rejected may still hold its connection — and a
|
|
1913
|
+
// replacement daemon opening another would be two of them answering
|
|
1914
|
+
// the same messages. Reported, and for a restart, counted as not
|
|
1915
|
+
// drained; a plain stop goes on closing either way.
|
|
1916
|
+
let channelsReleased = true;
|
|
1917
|
+
const drain = (async () => {
|
|
1918
|
+
await Promise.allSettled([...approvalEmissions]);
|
|
1919
|
+
const stopping = [...startedChannels];
|
|
1920
|
+
const stops = await Promise.allSettled(stopping.map((adapter) => adapter.stop()));
|
|
1921
|
+
stops.forEach((result, index) => {
|
|
1922
|
+
if (result.status === 'rejected') {
|
|
1923
|
+
channelsReleased = false;
|
|
1924
|
+
warn(`channel ${stopping[index]?.name} failed to stop: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`);
|
|
1925
|
+
}
|
|
1926
|
+
});
|
|
1927
|
+
startedChannels.length = 0;
|
|
1928
|
+
await Promise.allSettled([...inflight]);
|
|
1929
|
+
})();
|
|
1930
|
+
let drained = true;
|
|
1931
|
+
if (abortAfterMs === undefined) {
|
|
1932
|
+
await drain;
|
|
1933
|
+
}
|
|
1934
|
+
else if (await settlesWithin(drain, abortAfterMs)) {
|
|
1935
|
+
drained = channelsReleased;
|
|
1936
|
+
}
|
|
1937
|
+
else {
|
|
1938
|
+
warn(`restart: ${inflight.size} turn(s) still running after ${abortAfterMs}ms; aborting them`);
|
|
1939
|
+
// The watchdog's own mechanism: the runner fails each session with
|
|
1940
|
+
// the reason on the signal, saves it, and emits session.failed —
|
|
1941
|
+
// so the next daemon finds a finished turn, not an abandoned one.
|
|
1942
|
+
// Before the loop, so a turn queued behind one aborted here, which
|
|
1943
|
+
// starts as soon as that one settles, is aborted the same way.
|
|
1944
|
+
abortingTurns = true;
|
|
1945
|
+
for (const controller of turnControllers) {
|
|
1946
|
+
controller.abort(new RunAbortedError(RESTARTING_TURN_ERROR));
|
|
1947
|
+
}
|
|
1948
|
+
drained = await settlesWithin(drain, abortAfterMs);
|
|
1949
|
+
if (!drained) {
|
|
1950
|
+
warn(`restart: ${inflight.size} turn(s) did not stop after being aborted; shutting down around them`);
|
|
1951
|
+
}
|
|
1952
|
+
drained = drained && channelsReleased;
|
|
1953
|
+
}
|
|
1954
|
+
// After the turn drain: a firing's wrapper settles after its
|
|
1955
|
+
// dispatch does (it retires spent one-shots), so this is the last
|
|
1956
|
+
// writer of the schedule table. Bounded once the turn drain has given
|
|
1957
|
+
// up: the wrapper it would wait for is around the very turn that
|
|
1958
|
+
// ignored its abort.
|
|
1959
|
+
const schedulerDrain = scheduler.drain();
|
|
1960
|
+
if (drained || abortAfterMs === undefined) {
|
|
1961
|
+
await schedulerDrain;
|
|
1962
|
+
}
|
|
1963
|
+
else if (!(await settlesWithin(schedulerDrain, abortAfterMs))) {
|
|
1964
|
+
warn('restart: a scheduled firing did not stop after being aborted; shutting down around it');
|
|
1965
|
+
}
|
|
1966
|
+
// After the turns that might still be using them. A plugin holding a
|
|
1967
|
+
// browser is the reason this exists, and closing it out from under a
|
|
1968
|
+
// running screenshot would fail that turn rather than tidy up. Bounded
|
|
1969
|
+
// for a restart, whose likeliest reason is a plugin that misbehaves:
|
|
1970
|
+
// a dispose that never settles must not be what keeps the daemon from
|
|
1971
|
+
// coming back, and a plugin still holding on is one more thing the
|
|
1972
|
+
// host should not start a new daemon beside.
|
|
1973
|
+
const disposal = disposePlugins();
|
|
1974
|
+
if (abortAfterMs === undefined) {
|
|
1975
|
+
await disposal;
|
|
1976
|
+
}
|
|
1977
|
+
else if (await settlesWithin(disposal, abortAfterMs)) {
|
|
1978
|
+
// A dispose that rejected has already been named; what it may still
|
|
1979
|
+
// hold is the same problem as one that never answered.
|
|
1980
|
+
drained = drained && await disposal;
|
|
1981
|
+
}
|
|
1982
|
+
else {
|
|
1983
|
+
warn(`restart: a plugin did not release what it holds within ${abortAfterMs}ms; shutting down around it`);
|
|
1984
|
+
drained = false;
|
|
1985
|
+
}
|
|
1986
|
+
closeStores();
|
|
1987
|
+
log('stratusd stopped');
|
|
1988
|
+
return drained;
|
|
1989
|
+
};
|
|
1673
1990
|
const gateway = {
|
|
1674
1991
|
bus,
|
|
1675
1992
|
store,
|
|
1676
1993
|
async start() {
|
|
1677
|
-
await migrateLegacyMemory(env);
|
|
1678
|
-
// Before the roster and before channels: a turn must never arrive for
|
|
1679
|
-
// an agent whose soul lists a tool the daemon has not registered yet,
|
|
1680
|
-
// which would refuse the call as "not permitted" and blame the soul.
|
|
1681
|
-
// Skills first for the same reason — and before the plugins, so the
|
|
1682
|
-
// operator's bare ids win their aliases (see startSkills).
|
|
1683
|
-
await startSkills();
|
|
1684
|
-
await startPlugins();
|
|
1685
1994
|
try {
|
|
1995
|
+
await migrateLegacyMemory(env);
|
|
1996
|
+
// Before the roster and before channels: a turn must never arrive
|
|
1997
|
+
// for an agent whose soul lists a tool the daemon has not
|
|
1998
|
+
// registered yet, which would refuse the call as "not permitted"
|
|
1999
|
+
// and blame the soul. Skills first for the same reason — and before
|
|
2000
|
+
// the plugins, so the operator's bare ids win their aliases (see
|
|
2001
|
+
// startSkills).
|
|
2002
|
+
await startSkills();
|
|
2003
|
+
await startPlugins();
|
|
1686
2004
|
await startServing();
|
|
1687
2005
|
}
|
|
1688
2006
|
catch (error) {
|
|
1689
|
-
//
|
|
1690
|
-
//
|
|
1691
|
-
//
|
|
1692
|
-
//
|
|
1693
|
-
//
|
|
1694
|
-
//
|
|
1695
|
-
|
|
2007
|
+
// A `start()` that rejects never reaches its caller's shutdown path
|
|
2008
|
+
// — `stratus serve` awaits it *before* the try/finally that calls
|
|
2009
|
+
// `stop()` — so it is the shutdown, in full: a duplicate agent id
|
|
2010
|
+
// in the roster would otherwise leave a plugin's browser, socket,
|
|
2011
|
+
// or subscription held for the life of a process that is on its
|
|
2012
|
+
// way out; the two stores opened before any of this ran would keep
|
|
2013
|
+
// their descriptors, four more per attempt in a host that retries
|
|
2014
|
+
// a port it cannot bind; and a turn a channel that did come up had
|
|
2015
|
+
// already accepted must finish and save before the store it saves
|
|
2016
|
+
// to is closed, exactly as it would under a stop().
|
|
2017
|
+
await shutdown(undefined);
|
|
1696
2018
|
throw error;
|
|
1697
2019
|
}
|
|
2020
|
+
serving = true;
|
|
1698
2021
|
},
|
|
1699
2022
|
// Drain: channels stop taking messages, in-flight turns finish, new
|
|
1700
2023
|
// dispatches are refused, then the database closes. SIGTERM handling
|
|
1701
|
-
// in `stratus serve` calls this.
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
// so the drain below is over a set that can only shrink. A tick
|
|
1709
|
-
// already past its `stopping` check loses to `dispatch`'s own
|
|
1710
|
-
// refusal — the second gate is why this needs no handshake.
|
|
1711
|
-
scheduler.stop();
|
|
1712
|
-
// Deny what is parked before draining, or the drain waits out every
|
|
1713
|
-
// outstanding approval timeout — a shutdown would hang for as long as
|
|
1714
|
-
// the longest request had left. Each denial is a real decision the
|
|
1715
|
-
// turn continues from, so the drain below still finishes those turns.
|
|
1716
|
-
for (const parked of [...pendingApprovals.values()]) {
|
|
1717
|
-
parked.settle('deny', 'cancelled');
|
|
1718
|
-
}
|
|
1719
|
-
// Let those resolutions reach their subscribers before the channels
|
|
1720
|
-
// go down, or a channel that renders approvals never learns to
|
|
1721
|
-
// retract the buttons it is still showing.
|
|
1722
|
-
await Promise.allSettled([...approvalEmissions]);
|
|
1723
|
-
await Promise.allSettled(startedChannels.map((adapter) => adapter.stop()));
|
|
1724
|
-
startedChannels.length = 0;
|
|
1725
|
-
await Promise.allSettled([...inflight]);
|
|
1726
|
-
// After the turn drain: a firing's wrapper settles after its
|
|
1727
|
-
// dispatch does (it retires spent one-shots), so this is the last
|
|
1728
|
-
// writer of the schedule table.
|
|
1729
|
-
await scheduler.drain();
|
|
1730
|
-
// After the turns that might still be using them. A plugin holding a
|
|
1731
|
-
// browser is the reason this exists, and closing it out from under a
|
|
1732
|
-
// running screenshot would fail that turn rather than tidy up.
|
|
1733
|
-
await disposePlugins();
|
|
1734
|
-
scheduleStore.close();
|
|
1735
|
-
store.close();
|
|
1736
|
-
log('stratusd stopped');
|
|
2024
|
+
// in `stratus serve` calls this. Idempotent: a restart stops the
|
|
2025
|
+
// gateway itself before handing the process to the host, whose own
|
|
2026
|
+
// shutdown path then calls this again and must find nothing to do —
|
|
2027
|
+
// and a start() that failed has already run the same sequence.
|
|
2028
|
+
stop() {
|
|
2029
|
+
stopped ??= shutdown(undefined).then(() => undefined);
|
|
2030
|
+
return stopped;
|
|
1737
2031
|
},
|
|
1738
2032
|
dispatch,
|
|
1739
2033
|
async sessionRouting(sessionId) {
|
|
1740
2034
|
const session = await store.get(sessionId);
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
2035
|
+
if (!session) {
|
|
2036
|
+
return undefined;
|
|
2037
|
+
}
|
|
2038
|
+
// The reply, not the record: a channel that finishes a turn it did
|
|
2039
|
+
// not start needs the one message to post, and the transcript stays
|
|
2040
|
+
// the session's own. The latest turn's text only — a turn that
|
|
2041
|
+
// produced none has no reply, and an earlier turn's answer is not it.
|
|
2042
|
+
const reply = latestTurnReply(session);
|
|
2043
|
+
return {
|
|
2044
|
+
agentId: session.agent.id,
|
|
2045
|
+
metadata: session.metadata ?? {},
|
|
2046
|
+
...(reply !== undefined ? { reply } : {}),
|
|
2047
|
+
};
|
|
1744
2048
|
},
|
|
1745
2049
|
agents() {
|
|
1746
2050
|
return registry.list();
|
|
1747
2051
|
},
|
|
1748
2052
|
async reloadRoster() {
|
|
1749
|
-
await reloadsInOrder();
|
|
2053
|
+
await reloadsInOrder(loadRoster);
|
|
1750
2054
|
const roster = registry.list();
|
|
1751
2055
|
log(`roster reloaded — ${roster.length} agent(s): ${roster.map((agent) => agent.name).join(', ')}`);
|
|
1752
2056
|
return roster;
|
|
1753
2057
|
},
|
|
2058
|
+
async reloadSkills() {
|
|
2059
|
+
try {
|
|
2060
|
+
await reloadsInOrder(loadSkills);
|
|
2061
|
+
}
|
|
2062
|
+
catch (error) {
|
|
2063
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
2064
|
+
warn(`skills reload refused — ${reason}; the previously loaded skills are still serving`);
|
|
2065
|
+
throw new Error(`Skills were not reloaded: ${reason} The previously loaded skills are still serving; fix the file and reload again.`);
|
|
2066
|
+
}
|
|
2067
|
+
return gateway.skills();
|
|
2068
|
+
},
|
|
2069
|
+
restart(request = {}) {
|
|
2070
|
+
if (!options.onRestart) {
|
|
2071
|
+
throw new RestartUnsupportedError();
|
|
2072
|
+
}
|
|
2073
|
+
if (restartStatus) {
|
|
2074
|
+
return restartStatus;
|
|
2075
|
+
}
|
|
2076
|
+
if (stopping) {
|
|
2077
|
+
throw new Error('The gateway is already stopping, and a stop does not come back on its own.');
|
|
2078
|
+
}
|
|
2079
|
+
if (!serving) {
|
|
2080
|
+
throw new Error('stratusd is still starting; ask for a restart again once it is serving.');
|
|
2081
|
+
}
|
|
2082
|
+
const requested = request.drainTimeoutMs ?? options.restartDrainTimeoutMs ?? DEFAULT_RESTART_DRAIN_TIMEOUT_MS;
|
|
2083
|
+
// Checked here and not only at the HTTP route: a host calling this
|
|
2084
|
+
// directly with a negative or NaN window would get a timer that
|
|
2085
|
+
// fires at once, aborting every turn it meant to let finish. Refused
|
|
2086
|
+
// before anything is announced, so a bad argument is not a restart.
|
|
2087
|
+
if (!Number.isFinite(requested) || requested < 0) {
|
|
2088
|
+
throw new Error(`The restart drain window must be a non-negative number of milliseconds, not ${String(requested)}.`);
|
|
2089
|
+
}
|
|
2090
|
+
// Above Node's maximum timer delay a setTimeout fires at once, so a
|
|
2091
|
+
// window asked for as "very long" would abort every turn immediately
|
|
2092
|
+
// — the same clamp the approval timeout gets, for the same reason.
|
|
2093
|
+
const drainTimeoutMs = Math.min(requested, MAX_TIMER_DELAY_MS);
|
|
2094
|
+
if (requested > MAX_TIMER_DELAY_MS) {
|
|
2095
|
+
warn(`restart drain window ${requested}ms is above Node's maximum timer delay; using ${MAX_TIMER_DELAY_MS}ms (~24.8 days)`);
|
|
2096
|
+
}
|
|
2097
|
+
const status = {
|
|
2098
|
+
...(request.reason !== undefined ? { reason: request.reason } : {}),
|
|
2099
|
+
drainTimeoutMs,
|
|
2100
|
+
inflight: inflight.size,
|
|
2101
|
+
};
|
|
2102
|
+
restartStatus = status;
|
|
2103
|
+
// Said now, while stdout and the structured log both still work: a
|
|
2104
|
+
// process on its way down is the one whose last lines matter, and
|
|
2105
|
+
// the one least able to write them.
|
|
2106
|
+
log(`restart requested${status.reason ? ` (${status.reason})` : ''} — refusing new turns; `
|
|
2107
|
+
+ `${status.inflight} turn(s) in flight, draining for up to ${drainTimeoutMs}ms`);
|
|
2108
|
+
const onRestart = options.onRestart;
|
|
2109
|
+
const draining = shutdown(drainTimeoutMs);
|
|
2110
|
+
// The host's own shutdown path calls stop() next; it must find this
|
|
2111
|
+
// one rather than start a second.
|
|
2112
|
+
stopped = draining.then(() => undefined);
|
|
2113
|
+
void draining
|
|
2114
|
+
.then((drained) => onRestart({ ...(status.reason !== undefined ? { reason: status.reason } : {}), drained }))
|
|
2115
|
+
.catch((error) => {
|
|
2116
|
+
warn(`restart failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
2117
|
+
});
|
|
2118
|
+
return status;
|
|
2119
|
+
},
|
|
1754
2120
|
activeTurnId(sessionId) {
|
|
1755
2121
|
return activeTurns.get(sessionId);
|
|
1756
2122
|
},
|