@songsid/agend 2.1.4-beta.39 → 2.1.4-beta.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/doctor.d.ts CHANGED
@@ -3,7 +3,7 @@ import type { ServiceInfo } from "./service-installer.js";
3
3
  import { type NetworkFamilyState } from "./network-family.js";
4
4
  export type DoctorCheckStatus = "ok" | "warn" | "error";
5
5
  export interface DoctorCheck {
6
- section: "Prerequisites" | "Service" | "Fleet" | "MCP IPC";
6
+ section: "Prerequisites" | "Service" | "Fleet" | "Channel gateways" | "MCP IPC";
7
7
  status: DoctorCheckStatus;
8
8
  label: string;
9
9
  detail: string;
@@ -20,6 +20,19 @@ interface DoctorDeps {
20
20
  processAlive: (pid: number) => boolean;
21
21
  connectSocket: (path: string) => Promise<boolean>;
22
22
  networkFamily: () => NetworkFamilyState;
23
+ fetchFleetHealth: (port: number) => Promise<{
24
+ adapters?: {
25
+ details?: Record<string, {
26
+ status?: string;
27
+ isReady?: boolean;
28
+ reconnectCount?: number;
29
+ heartbeatAgeMs?: number | null;
30
+ shards?: Array<{
31
+ heartbeatAgeMs?: number | null;
32
+ }>;
33
+ }>;
34
+ };
35
+ }>;
23
36
  }
24
37
  export declare function connectUnixSocket(path: string, timeoutMs?: number): Promise<boolean>;
25
38
  /** Collect read-only diagnostics. Dependencies are injectable for deterministic tests. */
package/dist/doctor.js CHANGED
@@ -24,6 +24,10 @@ const defaultDeps = {
24
24
  },
25
25
  connectSocket: connectUnixSocket,
26
26
  networkFamily: getNetworkFamilyState,
27
+ fetchFleetHealth: async (port) => {
28
+ const response = await fetch(`http://127.0.0.1:${port}/health`, { signal: AbortSignal.timeout(3000) });
29
+ return await response.json();
30
+ },
27
31
  };
28
32
  function tmuxArgs(args) {
29
33
  const socket = getTmuxSocketName();
@@ -180,6 +184,42 @@ export async function collectDoctorReport(dataDir, service, overrides = {}) {
180
184
  }
181
185
  }
182
186
  add("Fleet", "ok", t("doctor.instances"), t("doctor.instance_counts", counts.running, counts.paused, counts.stopped, counts.crashed, names.length));
187
+ const rawChannels = Array.isArray(raw.channels)
188
+ ? raw.channels
189
+ : raw.channel ? [raw.channel] : [];
190
+ const hasDiscord = rawChannels.some((channel) => channel?.type === "discord");
191
+ if (!hasDiscord) {
192
+ // Telegram and local-only fleets have no Discord Gateway session to inspect.
193
+ }
194
+ else if (!fleetAlive) {
195
+ add("Channel gateways", "warn", t("doctor.gateway_health"), t("doctor.gateway_unknown"));
196
+ }
197
+ else {
198
+ try {
199
+ const healthPort = typeof raw.health_port === "number" ? raw.health_port : 19280;
200
+ const health = await deps.fetchFleetHealth(healthPort);
201
+ const details = health.adapters?.details ?? {};
202
+ const entries = Object.entries(details);
203
+ if (entries.length === 0) {
204
+ add("Channel gateways", "warn", t("doctor.gateway_health"), t("doctor.gateway_no_details"));
205
+ }
206
+ else {
207
+ for (const [id, detail] of entries) {
208
+ const ages = (detail.shards ?? [])
209
+ .map(shard => shard.heartbeatAgeMs)
210
+ .filter((age) => typeof age === "number");
211
+ const newestAge = typeof detail.heartbeatAgeMs === "number"
212
+ ? detail.heartbeatAgeMs
213
+ : ages.length > 0 ? Math.max(...ages) : null;
214
+ const healthy = detail.status === "connected" && detail.isReady === true;
215
+ add("Channel gateways", healthy ? "ok" : "error", id, t("doctor.gateway_detail", detail.status ?? "unknown", detail.reconnectCount ?? 0, newestAge == null ? "—" : `${Math.round(newestAge / 1000)}s`));
216
+ }
217
+ }
218
+ }
219
+ catch (err) {
220
+ add("Channel gateways", "warn", t("doctor.gateway_health"), t("doctor.gateway_unreachable", err.message));
221
+ }
222
+ }
183
223
  const ipcResults = await Promise.all(runningNames.map(async (name) => {
184
224
  const socketPath = join(dataDir, "instances", name, "channel.sock");
185
225
  return { name, socketPath, exists: existsSync(socketPath), connected: await deps.connectSocket(socketPath) };
@@ -202,6 +242,7 @@ export function formatDoctorReport(report) {
202
242
  Prerequisites: t("doctor.section.prerequisites"),
203
243
  Service: t("doctor.section.service"),
204
244
  Fleet: t("doctor.section.fleet"),
245
+ "Channel gateways": t("doctor.section.gateways"),
205
246
  "MCP IPC": t("doctor.section.ipc"),
206
247
  };
207
248
  const lines = ["", ` \x1b[1m${t("doctor.title")}\x1b[0m`];
@@ -1 +1 @@
1
- {"version":3,"file":"doctor.js","sourceRoot":"","sources":["../src/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,SAAS,EAAyB,MAAM,oBAAoB,CAAC;AACtE,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C,OAAO,EAAE,CAAC,EAAE,MAAM,aAAa,CAAC;AAChC,OAAO,EACL,uCAAuC,EACvC,qBAAqB,GAEtB,MAAM,qBAAqB,CAAC;AA0B7B,MAAM,WAAW,GAAe;IAC9B,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,QAAQ,EAAE,QAAQ,EAAE;IACpB,GAAG,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC/E,YAAY,EAAE,GAAG,CAAC,EAAE;QAClB,IAAI,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,aAAa,EAAE,iBAAiB;IAChC,aAAa,EAAE,qBAAqB;CACrC,CAAC;AAEF,SAAS,QAAQ,CAAC,IAAc;IAC9B,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;IACnC,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACnE,OAAO,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,OAAe,EAAE,IAAY;IACjD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,kBAAkB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACrG,OAAO,MAAM,CAAC,KAAK,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY,EAAE,SAAS,GAAG,IAAI;IACpE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;QAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,MAAM,GAAG,CAAC,KAAc,EAAE,EAAE;YAChC,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC,CAAC;QACF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;QACzD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAe,EACf,OAAoB,EACpB,YAAiC,EAAE;IAEnC,MAAM,IAAI,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,SAAS,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,GAAG,GAAG,CAAC,OAA+B,EAAE,MAAyB,EAAE,KAAa,EAAE,MAAc,EAAE,EAAE;QACxG,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC;IACF,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAC/C,IAAI,GAAG,GAA0C,EAAE,CAAC;IACpD,IAAI,CAAC;QACH,GAAG,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,uBAAuB,EAAG,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,CAAC;IAED,MAAM,cAAc,GAAG,GAAG,CAAC,QAAQ,EAAE,OAAO,IAAI,aAAa,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,cAAc,CAAC,CAAC,CAAC;IACrD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,CAAC;QACxD,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,cAAc,CAAC,CAAC;IACnD,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;QAC5B,2EAA2E;QAC3E,IAAI,EAAE,KAAK,MAAM;YAAE,SAAS;QAC5B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,GAAG,CAAC,eAAe,EAAE,OAAO,EAAE,CAAC,CAAC,sBAAsB,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC;YAC1F,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,eAAe,EAAE,IAAI,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,IAAI,CAAC,CAAC,yBAAyB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACrG,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,eAAe,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,uBAAuB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAG,CAAC,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC;IACrE,GAAG,CACD,eAAe,EACf,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAC9B,MAAM,EACN,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAChF,CAAC;IACF,GAAG,CACD,eAAe,EACf,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAC7B,MAAM,EACN,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,mBAAmB,CAAC,CACxC,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;IAC3C,MAAM,2BAA2B,GAAG,aAAa,CAAC,gBAAgB;WAC7D,aAAa,CAAC,gBAAgB,GAAG,uCAAuC,CAAC;IAC9E,GAAG,CACD,eAAe,EACf,2BAA2B,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAC3C,CAAC,CAAC,uBAAuB,CAAC,EAC1B,aAAa,CAAC,gBAAgB;QAC5B,CAAC,CAAC,2BAA2B;YAC3B,CAAC,CAAC,CAAC,CAAC,6BAA6B,EAAE,aAAa,CAAC,gBAAgB,EAAE,uCAAuC,CAAC;YAC3G,CAAC,CAAC,CAAC,CAAC,4BAA4B,EAAE,aAAa,CAAC,gBAAgB,CAAC;QACnE,CAAC,CAAC,CAAC,CAAC,gCAAgC,EAAE,aAAa,CAAC,gBAAgB,CAAC,CACxE,CAAC;IAEF,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,0BAA0B,CAAC,EAAE,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAClF,CAAC;SAAM,CAAC;QACN,MAAM,kBAAkB,GAAG,OAAO,CAAC,OAAO,KAAK,gBAAgB;YAC7D,CAAC,CAAC,CAAC,CAAC,iCAAiC,CAAC;YACtC,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC;QACpC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,0BAA0B,CAAC,EAAE,GAAG,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3F,GAAG,CACD,SAAS,EACT,OAAO,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EACxC,CAAC,CAAC,wBAAwB,CAAC,EAC3B,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAC1G,CAAC;QACF,GAAG,CACD,SAAS,EACT,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EACvC,CAAC,CAAC,uBAAuB,CAAC,EAC1B,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CACxH,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAC9B,GAAG,CACD,SAAS,EACT,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EACjD,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,wBAAwB;YAC/B,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;YACtB,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAC3B,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,QAAQ,IAAI,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACnE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC3E,CAAC;SAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QAC5B,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC5E,CAAC;SAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC,CAAC;IACpF,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,WAAW,GAAG,kBAAkB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAG,aAAa;QAC/B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;QAChE,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,YAAY,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC;IACrF,GAAG,CACD,OAAO,EACP,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EACnD,CAAC,CAAC,qBAAqB,CAAC,EACxB,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB,EAAE,WAAW,CAAC,CAClG,CAAC;IAEF,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC,uBAAuB,EAAE,UAAU,CAAC,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACjE,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;QACrD,IAAI,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;QACrD,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IACD,GAAG,CACD,OAAO,EACP,IAAI,EACJ,CAAC,CAAC,kBAAkB,CAAC,EACrB,CAAC,CAAC,wBAAwB,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CACzG,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,IAAI,EAAC,EAAE;QACjE,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACpE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/G,CAAC,CAAC,CAAC,CAAC;IACJ,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;IACvE,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/G,MAAM,WAAW,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,WAAW,CAAC,CAAC;IACjD,GAAG,CACD,SAAS,EACT,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EACzC,cAAc,EACd,WAAW,CAAC,MAAM,KAAK,CAAC;QACtB,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,SAAS,EAAE,YAAY,CAAC,MAAM,CAAC;QACpD,CAAC,CAAC,CAAC,CAAC,mBAAmB,EAAE,SAAS,EAAE,YAAY,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CACnF,CAAC;IAEF,OAAO;QACL,MAAM;QACN,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;QAC/D,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;KACjE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,MAAoB;IACrD,MAAM,aAAa,GAA2C;QAC5D,aAAa,EAAE,CAAC,CAAC,8BAA8B,CAAC;QAChD,OAAO,EAAE,CAAC,CAAC,wBAAwB,CAAC;QACpC,KAAK,EAAE,CAAC,CAAC,sBAAsB,CAAC;QAChC,SAAS,EAAE,CAAC,CAAC,oBAAoB,CAAC;KACnC,CAAC;IACF,MAAM,KAAK,GAAG,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAC3D,IAAI,OAA2C,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,KAAK,IAAI;YAChC,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM;gBACvB,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,kBAAkB,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,KAAK,CAAC,IAAI,CACR,EAAE,EACF,MAAM,CAAC,MAAM,KAAK,CAAC;QACjB,CAAC,CAAC,eAAe,CAAC,CAAC,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;QAC/D,CAAC,CAAC,eAAe,CAAC,CAAC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAC9E,EAAE,CACH,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
1
+ {"version":3,"file":"doctor.js","sourceRoot":"","sources":["../src/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,SAAS,EAAyB,MAAM,oBAAoB,CAAC;AACtE,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C,OAAO,EAAE,CAAC,EAAE,MAAM,aAAa,CAAC;AAChC,OAAO,EACL,uCAAuC,EACvC,qBAAqB,GAEtB,MAAM,qBAAqB,CAAC;AA6B7B,MAAM,WAAW,GAAe;IAC9B,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,QAAQ,EAAE,QAAQ,EAAE;IACpB,GAAG,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC/E,YAAY,EAAE,GAAG,CAAC,EAAE;QAClB,IAAI,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,aAAa,EAAE,iBAAiB;IAChC,aAAa,EAAE,qBAAqB;IACpC,gBAAgB,EAAE,KAAK,EAAC,IAAI,EAAC,EAAE;QAC7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,oBAAoB,IAAI,SAAS,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvG,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAS,CAAC;IACtC,CAAC;CACF,CAAC;AAEF,SAAS,QAAQ,CAAC,IAAc;IAC9B,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;IACnC,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACnE,OAAO,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,OAAe,EAAE,IAAY;IACjD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,kBAAkB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACrG,OAAO,MAAM,CAAC,KAAK,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY,EAAE,SAAS,GAAG,IAAI;IACpE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;QAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,MAAM,GAAG,CAAC,KAAc,EAAE,EAAE;YAChC,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC,CAAC;QACF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;QACzD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAe,EACf,OAAoB,EACpB,YAAiC,EAAE;IAEnC,MAAM,IAAI,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,SAAS,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,GAAG,GAAG,CAAC,OAA+B,EAAE,MAAyB,EAAE,KAAa,EAAE,MAAc,EAAE,EAAE;QACxG,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC;IACF,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAC/C,IAAI,GAAG,GAA0C,EAAE,CAAC;IACpD,IAAI,CAAC;QACH,GAAG,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,uBAAuB,EAAG,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,CAAC;IAED,MAAM,cAAc,GAAG,GAAG,CAAC,QAAQ,EAAE,OAAO,IAAI,aAAa,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,cAAc,CAAC,CAAC,CAAC;IACrD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,CAAC;QACxD,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,cAAc,CAAC,CAAC;IACnD,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;QAC5B,2EAA2E;QAC3E,IAAI,EAAE,KAAK,MAAM;YAAE,SAAS;QAC5B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,GAAG,CAAC,eAAe,EAAE,OAAO,EAAE,CAAC,CAAC,sBAAsB,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC;YAC1F,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,eAAe,EAAE,IAAI,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,IAAI,CAAC,CAAC,yBAAyB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACrG,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,eAAe,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,uBAAuB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAG,CAAC,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC;IACrE,GAAG,CACD,eAAe,EACf,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAC9B,MAAM,EACN,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAChF,CAAC;IACF,GAAG,CACD,eAAe,EACf,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAC7B,MAAM,EACN,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,mBAAmB,CAAC,CACxC,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;IAC3C,MAAM,2BAA2B,GAAG,aAAa,CAAC,gBAAgB;WAC7D,aAAa,CAAC,gBAAgB,GAAG,uCAAuC,CAAC;IAC9E,GAAG,CACD,eAAe,EACf,2BAA2B,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAC3C,CAAC,CAAC,uBAAuB,CAAC,EAC1B,aAAa,CAAC,gBAAgB;QAC5B,CAAC,CAAC,2BAA2B;YAC3B,CAAC,CAAC,CAAC,CAAC,6BAA6B,EAAE,aAAa,CAAC,gBAAgB,EAAE,uCAAuC,CAAC;YAC3G,CAAC,CAAC,CAAC,CAAC,4BAA4B,EAAE,aAAa,CAAC,gBAAgB,CAAC;QACnE,CAAC,CAAC,CAAC,CAAC,gCAAgC,EAAE,aAAa,CAAC,gBAAgB,CAAC,CACxE,CAAC;IAEF,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,0BAA0B,CAAC,EAAE,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAClF,CAAC;SAAM,CAAC;QACN,MAAM,kBAAkB,GAAG,OAAO,CAAC,OAAO,KAAK,gBAAgB;YAC7D,CAAC,CAAC,CAAC,CAAC,iCAAiC,CAAC;YACtC,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC;QACpC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,0BAA0B,CAAC,EAAE,GAAG,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3F,GAAG,CACD,SAAS,EACT,OAAO,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EACxC,CAAC,CAAC,wBAAwB,CAAC,EAC3B,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAC1G,CAAC;QACF,GAAG,CACD,SAAS,EACT,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EACvC,CAAC,CAAC,uBAAuB,CAAC,EAC1B,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CACxH,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAC9B,GAAG,CACD,SAAS,EACT,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EACjD,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,wBAAwB;YAC/B,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;YACtB,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAC3B,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,QAAQ,IAAI,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACnE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC3E,CAAC;SAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QAC5B,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC5E,CAAC;SAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC,CAAC;IACpF,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,WAAW,GAAG,kBAAkB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAG,aAAa;QAC/B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;QAChE,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,YAAY,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC;IACrF,GAAG,CACD,OAAO,EACP,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EACnD,CAAC,CAAC,qBAAqB,CAAC,EACxB,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB,EAAE,WAAW,CAAC,CAClG,CAAC;IAEF,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC,uBAAuB,EAAE,UAAU,CAAC,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACjE,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;QACrD,IAAI,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;QACrD,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IACD,GAAG,CACD,OAAO,EACP,IAAI,EACJ,CAAC,CAAC,kBAAkB,CAAC,EACrB,CAAC,CAAC,wBAAwB,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CACzG,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAE,GAAW,CAAC,QAAQ,CAAC;QACtD,CAAC,CAAE,GAAW,CAAC,QAAQ;QACvB,CAAC,CAAE,GAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,GAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,OAAY,EAAE,EAAE,CAAC,OAAO,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC;IACnF,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,6EAA6E;IAC/E,CAAC;SAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvB,GAAG,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAC3F,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC;YACjF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,EAAE,OAAO,IAAI,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACxC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,GAAG,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC;YAC9F,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;oBACnC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;yBAC/B,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC;yBAClC,MAAM,CAAC,CAAC,GAAG,EAAiB,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC;oBAC3D,MAAM,SAAS,GAAG,OAAO,MAAM,CAAC,cAAc,KAAK,QAAQ;wBACzD,CAAC,CAAC,MAAM,CAAC,cAAc;wBACvB,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC;oBACzE,GAAG,CACD,kBAAkB,EAClB,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EACxB,EAAE,EACF,CAAC,CAAC,uBAAuB,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,MAAM,CAAC,cAAc,IAAI,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CACjJ,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC,4BAA4B,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC,CAAC;QACvH,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,IAAI,EAAC,EAAE;QACjE,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACpE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/G,CAAC,CAAC,CAAC,CAAC;IACJ,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;IACvE,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/G,MAAM,WAAW,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,WAAW,CAAC,CAAC;IACjD,GAAG,CACD,SAAS,EACT,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EACzC,cAAc,EACd,WAAW,CAAC,MAAM,KAAK,CAAC;QACtB,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,SAAS,EAAE,YAAY,CAAC,MAAM,CAAC;QACpD,CAAC,CAAC,CAAC,CAAC,mBAAmB,EAAE,SAAS,EAAE,YAAY,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CACnF,CAAC;IAEF,OAAO;QACL,MAAM;QACN,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;QAC/D,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;KACjE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,MAAoB;IACrD,MAAM,aAAa,GAA2C;QAC5D,aAAa,EAAE,CAAC,CAAC,8BAA8B,CAAC;QAChD,OAAO,EAAE,CAAC,CAAC,wBAAwB,CAAC;QACpC,KAAK,EAAE,CAAC,CAAC,sBAAsB,CAAC;QAChC,kBAAkB,EAAE,CAAC,CAAC,yBAAyB,CAAC;QAChD,SAAS,EAAE,CAAC,CAAC,oBAAoB,CAAC;KACnC,CAAC;IACF,MAAM,KAAK,GAAG,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAC3D,IAAI,OAA2C,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,KAAK,IAAI;YAChC,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM;gBACvB,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,kBAAkB,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,KAAK,CAAC,IAAI,CACR,EAAE,EACF,MAAM,CAAC,MAAM,KAAK,CAAC;QACjB,CAAC,CAAC,eAAe,CAAC,CAAC,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;QAC/D,CAAC,CAAC,eAAe,CAAC,CAAC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAC9E,EAAE,CACH,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
@@ -6,7 +6,7 @@ import { EventLog } from "./event-log.js";
6
6
  import { AdapterWorld } from "./adapter-world.js";
7
7
  import { CostGuard } from "./cost-guard.js";
8
8
  import { IpcClient } from "./channel/ipc-bridge.js";
9
- import type { ChannelAdapter } from "./channel/types.js";
9
+ import type { AdapterHealthSnapshot, ChannelAdapter } from "./channel/types.js";
10
10
  import { type Logger } from "./logger.js";
11
11
  import { Scheduler } from "./scheduler/index.js";
12
12
  import type { FleetContext } from "./fleet-context.js";
@@ -20,6 +20,8 @@ import { type RawConfigPatch } from "./settings-api.js";
20
20
  import { type AgentEndpointContext } from "./agent-endpoint.js";
21
21
  import { ClassicChannelManager } from "./classic-channel-manager.js";
22
22
  import type { InstanceState } from "./backend/types.js";
23
+ import { StormWindow } from "./storm-window.js";
24
+ import { SpawnGate } from "./spawn-gate.js";
23
25
  export declare function resolveReplyThreadId(argsThreadId: unknown, instanceConfig?: InstanceConfig): string | undefined;
24
26
  /**
25
27
  * Pure warm-cap victim selection (extracted for testability). Given the current
@@ -69,6 +71,8 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
69
71
  private static sighupHandlerInstalled;
70
72
  private children;
71
73
  readonly lifecycle: InstanceLifecycle;
74
+ readonly stormWindow: StormWindow;
75
+ readonly spawnGate: SpawnGate;
72
76
  /** Live view of lifecycle.daemons — used throughout; not deprecated. */
73
77
  get daemons(): Map<string, import("./daemon.js").Daemon>;
74
78
  fleetConfig: FleetConfig | null;
@@ -176,6 +180,7 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
176
180
  private static readonly EVENT_LOG_RETENTION_DAYS;
177
181
  private watchdogTimer;
178
182
  private startedAt;
183
+ private stormOpenNotifyTimer;
179
184
  private mirrorBuffer;
180
185
  private mirrorTimer;
181
186
  private sseClients;
@@ -183,6 +188,11 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
183
188
  private viewToken;
184
189
  private healthServerListening;
185
190
  constructor(dataDir: string);
191
+ private spawnConcurrency;
192
+ /** Wire the one fleet-wide storm into notification and recovery surfaces. */
193
+ private bindStormWindowEvents;
194
+ private formatStormDelay;
195
+ stormSuppressed(kind: string): boolean;
186
196
  private handleSighup;
187
197
  private scheduleReconcile;
188
198
  /**
@@ -315,6 +325,7 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
315
325
  */
316
326
  private refreshInstanceExecutionState;
317
327
  private deliverWithIdleGate;
328
+ private holdDeliveryForStorm;
318
329
  /**
319
330
  * Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
320
331
  *
@@ -408,6 +419,7 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
408
419
  retryCount: number;
409
420
  lastError?: string;
410
421
  }>;
422
+ private bindAdapterHealth;
411
423
  /**
412
424
  * Real, checkable fleet health for `/health` and the operator.
413
425
  *
@@ -436,6 +448,7 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
436
448
  total: number;
437
449
  connected: number;
438
450
  states: Record<string, string>;
451
+ details: Record<string, AdapterHealthSnapshot>;
439
452
  };
440
453
  startupComplete: boolean;
441
454
  /** See process-memory.ts: the fleet process and the whole service cgroup are
@@ -1,6 +1,7 @@
1
1
  import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
2
2
  import { randomBytes } from "node:crypto";
3
3
  import { spawnSync } from "node:child_process";
4
+ import { freemem } from "node:os";
4
5
  import { createServer } from "node:http";
5
6
  import { join, dirname, basename } from "node:path";
6
7
  import { fileURLToPath } from "node:url";
@@ -59,6 +60,8 @@ import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
59
60
  import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
60
61
  import { RestartProgress } from "./restart-progress.js";
61
62
  import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
63
+ import { StormWindow } from "./storm-window.js";
64
+ import { SpawnGate } from "./spawn-gate.js";
62
65
  import { canUnlockAdvancedTips, DailyTipScheduler, selectTip, visibleTipLevels, } from "./tips.js";
63
66
  import { getTmuxSession } from "./config.js";
64
67
  export function resolveReplyThreadId(argsThreadId, instanceConfig) {
@@ -227,6 +230,8 @@ export class FleetManager {
227
230
  static sighupHandlerInstalled = false;
228
231
  children = new Map();
229
232
  lifecycle;
233
+ stormWindow;
234
+ spawnGate;
230
235
  /** Live view of lifecycle.daemons — used throughout; not deprecated. */
231
236
  get daemons() { return this.lifecycle.daemons; }
232
237
  fleetConfig = null;
@@ -351,6 +356,7 @@ export class FleetManager {
351
356
  static EVENT_LOG_RETENTION_DAYS = 30;
352
357
  watchdogTimer = null;
353
358
  startedAt = 0;
359
+ stormOpenNotifyTimer = null;
354
360
  // Mirror topic: buffer cross-instance messages, flush every 3s
355
361
  mirrorBuffer = [];
356
362
  mirrorTimer = null;
@@ -366,11 +372,63 @@ export class FleetManager {
366
372
  process.on("SIGHUP", () => FleetManager.signalTarget?.handleSighup());
367
373
  FleetManager.sighupHandlerInstalled = true;
368
374
  }
375
+ this.stormWindow = new StormWindow();
376
+ this.spawnGate = new SpawnGate({
377
+ storm: this.stormWindow,
378
+ concurrency: () => this.spawnConcurrency(),
379
+ staggerMs: () => this.fleetConfig?.defaults?.startup?.stagger_delay_ms ?? 500,
380
+ });
381
+ this.bindStormWindowEvents();
369
382
  this.lifecycle = new InstanceLifecycle(this);
370
383
  this.topicCommands = new TopicCommands(this);
371
384
  this.topicArchiver = new TopicArchiver(this);
372
385
  this.statuslineWatcher = new StatuslineWatcher(this);
373
386
  }
387
+ spawnConcurrency() {
388
+ const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
389
+ if (explicit != null)
390
+ return Math.max(1, Math.min(20, explicit));
391
+ const freeMemMB = Math.round(freemem() / (1024 * 1024));
392
+ return Math.max(2, Math.min(10, Math.floor(freeMemMB / 300)));
393
+ }
394
+ /** Wire the one fleet-wide storm into notification and recovery surfaces. */
395
+ bindStormWindowEvents() {
396
+ this.stormWindow.on("opened", (snapshot) => {
397
+ for (const [name, daemon] of this.daemons) {
398
+ if (!daemon.isPaused)
399
+ this.stormWindow.addAffected(name);
400
+ }
401
+ this.logger.error({ ...snapshot }, "tmux server storm opened — holding respawns and delivery");
402
+ if (this.stormOpenNotifyTimer)
403
+ clearTimeout(this.stormOpenNotifyTimer);
404
+ this.stormOpenNotifyTimer = setTimeout(() => {
405
+ this.stormOpenNotifyTimer = null;
406
+ const current = this.stormWindow.snapshot();
407
+ this.notifyFleetError(t("storm.opened", current.affected.length, this.formatStormDelay(current.backoffMs)));
408
+ }, 1_000);
409
+ this.stormOpenNotifyTimer.unref?.();
410
+ });
411
+ this.stormWindow.on("extended", (snapshot) => {
412
+ this.logger.error({ ...snapshot }, "tmux server storm repeated — backoff extended");
413
+ this.notifyFleetError(t("storm.extended", snapshot.crashCount, this.formatStormDelay(snapshot.backoffMs)));
414
+ });
415
+ this.stormWindow.on("recovery_due", (snapshot) => {
416
+ this.logger.warn({ ...snapshot }, "tmux storm backoff elapsed — rolling recovery started");
417
+ });
418
+ this.stormWindow.on("closed", (snapshot, reason) => {
419
+ const unresolved = snapshot.affected.filter(name => !snapshot.recovered.includes(name));
420
+ this.logger.info({ ...snapshot, reason, unresolved }, "tmux server storm closed");
421
+ this.notifyFleetError(t(reason === "timeout" ? "storm.timed_out" : "storm.recovered", snapshot.recovered.length, snapshot.affected.length, unresolved.length > 0 ? unresolved.join(", ") : t("storm.none")));
422
+ });
423
+ }
424
+ formatStormDelay(ms) {
425
+ if (ms >= 60_000)
426
+ return `${Math.round(ms / 60_000)}m`;
427
+ return `${Math.round(ms / 1_000)}s`;
428
+ }
429
+ stormSuppressed(kind) {
430
+ return this.stormWindow.shouldSuppress(kind);
431
+ }
374
432
  handleSighup() {
375
433
  this.logger.info("Received SIGHUP, hot-reloading config...");
376
434
  if (!this.startupComplete) {
@@ -1077,7 +1135,10 @@ export class FleetManager {
1077
1135
  });
1078
1136
  }
1079
1137
  async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch) {
1080
- if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1138
+ if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1139
+ return;
1140
+ await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
1141
+ if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1081
1142
  return;
1082
1143
  let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
1083
1144
  if (this.lifecycle.isPaused(instanceName)) {
@@ -1096,6 +1157,11 @@ export class FleetManager {
1096
1157
  this.logger.info({ instanceName }, "Pending delivery dropped by user cancel");
1097
1158
  return;
1098
1159
  }
1160
+ // A server crash can land while waitForInstanceIdle is pending. Re-check
1161
+ // immediately before the old timeout path would force text into a boot UI.
1162
+ await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
1163
+ if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1164
+ return;
1099
1165
  if (!idle) {
1100
1166
  this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
1101
1167
  }
@@ -1104,6 +1170,12 @@ export class FleetManager {
1104
1170
  return;
1105
1171
  this.lastDeliveryAt.set(instanceName, Date.now());
1106
1172
  }
1173
+ async holdDeliveryForStorm(instanceName, deliveryEpoch) {
1174
+ if (!this.stormWindow.isDeliveryHeld(instanceName))
1175
+ return;
1176
+ this.logger.warn({ instanceName, deliveryEpoch }, "Delivery held during tmux server recovery");
1177
+ await this.stormWindow.waitForDeliveryAllowed(instanceName);
1178
+ }
1107
1179
  /**
1108
1180
  * Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
1109
1181
  *
@@ -1347,86 +1419,27 @@ export class FleetManager {
1347
1419
  }
1348
1420
  if (runnableEntries.length === 0)
1349
1421
  return;
1350
- const raw = this.fleetConfig?.defaults?.startup;
1351
- const explicitConcurrency = raw?.concurrency;
1352
- const staggerMs = Math.max(0, Math.min(30_000, raw?.stagger_delay_ms ?? 500));
1353
- // Adaptive concurrency: if not explicitly set, estimate from available RAM.
1354
- // Each instance uses ~300MB (tmux + CLI process + model overhead).
1355
- const ESTIMATED_MB_PER_INSTANCE = 300;
1356
- const { freemem } = await import("node:os");
1357
- let concurrency;
1358
- if (explicitConcurrency != null) {
1359
- concurrency = Math.max(1, Math.min(20, explicitConcurrency));
1360
- }
1361
- else {
1362
- const freeMemMB = Math.round(freemem() / (1024 * 1024));
1363
- concurrency = Math.max(2, Math.min(10, Math.floor(freeMemMB / ESTIMATED_MB_PER_INSTANCE)));
1364
- this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: runnableEntries.length }, "Adaptive startup concurrency");
1365
- }
1366
- const byWorkDir = new Map();
1367
- for (const [name, config] of runnableEntries) {
1368
- const dir = config.working_directory;
1369
- if (!byWorkDir.has(dir))
1370
- byWorkDir.set(dir, []);
1371
- byWorkDir.get(dir).push([name, config]);
1372
- }
1373
- const groups = [...byWorkDir.values()];
1374
- let running = 0;
1375
- let idx = 0;
1376
- let lastStartAt = 0;
1377
- let pendingTimer = false;
1378
- await new Promise((resolve) => {
1379
- if (groups.length === 0) {
1380
- resolve();
1381
- return;
1422
+ if (this.fleetConfig?.defaults?.startup?.concurrency == null) {
1423
+ this.logger.info({
1424
+ concurrency: this.spawnConcurrency(),
1425
+ freeMemMB: Math.round(freemem() / (1024 * 1024)),
1426
+ totalInstances: runnableEntries.length,
1427
+ }, "Adaptive startup concurrency");
1428
+ }
1429
+ await Promise.all(runnableEntries.map(([name, config]) => this.spawnGate.run({
1430
+ instanceName: name,
1431
+ workingDirectory: config.working_directory,
1432
+ reason: "startup",
1433
+ }, async () => {
1434
+ try {
1435
+ await this.startInstance(name, config, topicMode);
1436
+ if (this.daemons.has(name))
1437
+ onReady?.(name);
1382
1438
  }
1383
- const startNext = () => {
1384
- if (pendingTimer)
1385
- return;
1386
- while (running < concurrency && idx < groups.length) {
1387
- // Re-check memory if adaptive (no explicit concurrency set)
1388
- if (explicitConcurrency == null && running > 0) {
1389
- const nowFreeMB = Math.round(freemem() / (1024 * 1024));
1390
- if (nowFreeMB < ESTIMATED_MB_PER_INSTANCE) {
1391
- this.logger.warn({ freeMemMB: nowFreeMB, remaining: groups.length - idx }, "Low memory — pausing instance startup");
1392
- // Wait and retry in 5s
1393
- pendingTimer = true;
1394
- setTimeout(() => { pendingTimer = false; startNext(); }, 5000);
1395
- return;
1396
- }
1397
- }
1398
- const now = Date.now();
1399
- const elapsed = now - lastStartAt;
1400
- if (lastStartAt > 0 && elapsed < staggerMs) {
1401
- pendingTimer = true;
1402
- setTimeout(() => { pendingTimer = false; startNext(); }, staggerMs - elapsed);
1403
- return;
1404
- }
1405
- const group = groups[idx++];
1406
- running++;
1407
- lastStartAt = Date.now();
1408
- (async () => {
1409
- for (const [name, config] of group) {
1410
- try {
1411
- await this.startInstance(name, config, topicMode);
1412
- if (this.daemons.has(name))
1413
- onReady?.(name);
1414
- }
1415
- catch (err) {
1416
- this.logger.error({ err, name }, "Failed to start instance");
1417
- }
1418
- }
1419
- })().finally(() => {
1420
- running--;
1421
- if (idx >= groups.length && running === 0)
1422
- resolve();
1423
- else
1424
- startNext();
1425
- });
1426
- }
1427
- };
1428
- startNext();
1429
- });
1439
+ catch (err) {
1440
+ this.logger.error({ err, name }, "Failed to start instance");
1441
+ }
1442
+ })));
1430
1443
  }
1431
1444
  runnableStartupCount(fleet, includeClassic) {
1432
1445
  const names = this.configuredStartupInstanceNames(fleet, includeClassic);
@@ -1490,7 +1503,12 @@ export class FleetManager {
1490
1503
  this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
1491
1504
  return inFlight;
1492
1505
  }
1493
- const run = this.doRestartSingleInstance(name, opts)
1506
+ const workingDirectory = this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name);
1507
+ const run = this.spawnGate.run({
1508
+ instanceName: name,
1509
+ workingDirectory,
1510
+ reason: "restart",
1511
+ }, () => this.doRestartSingleInstance(name, opts))
1494
1512
  .finally(() => this.restartsInFlight.delete(name));
1495
1513
  this.restartsInFlight.set(name, run);
1496
1514
  return run;
@@ -1508,6 +1526,7 @@ export class FleetManager {
1508
1526
  this.writeFreshStartMarker(name);
1509
1527
  const topicMode = this.fleetConfig?.channel?.mode === "topic";
1510
1528
  await this.startInstance(name, config, topicMode ?? false);
1529
+ this.stormWindow.markRecovered(name);
1511
1530
  return;
1512
1531
  }
1513
1532
  // Classic instance fallback
@@ -1520,6 +1539,7 @@ export class FleetManager {
1520
1539
  if (opts?.freshStart)
1521
1540
  this.writeFreshStartMarker(name);
1522
1541
  await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
1542
+ this.stormWindow.markRecovered(name);
1523
1543
  return;
1524
1544
  }
1525
1545
  throw new Error(`Instance not found: ${name}`);
@@ -2281,6 +2301,19 @@ export class FleetManager {
2281
2301
  getAdapterStates() {
2282
2302
  return this.adapterState;
2283
2303
  }
2304
+ bindAdapterHealth(adapter, adapterId) {
2305
+ adapter.on("gateway_health", (snapshot) => {
2306
+ const previous = this.adapterState.get(adapterId);
2307
+ const status = snapshot.status === "connected" ? "connected"
2308
+ : snapshot.status === "stopped" ? "failed"
2309
+ : "retrying";
2310
+ this.adapterState.set(adapterId, {
2311
+ status,
2312
+ retryCount: previous?.retryCount ?? 0,
2313
+ lastError: status === "connected" ? undefined : snapshot.lastReconnectReason ?? previous?.lastError,
2314
+ });
2315
+ });
2316
+ }
2284
2317
  /**
2285
2318
  * Real, checkable fleet health for `/health` and the operator.
2286
2319
  *
@@ -2310,10 +2343,17 @@ export class FleetManager {
2310
2343
  counts.stopped++;
2311
2344
  }
2312
2345
  const states = {};
2346
+ const details = {};
2313
2347
  let connected = 0;
2314
2348
  for (const [id, state] of this.adapterState) {
2315
- states[id] = state.status;
2316
- if (state.status === "connected")
2349
+ const snapshot = this.adapters.get(id)?.getHealthSnapshot?.();
2350
+ if (snapshot)
2351
+ details[id] = snapshot;
2352
+ const effectiveStatus = snapshot?.status === "connected" ? "connected"
2353
+ : snapshot && snapshot.status !== "stopped" ? "retrying"
2354
+ : state.status;
2355
+ states[id] = effectiveStatus;
2356
+ if (effectiveStatus === "connected")
2317
2357
  connected++;
2318
2358
  }
2319
2359
  const problems = [];
@@ -2321,9 +2361,9 @@ export class FleetManager {
2321
2361
  problems.push("no channel adapter is connected");
2322
2362
  if (counts.crashed > 0)
2323
2363
  problems.push(`${counts.crashed} instance(s) crashed`);
2324
- for (const [id, state] of this.adapterState) {
2325
- if (state.status !== "connected")
2326
- problems.push(`adapter ${id} is ${state.status}`);
2364
+ for (const [id, state] of Object.entries(states)) {
2365
+ if (state !== "connected")
2366
+ problems.push(`adapter ${id} is ${state}`);
2327
2367
  }
2328
2368
  if (!this.startupComplete)
2329
2369
  problems.push("startup has not completed");
@@ -2336,7 +2376,7 @@ export class FleetManager {
2336
2376
  status,
2337
2377
  uptime: Math.floor((Date.now() - this.startedAt) / 1000),
2338
2378
  instances: counts,
2339
- adapters: { total: this.adapterState.size, connected, states },
2379
+ adapters: { total: this.adapterState.size, connected, states, details },
2340
2380
  startupComplete: this.startupComplete,
2341
2381
  memory: readFleetMemory(),
2342
2382
  problems,
@@ -2365,6 +2405,7 @@ export class FleetManager {
2365
2405
  const world = new AdapterWorld(adapterId, this.adapter, accessManager, channelConfig);
2366
2406
  this.worlds.set(adapterId, world);
2367
2407
  this.adapters.set(adapterId, this.adapter);
2408
+ this.bindAdapterHealth(this.adapter, adapterId);
2368
2409
  this.adapter.on("message", safeHandler(async (msg) => {
2369
2410
  await this.handleInboundMessage(msg);
2370
2411
  }, this.logger, "adapter.message"));
@@ -2715,7 +2756,7 @@ export class FleetManager {
2715
2756
  }, this.logger, "adapter.handler_error"));
2716
2757
  this.adapter.on("error", (err) => {
2717
2758
  this.logger.error({ err }, "Primary adapter fatal error");
2718
- this.restartAdapter(this.adapter, "primary").catch(() => { });
2759
+ this.restartAdapter(this.adapter, adapterId).catch(() => { });
2719
2760
  });
2720
2761
  this.adapter.on("new_group_detected", safeHandler((data) => {
2721
2762
  const adminMsg = t("alert.bot_added", data.groupTitle, data.groupId, data.source);
@@ -2757,6 +2798,7 @@ export class FleetManager {
2757
2798
  const world = new AdapterWorld(adapterId, adapter, accessManager, channelConfig);
2758
2799
  this.worlds.set(adapterId, world);
2759
2800
  this.adapters.set(adapterId, adapter);
2801
+ this.bindAdapterHealth(adapter, adapterId);
2760
2802
  // Wire up event handlers (same as primary, routes through shared handleInboundMessage)
2761
2803
  adapter.on("message", safeHandler(async (msg) => {
2762
2804
  await this.handleInboundMessage(msg);
@@ -3030,10 +3072,6 @@ export class FleetManager {
3030
3072
  await this.handleInstallCliSlash(data, adapterId, adapter);
3031
3073
  }
3032
3074
  }, this.logger, `adapter[${adapterId}].slash_command`));
3033
- await adapter.start();
3034
- if (channelConfig.group_id) {
3035
- adapter.setChatId(String(channelConfig.group_id));
3036
- }
3037
3075
  adapter.on("started", safeHandler((username, userId) => {
3038
3076
  this.logger.info(`[${adapterId}] Bot @${username} polling started.`);
3039
3077
  const world = this.worlds.get(adapterId);
@@ -3053,6 +3091,11 @@ export class FleetManager {
3053
3091
  this.logger.error({ err, adapterId }, "Additional adapter fatal error");
3054
3092
  this.restartAdapter(adapter, adapterId).catch(() => { });
3055
3093
  });
3094
+ // Register lifecycle listeners before login; a fast ready/error must not be lost.
3095
+ await adapter.start();
3096
+ if (channelConfig.group_id) {
3097
+ adapter.setChatId(String(channelConfig.group_id));
3098
+ }
3056
3099
  this.logger.info({ adapterId, type: channelConfig.type }, "Additional adapter started");
3057
3100
  }
3058
3101
  /** Connect IPC to a single instance with all handlers */
@@ -3295,6 +3338,26 @@ export class FleetManager {
3295
3338
  const previous = this.adapterState.get(id);
3296
3339
  this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
3297
3340
  try {
3341
+ if (adapter.reconnectGateway) {
3342
+ try {
3343
+ // Discord requires a fresh Client after destroy(). Its adapter owns the
3344
+ // single-flight, generation fence and bounded IDENTIFY backoff, so all
3345
+ // watchdog/manual/error triggers must converge here instead of stop/start.
3346
+ await adapter.reconnectGateway(previous?.lastError ?? "fleet adapter restart");
3347
+ this.adapterState.set(id, { status: "connected", retryCount: 0 });
3348
+ this.logger.info({ id }, "Adapter gateway rebuilt successfully");
3349
+ }
3350
+ catch (err) {
3351
+ if (!this.ipcStoppingInstances.has("__fleet_stopping__")) {
3352
+ this.adapterState.set(id, {
3353
+ status: "failed",
3354
+ retryCount: previous?.retryCount ?? 0,
3355
+ lastError: err?.message ?? String(err),
3356
+ });
3357
+ }
3358
+ }
3359
+ return;
3360
+ }
3298
3361
  for (let attempt = 1;; attempt++) {
3299
3362
  if (this.ipcStoppingInstances.has("__fleet_stopping__"))
3300
3363
  return;
@@ -8529,6 +8592,15 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
8529
8592
  // await in the same tick as the signal handler, so no event can slip in.
8530
8593
  this.shuttingDown = true;
8531
8594
  this.ipcStoppingInstances.add("__fleet_stopping__");
8595
+ // Release held delivery promises before awaiting daemon shutdown, then
8596
+ // reject spawn work which has not started. Otherwise a storm backoff could
8597
+ // make `agend stop` wait forever for its own queue.
8598
+ this.stormWindow.shutdown();
8599
+ this.spawnGate.shutdown();
8600
+ if (this.stormOpenNotifyTimer) {
8601
+ clearTimeout(this.stormOpenNotifyTimer);
8602
+ this.stormOpenNotifyTimer = null;
8603
+ }
8532
8604
  sdNotifyBlocking("STOPPING=1");
8533
8605
  if (this.watchdogTimer) {
8534
8606
  clearInterval(this.watchdogTimer);