@songsid/agend 2.1.7-beta.3 → 2.1.7-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/agent-endpoint.d.ts +2 -0
  2. package/dist/agent-endpoint.js +47 -5
  3. package/dist/agent-endpoint.js.map +1 -1
  4. package/dist/backend/codex.d.ts +18 -0
  5. package/dist/backend/codex.js +68 -3
  6. package/dist/backend/codex.js.map +1 -1
  7. package/dist/backend/types.d.ts +9 -0
  8. package/dist/backend/types.js.map +1 -1
  9. package/dist/channel/mcp-operation-response.d.ts +3 -0
  10. package/dist/channel/mcp-operation-response.js +21 -0
  11. package/dist/channel/mcp-operation-response.js.map +1 -0
  12. package/dist/channel/mcp-operation-response.test.d.ts +1 -0
  13. package/dist/channel/mcp-operation-response.test.js +23 -0
  14. package/dist/channel/mcp-operation-response.test.js.map +1 -0
  15. package/dist/channel/mcp-server.js +20 -5
  16. package/dist/channel/mcp-server.js.map +1 -1
  17. package/dist/daemon.d.ts +23 -0
  18. package/dist/daemon.js +248 -24
  19. package/dist/daemon.js.map +1 -1
  20. package/dist/delivery-outbox.d.ts +90 -0
  21. package/dist/delivery-outbox.js +562 -0
  22. package/dist/delivery-outbox.js.map +1 -0
  23. package/dist/delivery-outbox.test.d.ts +1 -0
  24. package/dist/delivery-outbox.test.js +343 -0
  25. package/dist/delivery-outbox.test.js.map +1 -0
  26. package/dist/fleet-manager.d.ts +38 -1
  27. package/dist/fleet-manager.js +302 -36
  28. package/dist/fleet-manager.js.map +1 -1
  29. package/dist/instance-lifecycle.d.ts +5 -0
  30. package/dist/instance-lifecycle.js +38 -0
  31. package/dist/instance-lifecycle.js.map +1 -1
  32. package/dist/instructions.js +2 -0
  33. package/dist/instructions.js.map +1 -1
  34. package/dist/outbound-handlers.d.ts +22 -1
  35. package/dist/outbound-handlers.js +81 -7
  36. package/dist/outbound-handlers.js.map +1 -1
  37. package/dist/remote-credential-check.d.ts +39 -0
  38. package/dist/remote-credential-check.js +81 -0
  39. package/dist/remote-credential-check.js.map +1 -0
  40. package/dist/usage/providers.d.ts +11 -6
  41. package/dist/usage/providers.js +17 -10
  42. package/dist/usage/providers.js.map +1 -1
  43. package/package.json +1 -1
@@ -1,5 +1,5 @@
1
1
  import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
2
- import { randomBytes } from "node:crypto";
2
+ import { randomBytes, randomUUID } from "node:crypto";
3
3
  import { spawnSync } from "node:child_process";
4
4
  import { freemem, totalmem, cpus } from "node:os";
5
5
  import { createServer } from "node:http";
@@ -54,6 +54,12 @@ import { InstanceLifecycle, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } f
54
54
  import { TopicArchiver } from "./topic-archiver.js";
55
55
  import { StatuslineWatcher } from "./statusline-watcher.js";
56
56
  import { outboundHandlers } from "./outbound-handlers.js";
57
+ import { DeliveryOutbox } from "./delivery-outbox.js";
58
+ // The target Daemon can legitimately wait 30 minutes for a busy pane before
59
+ // deciding to defer before submission. This is an alert threshold only: the
60
+ // active manager/target generation pair owns the lane until the daemon reports
61
+ // a state transition or that generation is replaced.
62
+ export const DURABLE_DELIVERY_LANE_ALERT_MS = 35 * 60_000;
57
63
  import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
58
64
  import { handleViewRequest, isViewPath } from "./view-api.js";
59
65
  import { filterUsageProviders, formatDiscordUsageActivity, getUsageSnapshot, handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
@@ -290,6 +296,14 @@ export class FleetManager {
290
296
  spawnGate;
291
297
  /** Fleet-level backend reachability memory (fed by pty_error / startup panes). */
292
298
  backendOutage = new BackendOutageTracker();
299
+ /** Stable for this FleetManager OS-process lifetime; Daemon objects have their own boot IDs. */
300
+ managerBootId = randomUUID();
301
+ deliveryOutbox = null;
302
+ deliveryOutboxRecovered = false;
303
+ deliveryPumpScheduled = false;
304
+ deliveryPumpRunning = false;
305
+ deliveryPumpTimer = null;
306
+ activeDurableTargets = new Set();
293
307
  /** Live view of lifecycle.daemons — used throughout; not deprecated. */
294
308
  get daemons() { return this.lifecycle.daemons; }
295
309
  fleetConfig = null;
@@ -558,6 +572,226 @@ export class FleetManager {
558
572
  this.topicArchiver = new TopicArchiver(this);
559
573
  this.statuslineWatcher = new StatuslineWatcher(this);
560
574
  }
575
+ ensureDeliveryOutbox() {
576
+ if (this.deliveryOutbox)
577
+ return;
578
+ const outbox = new DeliveryOutbox(join(this.dataDir, "delivery-outbox.db"), this.managerBootId);
579
+ this.deliveryOutbox = outbox;
580
+ if (!this.deliveryOutboxRecovered) {
581
+ const recovered = outbox.recoverForBoot(this.managerBootId);
582
+ this.logger.info({ ...recovered }, "Recovered durable delivery outbox for this process boot");
583
+ if (recovered.uncertain > 0) {
584
+ this.notifyFleetError(`${recovered.uncertain} durable delivery outcome(s) became uncertain during process restart; inspect the sender's delivery notice before retrying.`);
585
+ }
586
+ this.deliveryOutboxRecovered = true;
587
+ }
588
+ outbox.on("admitted", () => this.scheduleDeliveryOutboxPump());
589
+ outbox.on("state", (event) => {
590
+ this.scheduleDeliveryOutboxPump();
591
+ if (event.deliveryId && (event.state === "failed" || event.state === "uncertain")) {
592
+ const row = outbox.get(event.deliveryId);
593
+ if (row && row.kind !== "delivery_outcome_notice" && row.kind !== "post_restart_outcome_notice") {
594
+ const notice = `Durable delivery ${row.state}: operation_id=${row.operationId}; target=${row.targetInstance}; delivery_id=${row.deliveryId}.`;
595
+ this.logger.error({ deliveryId: row.deliveryId, operationId: row.operationId, target: row.targetInstance, state: row.state }, notice);
596
+ this.notifyFleetError(notice);
597
+ }
598
+ }
599
+ });
600
+ outbox.on("generation_recovered", () => this.scheduleDeliveryOutboxPump());
601
+ outbox.on("expired", (event) => {
602
+ this.scheduleDeliveryOutboxPump();
603
+ if (event.count)
604
+ this.notifyFleetError(`${event.count} durable delivery row(s) expired before their target became available; sender outcome notice queued.`);
605
+ });
606
+ this.scheduleDeliveryOutboxPump();
607
+ }
608
+ /** Synchronous SQLite admission; the caller may acknowledge only after this returns. */
609
+ admitDurableDelivery(input) {
610
+ const outbox = this.deliveryOutbox;
611
+ if (!outbox)
612
+ throw new Error("Durable delivery store is unavailable");
613
+ const source = this.daemons.get(input.sourceInstance);
614
+ const sourceDaemonBootId = input.sourceDaemonBootId ?? source?.bootId;
615
+ if (!source || !sourceDaemonBootId || source.bootId !== sourceDaemonBootId) {
616
+ throw new Error("Source daemon generation is no longer current; outcome is unknown");
617
+ }
618
+ const sourceKey = `mcp:${input.sourceInstance}:${input.operationId}:${input.targetInstance}:${input.targetSession ?? ""}:${input.kind}`;
619
+ const admitted = outbox.admit({ ...input, sourceDaemonBootId, sourceKey });
620
+ if (admitted.inserted)
621
+ this.logger.info({
622
+ deliveryId: admitted.delivery.deliveryId,
623
+ operationId: input.operationId,
624
+ source: input.sourceInstance,
625
+ target: input.targetInstance,
626
+ kind: input.kind,
627
+ }, "Durable cross-instance delivery accepted");
628
+ this.scheduleDeliveryOutboxPump();
629
+ return {
630
+ deliveryId: admitted.delivery.deliveryId,
631
+ state: admitted.delivery.state,
632
+ duplicate: !admitted.inserted,
633
+ };
634
+ }
635
+ getDaemonBootId(instanceName) {
636
+ return this.daemons.get(instanceName)?.bootId;
637
+ }
638
+ markDurableResponseDelivered(sourceInstance, operationId) {
639
+ this.deliveryOutbox?.markResponseDelivered(sourceInstance, operationId);
640
+ }
641
+ /** Called after a replacement Daemon object is installed in lifecycle.daemons. */
642
+ onDaemonReady(name, daemonBootId) {
643
+ const outbox = this.deliveryOutbox;
644
+ if (!outbox)
645
+ return;
646
+ for (const prior of outbox.getUnansweredAccepted(name, daemonBootId)) {
647
+ try {
648
+ outbox.admitPostRestartOutcomeNotice(prior, daemonBootId);
649
+ }
650
+ catch (err) {
651
+ this.logger.error({ err, source: name, operationId: prior.operationId }, "Could not persist post-restart delivery outcome notice");
652
+ }
653
+ }
654
+ this.scheduleDeliveryOutboxPump();
655
+ }
656
+ scheduleDeliveryOutboxPump(delayMs = 0) {
657
+ if (!this.deliveryOutbox || this.shuttingDown)
658
+ return;
659
+ if (this.deliveryPumpTimer) {
660
+ if (delayMs > 0)
661
+ return;
662
+ clearTimeout(this.deliveryPumpTimer);
663
+ this.deliveryPumpTimer = null;
664
+ }
665
+ if (this.deliveryPumpRunning) {
666
+ this.deliveryPumpScheduled = true;
667
+ return;
668
+ }
669
+ this.deliveryPumpTimer = setTimeout(() => {
670
+ this.deliveryPumpTimer = null;
671
+ void this.runDeliveryOutboxPump();
672
+ }, Math.max(0, delayMs));
673
+ this.deliveryPumpTimer.unref?.();
674
+ }
675
+ async runDeliveryOutboxPump() {
676
+ const outbox = this.deliveryOutbox;
677
+ if (!outbox || this.shuttingDown || this.deliveryPumpRunning)
678
+ return;
679
+ this.deliveryPumpRunning = true;
680
+ this.deliveryPumpScheduled = false;
681
+ try {
682
+ outbox.expireStale();
683
+ while (this.activeDurableTargets.size < 8) {
684
+ const claimed = outbox.claimNext(this.managerBootId, target => this.isInstanceRestarting(target) ? null : this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets);
685
+ if (!claimed)
686
+ break;
687
+ this.activeDurableTargets.add(claimed.targetInstance);
688
+ void this.dispatchDurableDelivery(claimed).finally(() => {
689
+ this.activeDurableTargets.delete(claimed.targetInstance);
690
+ this.scheduleDeliveryOutboxPump();
691
+ });
692
+ }
693
+ }
694
+ catch (err) {
695
+ this.logger.error({ err }, "Durable delivery dispatcher failed; committed rows remain in SQLite");
696
+ this.scheduleDeliveryOutboxPump(5_000);
697
+ }
698
+ finally {
699
+ this.deliveryPumpRunning = false;
700
+ if (this.deliveryPumpScheduled)
701
+ this.scheduleDeliveryOutboxPump();
702
+ const nextRetryAt = outbox.nextRetryAt();
703
+ if (nextRetryAt && !this.deliveryPumpTimer) {
704
+ this.scheduleDeliveryOutboxPump(Math.max(1, Date.parse(nextRetryAt) - Date.now()));
705
+ }
706
+ const nextExpiryAt = outbox.nextExpiryAt();
707
+ if (nextExpiryAt && !this.deliveryPumpTimer) {
708
+ this.scheduleDeliveryOutboxPump(Math.max(1, Date.parse(nextExpiryAt) - Date.now()));
709
+ }
710
+ }
711
+ }
712
+ async dispatchDurableDelivery(claimed) {
713
+ const outbox = this.deliveryOutbox;
714
+ if (!outbox)
715
+ return;
716
+ const target = claimed.targetInstance;
717
+ const attempt = claimed.attemptNo;
718
+ const payload = { ...claimed.payload };
719
+ const rawMeta = payload.meta && typeof payload.meta === "object"
720
+ ? payload.meta
721
+ : {};
722
+ payload.meta = {
723
+ ...rawMeta,
724
+ delivery_id: claimed.deliveryId,
725
+ delivery_attempt: String(claimed.attemptNo),
726
+ from_instance: typeof rawMeta.from_instance === "string" && rawMeta.from_instance
727
+ ? rawMeta.from_instance
728
+ : claimed.sourceInstance,
729
+ correlation_id: claimed.correlationId ?? String(rawMeta.correlation_id ?? ""),
730
+ };
731
+ // Arm the visible liveness alert as soon as this generation owns the lane,
732
+ // including time spent waiting for a handoff/idle gate to return.
733
+ const laneReleased = this.waitForDurableLaneRelease(claimed);
734
+ try {
735
+ const sent = await this.deliverToInstance(target, payload, {
736
+ isCrossInstance: true,
737
+ waitForIdle: claimed.kind !== "steer",
738
+ });
739
+ if (!sent) {
740
+ const current = outbox.get(claimed.deliveryId);
741
+ if (current?.state === "delivering") {
742
+ outbox.retryBeforeBegin(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, "delivery was cancelled before target IPC handoff", Math.min(60_000, 1_000 * 2 ** Math.min(attempt - 1, 6)));
743
+ }
744
+ }
745
+ }
746
+ catch (err) {
747
+ const latest = outbox.get(claimed.deliveryId);
748
+ const reason = err instanceof Error ? err.message : String(err);
749
+ if (latest?.state === "submission_started") {
750
+ outbox.complete(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, "uncertain", reason);
751
+ }
752
+ else {
753
+ outbox.retryBeforeBegin(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, reason, Math.min(60_000, 1_000 * 2 ** Math.min(attempt - 1, 6)));
754
+ }
755
+ this.logger.warn({ deliveryId: claimed.deliveryId, target, attempt, err: reason }, "Durable delivery dispatch deferred");
756
+ }
757
+ await laneReleased;
758
+ }
759
+ async waitForDurableLaneRelease(claimed) {
760
+ const outbox = this.deliveryOutbox;
761
+ if (!outbox)
762
+ return;
763
+ const deliveryId = claimed.deliveryId;
764
+ const released = (row) => !row
765
+ || row.state === "queued" || row.state === "retry_wait"
766
+ || row.state === "delivered" || row.state === "failed"
767
+ || row.state === "uncertain" || row.state === "cancelled";
768
+ if (released(outbox.get(deliveryId)))
769
+ return;
770
+ await new Promise(resolve => {
771
+ const finish = () => {
772
+ clearTimeout(alertTimer);
773
+ outbox.off("state", check);
774
+ outbox.off("generation_recovered", check);
775
+ resolve();
776
+ };
777
+ const check = () => {
778
+ if (released(outbox.get(deliveryId)))
779
+ finish();
780
+ };
781
+ const alertTimer = setTimeout(() => {
782
+ const current = outbox.get(deliveryId);
783
+ if ((current?.state === "delivering" || current?.state === "submission_started")
784
+ && current.targetDaemonBootId === claimed.targetDaemonBootId
785
+ && current.managerBootId === claimed.managerBootId) {
786
+ this.notifyFleetError(`Durable delivery to ${claimed.targetInstance} is still waiting after ${Math.round(DURABLE_DELIVERY_LANE_ALERT_MS / 60_000)} minutes; the same daemon generation retains delivery ${deliveryId}'s lane until it reports or is replaced.`);
787
+ }
788
+ }, DURABLE_DELIVERY_LANE_ALERT_MS);
789
+ alertTimer.unref?.();
790
+ outbox.on("state", check);
791
+ outbox.on("generation_recovered", check);
792
+ check();
793
+ });
794
+ }
561
795
  spawnConcurrency() {
562
796
  const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
563
797
  if (explicit != null)
@@ -1509,16 +1743,16 @@ export class FleetManager {
1509
1743
  }
1510
1744
  async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch) {
1511
1745
  if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1512
- return;
1746
+ return false;
1513
1747
  await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
1514
1748
  if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1515
- return;
1749
+ return false;
1516
1750
  let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
1517
1751
  if (this.lifecycle.isPaused(instanceName)) {
1518
1752
  const wakeStartedAt = Date.now();
1519
1753
  await this.lifecycle.wake(instanceName, 30_000);
1520
1754
  if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1521
- return;
1755
+ return false;
1522
1756
  // Waking added one to the warm count — make room by evicting a different
1523
1757
  // LRU idle instance (never this one; it's about to work).
1524
1758
  this.enforceWarmCap(instanceName);
@@ -1528,20 +1762,23 @@ export class FleetManager {
1528
1762
  const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter, () => !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch));
1529
1763
  if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
1530
1764
  this.logger.info({ instanceName }, "Pending delivery dropped by user cancel");
1531
- return;
1765
+ return false;
1532
1766
  }
1533
1767
  // A server crash can land while waitForInstanceIdle is pending. Re-check
1534
1768
  // immediately before the old timeout path would force text into a boot UI.
1535
1769
  await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
1536
1770
  if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1537
- return;
1771
+ return false;
1538
1772
  if (!idle) {
1539
1773
  this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
1540
1774
  }
1541
- await this.sendWhenConnected(instanceName, payload, deliveryEpoch);
1542
- if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1543
- return;
1544
- this.lastDeliveryAt.set(instanceName, Date.now());
1775
+ const sent = await this.sendWhenConnected(instanceName, payload, deliveryEpoch);
1776
+ if (!sent)
1777
+ return false;
1778
+ if (this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
1779
+ this.lastDeliveryAt.set(instanceName, Date.now());
1780
+ }
1781
+ return true;
1545
1782
  }
1546
1783
  async holdDeliveryForStorm(instanceName, deliveryEpoch) {
1547
1784
  if (!this.stormWindow.isDeliveryHeld(instanceName))
@@ -1567,26 +1804,26 @@ export class FleetManager {
1567
1804
  */
1568
1805
  async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName)) {
1569
1806
  if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1570
- return;
1807
+ return false;
1571
1808
  const queued = this.ipcWaitTails.get(instanceName);
1572
1809
  if (!queued) {
1573
1810
  const ipc = this.instanceIpcClients.get(instanceName);
1574
1811
  if (ipc?.connected && ipc.send(payload))
1575
- return;
1812
+ return true;
1576
1813
  }
1577
1814
  const attempt = (queued ?? Promise.resolve())
1578
1815
  .catch(() => { })
1579
1816
  .then(() => {
1580
1817
  if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1581
- return;
1818
+ return false;
1582
1819
  return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch);
1583
1820
  });
1584
1821
  // The chain stores a settled-either-way promise so one failed delivery cannot
1585
1822
  // wedge every later one, and so `queued` above is safe to await unguarded.
1586
- const tail = attempt.catch(() => { });
1823
+ const tail = attempt.then(() => { }, () => { });
1587
1824
  this.ipcWaitTails.set(instanceName, tail);
1588
1825
  try {
1589
- await attempt;
1826
+ return await attempt;
1590
1827
  }
1591
1828
  finally {
1592
1829
  // Only the last waiter clears the chain; while a queue is still draining the
@@ -1602,12 +1839,12 @@ export class FleetManager {
1602
1839
  let warned = false;
1603
1840
  for (;;) {
1604
1841
  if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1605
- return;
1842
+ return false;
1606
1843
  // Re-read every round: a reconnect replaces the IpcClient object entirely,
1607
1844
  // so a cached reference would stay dead forever.
1608
1845
  const ipc = this.instanceIpcClients.get(instanceName);
1609
1846
  if (ipc?.connected && ipc.send(payload))
1610
- return;
1847
+ return true;
1611
1848
  if (Date.now() >= deadline) {
1612
1849
  throw new Error(`Instance '${instanceName}' IPC is unavailable`);
1613
1850
  }
@@ -1638,22 +1875,24 @@ export class FleetManager {
1638
1875
  if (this.lifecycle.isPaused(instanceName)) {
1639
1876
  await this.lifecycle.wake(instanceName, 30_000);
1640
1877
  if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1641
- return;
1878
+ return false;
1642
1879
  this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
1643
1880
  }
1644
- await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch);
1645
- if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
1646
- return;
1881
+ const sent = await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch);
1882
+ if (!sent)
1883
+ return false;
1647
1884
  // A cross-instance item arriving before the daemon observes this turn as
1648
1885
  // working must not trust the stale idle snapshot from before the send.
1649
- this.lastDeliveryAt.set(instanceName, Date.now());
1650
- return;
1886
+ if (this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
1887
+ this.lastDeliveryAt.set(instanceName, Date.now());
1888
+ }
1889
+ return true;
1651
1890
  }
1652
1891
  const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
1653
1892
  const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch));
1654
1893
  this.idleGatedDeliveryTails.set(instanceName, delivery);
1655
1894
  try {
1656
- await delivery;
1895
+ return await delivery;
1657
1896
  }
1658
1897
  finally {
1659
1898
  if (this.idleGatedDeliveryTails.get(instanceName) === delivery) {
@@ -2279,10 +2518,16 @@ export class FleetManager {
2279
2518
  workingDirectory,
2280
2519
  reason: "restart",
2281
2520
  }, () => this.doRestartSingleInstance(name, opts))
2282
- .finally(() => this.restartsInFlight.delete(name));
2521
+ .finally(() => {
2522
+ this.restartsInFlight.delete(name);
2523
+ this.scheduleDeliveryOutboxPump();
2524
+ });
2283
2525
  this.restartsInFlight.set(name, run);
2284
2526
  return run;
2285
2527
  }
2528
+ isInstanceRestarting(name) {
2529
+ return this.restartsInFlight.has(name);
2530
+ }
2286
2531
  async doRestartSingleInstance(name, opts) {
2287
2532
  if (this.configPath) {
2288
2533
  this.loadConfig(this.configPath);
@@ -2575,6 +2820,7 @@ export class FleetManager {
2575
2820
  this.shuttingDown = false;
2576
2821
  this.configPath = configPath;
2577
2822
  this.loadEnvFile();
2823
+ this.ensureDeliveryOutbox();
2578
2824
  // Rotate fleet.log if oversized (before any logging)
2579
2825
  rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
2580
2826
  const fleet = this.loadConfig(configPath);
@@ -5197,6 +5443,8 @@ export class FleetManager {
5197
5443
  const requestId = msg.requestId;
5198
5444
  const fleetRequestId = msg.fleetRequestId;
5199
5445
  const senderSessionName = msg.senderSessionName;
5446
+ const operationId = typeof msg.operationId === "string" ? msg.operationId : undefined;
5447
+ const sourceDaemonBootId = typeof msg.sourceDaemonBootId === "string" ? msg.sourceDaemonBootId : undefined;
5200
5448
  const respond = (result, error) => {
5201
5449
  const ipc = this.instanceIpcClients.get(instanceName);
5202
5450
  let sent = false;
@@ -5314,7 +5562,9 @@ export class FleetManager {
5314
5562
  // Dispatch fleet-specific tools via handler map
5315
5563
  const handler = outboundHandlers.get(tool);
5316
5564
  if (handler) {
5317
- await handler(this, args, respond, { instanceName, requestId, fleetRequestId, senderSessionName });
5565
+ await handler(this, args, respond, {
5566
+ instanceName, requestId, fleetRequestId, senderSessionName, operationId, sourceDaemonBootId,
5567
+ });
5318
5568
  }
5319
5569
  else {
5320
5570
  respond(null, `Unknown tool: ${tool}`);
@@ -6528,6 +6778,8 @@ export class FleetManager {
6528
6778
  * the chain drains, so idle messages hold no memory.
6529
6779
  */
6530
6780
  deliveryStatusChains = new Map();
6781
+ /** #725: in-flight CLI env probes, keyed by backend name. Coalesces concurrent /model requests. */
6782
+ pendingCliEnvProbes = new Map();
6531
6783
  reactMessageStatus(instanceName, chatId, messageId, emoji, threadId) {
6532
6784
  // React via the adapter BOUND to this instance — NOT the first discord world.
6533
6785
  // Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
@@ -6584,9 +6836,10 @@ export class FleetManager {
6584
6836
  return;
6585
6837
  // Thread-aware adapters (Discord) react where the thread is; Telegram
6586
6838
  // addresses the supergroup chat and ignores the thread part.
6587
- // Discord needs the old reaction removed before the new one is added.
6588
- // Telegram's setMessageReaction replaces the bot's one current reaction.
6589
- if (prev && prev !== emoji && adapter.unreact && !(adapter instanceof TelegramAdapter)) {
6839
+ // #972: add-only by default — only remove ❌ when leaving the failed
6840
+ // state. 👀/⏳/✅ stacking together is harmless; leaving ❌ visible
6841
+ // after a recovery would be misleading.
6842
+ if (prev === "❌" && adapter.unreact && !(adapter instanceof TelegramAdapter)) {
6590
6843
  // Best effort: a failed removal must not block the new status.
6591
6844
  await adapter.unreact(target, messageId, prev, threadId).catch(e => this.logger.debug({ err: e.message }, "Delivery status reaction removal failed"));
6592
6845
  }
@@ -9901,6 +10154,15 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
9901
10154
  * previous list, never stall the command that asked for it.
9902
10155
  */
9903
10156
  async probeBackendBounded(backend, opts = {}) {
10157
+ // #725: single-flight per backend — if a probe for this backend is already
10158
+ // in flight, return the same promise instead of starting a duplicate.
10159
+ // Refresh probes (refreshVendorCatalog=true) are always started fresh since
10160
+ // the caller explicitly asked for a new vendor fetch.
10161
+ if (!opts.refreshVendorCatalog) {
10162
+ const existing = this.pendingCliEnvProbes.get(backend);
10163
+ if (existing)
10164
+ return existing;
10165
+ }
9904
10166
  const work = this.probeBackend(backend, opts);
9905
10167
  work.catch(() => { });
9906
10168
  let timer;
@@ -9910,12 +10172,14 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
9910
10172
  resolve(null);
9911
10173
  }, CLI_ENV_PROBE_DEADLINE_MS);
9912
10174
  });
9913
- try {
9914
- return await Promise.race([work, deadline]);
9915
- }
9916
- finally {
10175
+ const bounded = Promise.race([work, deadline]).finally(() => {
9917
10176
  clearTimeout(timer);
9918
- }
10177
+ if (this.pendingCliEnvProbes.get(backend) === bounded)
10178
+ this.pendingCliEnvProbes.delete(backend);
10179
+ });
10180
+ if (!opts.refreshVendorCatalog)
10181
+ this.pendingCliEnvProbes.set(backend, bounded);
10182
+ return bounded;
9919
10183
  }
9920
10184
  /**
9921
10185
  * Resolve the effective model for a fleet or ClassicBot instance, plus where it
@@ -10025,8 +10289,10 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
10025
10289
  backends.add(ch.backend);
10026
10290
  if (backends.size === 0)
10027
10291
  backends.add("claude-code");
10292
+ // #724: use the bounded variant so background startup probes cannot hang
10293
+ // indefinitely — each is wrapped by CLI_ENV_PROBE_DEADLINE_MS.
10028
10294
  for (const b of backends)
10029
- void this.probeBackend(b);
10295
+ void this.probeBackendBounded(b);
10030
10296
  }
10031
10297
  /** Best-effort model list for `/model`: cached CLI env first, else live probe. Never throws. */
10032
10298
  async getModelOptions(instanceName, refresh = false, onLiveProbe) {
@@ -10448,7 +10714,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
10448
10714
  const cached = this.readCliEnv("claude-code");
10449
10715
  if (cached?.apiModels?.length)
10450
10716
  return cached.apiModels;
10451
- const env = await this.probeBackend("claude-code");
10717
+ const env = await this.probeBackendBounded("claude-code");
10452
10718
  return env?.apiModels ?? [];
10453
10719
  }
10454
10720
  /** Replace a consumed "/model" menu with the full account catalog tier. */