@songsid/agend 2.1.7-beta.3 → 2.1.7-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-endpoint.d.ts +2 -0
- package/dist/agent-endpoint.js +47 -5
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/codex.d.ts +18 -0
- package/dist/backend/codex.js +68 -3
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/types.d.ts +9 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/mcp-operation-response.d.ts +3 -0
- package/dist/channel/mcp-operation-response.js +21 -0
- package/dist/channel/mcp-operation-response.js.map +1 -0
- package/dist/channel/mcp-operation-response.test.d.ts +1 -0
- package/dist/channel/mcp-operation-response.test.js +23 -0
- package/dist/channel/mcp-operation-response.test.js.map +1 -0
- package/dist/channel/mcp-server.js +20 -5
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/daemon.d.ts +23 -0
- package/dist/daemon.js +248 -24
- package/dist/daemon.js.map +1 -1
- package/dist/delivery-outbox.d.ts +90 -0
- package/dist/delivery-outbox.js +562 -0
- package/dist/delivery-outbox.js.map +1 -0
- package/dist/delivery-outbox.test.d.ts +1 -0
- package/dist/delivery-outbox.test.js +343 -0
- package/dist/delivery-outbox.test.js.map +1 -0
- package/dist/fleet-manager.d.ts +38 -1
- package/dist/fleet-manager.js +302 -36
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +5 -0
- package/dist/instance-lifecycle.js +38 -0
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.js +2 -0
- package/dist/instructions.js.map +1 -1
- package/dist/outbound-handlers.d.ts +22 -1
- package/dist/outbound-handlers.js +81 -7
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/remote-credential-check.d.ts +39 -0
- package/dist/remote-credential-check.js +81 -0
- package/dist/remote-credential-check.js.map +1 -0
- package/dist/usage/providers.d.ts +11 -6
- package/dist/usage/providers.js +17 -10
- package/dist/usage/providers.js.map +1 -1
- package/package.json +1 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
|
|
2
|
-
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { randomBytes, randomUUID } from "node:crypto";
|
|
3
3
|
import { spawnSync } from "node:child_process";
|
|
4
4
|
import { freemem, totalmem, cpus } from "node:os";
|
|
5
5
|
import { createServer } from "node:http";
|
|
@@ -54,6 +54,12 @@ import { InstanceLifecycle, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } f
|
|
|
54
54
|
import { TopicArchiver } from "./topic-archiver.js";
|
|
55
55
|
import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
56
56
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
57
|
+
import { DeliveryOutbox } from "./delivery-outbox.js";
|
|
58
|
+
// The target Daemon can legitimately wait 30 minutes for a busy pane before
|
|
59
|
+
// deciding to defer before submission. This is an alert threshold only: the
|
|
60
|
+
// active manager/target generation pair owns the lane until the daemon reports
|
|
61
|
+
// a state transition or that generation is replaced.
|
|
62
|
+
export const DURABLE_DELIVERY_LANE_ALERT_MS = 35 * 60_000;
|
|
57
63
|
import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
|
|
58
64
|
import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
59
65
|
import { filterUsageProviders, formatDiscordUsageActivity, getUsageSnapshot, handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
|
|
@@ -290,6 +296,14 @@ export class FleetManager {
|
|
|
290
296
|
spawnGate;
|
|
291
297
|
/** Fleet-level backend reachability memory (fed by pty_error / startup panes). */
|
|
292
298
|
backendOutage = new BackendOutageTracker();
|
|
299
|
+
/** Stable for this FleetManager OS-process lifetime; Daemon objects have their own boot IDs. */
|
|
300
|
+
managerBootId = randomUUID();
|
|
301
|
+
deliveryOutbox = null;
|
|
302
|
+
deliveryOutboxRecovered = false;
|
|
303
|
+
deliveryPumpScheduled = false;
|
|
304
|
+
deliveryPumpRunning = false;
|
|
305
|
+
deliveryPumpTimer = null;
|
|
306
|
+
activeDurableTargets = new Set();
|
|
293
307
|
/** Live view of lifecycle.daemons — used throughout; not deprecated. */
|
|
294
308
|
get daemons() { return this.lifecycle.daemons; }
|
|
295
309
|
fleetConfig = null;
|
|
@@ -558,6 +572,226 @@ export class FleetManager {
|
|
|
558
572
|
this.topicArchiver = new TopicArchiver(this);
|
|
559
573
|
this.statuslineWatcher = new StatuslineWatcher(this);
|
|
560
574
|
}
|
|
575
|
+
ensureDeliveryOutbox() {
|
|
576
|
+
if (this.deliveryOutbox)
|
|
577
|
+
return;
|
|
578
|
+
const outbox = new DeliveryOutbox(join(this.dataDir, "delivery-outbox.db"), this.managerBootId);
|
|
579
|
+
this.deliveryOutbox = outbox;
|
|
580
|
+
if (!this.deliveryOutboxRecovered) {
|
|
581
|
+
const recovered = outbox.recoverForBoot(this.managerBootId);
|
|
582
|
+
this.logger.info({ ...recovered }, "Recovered durable delivery outbox for this process boot");
|
|
583
|
+
if (recovered.uncertain > 0) {
|
|
584
|
+
this.notifyFleetError(`${recovered.uncertain} durable delivery outcome(s) became uncertain during process restart; inspect the sender's delivery notice before retrying.`);
|
|
585
|
+
}
|
|
586
|
+
this.deliveryOutboxRecovered = true;
|
|
587
|
+
}
|
|
588
|
+
outbox.on("admitted", () => this.scheduleDeliveryOutboxPump());
|
|
589
|
+
outbox.on("state", (event) => {
|
|
590
|
+
this.scheduleDeliveryOutboxPump();
|
|
591
|
+
if (event.deliveryId && (event.state === "failed" || event.state === "uncertain")) {
|
|
592
|
+
const row = outbox.get(event.deliveryId);
|
|
593
|
+
if (row && row.kind !== "delivery_outcome_notice" && row.kind !== "post_restart_outcome_notice") {
|
|
594
|
+
const notice = `Durable delivery ${row.state}: operation_id=${row.operationId}; target=${row.targetInstance}; delivery_id=${row.deliveryId}.`;
|
|
595
|
+
this.logger.error({ deliveryId: row.deliveryId, operationId: row.operationId, target: row.targetInstance, state: row.state }, notice);
|
|
596
|
+
this.notifyFleetError(notice);
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
});
|
|
600
|
+
outbox.on("generation_recovered", () => this.scheduleDeliveryOutboxPump());
|
|
601
|
+
outbox.on("expired", (event) => {
|
|
602
|
+
this.scheduleDeliveryOutboxPump();
|
|
603
|
+
if (event.count)
|
|
604
|
+
this.notifyFleetError(`${event.count} durable delivery row(s) expired before their target became available; sender outcome notice queued.`);
|
|
605
|
+
});
|
|
606
|
+
this.scheduleDeliveryOutboxPump();
|
|
607
|
+
}
|
|
608
|
+
/** Synchronous SQLite admission; the caller may acknowledge only after this returns. */
|
|
609
|
+
admitDurableDelivery(input) {
|
|
610
|
+
const outbox = this.deliveryOutbox;
|
|
611
|
+
if (!outbox)
|
|
612
|
+
throw new Error("Durable delivery store is unavailable");
|
|
613
|
+
const source = this.daemons.get(input.sourceInstance);
|
|
614
|
+
const sourceDaemonBootId = input.sourceDaemonBootId ?? source?.bootId;
|
|
615
|
+
if (!source || !sourceDaemonBootId || source.bootId !== sourceDaemonBootId) {
|
|
616
|
+
throw new Error("Source daemon generation is no longer current; outcome is unknown");
|
|
617
|
+
}
|
|
618
|
+
const sourceKey = `mcp:${input.sourceInstance}:${input.operationId}:${input.targetInstance}:${input.targetSession ?? ""}:${input.kind}`;
|
|
619
|
+
const admitted = outbox.admit({ ...input, sourceDaemonBootId, sourceKey });
|
|
620
|
+
if (admitted.inserted)
|
|
621
|
+
this.logger.info({
|
|
622
|
+
deliveryId: admitted.delivery.deliveryId,
|
|
623
|
+
operationId: input.operationId,
|
|
624
|
+
source: input.sourceInstance,
|
|
625
|
+
target: input.targetInstance,
|
|
626
|
+
kind: input.kind,
|
|
627
|
+
}, "Durable cross-instance delivery accepted");
|
|
628
|
+
this.scheduleDeliveryOutboxPump();
|
|
629
|
+
return {
|
|
630
|
+
deliveryId: admitted.delivery.deliveryId,
|
|
631
|
+
state: admitted.delivery.state,
|
|
632
|
+
duplicate: !admitted.inserted,
|
|
633
|
+
};
|
|
634
|
+
}
|
|
635
|
+
getDaemonBootId(instanceName) {
|
|
636
|
+
return this.daemons.get(instanceName)?.bootId;
|
|
637
|
+
}
|
|
638
|
+
markDurableResponseDelivered(sourceInstance, operationId) {
|
|
639
|
+
this.deliveryOutbox?.markResponseDelivered(sourceInstance, operationId);
|
|
640
|
+
}
|
|
641
|
+
/** Called after a replacement Daemon object is installed in lifecycle.daemons. */
|
|
642
|
+
onDaemonReady(name, daemonBootId) {
|
|
643
|
+
const outbox = this.deliveryOutbox;
|
|
644
|
+
if (!outbox)
|
|
645
|
+
return;
|
|
646
|
+
for (const prior of outbox.getUnansweredAccepted(name, daemonBootId)) {
|
|
647
|
+
try {
|
|
648
|
+
outbox.admitPostRestartOutcomeNotice(prior, daemonBootId);
|
|
649
|
+
}
|
|
650
|
+
catch (err) {
|
|
651
|
+
this.logger.error({ err, source: name, operationId: prior.operationId }, "Could not persist post-restart delivery outcome notice");
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
this.scheduleDeliveryOutboxPump();
|
|
655
|
+
}
|
|
656
|
+
scheduleDeliveryOutboxPump(delayMs = 0) {
|
|
657
|
+
if (!this.deliveryOutbox || this.shuttingDown)
|
|
658
|
+
return;
|
|
659
|
+
if (this.deliveryPumpTimer) {
|
|
660
|
+
if (delayMs > 0)
|
|
661
|
+
return;
|
|
662
|
+
clearTimeout(this.deliveryPumpTimer);
|
|
663
|
+
this.deliveryPumpTimer = null;
|
|
664
|
+
}
|
|
665
|
+
if (this.deliveryPumpRunning) {
|
|
666
|
+
this.deliveryPumpScheduled = true;
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
669
|
+
this.deliveryPumpTimer = setTimeout(() => {
|
|
670
|
+
this.deliveryPumpTimer = null;
|
|
671
|
+
void this.runDeliveryOutboxPump();
|
|
672
|
+
}, Math.max(0, delayMs));
|
|
673
|
+
this.deliveryPumpTimer.unref?.();
|
|
674
|
+
}
|
|
675
|
+
async runDeliveryOutboxPump() {
|
|
676
|
+
const outbox = this.deliveryOutbox;
|
|
677
|
+
if (!outbox || this.shuttingDown || this.deliveryPumpRunning)
|
|
678
|
+
return;
|
|
679
|
+
this.deliveryPumpRunning = true;
|
|
680
|
+
this.deliveryPumpScheduled = false;
|
|
681
|
+
try {
|
|
682
|
+
outbox.expireStale();
|
|
683
|
+
while (this.activeDurableTargets.size < 8) {
|
|
684
|
+
const claimed = outbox.claimNext(this.managerBootId, target => this.isInstanceRestarting(target) ? null : this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets);
|
|
685
|
+
if (!claimed)
|
|
686
|
+
break;
|
|
687
|
+
this.activeDurableTargets.add(claimed.targetInstance);
|
|
688
|
+
void this.dispatchDurableDelivery(claimed).finally(() => {
|
|
689
|
+
this.activeDurableTargets.delete(claimed.targetInstance);
|
|
690
|
+
this.scheduleDeliveryOutboxPump();
|
|
691
|
+
});
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
catch (err) {
|
|
695
|
+
this.logger.error({ err }, "Durable delivery dispatcher failed; committed rows remain in SQLite");
|
|
696
|
+
this.scheduleDeliveryOutboxPump(5_000);
|
|
697
|
+
}
|
|
698
|
+
finally {
|
|
699
|
+
this.deliveryPumpRunning = false;
|
|
700
|
+
if (this.deliveryPumpScheduled)
|
|
701
|
+
this.scheduleDeliveryOutboxPump();
|
|
702
|
+
const nextRetryAt = outbox.nextRetryAt();
|
|
703
|
+
if (nextRetryAt && !this.deliveryPumpTimer) {
|
|
704
|
+
this.scheduleDeliveryOutboxPump(Math.max(1, Date.parse(nextRetryAt) - Date.now()));
|
|
705
|
+
}
|
|
706
|
+
const nextExpiryAt = outbox.nextExpiryAt();
|
|
707
|
+
if (nextExpiryAt && !this.deliveryPumpTimer) {
|
|
708
|
+
this.scheduleDeliveryOutboxPump(Math.max(1, Date.parse(nextExpiryAt) - Date.now()));
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
async dispatchDurableDelivery(claimed) {
|
|
713
|
+
const outbox = this.deliveryOutbox;
|
|
714
|
+
if (!outbox)
|
|
715
|
+
return;
|
|
716
|
+
const target = claimed.targetInstance;
|
|
717
|
+
const attempt = claimed.attemptNo;
|
|
718
|
+
const payload = { ...claimed.payload };
|
|
719
|
+
const rawMeta = payload.meta && typeof payload.meta === "object"
|
|
720
|
+
? payload.meta
|
|
721
|
+
: {};
|
|
722
|
+
payload.meta = {
|
|
723
|
+
...rawMeta,
|
|
724
|
+
delivery_id: claimed.deliveryId,
|
|
725
|
+
delivery_attempt: String(claimed.attemptNo),
|
|
726
|
+
from_instance: typeof rawMeta.from_instance === "string" && rawMeta.from_instance
|
|
727
|
+
? rawMeta.from_instance
|
|
728
|
+
: claimed.sourceInstance,
|
|
729
|
+
correlation_id: claimed.correlationId ?? String(rawMeta.correlation_id ?? ""),
|
|
730
|
+
};
|
|
731
|
+
// Arm the visible liveness alert as soon as this generation owns the lane,
|
|
732
|
+
// including time spent waiting for a handoff/idle gate to return.
|
|
733
|
+
const laneReleased = this.waitForDurableLaneRelease(claimed);
|
|
734
|
+
try {
|
|
735
|
+
const sent = await this.deliverToInstance(target, payload, {
|
|
736
|
+
isCrossInstance: true,
|
|
737
|
+
waitForIdle: claimed.kind !== "steer",
|
|
738
|
+
});
|
|
739
|
+
if (!sent) {
|
|
740
|
+
const current = outbox.get(claimed.deliveryId);
|
|
741
|
+
if (current?.state === "delivering") {
|
|
742
|
+
outbox.retryBeforeBegin(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, "delivery was cancelled before target IPC handoff", Math.min(60_000, 1_000 * 2 ** Math.min(attempt - 1, 6)));
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
catch (err) {
|
|
747
|
+
const latest = outbox.get(claimed.deliveryId);
|
|
748
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
749
|
+
if (latest?.state === "submission_started") {
|
|
750
|
+
outbox.complete(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, "uncertain", reason);
|
|
751
|
+
}
|
|
752
|
+
else {
|
|
753
|
+
outbox.retryBeforeBegin(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, reason, Math.min(60_000, 1_000 * 2 ** Math.min(attempt - 1, 6)));
|
|
754
|
+
}
|
|
755
|
+
this.logger.warn({ deliveryId: claimed.deliveryId, target, attempt, err: reason }, "Durable delivery dispatch deferred");
|
|
756
|
+
}
|
|
757
|
+
await laneReleased;
|
|
758
|
+
}
|
|
759
|
+
async waitForDurableLaneRelease(claimed) {
|
|
760
|
+
const outbox = this.deliveryOutbox;
|
|
761
|
+
if (!outbox)
|
|
762
|
+
return;
|
|
763
|
+
const deliveryId = claimed.deliveryId;
|
|
764
|
+
const released = (row) => !row
|
|
765
|
+
|| row.state === "queued" || row.state === "retry_wait"
|
|
766
|
+
|| row.state === "delivered" || row.state === "failed"
|
|
767
|
+
|| row.state === "uncertain" || row.state === "cancelled";
|
|
768
|
+
if (released(outbox.get(deliveryId)))
|
|
769
|
+
return;
|
|
770
|
+
await new Promise(resolve => {
|
|
771
|
+
const finish = () => {
|
|
772
|
+
clearTimeout(alertTimer);
|
|
773
|
+
outbox.off("state", check);
|
|
774
|
+
outbox.off("generation_recovered", check);
|
|
775
|
+
resolve();
|
|
776
|
+
};
|
|
777
|
+
const check = () => {
|
|
778
|
+
if (released(outbox.get(deliveryId)))
|
|
779
|
+
finish();
|
|
780
|
+
};
|
|
781
|
+
const alertTimer = setTimeout(() => {
|
|
782
|
+
const current = outbox.get(deliveryId);
|
|
783
|
+
if ((current?.state === "delivering" || current?.state === "submission_started")
|
|
784
|
+
&& current.targetDaemonBootId === claimed.targetDaemonBootId
|
|
785
|
+
&& current.managerBootId === claimed.managerBootId) {
|
|
786
|
+
this.notifyFleetError(`Durable delivery to ${claimed.targetInstance} is still waiting after ${Math.round(DURABLE_DELIVERY_LANE_ALERT_MS / 60_000)} minutes; the same daemon generation retains delivery ${deliveryId}'s lane until it reports or is replaced.`);
|
|
787
|
+
}
|
|
788
|
+
}, DURABLE_DELIVERY_LANE_ALERT_MS);
|
|
789
|
+
alertTimer.unref?.();
|
|
790
|
+
outbox.on("state", check);
|
|
791
|
+
outbox.on("generation_recovered", check);
|
|
792
|
+
check();
|
|
793
|
+
});
|
|
794
|
+
}
|
|
561
795
|
spawnConcurrency() {
|
|
562
796
|
const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
|
|
563
797
|
if (explicit != null)
|
|
@@ -1509,16 +1743,16 @@ export class FleetManager {
|
|
|
1509
1743
|
}
|
|
1510
1744
|
async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch) {
|
|
1511
1745
|
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1512
|
-
return;
|
|
1746
|
+
return false;
|
|
1513
1747
|
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
1514
1748
|
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1515
|
-
return;
|
|
1749
|
+
return false;
|
|
1516
1750
|
let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
|
|
1517
1751
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
1518
1752
|
const wakeStartedAt = Date.now();
|
|
1519
1753
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
1520
1754
|
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1521
|
-
return;
|
|
1755
|
+
return false;
|
|
1522
1756
|
// Waking added one to the warm count — make room by evicting a different
|
|
1523
1757
|
// LRU idle instance (never this one; it's about to work).
|
|
1524
1758
|
this.enforceWarmCap(instanceName);
|
|
@@ -1528,20 +1762,23 @@ export class FleetManager {
|
|
|
1528
1762
|
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter, () => !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch));
|
|
1529
1763
|
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
|
|
1530
1764
|
this.logger.info({ instanceName }, "Pending delivery dropped by user cancel");
|
|
1531
|
-
return;
|
|
1765
|
+
return false;
|
|
1532
1766
|
}
|
|
1533
1767
|
// A server crash can land while waitForInstanceIdle is pending. Re-check
|
|
1534
1768
|
// immediately before the old timeout path would force text into a boot UI.
|
|
1535
1769
|
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
1536
1770
|
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1537
|
-
return;
|
|
1771
|
+
return false;
|
|
1538
1772
|
if (!idle) {
|
|
1539
1773
|
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
1540
1774
|
}
|
|
1541
|
-
await this.sendWhenConnected(instanceName, payload, deliveryEpoch);
|
|
1542
|
-
if (!
|
|
1543
|
-
return;
|
|
1544
|
-
this.
|
|
1775
|
+
const sent = await this.sendWhenConnected(instanceName, payload, deliveryEpoch);
|
|
1776
|
+
if (!sent)
|
|
1777
|
+
return false;
|
|
1778
|
+
if (this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
|
|
1779
|
+
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
1780
|
+
}
|
|
1781
|
+
return true;
|
|
1545
1782
|
}
|
|
1546
1783
|
async holdDeliveryForStorm(instanceName, deliveryEpoch) {
|
|
1547
1784
|
if (!this.stormWindow.isDeliveryHeld(instanceName))
|
|
@@ -1567,26 +1804,26 @@ export class FleetManager {
|
|
|
1567
1804
|
*/
|
|
1568
1805
|
async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName)) {
|
|
1569
1806
|
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1570
|
-
return;
|
|
1807
|
+
return false;
|
|
1571
1808
|
const queued = this.ipcWaitTails.get(instanceName);
|
|
1572
1809
|
if (!queued) {
|
|
1573
1810
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1574
1811
|
if (ipc?.connected && ipc.send(payload))
|
|
1575
|
-
return;
|
|
1812
|
+
return true;
|
|
1576
1813
|
}
|
|
1577
1814
|
const attempt = (queued ?? Promise.resolve())
|
|
1578
1815
|
.catch(() => { })
|
|
1579
1816
|
.then(() => {
|
|
1580
1817
|
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1581
|
-
return;
|
|
1818
|
+
return false;
|
|
1582
1819
|
return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch);
|
|
1583
1820
|
});
|
|
1584
1821
|
// The chain stores a settled-either-way promise so one failed delivery cannot
|
|
1585
1822
|
// wedge every later one, and so `queued` above is safe to await unguarded.
|
|
1586
|
-
const tail = attempt.
|
|
1823
|
+
const tail = attempt.then(() => { }, () => { });
|
|
1587
1824
|
this.ipcWaitTails.set(instanceName, tail);
|
|
1588
1825
|
try {
|
|
1589
|
-
await attempt;
|
|
1826
|
+
return await attempt;
|
|
1590
1827
|
}
|
|
1591
1828
|
finally {
|
|
1592
1829
|
// Only the last waiter clears the chain; while a queue is still draining the
|
|
@@ -1602,12 +1839,12 @@ export class FleetManager {
|
|
|
1602
1839
|
let warned = false;
|
|
1603
1840
|
for (;;) {
|
|
1604
1841
|
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1605
|
-
return;
|
|
1842
|
+
return false;
|
|
1606
1843
|
// Re-read every round: a reconnect replaces the IpcClient object entirely,
|
|
1607
1844
|
// so a cached reference would stay dead forever.
|
|
1608
1845
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1609
1846
|
if (ipc?.connected && ipc.send(payload))
|
|
1610
|
-
return;
|
|
1847
|
+
return true;
|
|
1611
1848
|
if (Date.now() >= deadline) {
|
|
1612
1849
|
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
1613
1850
|
}
|
|
@@ -1638,22 +1875,24 @@ export class FleetManager {
|
|
|
1638
1875
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
1639
1876
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
1640
1877
|
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1641
|
-
return;
|
|
1878
|
+
return false;
|
|
1642
1879
|
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
1643
1880
|
}
|
|
1644
|
-
await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch);
|
|
1645
|
-
if (!
|
|
1646
|
-
return;
|
|
1881
|
+
const sent = await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch);
|
|
1882
|
+
if (!sent)
|
|
1883
|
+
return false;
|
|
1647
1884
|
// A cross-instance item arriving before the daemon observes this turn as
|
|
1648
1885
|
// working must not trust the stale idle snapshot from before the send.
|
|
1649
|
-
this.
|
|
1650
|
-
|
|
1886
|
+
if (this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
|
|
1887
|
+
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
1888
|
+
}
|
|
1889
|
+
return true;
|
|
1651
1890
|
}
|
|
1652
1891
|
const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
|
|
1653
1892
|
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch));
|
|
1654
1893
|
this.idleGatedDeliveryTails.set(instanceName, delivery);
|
|
1655
1894
|
try {
|
|
1656
|
-
await delivery;
|
|
1895
|
+
return await delivery;
|
|
1657
1896
|
}
|
|
1658
1897
|
finally {
|
|
1659
1898
|
if (this.idleGatedDeliveryTails.get(instanceName) === delivery) {
|
|
@@ -2279,10 +2518,16 @@ export class FleetManager {
|
|
|
2279
2518
|
workingDirectory,
|
|
2280
2519
|
reason: "restart",
|
|
2281
2520
|
}, () => this.doRestartSingleInstance(name, opts))
|
|
2282
|
-
.finally(() =>
|
|
2521
|
+
.finally(() => {
|
|
2522
|
+
this.restartsInFlight.delete(name);
|
|
2523
|
+
this.scheduleDeliveryOutboxPump();
|
|
2524
|
+
});
|
|
2283
2525
|
this.restartsInFlight.set(name, run);
|
|
2284
2526
|
return run;
|
|
2285
2527
|
}
|
|
2528
|
+
isInstanceRestarting(name) {
|
|
2529
|
+
return this.restartsInFlight.has(name);
|
|
2530
|
+
}
|
|
2286
2531
|
async doRestartSingleInstance(name, opts) {
|
|
2287
2532
|
if (this.configPath) {
|
|
2288
2533
|
this.loadConfig(this.configPath);
|
|
@@ -2575,6 +2820,7 @@ export class FleetManager {
|
|
|
2575
2820
|
this.shuttingDown = false;
|
|
2576
2821
|
this.configPath = configPath;
|
|
2577
2822
|
this.loadEnvFile();
|
|
2823
|
+
this.ensureDeliveryOutbox();
|
|
2578
2824
|
// Rotate fleet.log if oversized (before any logging)
|
|
2579
2825
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
2580
2826
|
const fleet = this.loadConfig(configPath);
|
|
@@ -5197,6 +5443,8 @@ export class FleetManager {
|
|
|
5197
5443
|
const requestId = msg.requestId;
|
|
5198
5444
|
const fleetRequestId = msg.fleetRequestId;
|
|
5199
5445
|
const senderSessionName = msg.senderSessionName;
|
|
5446
|
+
const operationId = typeof msg.operationId === "string" ? msg.operationId : undefined;
|
|
5447
|
+
const sourceDaemonBootId = typeof msg.sourceDaemonBootId === "string" ? msg.sourceDaemonBootId : undefined;
|
|
5200
5448
|
const respond = (result, error) => {
|
|
5201
5449
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
5202
5450
|
let sent = false;
|
|
@@ -5314,7 +5562,9 @@ export class FleetManager {
|
|
|
5314
5562
|
// Dispatch fleet-specific tools via handler map
|
|
5315
5563
|
const handler = outboundHandlers.get(tool);
|
|
5316
5564
|
if (handler) {
|
|
5317
|
-
await handler(this, args, respond, {
|
|
5565
|
+
await handler(this, args, respond, {
|
|
5566
|
+
instanceName, requestId, fleetRequestId, senderSessionName, operationId, sourceDaemonBootId,
|
|
5567
|
+
});
|
|
5318
5568
|
}
|
|
5319
5569
|
else {
|
|
5320
5570
|
respond(null, `Unknown tool: ${tool}`);
|
|
@@ -6528,6 +6778,8 @@ export class FleetManager {
|
|
|
6528
6778
|
* the chain drains, so idle messages hold no memory.
|
|
6529
6779
|
*/
|
|
6530
6780
|
deliveryStatusChains = new Map();
|
|
6781
|
+
/** #725: in-flight CLI env probes, keyed by backend name. Coalesces concurrent /model requests. */
|
|
6782
|
+
pendingCliEnvProbes = new Map();
|
|
6531
6783
|
reactMessageStatus(instanceName, chatId, messageId, emoji, threadId) {
|
|
6532
6784
|
// React via the adapter BOUND to this instance — NOT the first discord world.
|
|
6533
6785
|
// Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
|
|
@@ -6584,9 +6836,10 @@ export class FleetManager {
|
|
|
6584
6836
|
return;
|
|
6585
6837
|
// Thread-aware adapters (Discord) react where the thread is; Telegram
|
|
6586
6838
|
// addresses the supergroup chat and ignores the thread part.
|
|
6587
|
-
//
|
|
6588
|
-
//
|
|
6589
|
-
|
|
6839
|
+
// #972: add-only by default — only remove ❌ when leaving the failed
|
|
6840
|
+
// state. 👀/⏳/✅ stacking together is harmless; leaving ❌ visible
|
|
6841
|
+
// after a recovery would be misleading.
|
|
6842
|
+
if (prev === "❌" && adapter.unreact && !(adapter instanceof TelegramAdapter)) {
|
|
6590
6843
|
// Best effort: a failed removal must not block the new status.
|
|
6591
6844
|
await adapter.unreact(target, messageId, prev, threadId).catch(e => this.logger.debug({ err: e.message }, "Delivery status reaction removal failed"));
|
|
6592
6845
|
}
|
|
@@ -9901,6 +10154,15 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9901
10154
|
* previous list, never stall the command that asked for it.
|
|
9902
10155
|
*/
|
|
9903
10156
|
async probeBackendBounded(backend, opts = {}) {
|
|
10157
|
+
// #725: single-flight per backend — if a probe for this backend is already
|
|
10158
|
+
// in flight, return the same promise instead of starting a duplicate.
|
|
10159
|
+
// Refresh probes (refreshVendorCatalog=true) are always started fresh since
|
|
10160
|
+
// the caller explicitly asked for a new vendor fetch.
|
|
10161
|
+
if (!opts.refreshVendorCatalog) {
|
|
10162
|
+
const existing = this.pendingCliEnvProbes.get(backend);
|
|
10163
|
+
if (existing)
|
|
10164
|
+
return existing;
|
|
10165
|
+
}
|
|
9904
10166
|
const work = this.probeBackend(backend, opts);
|
|
9905
10167
|
work.catch(() => { });
|
|
9906
10168
|
let timer;
|
|
@@ -9910,12 +10172,14 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9910
10172
|
resolve(null);
|
|
9911
10173
|
}, CLI_ENV_PROBE_DEADLINE_MS);
|
|
9912
10174
|
});
|
|
9913
|
-
|
|
9914
|
-
return await Promise.race([work, deadline]);
|
|
9915
|
-
}
|
|
9916
|
-
finally {
|
|
10175
|
+
const bounded = Promise.race([work, deadline]).finally(() => {
|
|
9917
10176
|
clearTimeout(timer);
|
|
9918
|
-
|
|
10177
|
+
if (this.pendingCliEnvProbes.get(backend) === bounded)
|
|
10178
|
+
this.pendingCliEnvProbes.delete(backend);
|
|
10179
|
+
});
|
|
10180
|
+
if (!opts.refreshVendorCatalog)
|
|
10181
|
+
this.pendingCliEnvProbes.set(backend, bounded);
|
|
10182
|
+
return bounded;
|
|
9919
10183
|
}
|
|
9920
10184
|
/**
|
|
9921
10185
|
* Resolve the effective model for a fleet or ClassicBot instance, plus where it
|
|
@@ -10025,8 +10289,10 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10025
10289
|
backends.add(ch.backend);
|
|
10026
10290
|
if (backends.size === 0)
|
|
10027
10291
|
backends.add("claude-code");
|
|
10292
|
+
// #724: use the bounded variant so background startup probes cannot hang
|
|
10293
|
+
// indefinitely — each is wrapped by CLI_ENV_PROBE_DEADLINE_MS.
|
|
10028
10294
|
for (const b of backends)
|
|
10029
|
-
void this.
|
|
10295
|
+
void this.probeBackendBounded(b);
|
|
10030
10296
|
}
|
|
10031
10297
|
/** Best-effort model list for `/model`: cached CLI env first, else live probe. Never throws. */
|
|
10032
10298
|
async getModelOptions(instanceName, refresh = false, onLiveProbe) {
|
|
@@ -10448,7 +10714,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10448
10714
|
const cached = this.readCliEnv("claude-code");
|
|
10449
10715
|
if (cached?.apiModels?.length)
|
|
10450
10716
|
return cached.apiModels;
|
|
10451
|
-
const env = await this.
|
|
10717
|
+
const env = await this.probeBackendBounded("claude-code");
|
|
10452
10718
|
return env?.apiModels ?? [];
|
|
10453
10719
|
}
|
|
10454
10720
|
/** Replace a consumed "/model" menu with the full account catalog tier. */
|