@songsid/agend 2.1.8 → 2.1.10-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/README.zh-TW.md +11 -5
- package/dist/backend/claude-code.d.ts +35 -5
- package/dist/backend/claude-code.js +288 -17
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +13 -0
- package/dist/backend/codex.js +74 -2
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/credential-profile.d.ts +9 -0
- package/dist/backend/credential-profile.js +20 -0
- package/dist/backend/credential-profile.js.map +1 -1
- package/dist/backend/grok.js +19 -0
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +0 -2
- package/dist/backend/kiro.js +8 -7
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/muse.d.ts +6 -0
- package/dist/backend/muse.js +7 -1
- package/dist/backend/muse.js.map +1 -1
- package/dist/backend/types.d.ts +1 -1
- package/dist/channel/adapters/telegram.js +4 -3
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/mcp-operation-response.js +5 -4
- package/dist/channel/mcp-operation-response.js.map +1 -1
- package/dist/channel/mcp-server.js +9 -0
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +1 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/types.d.ts +7 -0
- package/dist/cli.js +150 -52
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +24 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +1 -0
- package/dist/config.js +27 -1
- package/dist/config.js.map +1 -1
- package/dist/cross-instance-envelope.d.ts +7 -0
- package/dist/cross-instance-envelope.js +19 -1
- package/dist/cross-instance-envelope.js.map +1 -1
- package/dist/daemon.d.ts +112 -3
- package/dist/daemon.js +360 -41
- package/dist/daemon.js.map +1 -1
- package/dist/delivery-outbox.d.ts +9 -1
- package/dist/delivery-outbox.js +43 -27
- package/dist/delivery-outbox.js.map +1 -1
- package/dist/fleet-manager.d.ts +133 -5
- package/dist/fleet-manager.js +525 -86
- package/dist/fleet-manager.js.map +1 -1
- package/dist/forged-envelope-scan.d.ts +160 -0
- package/dist/forged-envelope-scan.js +309 -0
- package/dist/forged-envelope-scan.js.map +1 -0
- package/dist/general-knowledge/skills/persona-emoji/SKILL.md +5 -2
- package/dist/instance-lifecycle.d.ts +90 -6
- package/dist/instance-lifecycle.js +238 -30
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +180 -8
- package/dist/locale.js.map +1 -1
- package/dist/login-controller.d.ts +38 -0
- package/dist/login-controller.js +236 -8
- package/dist/login-controller.js.map +1 -1
- package/dist/login-flows.d.ts +25 -1
- package/dist/login-flows.js +10 -2
- package/dist/login-flows.js.map +1 -1
- package/dist/ls-rows.d.ts +17 -0
- package/dist/ls-rows.js +73 -0
- package/dist/ls-rows.js.map +1 -1
- package/dist/outbound-handlers.d.ts +3 -0
- package/dist/outbound-handlers.js +17 -19
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +2 -0
- package/dist/outbound-schemas.js +1 -1
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/paths.js +4 -2
- package/dist/paths.js.map +1 -1
- package/dist/pause-marker.d.ts +10 -1
- package/dist/pause-marker.js +26 -2
- package/dist/pause-marker.js.map +1 -1
- package/dist/routing-engine.d.ts +17 -2
- package/dist/routing-engine.js +48 -5
- package/dist/routing-engine.js.map +1 -1
- package/dist/service-installer.d.ts +8 -0
- package/dist/service-installer.js +36 -1
- package/dist/service-installer.js.map +1 -1
- package/dist/settings-api.d.ts +2 -0
- package/dist/settings-api.js +3 -1
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +3 -1
- package/dist/setup-wizard.js.map +1 -1
- package/dist/status-emojis.d.ts +9 -1
- package/dist/status-emojis.js +11 -1
- package/dist/status-emojis.js.map +1 -1
- package/dist/target-queue-worker.d.ts +67 -0
- package/dist/target-queue-worker.js +76 -0
- package/dist/target-queue-worker.js.map +1 -0
- package/dist/tool-permissions.js +7 -2
- package/dist/tool-permissions.js.map +1 -1
- package/dist/topic-commands.js +23 -19
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-sources.d.ts +30 -0
- package/dist/transcript-sources.js +78 -0
- package/dist/transcript-sources.js.map +1 -1
- package/dist/tunnel/cloudflared.d.ts +23 -5
- package/dist/tunnel/cloudflared.js +32 -13
- package/dist/tunnel/cloudflared.js.map +1 -1
- package/dist/tunnel/manager.d.ts +6 -0
- package/dist/tunnel/manager.js +60 -22
- package/dist/tunnel/manager.js.map +1 -1
- package/dist/tunnel/public-fetch.d.ts +30 -0
- package/dist/tunnel/public-fetch.js +156 -0
- package/dist/tunnel/public-fetch.js.map +1 -0
- package/dist/tunnel/types.d.ts +8 -2
- package/dist/tunnel/types.js +8 -2
- package/dist/tunnel/types.js.map +1 -1
- package/dist/types.d.ts +50 -1
- package/dist/types.js +9 -1
- package/dist/types.js.map +1 -1
- package/dist/ui/settings.html +2 -2
- package/dist/ui/view.html +14 -0
- package/dist/ui/web-terminal/terminal.html +1 -0
- package/dist/wake-coordinator.d.ts +128 -0
- package/dist/wake-coordinator.js +337 -0
- package/dist/wake-coordinator.js.map +1 -0
- package/dist/web-api.d.ts +3 -1
- package/dist/web-api.js +1 -1
- package/dist/web-api.js.map +1 -1
- package/dist/web-host-guard.d.ts +47 -0
- package/dist/web-host-guard.js +75 -0
- package/dist/web-host-guard.js.map +1 -0
- package/dist/web-terminal-http.d.ts +34 -0
- package/dist/web-terminal-http.js +63 -0
- package/dist/web-terminal-http.js.map +1 -1
- package/dist/web-terminal.d.ts +2 -0
- package/dist/web-terminal.js +3 -6
- package/dist/web-terminal.js.map +1 -1
- package/package.json +1 -1
- package/templates/systemd.service.ejs +7 -0
package/dist/fleet-manager.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
|
|
1
|
+
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync, accessSync, constants as fsConstants } from "node:fs";
|
|
2
2
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
3
3
|
import { spawnSync } from "node:child_process";
|
|
4
|
-
import { freemem, totalmem, cpus } from "node:os";
|
|
4
|
+
import { freemem, totalmem, cpus, homedir } from "node:os";
|
|
5
5
|
import { createServer } from "node:http";
|
|
6
6
|
import { Worker } from "node:worker_threads";
|
|
7
|
-
import { join, dirname, basename } from "node:path";
|
|
7
|
+
import { join, dirname, basename, delimiter, isAbsolute } from "node:path";
|
|
8
8
|
import { fileURLToPath } from "node:url";
|
|
9
9
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
10
10
|
import { beginFullRestartProgress as persistFullRestartProgress, beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, updateProgressOperation, } from "./update-marker.js";
|
|
@@ -41,7 +41,7 @@ import { WebhookEmitter } from "./webhook-emitter.js";
|
|
|
41
41
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
42
42
|
import { safeHandler } from "./safe-async.js";
|
|
43
43
|
import { RoutingEngine } from "./routing-engine.js";
|
|
44
|
-
import { InstanceLifecycle, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } from "./instance-lifecycle.js";
|
|
44
|
+
import { InstanceLifecycle, SupersededStartError, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } from "./instance-lifecycle.js";
|
|
45
45
|
import { TopicArchiver } from "./topic-archiver.js";
|
|
46
46
|
import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
47
47
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
@@ -66,7 +66,10 @@ import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBac
|
|
|
66
66
|
import { assertExplicitInstanceRemoval } from "./instance-removal.js";
|
|
67
67
|
import { validateFleetConfig } from "./config-validator.js";
|
|
68
68
|
import { readLastInboundAt } from "./daemon.js";
|
|
69
|
-
import { clearPausedMarker } from "./pause-marker.js";
|
|
69
|
+
import { clearPausedMarker, readPausedAt, readPauseReason, writePausedMarker } from "./pause-marker.js";
|
|
70
|
+
import { DEFAULT_WARM_OVERFLOW, WakeCoordinator } from "./wake-coordinator.js";
|
|
71
|
+
import { TargetQueueWorker } from "./target-queue-worker.js";
|
|
72
|
+
import { resolveDeliveryWorkerMode } from "./types.js";
|
|
70
73
|
import { isFleetStartCommandLine, readProcessCommandLine, releaseProcessFleetLock } from "./fleet-lock.js";
|
|
71
74
|
import { isSetupComplete, markSetupComplete } from "./setup-marker.js";
|
|
72
75
|
import { manualCleanupMessage, reapStaleTunnel } from "./tunnel/lease.js";
|
|
@@ -74,6 +77,7 @@ import { buildToolPermissionsNotice } from "./tool-permissions-notice.js";
|
|
|
74
77
|
import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
|
|
75
78
|
import { mayUseTool, resolveToolSet, scheduleOpRefusal, toolForIpcType, toolRefusedMessage, } from "./tool-permissions.js";
|
|
76
79
|
import { decideWebGate, loadOrCreateWebToken, readWebToken } from "./web-auth.js";
|
|
80
|
+
import { allowedHostNames, applyWebSecurityHeaders, hostnameOf, isHostAllowed, WEB_HOST_REJECTED_MESSAGE } from "./web-host-guard.js";
|
|
77
81
|
import { fleetLevelDifferences, fleetLevelSignature } from "./fleet-level-config.js";
|
|
78
82
|
import { checkSelfRestartAllowance, recordSelfRestartAttempt } from "./self-restart-limit.js";
|
|
79
83
|
import { SecretStore } from "./secret-store.js";
|
|
@@ -286,6 +290,14 @@ export class FleetManager {
|
|
|
286
290
|
managerBootId = randomUUID();
|
|
287
291
|
deliveryOutbox = null;
|
|
288
292
|
deliveryOutboxRecovered = false;
|
|
293
|
+
/** Phase 2b: wakes paused targets for queued durable work (delivery_worker ≠ off). */
|
|
294
|
+
wakeCoordinator = null;
|
|
295
|
+
/**
|
|
296
|
+
* Phase 2c: targets whose durable lane a TargetQueueWorker owns
|
|
297
|
+
* (delivery_worker: on). Granted and released only in the pump's
|
|
298
|
+
* synchronous section and only while the lane is empty; the pump skips them.
|
|
299
|
+
*/
|
|
300
|
+
queueWorkers = new Map();
|
|
289
301
|
deliveryPumpScheduled = false;
|
|
290
302
|
deliveryPumpRunning = false;
|
|
291
303
|
deliveryPumpTimer = null;
|
|
@@ -570,13 +582,19 @@ export class FleetManager {
|
|
|
570
582
|
this.logger.info({ ...recovered }, "Recovered durable delivery outbox for this process boot");
|
|
571
583
|
this.deliveryOutboxRecovered = true;
|
|
572
584
|
}
|
|
573
|
-
outbox.on("admitted", () => this.scheduleDeliveryOutboxPump());
|
|
585
|
+
outbox.on("admitted", () => { this.scheduleDeliveryOutboxPump(); this.wakeCoordinator?.kick(); });
|
|
574
586
|
outbox.on("state", (event) => {
|
|
575
587
|
this.scheduleDeliveryOutboxPump();
|
|
588
|
+
this.wakeCoordinator?.kick();
|
|
589
|
+
if (event.deliveryId && event.state === "delivered") {
|
|
590
|
+
const target = outbox.get(event.deliveryId)?.targetInstance;
|
|
591
|
+
if (target)
|
|
592
|
+
this.wakeCoordinator?.noteDelivered(target);
|
|
593
|
+
}
|
|
576
594
|
if (event.deliveryId && (event.state === "failed" || event.state === "uncertain")) {
|
|
577
595
|
const row = outbox.get(event.deliveryId);
|
|
578
596
|
if (row && row.kind !== "delivery_outcome_notice" && row.kind !== "post_restart_outcome_notice") {
|
|
579
|
-
const notice = `
|
|
597
|
+
const notice = t(`delivery.fleet_${row.state}`, row.targetInstance, row.operationId, row.deliveryId);
|
|
580
598
|
this.logger.error({ deliveryId: row.deliveryId, operationId: row.operationId, target: row.targetInstance, state: row.state }, notice);
|
|
581
599
|
this.notifyFleetError(notice);
|
|
582
600
|
}
|
|
@@ -591,10 +609,13 @@ export class FleetManager {
|
|
|
591
609
|
outbox.on("content_digest_mismatch", (event) => {
|
|
592
610
|
this.logger.warn(event, "Delivered text differs from the admitted text (content digest mismatch)");
|
|
593
611
|
});
|
|
612
|
+
this.wakeCoordinator = this.createWakeCoordinator();
|
|
613
|
+
this.wakeCoordinator.start();
|
|
594
614
|
outbox.on("expired", (event) => {
|
|
595
615
|
this.scheduleDeliveryOutboxPump();
|
|
616
|
+
this.wakeCoordinator?.kick();
|
|
596
617
|
if (event.count)
|
|
597
|
-
this.notifyFleetError(
|
|
618
|
+
this.notifyFleetError(t("delivery.expired", event.count, event.uncertain ?? 0));
|
|
598
619
|
});
|
|
599
620
|
this.scheduleDeliveryOutboxPump();
|
|
600
621
|
}
|
|
@@ -619,6 +640,9 @@ export class FleetManager {
|
|
|
619
640
|
kind: input.kind,
|
|
620
641
|
}, "Durable cross-instance delivery accepted");
|
|
621
642
|
this.scheduleDeliveryOutboxPump();
|
|
643
|
+
// Phase 2b: a paused wake_only target is woken by the coordinator, never by
|
|
644
|
+
// the pump; admission must tell it, or the work waits for the watchdog.
|
|
645
|
+
this.wakeCoordinator?.kick();
|
|
622
646
|
return {
|
|
623
647
|
deliveryId: admitted.delivery.deliveryId,
|
|
624
648
|
state: admitted.delivery.state,
|
|
@@ -681,7 +705,7 @@ export class FleetManager {
|
|
|
681
705
|
}
|
|
682
706
|
}
|
|
683
707
|
scheduleDeliveryOutboxPump(delayMs = 0) {
|
|
684
|
-
if (!this.deliveryOutbox || this.shuttingDown)
|
|
708
|
+
if (!this.deliveryOutbox || this.shuttingDown || !this.deliveryOutbox.isOpen)
|
|
685
709
|
return;
|
|
686
710
|
if (this.deliveryPumpTimer) {
|
|
687
711
|
if (delayMs > 0)
|
|
@@ -701,14 +725,27 @@ export class FleetManager {
|
|
|
701
725
|
}
|
|
702
726
|
async runDeliveryOutboxPump() {
|
|
703
727
|
const outbox = this.deliveryOutbox;
|
|
704
|
-
|
|
728
|
+
// A closed database is never queried: this runs from timers, where a throw
|
|
729
|
+
// is an unhandled rejection. Checking here covers the finally block too:
|
|
730
|
+
// the body below never awaits, so the database cannot close in between.
|
|
731
|
+
if (!outbox || this.shuttingDown || this.deliveryPumpRunning || !outbox.isOpen)
|
|
705
732
|
return;
|
|
706
733
|
this.deliveryPumpRunning = true;
|
|
707
734
|
this.deliveryPumpScheduled = false;
|
|
708
735
|
try {
|
|
709
|
-
outbox.expireStale(
|
|
736
|
+
outbox.expireStale(undefined, undefined, target => {
|
|
737
|
+
const failure = this.wakeCoordinator?.wakeFailure(target);
|
|
738
|
+
return failure ? `target could not be woken (${failure})` : undefined;
|
|
739
|
+
});
|
|
740
|
+
this.reconcileQueueWorkers(outbox);
|
|
710
741
|
while (this.activeDurableTargets.size < 8) {
|
|
711
|
-
const claimed = outbox.claimNext(this.managerBootId,
|
|
742
|
+
const claimed = outbox.claimNext(this.managerBootId,
|
|
743
|
+
// Phase 2b: a wake_only target is claimed only once it is awake; the
|
|
744
|
+
// wake coordinator, not this pump, wakes it (no claim, no attempt).
|
|
745
|
+
// Phase 2c: a lane a queue worker owns is never claimed here.
|
|
746
|
+
target => this.queueWorkers.has(target) || this.isInstanceRestarting(target) || this.wakeCoordinator?.blocksClaim(target)
|
|
747
|
+
? null
|
|
748
|
+
: this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets);
|
|
712
749
|
if (!claimed)
|
|
713
750
|
break;
|
|
714
751
|
this.activeDurableTargets.add(claimed.targetInstance);
|
|
@@ -736,7 +773,67 @@ export class FleetManager {
|
|
|
736
773
|
}
|
|
737
774
|
}
|
|
738
775
|
}
|
|
739
|
-
|
|
776
|
+
/**
|
|
777
|
+
* Phase 2c ownership (design §1.5-A), called synchronously from the pump
|
|
778
|
+
* before its claim loop, so a grant and the pump's own claims cannot
|
|
779
|
+
* interleave. A worker is granted a target's lane only when the target is
|
|
780
|
+
* `on` and its lane is empty: no row in flight in the outbox and no pump
|
|
781
|
+
* dispatch holding it. It gives the lane back only when it is idle and the
|
|
782
|
+
* target is no longer `on` (or has nothing pending). A flag change never
|
|
783
|
+
* takes a lane mid-drain.
|
|
784
|
+
*/
|
|
785
|
+
reconcileQueueWorkers(outbox) {
|
|
786
|
+
const pendingTargets = new Set(outbox.listPending().map(row => row.targetInstance));
|
|
787
|
+
for (const [target, worker] of this.queueWorkers) {
|
|
788
|
+
const keep = this.deliveryWorkerMode(target) === "on" && pendingTargets.has(target);
|
|
789
|
+
if (!keep && !worker.busy && !worker.inFlight)
|
|
790
|
+
this.queueWorkers.delete(target);
|
|
791
|
+
}
|
|
792
|
+
for (const target of pendingTargets) {
|
|
793
|
+
if (this.queueWorkers.has(target) || this.deliveryWorkerMode(target) !== "on")
|
|
794
|
+
continue;
|
|
795
|
+
const laneEmpty = !this.activeDurableTargets.has(target)
|
|
796
|
+
&& outbox.countForTarget(target, ["delivering", "submission_started"]) === 0;
|
|
797
|
+
if (laneEmpty)
|
|
798
|
+
this.queueWorkers.set(target, this.createQueueWorker(target));
|
|
799
|
+
}
|
|
800
|
+
for (const worker of this.queueWorkers.values())
|
|
801
|
+
void worker.drain();
|
|
802
|
+
}
|
|
803
|
+
createQueueWorker(target) {
|
|
804
|
+
const worker = new TargetQueueWorker(target, {
|
|
805
|
+
owns: () => this.queueWorkers.get(target) === worker,
|
|
806
|
+
wanted: () => this.deliveryWorkerMode(target) === "on",
|
|
807
|
+
available: () => !this.shuttingDown && this.deliveryOutbox?.isOpen === true,
|
|
808
|
+
blocked: () => this.isInstanceRestarting(target) || this.wakeCoordinator?.blocksClaim(target) === true,
|
|
809
|
+
daemonBootId: () => this.daemons.get(target)?.bootId ?? null,
|
|
810
|
+
claim: bootId => this.deliveryOutbox?.claimNext(this.managerBootId, candidate => candidate === target ? bootId : null, new Set()),
|
|
811
|
+
tryAcquireBudget: () => {
|
|
812
|
+
if (this.activeDurableTargets.size >= 8)
|
|
813
|
+
return false;
|
|
814
|
+
this.activeDurableTargets.add(target);
|
|
815
|
+
return true;
|
|
816
|
+
},
|
|
817
|
+
releaseBudget: () => { this.activeDurableTargets.delete(target); },
|
|
818
|
+
dispatch: async (claimed) => {
|
|
819
|
+
try {
|
|
820
|
+
await this.dispatchDurableDelivery(claimed, { holdLaneOnTransportError: true });
|
|
821
|
+
}
|
|
822
|
+
finally {
|
|
823
|
+
// The row's state event can fire before the hand-off settles; the
|
|
824
|
+
// pump pass it starts then still sees this worker busy and keeps it
|
|
825
|
+
// as owner. One kick after the dispatch has really settled lets the
|
|
826
|
+
// pump release an owner that is no longer wanted (or claim the next
|
|
827
|
+
// row). The no-claim path never kicks, so no retry hot loop.
|
|
828
|
+
this.scheduleDeliveryOutboxPump();
|
|
829
|
+
}
|
|
830
|
+
},
|
|
831
|
+
kickCoordinator: () => this.wakeCoordinator?.kick(),
|
|
832
|
+
logger: this.logger,
|
|
833
|
+
});
|
|
834
|
+
return worker;
|
|
835
|
+
}
|
|
836
|
+
async dispatchDurableDelivery(claimed, opts = {}) {
|
|
740
837
|
const outbox = this.deliveryOutbox;
|
|
741
838
|
if (!outbox)
|
|
742
839
|
return;
|
|
@@ -762,7 +859,7 @@ export class FleetManager {
|
|
|
762
859
|
content: payload.content,
|
|
763
860
|
delivery_id: claimed.deliveryId,
|
|
764
861
|
delivery_attempt: String(claimed.attemptNo),
|
|
765
|
-
}, { waitForIdle: false });
|
|
862
|
+
}, { waitForIdle: false, noInlineWake: this.deliveryWorkerMode(target) !== "off" });
|
|
766
863
|
}
|
|
767
864
|
else {
|
|
768
865
|
payload.meta = {
|
|
@@ -777,6 +874,7 @@ export class FleetManager {
|
|
|
777
874
|
sent = await this.deliverToInstance(target, payload, {
|
|
778
875
|
isCrossInstance: true,
|
|
779
876
|
waitForIdle: claimed.kind !== "steer",
|
|
877
|
+
noInlineWake: this.deliveryWorkerMode(target) !== "off",
|
|
780
878
|
});
|
|
781
879
|
}
|
|
782
880
|
if (!sent) {
|
|
@@ -789,7 +887,10 @@ export class FleetManager {
|
|
|
789
887
|
catch (err) {
|
|
790
888
|
const latest = outbox.get(claimed.deliveryId);
|
|
791
889
|
const reason = err instanceof Error ? err.message : String(err);
|
|
792
|
-
if (latest?.state === "submission_started") {
|
|
890
|
+
if (opts.holdLaneOnTransportError && (latest?.state === "submission_started" || latest?.state === "reconciliation_pending")) {
|
|
891
|
+
this.logger.warn({ deliveryId: claimed.deliveryId, target, attempt, err: reason }, "Transport failed after the submission began; holding the lane for the daemon's verdict or its generation's reconciliation");
|
|
892
|
+
}
|
|
893
|
+
else if (latest?.state === "submission_started") {
|
|
793
894
|
outbox.complete(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, "uncertain", reason);
|
|
794
895
|
}
|
|
795
896
|
else {
|
|
@@ -826,7 +927,7 @@ export class FleetManager {
|
|
|
826
927
|
if ((current?.state === "delivering" || current?.state === "submission_started")
|
|
827
928
|
&& current.targetDaemonBootId === claimed.targetDaemonBootId
|
|
828
929
|
&& current.managerBootId === claimed.managerBootId) {
|
|
829
|
-
this.notifyFleetError(
|
|
930
|
+
this.notifyFleetError(t("delivery.lane_waiting", claimed.targetInstance, Math.round(DURABLE_DELIVERY_LANE_ALERT_MS / 60_000), deliveryId));
|
|
830
931
|
}
|
|
831
932
|
}, DURABLE_DELIVERY_LANE_ALERT_MS);
|
|
832
933
|
alertTimer.unref?.();
|
|
@@ -1688,13 +1789,15 @@ export class FleetManager {
|
|
|
1688
1789
|
exclude,
|
|
1689
1790
|
isEvicting: name => this.warmCapEvicting.has(name),
|
|
1690
1791
|
isGeneral: name => isGeneralInstance(this.fleetConfig, name),
|
|
1691
|
-
|
|
1792
|
+
// A work lease (Phase 2a) keeps a just-woken or mid-submission instance
|
|
1793
|
+
// out of the eviction candidates; it is not idle in the sense that matters.
|
|
1794
|
+
isIdle: name => this.getInstanceExecutionState(name) === "idle" && !this.lifecycle.hasWorkLease(name),
|
|
1692
1795
|
lastInboundAt: name => readLastInboundAt(this.getInstanceDir(name)) ?? 0,
|
|
1693
1796
|
});
|
|
1694
1797
|
for (const victim of victims) {
|
|
1695
1798
|
this.warmCapEvicting.add(victim);
|
|
1696
1799
|
this.logger.info({ instance: victim, warm: warm.length, cap }, "warm_cap exceeded — auto-pausing LRU idle instance");
|
|
1697
|
-
this.lifecycle.pause(victim)
|
|
1800
|
+
this.lifecycle.pause(victim, "warm_cap")
|
|
1698
1801
|
.catch(err => this.logger.warn({ err, instance: victim }, "warm_cap auto-pause failed"))
|
|
1699
1802
|
.finally(() => this.warmCapEvicting.delete(victim));
|
|
1700
1803
|
}
|
|
@@ -1787,16 +1890,22 @@ export class FleetManager {
|
|
|
1787
1890
|
finish(false);
|
|
1788
1891
|
});
|
|
1789
1892
|
}
|
|
1790
|
-
async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch) {
|
|
1893
|
+
async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch, noInlineWake = false) {
|
|
1791
1894
|
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1792
1895
|
return false;
|
|
1793
1896
|
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
1794
1897
|
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1795
1898
|
return false;
|
|
1796
1899
|
let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
|
|
1900
|
+
if (this.lifecycle.isPaused(instanceName) && noInlineWake) {
|
|
1901
|
+
// Paused after its row was claimed (an operator or auth pause): hand the
|
|
1902
|
+
// row back; the coordinator decides whether and when to wake it.
|
|
1903
|
+
this.wakeCoordinator?.kick();
|
|
1904
|
+
return false;
|
|
1905
|
+
}
|
|
1797
1906
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
1798
1907
|
const wakeStartedAt = Date.now();
|
|
1799
|
-
await this.
|
|
1908
|
+
await this.explicitWake(instanceName, 30_000);
|
|
1800
1909
|
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1801
1910
|
return false;
|
|
1802
1911
|
// Waking added one to the warm count — make room by evicting a different
|
|
@@ -1919,7 +2028,11 @@ export class FleetManager {
|
|
|
1919
2028
|
?? ((options.isCrossInstance ?? inferredCrossInstance) || payload.type === "fleet_schedule_trigger");
|
|
1920
2029
|
if (!waitForIdle) {
|
|
1921
2030
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
1922
|
-
|
|
2031
|
+
if (options.noInlineWake) {
|
|
2032
|
+
this.wakeCoordinator?.kick();
|
|
2033
|
+
return false;
|
|
2034
|
+
}
|
|
2035
|
+
await this.explicitWake(instanceName, 30_000);
|
|
1923
2036
|
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1924
2037
|
return false;
|
|
1925
2038
|
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
@@ -1935,7 +2048,7 @@ export class FleetManager {
|
|
|
1935
2048
|
return true;
|
|
1936
2049
|
}
|
|
1937
2050
|
const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
|
|
1938
|
-
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch));
|
|
2051
|
+
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch, options.noInlineWake === true));
|
|
1939
2052
|
this.idleGatedDeliveryTails.set(instanceName, delivery);
|
|
1940
2053
|
try {
|
|
1941
2054
|
return await delivery;
|
|
@@ -1951,9 +2064,117 @@ export class FleetManager {
|
|
|
1951
2064
|
const allowed = this.getChannelConfig(adapterId)?.access?.allowed_users ?? [];
|
|
1952
2065
|
return allowed.some(entry => String(entry) === String(userId));
|
|
1953
2066
|
}
|
|
2067
|
+
/** Phase 2: delivery_worker for a target (instance override → fleet default → off). */
|
|
2068
|
+
deliveryWorkerMode(target) {
|
|
2069
|
+
return resolveDeliveryWorkerMode(this.fleetConfig, target);
|
|
2070
|
+
}
|
|
2071
|
+
/** Active instances mid-restart: they keep their warm slot through the stop → replacement gap. */
|
|
2072
|
+
restartsHoldingSlot = new Set();
|
|
2073
|
+
/**
|
|
2074
|
+
* Instances counted against the wake coordinator's hard cap: running
|
|
2075
|
+
* daemons, plus active instances whose restart is between stopping the old
|
|
2076
|
+
* daemon and publishing the replacement.
|
|
2077
|
+
*/
|
|
2078
|
+
warmInstanceNames() {
|
|
2079
|
+
const names = new Set([...this.daemons.keys()].filter(name => this.getInstanceStatus(name) === "running"));
|
|
2080
|
+
for (const name of this.restartsHoldingSlot)
|
|
2081
|
+
names.add(name);
|
|
2082
|
+
return [...names];
|
|
2083
|
+
}
|
|
2084
|
+
createWakeCoordinator() {
|
|
2085
|
+
return new WakeCoordinator({
|
|
2086
|
+
mode: target => this.deliveryWorkerMode(target),
|
|
2087
|
+
available: () => !this.shuttingDown && this.deliveryOutbox?.isOpen === true,
|
|
2088
|
+
listPending: () => this.deliveryOutbox?.listPending() ?? [],
|
|
2089
|
+
isPaused: target => this.lifecycle.isPaused(target),
|
|
2090
|
+
pauseReason: target => readPauseReason(this.getInstanceDir(target)),
|
|
2091
|
+
isRestarting: target => this.isInstanceRestarting(target),
|
|
2092
|
+
wake: async (target) => {
|
|
2093
|
+
await this.lifecycle.wake(target, 30_000, undefined, { source: "coordinator" });
|
|
2094
|
+
// The soft cap still applies: evict a different, unleased idle instance.
|
|
2095
|
+
this.enforceWarmCap(target);
|
|
2096
|
+
},
|
|
2097
|
+
residentNames: () => this.warmInstanceNames(),
|
|
2098
|
+
warmCap: () => this.fleetConfig?.defaults?.warm_cap ?? 0,
|
|
2099
|
+
warmOverflow: () => this.fleetConfig?.defaults?.warm_overflow ?? DEFAULT_WARM_OVERFLOW,
|
|
2100
|
+
wokeAt: target => this.lifecycle.wokeAtFor(target),
|
|
2101
|
+
notAcceptingReason: target => this.daemons.get(target)?.notAcceptingReason?.() ?? null,
|
|
2102
|
+
notifyTarget: (target, text) => { this.notifyInstanceTopic(target, text); },
|
|
2103
|
+
notifySender: (source, text) => { this.notifyInstanceTopic(source, text); },
|
|
2104
|
+
kickPump: () => this.scheduleDeliveryOutboxPump(),
|
|
2105
|
+
now: () => Date.now(),
|
|
2106
|
+
logger: this.logger,
|
|
2107
|
+
});
|
|
2108
|
+
}
|
|
2109
|
+
/** LifecycleContext: an operator or user wake clears the coordinator's park/backoff for it. */
|
|
2110
|
+
onExternalWake(name) {
|
|
2111
|
+
this.wakeCoordinator?.noteExternalWake(name);
|
|
2112
|
+
}
|
|
2113
|
+
/**
|
|
2114
|
+
* Phase 2b shared warm-slot admission (design §1.3, §1.6) for every wake
|
|
2115
|
+
* that is not the coordinator's own: an operator's wake, a user's message,
|
|
2116
|
+
* an explicit start or a restart of a paused instance. With delivery_worker
|
|
2117
|
+
* off, or for a target that is not paused (an active restart is already
|
|
2118
|
+
* counted), nothing is reserved. A target already being woken is joined —
|
|
2119
|
+
* its flight holds the slot — rather than reserved twice. Otherwise a slot
|
|
2120
|
+
* is taken under the hard cap, an unleased idle instance is paused to make
|
|
2121
|
+
* one, and if none can be, the wake is refused instead of exceeding the cap.
|
|
2122
|
+
*/
|
|
2123
|
+
async acquireWakeSlot(name) {
|
|
2124
|
+
const coordinator = this.wakeCoordinator;
|
|
2125
|
+
if (!coordinator || this.deliveryWorkerMode(name) === "off" || !this.lifecycle.isPaused(name))
|
|
2126
|
+
return null;
|
|
2127
|
+
if (coordinator.isWaking(name))
|
|
2128
|
+
return null;
|
|
2129
|
+
let token = coordinator.tryReserve(name);
|
|
2130
|
+
if (!token) {
|
|
2131
|
+
const victim = this.lruUnleasedIdle(name);
|
|
2132
|
+
if (victim) {
|
|
2133
|
+
await this.lifecycle.pause(victim, "warm_cap");
|
|
2134
|
+
if (coordinator.isWaking(name))
|
|
2135
|
+
return null;
|
|
2136
|
+
token = coordinator.tryReserve(name);
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
if (!token) {
|
|
2140
|
+
throw new Error(`No warm slot is free for '${name}' (warm_cap + overflow reached and no idle instance can be paused); try again when one goes idle`);
|
|
2141
|
+
}
|
|
2142
|
+
return token;
|
|
2143
|
+
}
|
|
2144
|
+
releaseWakeSlot(name, token) {
|
|
2145
|
+
if (!token || !this.wakeCoordinator)
|
|
2146
|
+
return;
|
|
2147
|
+
this.wakeCoordinator.release(name, token);
|
|
2148
|
+
this.wakeCoordinator.kick();
|
|
2149
|
+
}
|
|
2150
|
+
/**
|
|
2151
|
+
* An operator's or user's wake (/wake, wake_instance, start_instance on a
|
|
2152
|
+
* paused instance, Settings, a channel message). With delivery_worker off it
|
|
2153
|
+
* is exactly the old lifecycle wake; otherwise it goes through the shared
|
|
2154
|
+
* slot admission and joins any wake already in flight for the target.
|
|
2155
|
+
*/
|
|
2156
|
+
async explicitWake(name, timeoutMs = 30_000) {
|
|
2157
|
+
const token = await this.acquireWakeSlot(name);
|
|
2158
|
+
try {
|
|
2159
|
+
await this.lifecycle.wake(name, timeoutMs);
|
|
2160
|
+
}
|
|
2161
|
+
finally {
|
|
2162
|
+
this.releaseWakeSlot(name, token);
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
2165
|
+
/** The least-recently-active idle, unleased, non-general running instance other than `exclude`. */
|
|
2166
|
+
lruUnleasedIdle(exclude) {
|
|
2167
|
+
return this.warmInstanceNames()
|
|
2168
|
+
.filter(name => name !== exclude
|
|
2169
|
+
&& !this.warmCapEvicting.has(name)
|
|
2170
|
+
&& !isGeneralInstance(this.fleetConfig, name)
|
|
2171
|
+
&& this.getInstanceExecutionState(name) === "idle"
|
|
2172
|
+
&& !this.lifecycle.hasWorkLease(name))
|
|
2173
|
+
.sort((a, b) => (readLastInboundAt(this.getInstanceDir(a)) ?? 0) - (readLastInboundAt(this.getInstanceDir(b)) ?? 0))[0];
|
|
2174
|
+
}
|
|
1954
2175
|
async changeInstancePauseState(name, action) {
|
|
1955
2176
|
if (action === "wake") {
|
|
1956
|
-
await this.
|
|
2177
|
+
await this.explicitWake(name, 30_000);
|
|
1957
2178
|
this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
|
|
1958
2179
|
return "awake";
|
|
1959
2180
|
}
|
|
@@ -2072,12 +2293,14 @@ export class FleetManager {
|
|
|
2072
2293
|
* and reconcile calls leave this false so a persisted pause remains paused
|
|
2073
2294
|
* across a fleet restart.
|
|
2074
2295
|
*/
|
|
2075
|
-
resumePaused = false
|
|
2296
|
+
resumePaused = false,
|
|
2297
|
+
/** Phase 2a: the caller's transition, for a start made from inside a restart or wake. */
|
|
2298
|
+
transition) {
|
|
2076
2299
|
// Any start supersedes a pending automatic retry (it would otherwise fire
|
|
2077
2300
|
// into a running instance — harmless, but noisy — or race this start).
|
|
2078
2301
|
this.cancelStartupRetry(name);
|
|
2079
2302
|
if (resumePaused && this.lifecycle.isPaused(name)) {
|
|
2080
|
-
await this.
|
|
2303
|
+
await this.explicitWake(name, 30_000);
|
|
2081
2304
|
// A successful wake clears the persisted pause marker and produces a
|
|
2082
2305
|
// fresh instance_state snapshot. Drop any stale process error left by a
|
|
2083
2306
|
// pre-pause crash so /api/fleet and `agend ls` converge on running.
|
|
@@ -2104,6 +2327,10 @@ export class FleetManager {
|
|
|
2104
2327
|
this.logger.info({ name }, "Instance already running, skipping");
|
|
2105
2328
|
return;
|
|
2106
2329
|
}
|
|
2330
|
+
// An explicit start (CLI/API) begins a fresh idle window; boot and
|
|
2331
|
+
// reconcile starts keep the persisted last-inbound seed (Phase 2a).
|
|
2332
|
+
if (resumePaused)
|
|
2333
|
+
this.lifecycle.markActivitySeedNow(name);
|
|
2107
2334
|
const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
2108
2335
|
if (config.general_topic) {
|
|
2109
2336
|
this.ensureGeneralInstructions(config.working_directory, backend, name);
|
|
@@ -2127,7 +2354,7 @@ export class FleetManager {
|
|
|
2127
2354
|
kind,
|
|
2128
2355
|
backend,
|
|
2129
2356
|
model: this.resolveInstanceModel(name).display,
|
|
2130
|
-
});
|
|
2357
|
+
}, transition);
|
|
2131
2358
|
// Only clear a stale process status after a real start succeeded. Clearing
|
|
2132
2359
|
// it before lifecycle.start() can turn a crash-loop daemon's dead pane into
|
|
2133
2360
|
// a falsely running instance when lifecycle.start() returns early.
|
|
@@ -2141,18 +2368,18 @@ export class FleetManager {
|
|
|
2141
2368
|
this.requestDiscordUsagePresenceRefresh();
|
|
2142
2369
|
}
|
|
2143
2370
|
/** Recreate a daemon for a marker-only paused instance after an explicit wake/delivery. */
|
|
2144
|
-
async startPersistedPausedInstance(name) {
|
|
2371
|
+
async startPersistedPausedInstance(name, transition) {
|
|
2145
2372
|
const topicMode = this.fleetConfig?.channel?.mode === "topic"
|
|
2146
2373
|
|| !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
|
|
2147
2374
|
const fleetConfig = this.fleetConfig?.instances[name];
|
|
2148
2375
|
if (fleetConfig) {
|
|
2149
|
-
await this.startInstance(name, fleetConfig, topicMode);
|
|
2376
|
+
await this.startInstance(name, fleetConfig, topicMode, "fleet-topic", false, transition);
|
|
2150
2377
|
return;
|
|
2151
2378
|
}
|
|
2152
2379
|
const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
|
|
2153
2380
|
if (!channel || !this.classicChannels)
|
|
2154
2381
|
throw new Error(`Paused instance '${name}' is no longer configured`);
|
|
2155
|
-
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
2382
|
+
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after), transition);
|
|
2156
2383
|
}
|
|
2157
2384
|
/**
|
|
2158
2385
|
* Start instances with configurable concurrency and stagger delay.
|
|
@@ -2201,6 +2428,10 @@ export class FleetManager {
|
|
|
2201
2428
|
return this.daemons.has(name);
|
|
2202
2429
|
}
|
|
2203
2430
|
catch (err) {
|
|
2431
|
+
// Superseded by a stop/restart (Phase 2a): that transition owns the
|
|
2432
|
+
// outcome. Retrying would start an instance the operator just stopped.
|
|
2433
|
+
if (err instanceof SupersededStartError)
|
|
2434
|
+
return false;
|
|
2204
2435
|
this.logger.error({ err, name }, `Failed to start ${what}`);
|
|
2205
2436
|
this.scheduleStartupRetry(name, 0);
|
|
2206
2437
|
return false;
|
|
@@ -2322,6 +2553,9 @@ export class FleetManager {
|
|
|
2322
2553
|
}
|
|
2323
2554
|
}
|
|
2324
2555
|
catch (err) {
|
|
2556
|
+
// Superseded by a stop/restart: that transition owns the outcome.
|
|
2557
|
+
if (err instanceof SupersededStartError)
|
|
2558
|
+
return;
|
|
2325
2559
|
this.logger.error({ err, name, attempt: attempt + 1 }, "Automatic startup retry failed");
|
|
2326
2560
|
this.scheduleStartupRetry(name, attempt + 1);
|
|
2327
2561
|
}
|
|
@@ -2362,6 +2596,8 @@ export class FleetManager {
|
|
|
2362
2596
|
return this.daemons.has(ch.instanceName);
|
|
2363
2597
|
}
|
|
2364
2598
|
catch (err) {
|
|
2599
|
+
if (err instanceof SupersededStartError)
|
|
2600
|
+
return false;
|
|
2365
2601
|
this.logger.warn({ err, instanceName: ch.instanceName }, `Failed to start ${what}`);
|
|
2366
2602
|
this.scheduleStartupRetry(ch.instanceName, 0);
|
|
2367
2603
|
return false;
|
|
@@ -2518,7 +2754,7 @@ export class FleetManager {
|
|
|
2518
2754
|
}
|
|
2519
2755
|
return false;
|
|
2520
2756
|
}
|
|
2521
|
-
async stopInstance(name) {
|
|
2757
|
+
async stopInstance(name, transition) {
|
|
2522
2758
|
this.explicitStopGeneration.set(name, (this.explicitStopGeneration.get(name) ?? 0) + 1);
|
|
2523
2759
|
this.cancelStartupRetry(name);
|
|
2524
2760
|
this.failoverActive.delete(name);
|
|
@@ -2537,7 +2773,7 @@ export class FleetManager {
|
|
|
2537
2773
|
// the hand-off snapshotted it).
|
|
2538
2774
|
const run = (async () => {
|
|
2539
2775
|
try {
|
|
2540
|
-
await this.lifecycle.stop(name);
|
|
2776
|
+
await this.lifecycle.stop(name, transition);
|
|
2541
2777
|
}
|
|
2542
2778
|
finally {
|
|
2543
2779
|
this.clearNoncePromptsForInstance(name);
|
|
@@ -2559,11 +2795,16 @@ export class FleetManager {
|
|
|
2559
2795
|
return inFlight;
|
|
2560
2796
|
}
|
|
2561
2797
|
const workingDirectory = this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name);
|
|
2562
|
-
|
|
2798
|
+
// Phase 2a: invalidate synchronously so a start still spawning does not
|
|
2799
|
+
// publish, then queue behind it as this instance's next transition. The
|
|
2800
|
+
// spawn gate is taken inside the transition, never the other way round, so
|
|
2801
|
+
// the start being waited for can still get its own gate slot.
|
|
2802
|
+
this.lifecycle.invalidate(name);
|
|
2803
|
+
const run = this.lifecycle.runTransition(name, transition => this.spawnGate.run({
|
|
2563
2804
|
instanceName: name,
|
|
2564
2805
|
workingDirectory,
|
|
2565
2806
|
reason: "restart",
|
|
2566
|
-
}, () => this.doRestartSingleInstance(name, opts))
|
|
2807
|
+
}, () => this.doRestartSingleInstance(name, opts, transition)))
|
|
2567
2808
|
.finally(() => {
|
|
2568
2809
|
this.restartsInFlight.delete(name);
|
|
2569
2810
|
this.scheduleDeliveryOutboxPump();
|
|
@@ -2574,19 +2815,73 @@ export class FleetManager {
|
|
|
2574
2815
|
isInstanceRestarting(name) {
|
|
2575
2816
|
return this.restartsInFlight.has(name);
|
|
2576
2817
|
}
|
|
2577
|
-
|
|
2818
|
+
/** InstanceLifecycleContext: an explicit wake joins the restart in flight. */
|
|
2819
|
+
restartInFlight(name) {
|
|
2820
|
+
return this.restartsInFlight.get(name);
|
|
2821
|
+
}
|
|
2822
|
+
/**
|
|
2823
|
+
* Phase 2a: restarting a paused instance is a wake. The stop leaves the
|
|
2824
|
+
* pause marker in place, and the start used to skip on it ("Persisted
|
|
2825
|
+
* paused instance"), turning a resident pause into a marker-only one that
|
|
2826
|
+
* nothing would wake. Clear it before the start; if the start fails, put it
|
|
2827
|
+
* back so the instance stays wakeable instead of becoming plain stopped.
|
|
2828
|
+
*/
|
|
2829
|
+
async startAfterRestart(name, wasPaused, pausedAt, reason, start) {
|
|
2830
|
+
const instanceDir = this.getInstanceDir(name);
|
|
2831
|
+
if (wasPaused)
|
|
2832
|
+
clearPausedMarker(instanceDir);
|
|
2833
|
+
this.lifecycle.markActivitySeedNow(name);
|
|
2834
|
+
try {
|
|
2835
|
+
await start();
|
|
2836
|
+
}
|
|
2837
|
+
catch (err) {
|
|
2838
|
+
if (wasPaused && !this.daemons.has(name))
|
|
2839
|
+
writePausedMarker(instanceDir, pausedAt ?? Date.now(), reason);
|
|
2840
|
+
throw err;
|
|
2841
|
+
}
|
|
2842
|
+
finally {
|
|
2843
|
+
this.lifecycle.clearActivitySeedNow(name);
|
|
2844
|
+
}
|
|
2845
|
+
}
|
|
2846
|
+
async doRestartSingleInstance(name, opts, transition) {
|
|
2578
2847
|
if (this.configPath) {
|
|
2579
2848
|
this.loadConfig(this.configPath);
|
|
2580
2849
|
this.routing.rebuild(this.fleetConfig);
|
|
2581
2850
|
this.reregisterClassicChannels();
|
|
2582
2851
|
}
|
|
2583
2852
|
const config = this.fleetConfig?.instances[name];
|
|
2853
|
+
const wasPaused = this.lifecycle.isPaused(name);
|
|
2854
|
+
// A paused instance's restart is a wake: it takes a warm slot like any other
|
|
2855
|
+
// (refused before anything is stopped when none can be had). An active
|
|
2856
|
+
// restart is already counted and reserves nothing.
|
|
2857
|
+
const slot = await this.acquireWakeSlot(name);
|
|
2858
|
+
// An active instance keeps the slot it already occupies across its own
|
|
2859
|
+
// stop → replacement gap: counted as warm until the replacement is
|
|
2860
|
+
// published or the restart settles, so no other target can take it.
|
|
2861
|
+
if (!wasPaused)
|
|
2862
|
+
this.restartsHoldingSlot.add(name);
|
|
2863
|
+
try {
|
|
2864
|
+
await this.doRestartWithSlot(name, opts, transition, config, wasPaused);
|
|
2865
|
+
// An explicit restart of a paused instance is an operator wake (§1.6):
|
|
2866
|
+
// its success lifts the coordinator's park/backoff (the level stays).
|
|
2867
|
+
// Automatic restarts never do.
|
|
2868
|
+
if (wasPaused && opts?.explicit && this.daemons.has(name))
|
|
2869
|
+
this.wakeCoordinator?.noteExternalWake(name);
|
|
2870
|
+
}
|
|
2871
|
+
finally {
|
|
2872
|
+
this.restartsHoldingSlot.delete(name);
|
|
2873
|
+
this.releaseWakeSlot(name, slot);
|
|
2874
|
+
}
|
|
2875
|
+
}
|
|
2876
|
+
async doRestartWithSlot(name, opts, transition, config, wasPaused) {
|
|
2877
|
+
const pausedAt = wasPaused ? (this.lifecycle.getLastPausedAt(name) ?? readPausedAt(this.getInstanceDir(name))) : null;
|
|
2878
|
+
const pauseReason = wasPaused ? readPauseReason(this.getInstanceDir(name)) : null;
|
|
2584
2879
|
if (config) {
|
|
2585
|
-
await this.stopInstance(name);
|
|
2880
|
+
await this.stopInstance(name, transition);
|
|
2586
2881
|
if (opts?.freshStart)
|
|
2587
2882
|
this.writeFreshStartMarker(name);
|
|
2588
2883
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
2589
|
-
await this.startInstance(name, config, topicMode ?? false);
|
|
2884
|
+
await this.startAfterRestart(name, wasPaused, pausedAt, pauseReason, () => this.startInstance(name, config, topicMode ?? false, "fleet-topic", false, transition));
|
|
2590
2885
|
this.stormWindow.markRecovered(name);
|
|
2591
2886
|
return;
|
|
2592
2887
|
}
|
|
@@ -2595,11 +2890,11 @@ export class FleetManager {
|
|
|
2595
2890
|
if (channelId) {
|
|
2596
2891
|
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
2597
2892
|
const adapterId = this.classicChannels.getAdapterIdByInstance(name);
|
|
2598
|
-
await this.stopInstance(name);
|
|
2893
|
+
await this.stopInstance(name, transition);
|
|
2599
2894
|
await new Promise(r => setTimeout(r, 1000)); // let tmux clean up
|
|
2600
2895
|
if (opts?.freshStart)
|
|
2601
2896
|
this.writeFreshStartMarker(name);
|
|
2602
|
-
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
2897
|
+
await this.startAfterRestart(name, wasPaused, pausedAt, pauseReason, () => this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after), transition));
|
|
2603
2898
|
this.stormWindow.markRecovered(name);
|
|
2604
2899
|
return;
|
|
2605
2900
|
}
|
|
@@ -4479,6 +4774,7 @@ export class FleetManager {
|
|
|
4479
4774
|
const targetInstance = String(msg.targetInstance ?? name);
|
|
4480
4775
|
const correlationId = String(msg.correlationId ?? "unknown");
|
|
4481
4776
|
const error = String(msg.error ?? "unknown delivery failure");
|
|
4777
|
+
const displayError = error.replace(/^delivery failed: phase=(.*); proof=(.*)$/, (_match, phase, proof) => t("delivery.pane_failure", phase, proof));
|
|
4482
4778
|
const senderInstance = this.instanceIpcClients.has(senderSession)
|
|
4483
4779
|
? senderSession
|
|
4484
4780
|
: this.sessionRegistry.get(senderSession);
|
|
@@ -4487,11 +4783,11 @@ export class FleetManager {
|
|
|
4487
4783
|
from: senderSession, correlation_id: correlationId, error,
|
|
4488
4784
|
});
|
|
4489
4785
|
if (senderInstance) {
|
|
4490
|
-
this.notifyInstanceTopic(senderInstance, t("cross_instance.delivery_failed_notice", targetInstance,
|
|
4786
|
+
this.notifyInstanceTopic(senderInstance, t("cross_instance.delivery_failed_notice", targetInstance, displayError, correlationId));
|
|
4491
4787
|
void this.deliverToInstance(senderInstance, {
|
|
4492
4788
|
type: "fleet_inbound",
|
|
4493
4789
|
targetSession: senderSession,
|
|
4494
|
-
content: t("cross_instance.delivery_failed_agent", targetInstance,
|
|
4790
|
+
content: t("cross_instance.delivery_failed_agent", targetInstance, displayError, correlationId),
|
|
4495
4791
|
meta: {
|
|
4496
4792
|
chat_id: "",
|
|
4497
4793
|
message_id: `delivery-failed-${Date.now()}`,
|
|
@@ -4767,7 +5063,7 @@ export class FleetManager {
|
|
|
4767
5063
|
*/
|
|
4768
5064
|
governingAccess(msg, threadId) {
|
|
4769
5065
|
if (threadId && !this.classicChannels?.hasChannel(threadId)) {
|
|
4770
|
-
const target = this.
|
|
5066
|
+
const target = this.resolveInboundTarget(msg, threadId);
|
|
4771
5067
|
if (target) {
|
|
4772
5068
|
const owner = this.getInstanceAdapterId(target.name);
|
|
4773
5069
|
if (owner)
|
|
@@ -4803,7 +5099,7 @@ export class FleetManager {
|
|
|
4803
5099
|
return null;
|
|
4804
5100
|
if (this.classicChannels?.hasChannel(threadId))
|
|
4805
5101
|
return null;
|
|
4806
|
-
const target = this.
|
|
5102
|
+
const target = this.resolveInboundTarget(msg, threadId);
|
|
4807
5103
|
if (!target)
|
|
4808
5104
|
return null;
|
|
4809
5105
|
const ownerAdapterId = this.getInstanceAdapterId(target.name);
|
|
@@ -4835,7 +5131,7 @@ export class FleetManager {
|
|
|
4835
5131
|
return "classic: collab off";
|
|
4836
5132
|
return null;
|
|
4837
5133
|
}
|
|
4838
|
-
const target = this.
|
|
5134
|
+
const target = this.resolveInboundTarget(msg, threadId);
|
|
4839
5135
|
if (!target)
|
|
4840
5136
|
return "fleet topic: no instance routed for this thread";
|
|
4841
5137
|
// Fleet topic: this gate lets the copy through when the adapter is open OR
|
|
@@ -4868,8 +5164,42 @@ export class FleetManager {
|
|
|
4868
5164
|
return;
|
|
4869
5165
|
this.logger.warn({ statePath, configured, inEffect: accessManager.getMode() }, `access mode "${accessManager.getMode()}" comes from the state file and overrides "${configured}" in fleet.yaml; delete ${statePath} to go back to the configured value`);
|
|
4870
5166
|
}
|
|
5167
|
+
/**
|
|
5168
|
+
* #1085: the thread id inbound routing may use. Telegram topic ids are small
|
|
5169
|
+
* integers numbered per group, but the routing table is keyed by topic id
|
|
5170
|
+
* alone — so a message in topic 30 of ANY group the bot is in resolved to
|
|
5171
|
+
* the fleet instance that owns topic 30 of the fleet's own group (and a group
|
|
5172
|
+
* that never ran /start was bound by coincidence). A Telegram topic counts
|
|
5173
|
+
* as a fleet topic only in the fleet's group: `chat_id == group_id` of the
|
|
5174
|
+
* receiving bot's channel. From any other group the message is a ClassicBot
|
|
5175
|
+
* candidate keyed by its chat id (served only if /start registered it,
|
|
5176
|
+
* ignored otherwise). Discord channel ids are global, so they are unaffected.
|
|
5177
|
+
*/
|
|
5178
|
+
inboundRouteThreadId(msg) {
|
|
5179
|
+
const raw = msg.threadId || undefined;
|
|
5180
|
+
if (!raw || msg.source !== "telegram")
|
|
5181
|
+
return raw;
|
|
5182
|
+
const fleetGroupId = this.getChannelConfig(msg.adapterId)?.group_id;
|
|
5183
|
+
if (fleetGroupId != null && String(fleetGroupId) === msg.chatId)
|
|
5184
|
+
return raw;
|
|
5185
|
+
this.logger.debug({ adapterId: msg.adapterId, chatId: msg.chatId, threadId: raw }, "Telegram topic outside the fleet group — not a fleet topic (#1085)");
|
|
5186
|
+
return undefined;
|
|
5187
|
+
}
|
|
5188
|
+
/**
|
|
5189
|
+
* #1085, defense in depth: a routed Telegram target must belong to the group
|
|
5190
|
+
* the message came from — its own world's group_id. Two Telegram fleet
|
|
5191
|
+
* worlds can number topics alike; a coincidence is dropped, never delivered.
|
|
5192
|
+
*/
|
|
5193
|
+
resolveInboundTarget(msg, threadId) {
|
|
5194
|
+
if (msg.source !== "telegram")
|
|
5195
|
+
return this.routing.resolve(threadId);
|
|
5196
|
+
return this.routing.resolveAll(threadId).find(target => {
|
|
5197
|
+
const ownerGroupId = this.getChannelConfig(this.getInstanceAdapterId(target.name))?.group_id;
|
|
5198
|
+
return ownerGroupId != null && String(ownerGroupId) === msg.chatId;
|
|
5199
|
+
});
|
|
5200
|
+
}
|
|
4871
5201
|
async handleInboundMessage(msg) {
|
|
4872
|
-
const threadId = msg
|
|
5202
|
+
const threadId = this.inboundRouteThreadId(msg);
|
|
4873
5203
|
this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
|
|
4874
5204
|
// Ownership and per-adapter filtering both run before the dedup claim below.
|
|
4875
5205
|
// When several bots share a guild they all receive the same message; a copy
|
|
@@ -4925,7 +5255,7 @@ export class FleetManager {
|
|
|
4925
5255
|
// the message (the @mention filter downstream decides who actually forwards).
|
|
4926
5256
|
// Key the dedup per-adapter there so a sibling bot's copy isn't dropped.
|
|
4927
5257
|
if (msg.messageId) {
|
|
4928
|
-
const classicCid =
|
|
5258
|
+
const classicCid = threadId || msg.chatId;
|
|
4929
5259
|
const isClassicMsg = this.classicChannels?.hasChannel(classicCid) ?? false;
|
|
4930
5260
|
const dedupKey = isClassicMsg
|
|
4931
5261
|
? `${msg.source}:${msg.chatId}:${msg.messageId}:${msg.adapterId ?? ""}`
|
|
@@ -5214,7 +5544,7 @@ export class FleetManager {
|
|
|
5214
5544
|
// TG ClassicBot: group requires @mention, private chat forwards directly.
|
|
5215
5545
|
if (!isPrivateChat && !isBotMentioned) {
|
|
5216
5546
|
// No trigger: save attachments + react, log, but don't forward to agent
|
|
5217
|
-
const syntheticMsg = { ...msg, threadId: chatId, text: rawText.startsWith("/") ? "" : rawText };
|
|
5547
|
+
const syntheticMsg = { ...msg, threadId: chatId, transportThreadId: msg.threadId || undefined, text: rawText.startsWith("/") ? "" : rawText };
|
|
5218
5548
|
await this.handleClassicChannelMessage(classicName, syntheticMsg);
|
|
5219
5549
|
return;
|
|
5220
5550
|
}
|
|
@@ -5228,7 +5558,7 @@ export class FleetManager {
|
|
|
5228
5558
|
await msgAdapter?.sendText(chatId, t("cmd.admin_required", "/raw"));
|
|
5229
5559
|
return;
|
|
5230
5560
|
}
|
|
5231
|
-
const syntheticMsg = { ...msg, threadId: chatId, text: `/chat ${cleanText}` };
|
|
5561
|
+
const syntheticMsg = { ...msg, threadId: chatId, transportThreadId: msg.threadId || undefined, text: `/chat ${cleanText}` };
|
|
5232
5562
|
await this.handleClassicChannelMessage(classicName, syntheticMsg);
|
|
5233
5563
|
return;
|
|
5234
5564
|
}
|
|
@@ -5315,7 +5645,7 @@ export class FleetManager {
|
|
|
5315
5645
|
await this.handleClassicChannelMessage(classicName, msg);
|
|
5316
5646
|
return;
|
|
5317
5647
|
}
|
|
5318
|
-
const target = this.
|
|
5648
|
+
const target = this.resolveInboundTarget(msg, threadId);
|
|
5319
5649
|
if (!target) {
|
|
5320
5650
|
// Only show unbound message for actual forum topics (same group, has threadId)
|
|
5321
5651
|
const adapterGroupId = String(this.getChannelConfig(msg.adapterId)?.group_id ?? "");
|
|
@@ -5555,8 +5885,18 @@ export class FleetManager {
|
|
|
5555
5885
|
const classicChannelId = this.classicChannels?.getChannelIdByInstance(senderInstanceName ?? instanceName);
|
|
5556
5886
|
if (classicChannelId) {
|
|
5557
5887
|
args.chat_id = classicChannelId;
|
|
5558
|
-
|
|
5559
|
-
|
|
5888
|
+
// #1085: a Telegram forum ClassicBot replies in the topic it was asked
|
|
5889
|
+
// in. Any other thread_id (threadless Telegram, where it echoes the chat
|
|
5890
|
+
// id, or a Discord channel) is still cleared, as before.
|
|
5891
|
+
const topic = typeof args.thread_id === "string" ? args.thread_id : "";
|
|
5892
|
+
const telegram = (contextWorld?.adapter ?? outAdapter)?.type === "telegram";
|
|
5893
|
+
if (telegram && topic && topic !== classicChannelId) {
|
|
5894
|
+
threadId = topic;
|
|
5895
|
+
}
|
|
5896
|
+
else {
|
|
5897
|
+
delete args.thread_id;
|
|
5898
|
+
threadId = undefined;
|
|
5899
|
+
}
|
|
5560
5900
|
}
|
|
5561
5901
|
// Reply dedup: retries land here when the agent was told a send failed
|
|
5562
5902
|
// (daemon budget elapsed, shell tool killed) while the adapter send was
|
|
@@ -6002,10 +6342,15 @@ export class FleetManager {
|
|
|
6002
6342
|
?? this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName)?.displayName
|
|
6003
6343
|
?? instanceName;
|
|
6004
6344
|
}
|
|
6345
|
+
/** Inherited keys such as __proto__ must never resolve to a mutable instance. */
|
|
6346
|
+
ownInstanceConfig(instanceName) {
|
|
6347
|
+
const instances = this.fleetConfig?.instances;
|
|
6348
|
+
return instances && Object.hasOwn(instances, instanceName) ? instances[instanceName] : undefined;
|
|
6349
|
+
}
|
|
6005
6350
|
/** Persist identity to the instance's actual config store. Classic instances
|
|
6006
6351
|
* are registry rows in classicBot.yaml, not fleet.yaml instance entries. */
|
|
6007
6352
|
setInstanceDisplayName(instanceName, displayName) {
|
|
6008
|
-
const fleetInstance = this.
|
|
6353
|
+
const fleetInstance = this.ownInstanceConfig(instanceName);
|
|
6009
6354
|
if (fleetInstance) {
|
|
6010
6355
|
fleetInstance.display_name = displayName;
|
|
6011
6356
|
this.saveFleetConfig();
|
|
@@ -6014,7 +6359,7 @@ export class FleetManager {
|
|
|
6014
6359
|
return this.classicChannels?.setDisplayNameByInstance(instanceName, displayName) ?? false;
|
|
6015
6360
|
}
|
|
6016
6361
|
setInstanceDescription(instanceName, description) {
|
|
6017
|
-
const fleetInstance = this.
|
|
6362
|
+
const fleetInstance = this.ownInstanceConfig(instanceName);
|
|
6018
6363
|
if (fleetInstance) {
|
|
6019
6364
|
fleetInstance.description = description;
|
|
6020
6365
|
this.saveFleetConfig();
|
|
@@ -6623,7 +6968,7 @@ export class FleetManager {
|
|
|
6623
6968
|
quarantineMissingTopic(threadId, target, evidence) {
|
|
6624
6969
|
if (!isProbeableRouteTarget(target) || this.routing.resolve(threadId) !== target)
|
|
6625
6970
|
return;
|
|
6626
|
-
this.routing.unregister(threadId);
|
|
6971
|
+
this.routing.unregister(threadId, target.name);
|
|
6627
6972
|
this.logger.error({ instanceName: target.name, threadId, ...evidence }, "Topic is confirmed missing — route quarantined; instance configuration and user data were retained");
|
|
6628
6973
|
this.notifyFleetError(t("fleet.topic_quarantined", target.name, threadId));
|
|
6629
6974
|
}
|
|
@@ -6970,19 +7315,26 @@ export class FleetManager {
|
|
|
6970
7315
|
* Settings picker (#1005/#1021), so an agent and an operator see one truth.
|
|
6971
7316
|
*/
|
|
6972
7317
|
async listEmojisFor(instanceName, refresh = false) {
|
|
6973
|
-
|
|
6974
|
-
|
|
7318
|
+
// Seeing is not setting: a ClassicBot instance has no per-instance stamp,
|
|
7319
|
+
// but it reacts in its own channel like any bot and must be able to see
|
|
7320
|
+
// the emojis it can use (only set_persona_emoji refuses it).
|
|
7321
|
+
// Own entries only: an inherited name (constructor, __proto__, toString…)
|
|
7322
|
+
// is not a fleet instance, as set_persona_emoji already holds.
|
|
7323
|
+
const self = this.ownInstanceConfig(instanceName);
|
|
7324
|
+
const classic = !self && this.isClassicPersonaTarget(instanceName);
|
|
7325
|
+
if (!self && !classic)
|
|
6975
7326
|
return { error: this.personaEmojiMissing(instanceName) };
|
|
6976
7327
|
const worldId = this.getInstanceAdapterId(instanceName);
|
|
6977
7328
|
const { platform } = this.resolveStatusEmojisFor(instanceName);
|
|
6978
7329
|
const channel = (worldId ? this.worlds.get(worldId)?.channelConfig : undefined) ?? this.getChannelConfig(worldId ?? undefined);
|
|
6979
|
-
const current = previewStatusEmojis({ platform, platformConfig: channel?.options?.status_emojis, instanceConfig: self
|
|
7330
|
+
const current = previewStatusEmojis({ platform, platformConfig: channel?.options?.status_emojis, instanceConfig: self?.status_emojis });
|
|
6980
7331
|
const out = {
|
|
6981
7332
|
platform: platform ?? null,
|
|
6982
7333
|
statuses: current.entries.map(e => ({ status: e.key, value: e.value, source: e.source })),
|
|
6983
7334
|
standard: platform === "telegram"
|
|
6984
7335
|
? { reactions: [...TELEGRAM_REACTION_EMOJIS], note: "Telegram reacts only with these; progress_prefix may be any single emoji" }
|
|
6985
7336
|
: { suggestions: STATUS_EMOJI_SUGGESTIONS, note: "any single emoji works" },
|
|
7337
|
+
...(classic ? { note: "ClassicBot instances have no per-instance status emojis: the statuses above are the connection's, which an operator sets in Settings. You can still use any emoji listed here in your own reactions." } : {}),
|
|
6986
7338
|
};
|
|
6987
7339
|
if (platform === "discord" && worldId) {
|
|
6988
7340
|
const listed = await this.listGuildEmojis(worldId, refresh);
|
|
@@ -7013,9 +7365,10 @@ export class FleetManager {
|
|
|
7013
7365
|
* from that listed emoji's numeric id. Static PNG, size-capped, cached by id.
|
|
7014
7366
|
*/
|
|
7015
7367
|
async previewEmojis(instanceName, args) {
|
|
7016
|
-
|
|
7017
|
-
if (!
|
|
7368
|
+
// Seeing, like list_emojis: ClassicBot instances may preview too.
|
|
7369
|
+
if (!this.ownInstanceConfig(instanceName) && !this.isClassicPersonaTarget(instanceName)) {
|
|
7018
7370
|
return { error: this.personaEmojiMissing(instanceName) };
|
|
7371
|
+
}
|
|
7019
7372
|
const wanted = args.emojis;
|
|
7020
7373
|
if (!Array.isArray(wanted) || wanted.length === 0 || wanted.some(e => typeof e !== "string")) {
|
|
7021
7374
|
return { error: "emojis is required: a list of <:name:id> values from list_emojis" };
|
|
@@ -7110,7 +7463,7 @@ export class FleetManager {
|
|
|
7110
7463
|
* An empty emoji removes that status's override.
|
|
7111
7464
|
*/
|
|
7112
7465
|
async setPersonaEmoji(instanceName, args) {
|
|
7113
|
-
const self = this.
|
|
7466
|
+
const self = this.ownInstanceConfig(instanceName);
|
|
7114
7467
|
if (!self)
|
|
7115
7468
|
return { error: this.personaEmojiMissing(instanceName) };
|
|
7116
7469
|
const status = (args.status ?? "delivered");
|
|
@@ -7155,8 +7508,9 @@ export class FleetManager {
|
|
|
7155
7508
|
// Re-read after the validation above, which may have waited on Discord:
|
|
7156
7509
|
// another set_persona_emoji or a Settings save can have changed another
|
|
7157
7510
|
// status, or replaced this instance's config object, meanwhile. Change
|
|
7158
|
-
// only this one key on what is there now.
|
|
7159
|
-
|
|
7511
|
+
// only this one key on what is there now. Ownership must still hold:
|
|
7512
|
+
// removal during the await must not fall through to an inherited entry.
|
|
7513
|
+
const current = this.ownInstanceConfig(instanceName);
|
|
7160
7514
|
if (!current)
|
|
7161
7515
|
return { error: this.personaEmojiMissing(instanceName) };
|
|
7162
7516
|
const map = { ...(current.status_emojis ?? {}) };
|
|
@@ -7173,6 +7527,10 @@ export class FleetManager {
|
|
|
7173
7527
|
const resolved = this.resolveStatusEmojisFor(instanceName);
|
|
7174
7528
|
return { status, value: value ?? null, now: resolved[status], status_emojis: current.status_emojis ?? null };
|
|
7175
7529
|
}
|
|
7530
|
+
/** A ClassicBot instance: it may see (list/preview) emojis, never set a per-instance stamp. */
|
|
7531
|
+
isClassicPersonaTarget(instanceName) {
|
|
7532
|
+
return this.classicChannels?.getAll().some(c => c.instanceName === instanceName) ?? false;
|
|
7533
|
+
}
|
|
7176
7534
|
personaEmojiMissing(instanceName) {
|
|
7177
7535
|
return this.classicChannels?.getAll().some(c => c.instanceName === instanceName)
|
|
7178
7536
|
? "ClassicBot instances have no per-instance status emojis; an operator sets the connection's in Settings"
|
|
@@ -7234,6 +7592,16 @@ export class FleetManager {
|
|
|
7234
7592
|
const resolved = this.resolveStatusEmojisFor(instanceName, adapterId, adapter);
|
|
7235
7593
|
return reactionForm(resolved.platform, resolved.received);
|
|
7236
7594
|
}
|
|
7595
|
+
/**
|
|
7596
|
+
* The stamp for an inbound photo / file a classic bot saved (#1080): the
|
|
7597
|
+
* instance's `status_emojis.photo|attachment`, then its connection's, then
|
|
7598
|
+
* the built-in (📸/📎, or 👌/👍 on Telegram — what these paths always stamped).
|
|
7599
|
+
* Anything else a saved attachment can be counts as a file.
|
|
7600
|
+
*/
|
|
7601
|
+
savedAttachmentReactionFor(instanceName, adapter, adapterId, kind) {
|
|
7602
|
+
const resolved = this.resolveStatusEmojisFor(instanceName, adapterId, adapter);
|
|
7603
|
+
return reactionForm(resolved.platform, kind === "photo" ? resolved.photo : resolved.attachment);
|
|
7604
|
+
}
|
|
7237
7605
|
reactMessageStatus(instanceName, chatId, messageId, status, threadId) {
|
|
7238
7606
|
// React via the adapter BOUND to this instance — NOT the first discord world.
|
|
7239
7607
|
// Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
|
|
@@ -9053,6 +9421,7 @@ export class FleetManager {
|
|
|
9053
9421
|
releaseWindow: claim => { this.loginWindow.release(claim); },
|
|
9054
9422
|
isClaimCurrent: claim => this.loginWindow.isCurrent(claim),
|
|
9055
9423
|
windowBusyMessage: () => this.loginWindow.busyMessage(),
|
|
9424
|
+
tunnelDataDir: () => this.dataDir,
|
|
9056
9425
|
postButtons: async ({ prefix, instanceName, chat, message, choices, expiredText }) => {
|
|
9057
9426
|
await this.postNonceButtonPrompt({
|
|
9058
9427
|
prefix, alertType: "login", instanceName,
|
|
@@ -9570,7 +9939,7 @@ export class FleetManager {
|
|
|
9570
9939
|
}
|
|
9571
9940
|
/** Re-login confirmation (auth pre-check said credentials still work). */
|
|
9572
9941
|
async handleLoginConfirm(data, callbackAdapterId, receivingAdapter) {
|
|
9573
|
-
const claimed = this.consumeNonceCallback(LOGIN_CONFIRM_CALLBACK_PREFIX, /^login-confirm:([0-9a-f]+):(go|go-relogin|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
9942
|
+
const claimed = this.consumeNonceCallback(LOGIN_CONFIRM_CALLBACK_PREFIX, /^login-confirm:([0-9a-f]+):(go|go-relogin|go-tunnel|go-relogin-tunnel|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
9574
9943
|
if (claimed === null)
|
|
9575
9944
|
return false;
|
|
9576
9945
|
if (claimed === "consumed")
|
|
@@ -9588,7 +9957,12 @@ export class FleetManager {
|
|
|
9588
9957
|
chatId: entry.chatId,
|
|
9589
9958
|
threadId: entry.threadId,
|
|
9590
9959
|
userId: data.userId,
|
|
9591
|
-
}, {
|
|
9960
|
+
}, {
|
|
9961
|
+
skipAuthCheck: true,
|
|
9962
|
+
tokenPresent: action === "go-relogin" || action === "go-relogin-tunnel",
|
|
9963
|
+
// The consent is the button itself: only the explicit "Open public link" actions carry it.
|
|
9964
|
+
tunnel: action === "go-tunnel" || action === "go-relogin-tunnel",
|
|
9965
|
+
});
|
|
9592
9966
|
if (text)
|
|
9593
9967
|
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
9594
9968
|
return true;
|
|
@@ -9683,10 +10057,16 @@ export class FleetManager {
|
|
|
9683
10057
|
}
|
|
9684
10058
|
// The installer may only have added the binary to a profile PATH; a
|
|
9685
10059
|
// fresh login shell sees that, the fleet process's PATH may not.
|
|
9686
|
-
|
|
10060
|
+
const installedAt = this.locateBinaryOnLoginShell(info.binary) ?? this.locateInInstallerBinDirs(info);
|
|
10061
|
+
if (!installedAt) {
|
|
9687
10062
|
await chat.adapter.sendText(chat.chatId, t("install.verify_failed", backend, info.binary), { threadId: chat.threadId }).catch(() => { });
|
|
9688
10063
|
return;
|
|
9689
10064
|
}
|
|
10065
|
+
// #1059: make it reachable from this process too. `/login` lists what
|
|
10066
|
+
// `which` finds on the fleet's own PATH, and instances resolve their
|
|
10067
|
+
// binary the same way, so a binary only a login shell could see was
|
|
10068
|
+
// installed, reported as verified, and then offered nowhere.
|
|
10069
|
+
this.adoptBinaryDirectory(installedAt, backend);
|
|
9690
10070
|
if (!LOGIN_FLOWS[backend]) {
|
|
9691
10071
|
await chat.adapter.sendText(chat.chatId, t("install.success_no_login", backend), { threadId: chat.threadId }).catch(() => { });
|
|
9692
10072
|
return;
|
|
@@ -9746,14 +10126,54 @@ export class FleetManager {
|
|
|
9746
10126
|
return t("install.cancelled", backend);
|
|
9747
10127
|
}
|
|
9748
10128
|
/** `command -v` on a login shell, so PATH additions from rc files count. */
|
|
9749
|
-
|
|
10129
|
+
/**
|
|
10130
|
+
* The absolute path a fresh login shell resolves `binary` to, or null. Only
|
|
10131
|
+
* a real executable file counts: `command -v` also answers with an alias
|
|
10132
|
+
* definition or a function name, neither of which a spawn could run.
|
|
10133
|
+
*/
|
|
10134
|
+
locateBinaryOnLoginShell(binary) {
|
|
9750
10135
|
try {
|
|
9751
|
-
const result = spawnSync("bash", ["-lc", `command -v ${binary}`], { timeout: 10_000, stdio: "pipe" });
|
|
9752
|
-
|
|
10136
|
+
const result = spawnSync("bash", ["-lc", `command -v ${binary}`], { timeout: 10_000, stdio: "pipe", encoding: "utf8" });
|
|
10137
|
+
if (result.status !== 0)
|
|
10138
|
+
return null;
|
|
10139
|
+
const path = String(result.stdout ?? "").trim().split("\n").pop()?.trim() ?? "";
|
|
10140
|
+
if (!isAbsolute(path))
|
|
10141
|
+
return null;
|
|
10142
|
+
accessSync(path, fsConstants.X_OK);
|
|
10143
|
+
return statSync(path).isFile() ? path : null;
|
|
9753
10144
|
}
|
|
9754
10145
|
catch {
|
|
9755
|
-
return
|
|
10146
|
+
return null;
|
|
10147
|
+
}
|
|
10148
|
+
}
|
|
10149
|
+
/**
|
|
10150
|
+
* #1092: the binary in one of the directories its own installer installs to,
|
|
10151
|
+
* when no login shell can see it (the installer recorded its PATH in an
|
|
10152
|
+
* interactive-only rc file). The same executable-file check as the login
|
|
10153
|
+
* shell lookup; a symlink to an executable counts, as codex's launcher is one.
|
|
10154
|
+
*/
|
|
10155
|
+
locateInInstallerBinDirs(info) {
|
|
10156
|
+
for (const dir of info.binDirs?.(process.env, homedir()) ?? []) {
|
|
10157
|
+
if (!isAbsolute(dir))
|
|
10158
|
+
continue;
|
|
10159
|
+
const path = join(dir, info.binary);
|
|
10160
|
+
try {
|
|
10161
|
+
accessSync(path, fsConstants.X_OK);
|
|
10162
|
+
if (statSync(path).isFile())
|
|
10163
|
+
return path;
|
|
10164
|
+
}
|
|
10165
|
+
catch { /* not there */ }
|
|
9756
10166
|
}
|
|
10167
|
+
return null;
|
|
10168
|
+
}
|
|
10169
|
+
/** Put an installed binary's directory on the fleet's PATH (#1059), once. */
|
|
10170
|
+
adoptBinaryDirectory(binaryPath, backend) {
|
|
10171
|
+
const dir = dirname(binaryPath);
|
|
10172
|
+
const entries = (process.env.PATH ?? "").split(delimiter).filter(Boolean);
|
|
10173
|
+
if (entries.includes(dir))
|
|
10174
|
+
return;
|
|
10175
|
+
process.env.PATH = [dir, ...entries].join(delimiter);
|
|
10176
|
+
this.logger.info({ backend, dir }, "Added the installed CLI's directory to the fleet PATH");
|
|
9757
10177
|
}
|
|
9758
10178
|
/** "Sign in now?" button after a successful install. */
|
|
9759
10179
|
async handleInstallLoginConfirm(data, callbackAdapterId, receivingAdapter) {
|
|
@@ -10325,9 +10745,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10325
10745
|
const reactAdapter = this.worlds.get(msg.adapterId ?? "")?.adapter ?? this.adapter;
|
|
10326
10746
|
const noMentionReactChatId = msg.threadId ?? msg.chatId;
|
|
10327
10747
|
if (reactAdapter && noMentionReactChatId && msg.messageId) {
|
|
10328
|
-
const emoji = msg.
|
|
10329
|
-
? (saved.kind === "photo" ? "👌" : "👍")
|
|
10330
|
-
: (saved.kind === "photo" ? "📸" : "📎");
|
|
10748
|
+
const emoji = this.savedAttachmentReactionFor(instanceName, reactAdapter, msg.adapterId, saved.kind);
|
|
10331
10749
|
reactAdapter.react(noMentionReactChatId, msg.messageId, emoji)
|
|
10332
10750
|
.catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
|
|
10333
10751
|
}
|
|
@@ -10358,9 +10776,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10358
10776
|
return;
|
|
10359
10777
|
// Attachments already saved at the top of the collab block.
|
|
10360
10778
|
if (saved && classicAdapter && collabReactChatId && msg.messageId) {
|
|
10361
|
-
const emoji = msg.
|
|
10362
|
-
? (saved.kind === "photo" ? "👌" : "👍")
|
|
10363
|
-
: (saved.kind === "photo" ? "📸" : "📎");
|
|
10779
|
+
const emoji = this.savedAttachmentReactionFor(instanceName, classicAdapter, msg.adapterId, saved.kind);
|
|
10364
10780
|
classicAdapter.react(collabReactChatId, msg.messageId, emoji)
|
|
10365
10781
|
.catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
|
|
10366
10782
|
}
|
|
@@ -10408,10 +10824,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10408
10824
|
const reactAdapter = this.worlds.get(msg.adapterId ?? "")?.adapter ?? this.adapter;
|
|
10409
10825
|
const reactChatId = msg.threadId ?? msg.chatId;
|
|
10410
10826
|
if (saved && reactAdapter && reactChatId && msg.messageId) {
|
|
10411
|
-
|
|
10412
|
-
const emoji = msg.source === "telegram"
|
|
10413
|
-
? (saved.kind === "photo" ? "👌" : "👍")
|
|
10414
|
-
: (saved.kind === "photo" ? "📸" : "📎");
|
|
10827
|
+
const emoji = this.savedAttachmentReactionFor(instanceName, reactAdapter, msg.adapterId, saved.kind);
|
|
10415
10828
|
reactAdapter.react(reactChatId, msg.messageId, emoji)
|
|
10416
10829
|
.catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
|
|
10417
10830
|
}
|
|
@@ -10457,9 +10870,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10457
10870
|
classicMsgAdapter.react(reactChatId, msg.messageId, this.receivedReactionFor(instanceName, classicMsgAdapter, msg.adapterId))
|
|
10458
10871
|
.catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
|
|
10459
10872
|
if (saved) {
|
|
10460
|
-
const savedEmoji = msg.
|
|
10461
|
-
? (saved.kind === "photo" ? "👌" : "👍")
|
|
10462
|
-
: (saved.kind === "photo" ? "📸" : "📎");
|
|
10873
|
+
const savedEmoji = this.savedAttachmentReactionFor(instanceName, classicMsgAdapter, msg.adapterId, saved.kind);
|
|
10463
10874
|
classicMsgAdapter.react(reactChatId, msg.messageId, savedEmoji)
|
|
10464
10875
|
.catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
|
|
10465
10876
|
}
|
|
@@ -10515,6 +10926,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10515
10926
|
}
|
|
10516
10927
|
/** Forward a message to a classic channel instance with chat log context */
|
|
10517
10928
|
async forwardToClassicInstance(instanceName, text, msg, extraMeta) {
|
|
10929
|
+
// #1085: a Telegram forum ClassicBot is keyed by chat id (threadId), but the
|
|
10930
|
+
// reply context must carry the topic the message was really asked in.
|
|
10931
|
+
const replyThreadId = msg.transportThreadId ?? msg.threadId;
|
|
10518
10932
|
// Resolve the channel/adapter from the instance itself so per-channel context
|
|
10519
10933
|
// config is correct even for a same-channel second bot.
|
|
10520
10934
|
const ctxAdapterId = this.classicChannels?.getAdapterIdByInstance(instanceName);
|
|
@@ -10530,7 +10944,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10530
10944
|
user: msg.username,
|
|
10531
10945
|
user_id: msg.userId,
|
|
10532
10946
|
ts: msg.timestamp.toISOString(),
|
|
10533
|
-
thread_id:
|
|
10947
|
+
thread_id: replyThreadId ?? "",
|
|
10534
10948
|
...(msg.adapterId ? { adapter_id: msg.adapterId } : {}),
|
|
10535
10949
|
source: msg.source,
|
|
10536
10950
|
...extraMeta,
|
|
@@ -10567,7 +10981,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10567
10981
|
}
|
|
10568
10982
|
this.lastInboundUser.set(instanceName, msg.username);
|
|
10569
10983
|
this.logger.info(`${msg.username} → ${instanceName} (classic): ${text.slice(0, 100)}`);
|
|
10570
|
-
this.trackInboundMsg(instanceName, msg);
|
|
10984
|
+
this.trackInboundMsg(instanceName, { ...msg, threadId: replyThreadId });
|
|
10571
10985
|
void this.sendCancelButton(instanceName);
|
|
10572
10986
|
}
|
|
10573
10987
|
/** Paste raw text directly to a classic instance's CLI (no [user:] wrapping) */
|
|
@@ -11801,7 +12215,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
11801
12215
|
await pending.complete(warning ? `${warning}\n\n${reply}` : reply, pending.messageId);
|
|
11802
12216
|
}
|
|
11803
12217
|
/** Start a classic channel instance with lightweight config */
|
|
11804
|
-
async startClassicInstance(instanceName, backend, preTaskCommand, model, autoPauseAfter) {
|
|
12218
|
+
async startClassicInstance(instanceName, backend, preTaskCommand, model, autoPauseAfter, transition) {
|
|
11805
12219
|
if (this.daemons.has(instanceName))
|
|
11806
12220
|
return;
|
|
11807
12221
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
@@ -11828,7 +12242,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
11828
12242
|
...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
|
|
11829
12243
|
};
|
|
11830
12244
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
11831
|
-
await this.startInstance(instanceName, config, topicMode, "classic");
|
|
12245
|
+
await this.startInstance(instanceName, config, topicMode, "classic", false, transition);
|
|
11832
12246
|
}
|
|
11833
12247
|
/** Handle /start slash command — register classic channel */
|
|
11834
12248
|
async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
|
|
@@ -11940,6 +12354,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
11940
12354
|
clearInterval(this.replyObligationTimer);
|
|
11941
12355
|
this.replyObligationTimer = null;
|
|
11942
12356
|
}
|
|
12357
|
+
this.wakeCoordinator?.stop();
|
|
11943
12358
|
if (this.logRotateTimer) {
|
|
11944
12359
|
clearInterval(this.logRotateTimer);
|
|
11945
12360
|
this.logRotateTimer = null;
|
|
@@ -13766,6 +14181,20 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
13766
14181
|
return false; // identical
|
|
13767
14182
|
}
|
|
13768
14183
|
// ── Health HTTP endpoint ─────────────────────────────────────────────
|
|
14184
|
+
/** Distinct rejected Host names already logged; bounded so a scanner cannot grow it. */
|
|
14185
|
+
rejectedHostsLogged = new Set();
|
|
14186
|
+
/**
|
|
14187
|
+
* Say once per name why a request was refused, so a reverse-proxy deployment
|
|
14188
|
+
* that stopped working after the Host check is diagnosable from fleet.log.
|
|
14189
|
+
* Only the parsed host name is logged, never the raw header.
|
|
14190
|
+
*/
|
|
14191
|
+
noteRejectedHost(header) {
|
|
14192
|
+
const name = typeof header === "string" ? (hostnameOf(header) ?? "(malformed)") : "(missing)";
|
|
14193
|
+
if (this.rejectedHostsLogged.has(name) || this.rejectedHostsLogged.size >= 32)
|
|
14194
|
+
return;
|
|
14195
|
+
this.rejectedHostsLogged.add(name);
|
|
14196
|
+
this.logger.warn({ host: name }, "Web request refused: Host is not allowed (add it to web.allowed_hosts if this is a proxy you run)");
|
|
14197
|
+
}
|
|
13769
14198
|
startHealthServer(port) {
|
|
13770
14199
|
this.startedAt = Date.now();
|
|
13771
14200
|
this.healthServerListening = false;
|
|
@@ -13781,6 +14210,16 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
13781
14210
|
// Authorization now depends on a cookie, so a shared cache (a tunnel, a
|
|
13782
14211
|
// corporate proxy) must not serve one visitor's response to another.
|
|
13783
14212
|
res.setHeader("Vary", "Cookie");
|
|
14213
|
+
applyWebSecurityHeaders(res);
|
|
14214
|
+
// Before any route, /health and /agent included: loopback binding does not
|
|
14215
|
+
// stop DNS rebinding, and the Host the browser sends is the one thing a
|
|
14216
|
+
// rebinding page cannot change.
|
|
14217
|
+
if (!isHostAllowed(req.headers.host, allowedHostNames(this.fleetConfig))) {
|
|
14218
|
+
this.noteRejectedHost(req.headers.host);
|
|
14219
|
+
res.writeHead(403);
|
|
14220
|
+
res.end(JSON.stringify({ error: WEB_HOST_REJECTED_MESSAGE }));
|
|
14221
|
+
return;
|
|
14222
|
+
}
|
|
13784
14223
|
const requestPath = new URL(req.url ?? "/", `http://localhost:${port}`).pathname;
|
|
13785
14224
|
// Browsers request this automatically and AgEnD does not ship an icon.
|
|
13786
14225
|
// It is neither user data nor an API route, so do not turn the harmless
|
|
@@ -14001,7 +14440,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
14001
14440
|
this.logger.info({ name }, "Instance restart requested via HTTP");
|
|
14002
14441
|
(async () => {
|
|
14003
14442
|
try {
|
|
14004
|
-
await this.restartSingleInstance(name);
|
|
14443
|
+
await this.restartSingleInstance(name, { explicit: true });
|
|
14005
14444
|
this.logger.info({ name }, "Instance restarted");
|
|
14006
14445
|
this.emitSseEvent("status", this.getUiStatus());
|
|
14007
14446
|
res.writeHead(200);
|