@songsid/agend 2.1.5 → 2.1.6-beta.2

package/dist/fleet-manager.js

@@ -5,7 +5,6 @@ import { freemem, totalmem, cpus } from "node:os";
 import { createServer } from "node:http";
 import { join, dirname, basename } from "node:path";
 import { fileURLToPath } from "node:url";
-import { isDeepStrictEqual } from "node:util";
 import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
 import { beginFullRestartProgress as persistFullRestartProgress, beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, updateProgressOperation, } from "./update-marker.js";
 import { formatUpdateProgress } from "./update-progress.js";
@@ -59,9 +58,18 @@ import { assertExplicitInstanceRemoval } from "./instance-removal.js";
 import { validateFleetConfig } from "./config-validator.js";
 import { readLastInboundAt } from "./daemon.js";
 import { clearPausedMarker } from "./pause-marker.js";
-import { releaseProcessFleetLock } from "./fleet-lock.js";
+import { isFleetStartCommandLine, readProcessCommandLine, releaseProcessFleetLock } from "./fleet-lock.js";
+import { isSetupComplete, markSetupComplete } from "./setup-marker.js";
+import { manualCleanupMessage, reapStaleTunnel } from "./tunnel/lease.js";
 import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
-import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
+import { decideWebGate, loadOrCreateWebToken, readWebToken } from "./web-auth.js";
+import { fleetLevelDifferences, fleetLevelSignature } from "./fleet-level-config.js";
+import { checkSelfRestartAllowance, recordSelfRestartAttempt } from "./self-restart-limit.js";
+/** A self-restart is a whole service restart; 120s is the apply budget, not this. */
+const SELF_RESTART_DEADLINE_MS = 300_000;
+import { APPLY_FLEET_TARGET, ApplyJobStore, viewOf, } from "./apply-job.js";
+import { instanceCredentialProfile } from "./backend/credential-profile.js";
+import { classifyInstanceChange, CLASSIC_HOT_CONFIG_KEYS, HOT_INSTANCE_CONFIG_KEYS, hotConfigUpdate, } from "./instance-config-impact.js";
 import { formatRestartProgressCompletion, RESTART_PROGRESS_TERMINAL_TIMEOUT_MS, RestartProgress, } from "./restart-progress.js";
 import { launchFullRestartHelper } from "./full-restart.js";
 import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
@@ -179,31 +187,6 @@ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
  * emoji never changes the documented delivery-state protocol.
  */
 const IGNORED_REACTION_EMOJIS = new Set(["📷"]);
-const HOT_INSTANCE_CONFIG_KEYS = new Set([
-    "tool_progress",
-    "reply_completion_guard",
-    "mcp_proxy_reply",
-    "auto_pause_after",
-    "warm_cap",
-    "display_name",
-    "description",
-    "tags",
-    "log_level",
-]);
-function splitHotColdConfig(config) {
-    const hot = {};
-    const cold = {};
-    for (const [key, value] of Object.entries(config)) {
-        (HOT_INSTANCE_CONFIG_KEYS.has(key) ? hot : cold)[key] = value;
-    }
-    return { hot, cold };
-}
-function hotConfigUpdate(config) {
-    const update = {};
-    for (const key of HOT_INSTANCE_CONFIG_KEYS)
-        update[key] = config[key] ?? null;
-    return update;
-}
 /**
  * How long a delivery waits out a disconnected instance IPC before giving up.
  *
@@ -383,6 +366,10 @@ export class FleetManager {
     // reply, on cancel, or when a newer button supersedes it for the same
     // instance. Per-button tracking means a failed delete never strands a button.
     cancelButtons = new Map();
+    /** Latest publication generation per instance. Late results from older
+     * generations are retired without touching the current button. */
+    cancelButtonPublications = new Map();
+    nextCancelButtonPublicationGeneration = 0;
     /** Pending idle-edge retirement, one timer per instance. */
     cancelButtonIdleRetireTimers = new Map();
     /** Duplicate-reply suppression across both the MCP and HTTP reply paths. */
@@ -468,7 +455,29 @@ export class FleetManager {
     mirrorTimer = null;
     // Web UI: SSE clients + auth token
     sseClients = new Set();
-    webToken = null;
+    /**
+     * Read from disk on every access rather than cached at startup: `agend
+     * web-token rotate` runs in a separate process, and a cached copy would keep
+     * authorizing revoked links and cookies until the fleet restarted.
+     */
+    get webToken() { return readWebToken(this.dataDir); }
+    /**
+     * Set while a Settings apply job is driving the reconcile. The reconcile
+     * stays the single doer; it just says out loud what it is doing to whom, so
+     * the job's rows are the work rather than a prediction of it.
+     */
+    applyJobStoreCache = null;
+    /** The apply that currently owns the reconcile slot, reserved synchronously
+     * so a second request cannot slip in before the first one starts working. */
+    activeApplyJobId = null;
+    /** The fleet-level signature this process actually came up on. */
+    appliedFleetLevel = null;
+    /** The config behind that signature, kept so a "needs restart" log can name
+     * which keys moved rather than just asserting that something did. */
+    startupFleetConfig = null;
+    /** Set when the file on disk and the in-memory config disagree on a
+     * startup-only key at startup. See checkStartupSignatureConsistency(). */
+    fleetSignatureMismatch = null;
     viewToken = null;
     healthServerListening = false;
     constructor(dataDir) {
@@ -544,26 +553,48 @@ export class FleetManager {
         this.scheduleReconcile();
     }
     scheduleReconcile() {
-        if (this.reconcileInFlight) {
+        const started = this.startExclusiveReconcile();
+        if (!started) {
             this.reloadPending = true;
             this.logger.info("Config reconciliation already running — coalesced reload request");
             return;
         }
-        this.reloadPending = false;
-        this.reconcileInFlight = this.reconcileInstances()
-            .catch(err => {
+        started.catch(err => {
             // Almost always a YAML parse error. Log-only meant the user edited
             // fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
             this.logger.error({ err }, "SIGHUP config reload failed");
             const message = err instanceof Error ? err.message : String(err);
             this.notifyFleetError(t("fleet.reload_failed", message));
-        })
+        });
+    }
+    /**
+     * Take the reconcile slot, or refuse.
+     *
+     * Only one reconcile may touch lifecycle and config at a time — two of them
+     * stop and start the same instance in parallel. SIGHUP and a Settings apply
+     * are the same operation from two entrances, so they share the one slot: the
+     * signal coalesces into a pending replay, the apply is told the fleet is busy.
+     *
+     * The returned promise is the caller's to handle; the stored one is already
+     * handled, so a rejection never escapes as an unhandled rejection.
+     */
+    startExclusiveReconcile(observer) {
+        if (this.reconcileInFlight)
+            return null;
+        this.reloadPending = false;
+        let settle;
+        const caller = new Promise((resolve, reject) => {
+            settle = (err, outcome) => (err ? reject(err instanceof Error ? err : new Error(String(err))) : resolve(outcome ?? {}));
+        });
+        this.reconcileInFlight = this.reconcileInstances(observer)
+            .then(outcome => settle(null, outcome), err => settle(err))
             .finally(() => {
             this.reconcileInFlight = null;
             if (this.reloadPending && this.startupComplete) {
                 this.scheduleReconcile();
             }
         });
+        return caller;
     }
     /**
      * Is the fleet going down (or coming back up) on purpose?
@@ -579,6 +610,38 @@ export class FleetManager {
     }
     finishStartup() {
         this.startupComplete = true;
+        // Resolve whatever a previous run — or a setup host that crashed — left
+        // behind. A tunnel nobody is tracking is a public entrance nobody is
+        // watching, and the fleet starting is the moment there is finally a process
+        // around to notice. Never throws: a lease that cannot be resolved blocks
+        // the next tunnel and says so, it does not block the fleet.
+        void reapStaleTunnel(this.dataDir)
+            .then(outcome => {
+            if (outcome.kind === "manual")
+                this.logger.warn({ tunnel: outcome }, manualCleanupMessage(outcome));
+            else if (outcome.kind === "reaped")
+                this.logger.info({ how: outcome.how, pid: outcome.pid }, "Reaped a leftover tunnel");
+        })
+            .catch(err => this.logger.warn({ err }, "Tunnel reaper failed"));
+        // An existing installation has never written the setup marker — it predates
+        // it — so `agend setup` would open a pre-fleet form for a fleet that plainly
+        // exists. A fleet that just came up on a config with agents in it is proof
+        // enough that setup happened.
+        if (Object.keys(this.fleetConfig?.instances ?? {}).length > 0 && !isSetupComplete(this.dataDir)) {
+            markSetupComplete(this.dataDir);
+        }
+        // After slimFleetConfigAtStartup() and the general/topic fixups, all of
+        // which may rewrite fleet.yaml — the baseline has to be what this process
+        // is actually running, compared against what a reconcile would load.
+        this.appliedFleetLevel = this.fleetLevelSignature();
+        this.startupFleetConfig = this.fleetConfig ? structuredClone(this.fleetConfig) : null;
+        this.checkStartupSignatureConsistency();
+        // A job from the process that just died cannot still be running here. The
+        // restart applied the saved config to every instance, so its open rows are
+        // finished — by the restart, which is what the user needs told.
+        for (const settled of this.applyJobs.settleAfterRestart()) {
+            this.logger.info({ jobId: settled.id }, "Settings apply job settled by fleet restart");
+        }
         // We are the post-update fleet: the update is over by definition. Clearing
         // it here (rather than in the update command, which exits before the new
         // fleet is up) is what keeps the quiet window from outliving the restart.
@@ -1112,31 +1175,43 @@ export class FleetManager {
      */
     getActiveUsageProviderIds() {
         const providers = new Set();
-        for (const backend of this.getActiveBackendIds()) {
+        // Per instance, not per backend: two agents on two kiro subscriptions are
+        // two rows, and filtering by the bare backend id would hide both.
+        for (const [name, backend, profile] of this.activeBackendBindings()) {
+            void name;
             const provider = usageProviderIdForBackend(backend);
-            if (provider)
-                providers.add(provider);
+            if (!provider)
+                continue;
+            providers.add(profile ? `${provider}:${profile}` : provider);
         }
         return providers;
     }
-    /** Effective backends with a running or persisted-paused fleet/Classic instance. */
-    getActiveBackendIds() {
-        const backends = new Set();
-        const add = (name, backend) => {
+    /** `[instance, effective backend, credential profile]` for everything that is
+     * running or paused — the one place both usage views agree on who is live. */
+    activeBackendBindings() {
+        const bindings = [];
+        const add = (name, backend, profile) => {
             const status = this.getInstanceStatus(name);
             if (status !== "running" && status !== "paused")
                 return;
             if (backend)
-                backends.add(backend);
+                bindings.push([name, backend, profile]);
         };
         for (const [name, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
             // loadFleetConfig() has already merged the fleet default into each row.
-            add(name, config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
+            const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
+            add(name, backend, instanceCredentialProfile(config, this.fleetConfig?.defaults, backend));
         }
         for (const channel of this.classicChannels?.getAll() ?? []) {
-            add(channel.instanceName, this.classicChannels?.getBackendByInstance(channel.instanceName, this.fleetConfig?.defaults?.backend));
+            const backend = this.classicChannels?.getBackendByInstance(channel.instanceName, this.fleetConfig?.defaults?.backend);
+            // Classic channels carry no backend_options, so they run the shared login.
+            add(channel.instanceName, backend, null);
         }
-        return backends;
+        return bindings;
+    }
+    /** Effective backends with a running or persisted-paused fleet/Classic instance. */
+    getActiveBackendIds() {
+        return new Set(this.activeBackendBindings().map(([, backend]) => backend));
     }
     isClassicInstance(name) {
         return this.classicChannels?.getAll().some(channel => channel.instanceName === name) ?? false;
@@ -1181,12 +1256,17 @@ export class FleetManager {
     }
     scheduleIdleButtonRetirement(name) {
         this.cancelIdleButtonRetirement(name);
+        // Bind this edge to the publication that was current when idle was
+        // observed. A later inbound may start a new generation before this timer
+        // fires; the old edge must not mark that newer button for retirement.
+        const publication = this.cancelButtonPublications.get(name);
         const timer = setTimeout(() => {
             // Ignore a superseded timer even if it was already queued to run.
             if (this.cancelButtonIdleRetireTimers.get(name) !== timer)
                 return;
             this.cancelButtonIdleRetireTimers.delete(name);
             if (this.getInstanceExecutionState(name) === "idle") {
+                this.markCancelButtonPublicationForRetirement(name, publication);
                 this.retireInstanceButtons(name);
             }
         }, CANCEL_BTN_IDLE_RETIRE_GRACE_MS);
@@ -1539,7 +1619,7 @@ export class FleetManager {
         if (!wasRunning)
             return;
         const hotOnly = changedFields.length > 0
-            && changedFields.every(field => field === "tool_progress" || field === "reply_completion_guard");
+            && changedFields.every(field => CLASSIC_HOT_CONFIG_KEYS.has(field));
         if (hotOnly) {
             this.applyHotConfigUpdate(instanceName, this.classicBehaviorUpdate(instanceName));
             this.logger.info({ instanceName, fields: changedFields }, "Classic instance hot config reloaded");
@@ -2165,7 +2245,9 @@ export class FleetManager {
     }
     /** Initialize auth before any adapter can answer /dashboard. */
     initializeWebAuthTokens() {
-        this.webToken = loadOrCreateWebToken(this.dataDir);
+        // Creates web.token if absent; the value is then read back per request by
+        // the `webToken` getter, so nothing is cached here.
+        loadOrCreateWebToken(this.dataDir);
         this.viewToken = randomBytes(24).toString("hex");
         const viewTokenPath = join(this.dataDir, "view.token");
         writeFileSync(viewTokenPath, this.viewToken, { encoding: "utf8", mode: 0o600 });
@@ -7098,6 +7180,30 @@ export class FleetManager {
         }
         return false;
     }
+    beginCancelButtonPublication(instanceName, correlationId) {
+        // A new handoff supersedes any idle timer left by the previous turn. If the
+        // instance is still idle after this publication, the post-await level check
+        // below starts a fresh grace period for this generation.
+        this.cancelIdleButtonRetirement(instanceName);
+        const publication = {
+            generation: ++this.nextCancelButtonPublicationGeneration,
+            correlationId,
+            inFlight: true,
+            retirePending: false,
+        };
+        this.cancelButtonPublications.set(instanceName, publication);
+        return publication;
+    }
+    /** Remember a clear/cancel/idle decision that arrived before notifyAlert
+     * returned a message id. `expected` fences an idle timer to its generation. */
+    markCancelButtonPublicationForRetirement(instanceName, expected) {
+        const publication = this.cancelButtonPublications.get(instanceName);
+        if (!publication?.inFlight)
+            return;
+        if (expected && publication !== expected)
+            return;
+        publication.retirePending = true;
+    }
     async sendCancelButton(instanceName, correlationId, preserveProgress = false) {
         // Post first, retire after (see the tail of this method). Retiring up front
         // meant that from the delete until the new message came back — a chat API
@@ -7136,6 +7242,7 @@ export class FleetManager {
             this.logger.warn({ instanceName, topicId, groupId }, "Cannot address cancel button (no chat id resolved)");
             return;
         }
+        const publication = this.beginCancelButtonPublication(instanceName, correlationId);
         try {
             const sent = await adapter.notifyAlert(chatId, {
                 type: "cancel",
@@ -7143,6 +7250,7 @@ export class FleetManager {
                 message: "👀 處理中…",
                 choices: [{ id: `cancel:${instanceName}`, label: t("cancel.button") }],
             }, threadId ? { threadId } : undefined);
+            publication.inFlight = false;
             const entry = {
                 instanceName,
                 adapterId,
@@ -7164,6 +7272,16 @@ export class FleetManager {
                 // empty even if the daemon's reset broadcast is still in flight.
                 toolProgress: preserveProgress ? this.instanceProgress.get(instanceName) : undefined,
             };
+            // A newer send started while this API call was pending. Track this late
+            // message just long enough for the normal bounded delete/retry machinery
+            // to remove it; critically, do not sweep the newer generation.
+            if (this.cancelButtonPublications.get(instanceName) !== publication) {
+                this.cancelButtons.set(sent.messageId, entry);
+                this.persistCancelButtons();
+                this.logger.debug({ instanceName, messageId: sent.messageId, generation: publication.generation }, "Retiring superseded cancel-button publication");
+                this.retireButton(entry);
+                return;
+            }
             this.startProgressTicker(entry);
             // Idle-check backstop: every 5min, if the instance is idle, retire the
             // button. Covers turns that end without hitting a clear trigger (reply /
@@ -7200,8 +7318,30 @@ export class FleetManager {
             }
             this.persistCancelButtons();
             this.logger.info({ instanceName, messageId: sent.messageId }, "Cancel button sent");
+            // Retirement is normally edge-triggered, but the edge (or an explicit
+            // reply/cancel clear) may have happened while notifyAlert was in flight.
+            // Reconcile the level after publication so a late button cannot resurrect
+            // on an already-idle instance. Preserve the existing two-second grace:
+            // a working transition cancels this timer just as it does for a normal
+            // idle edge.
+            if (publication.retirePending) {
+                this.retireButton(entry);
+            }
+            else if (this.getInstanceExecutionState(instanceName) === "idle") {
+                this.scheduleIdleButtonRetirement(instanceName);
+            }
         }
         catch (e) {
+            if (this.cancelButtonPublications.get(instanceName) === publication) {
+                this.cancelButtonPublications.delete(instanceName);
+                // beginCancelButtonPublication cancelled the previous turn's idle
+                // timer so it could not act on this generation. If the provider POST
+                // failed while the instance remained idle, restore that retirement
+                // opportunity for any older button still on screen.
+                if (this.getInstanceExecutionState(instanceName) === "idle") {
+                    this.scheduleIdleButtonRetirement(instanceName);
+                }
+            }
             this.logger.warn({ err: e.message, instanceName }, "Failed to send cancel button");
         }
     }
@@ -7554,6 +7694,7 @@ export class FleetManager {
     }
     /** Retire all cancel buttons for an instance — on reply or cancel. */
     clearCancelButton(instanceName) {
+        this.markCancelButtonPublicationForRetirement(instanceName);
         this.retireInstanceButtons(instanceName);
     }
     /** Retire the cross-instance button matching a delegate→report correlation id.
@@ -7562,6 +7703,11 @@ export class FleetManager {
     clearCancelButtonByCorrelation(correlationId) {
         if (!correlationId)
             return;
+        for (const [instanceName, publication] of this.cancelButtonPublications) {
+            if (publication.inFlight && publication.correlationId === correlationId) {
+                this.markCancelButtonPublicationForRetirement(instanceName, publication);
+            }
+        }
         for (const e of [...this.cancelButtons.values()]) {
             if (e.correlationId === correlationId)
                 this.retireButton(e);
@@ -10299,6 +10445,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 clearInterval(entry.progressTimer);
         }
         this.cancelButtons.clear();
+        this.cancelButtonPublications.clear();
         for (const timer of this.cancelButtonIdleRetireTimers.values())
             clearTimeout(timer);
         this.cancelButtonIdleRetireTimers.clear();
@@ -10512,9 +10659,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
      * Whitelisted runtime fields are pushed into live daemons; all other instance
      * fields, plus cold fleet-level settings, retain restart semantics.
      */
-    async reconcileInstances() {
+    async reconcileInstances(observe) {
         if (!this.configPath)
-            return;
+            return {};
         const oldConfig = this.fleetConfig;
         const previousRawConfig = this.rawFleetConfig;
         const previousRawDocument = this.rawFleetDocument;
@@ -10557,7 +10704,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                     ? t("fleet.reload_removed_all", oldCount)
                     : t("fleet.reload_removed_half", oldCount, newCount);
             this.notifyFleetError(t("fleet.reload_rejected", why));
-            return;
+            // Reported, not swallowed: an apply job whose config was refused used to
+            // mark every row done and tell the user "changes applied".
+            return { rejected: why };
         }
         // Classic behavior settings share the fleet defaults but are not entries
         // in fleet.yaml. Snapshot the old effective chain before reloading the
@@ -10588,29 +10737,31 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
         this.scheduler?.reload();
         const newInstances = this.fleetConfig.instances;
         const topicMode = this.fleetConfig?.channel?.mode === "topic";
-        // Detect fleet-level changes which still need a restart. Hot defaults are
-        // reconciled below and must not produce a misleading restart warning.
-        const oldDefaultColdWithFleetFields = oldConfig?.defaults
-            ? splitHotColdConfig(oldConfig.defaults).cold
-            : {};
-        const newDefaultColdWithFleetFields = this.fleetConfig?.defaults
-            ? splitHotColdConfig(this.fleetConfig.defaults).cold
-            : {};
-        // `tips` is a fleet-owned hot switch; it is read at post time and never
-        // belongs in a daemon config or a "restart required" warning.
-        const { tips: _oldTips, ...oldDefaultCold } = oldDefaultColdWithFleetFields;
-        const { tips: _newTips, ...newDefaultCold } = newDefaultColdWithFleetFields;
-        const oldFleetLevel = JSON.stringify({ channel: oldConfig?.channel, defaults: oldDefaultCold });
-        const newFleetLevel = JSON.stringify({ channel: this.fleetConfig?.channel, defaults: newDefaultCold });
-        if (oldFleetLevel !== newFleetLevel) {
-            this.logger.warn("Fleet-level config changed (channel/defaults) — use /restart for full effect");
+        // Only what a fresh process can adopt, and only relative to the signature
+        // this process came up on: a Settings edit mutates this.fleetConfig in place
+        // before the reload, so comparing the pre-load copy sees nothing at all.
+        const newFleetLevel = this.fleetLevelSignature();
+        if (this.appliedFleetLevel !== null && this.appliedFleetLevel !== newFleetLevel) {
+            this.logger.warn({
+                keys: fleetLevelDifferences(this.startupFleetConfig, this.fleetConfig),
+            }, "Fleet-level config changed — restart AgEnD for it to take effect");
+            // Terminal, and deliberately not "done": this reconcile cannot adopt a
+            // fleet-level change, and saying otherwise would claim AgEnD is running
+            // on a configuration it is not running on.
+            observe?.(APPLY_FLEET_TARGET, "restart", "restart-required");
         }
         // Stop removed instances (skip classic bot instances — they're managed by classicBot.yaml)
         const classicNames = new Set(this.classicChannels?.getAll().map(ch => ch.instanceName) ?? []);
         for (const name of this.daemons.keys()) {
             if (!(name in newInstances) && !classicNames.has(name)) {
                 this.logger.info({ name }, "Instance removed from config — stopping");
-                await this.stopInstance(name).catch(err => this.logger.error({ err, name }, "Failed to stop removed instance"));
+                observe?.(name, "restart", "running");
+                await this.stopInstance(name)
+                    .then(() => observe?.(name, "restart", "done"))
+                    .catch(err => {
+                    observe?.(name, "restart", "failed", err.message);
+                    this.logger.error({ err, name }, "Failed to stop removed instance");
+                });
             }
         }
         // Start new + reconcile modified instances. Hot values are always sent as a
@@ -10620,20 +10771,23 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             if (!this.daemons.has(name)) {
                 // New instance — startInstance already calls connectIpcToInstance
                 this.logger.info({ name }, "New instance in config — starting");
+                observe?.(name, "restart", "running");
                 await this.startInstanceUnattended(name, config, topicMode, "new instance");
+                observe?.(name, "restart", "done");
             }
             else if (oldConfig?.instances[name]) {
                 const daemon = this.daemons.get(name);
                 const runtimeConfig = daemon.getConfigSnapshot?.() ?? oldConfig.instances[name];
-                const oldParts = splitHotColdConfig(runtimeConfig);
-                const newParts = splitHotColdConfig(config);
-                // Every field not explicitly classified hot is cold by default.
-                if (!isDeepStrictEqual(oldParts.cold, newParts.cold)) {
+                const change = classifyInstanceChange(runtimeConfig, config);
+                if (change === "restart") {
                     this.logger.info({ name }, "Instance config changed — restarting");
+                    observe?.(name, "restart", "running");
                     await this.stopInstance(name).catch(() => { });
                     await this.startInstanceUnattended(name, config, topicMode, "modified instance");
+                    observe?.(name, "restart", "done");
                 }
-                else if (!isDeepStrictEqual(oldParts.hot, newParts.hot)) {
+                else if (change === "hot") {
+                    observe?.(name, "hot", "running");
                     const update = hotConfigUpdate(config);
                     const ipc = this.instanceIpcClients.get(name);
                     const sent = ipc?.connected === true && ipc.send({ type: "config_update", config: update });
@@ -10644,6 +10798,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                         this.logger.warn({ name }, "Config-update IPC unavailable — applied hot config in-process");
                     }
                     this.logger.info({ name, fields: [...HOT_INSTANCE_CONFIG_KEYS] }, "Instance hot config reloaded");
+                    observe?.(name, "hot", "done");
                 }
             }
         }
@@ -10661,25 +10816,362 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 const autoPauseAfter = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
                 if (old.backend !== backend || old.model !== model || old.autoPauseAfter !== autoPauseAfter) {
                     this.logger.info({ instanceName: ch.instanceName }, "Classic cold config changed — restarting");
+                    observe?.(ch.instanceName, "restart", "running");
                     await this.stopInstance(ch.instanceName).catch(() => { });
                     await this.startClassicInstanceUnattended(ch, "classic instance after fleet reload");
+                    observe?.(ch.instanceName, "restart", "done");
                     continue;
                 }
                 const toolProgress = this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress);
                 const replyCompletionGuard = this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard);
                 if (old.toolProgress !== toolProgress || old.replyCompletionGuard !== replyCompletionGuard) {
+                    observe?.(ch.instanceName, "hot", "running");
                     this.applyHotConfigUpdate(ch.instanceName, {
                         tool_progress: toolProgress,
                         reply_completion_guard: replyCompletionGuard,
                     });
                     this.logger.info({ instanceName: ch.instanceName }, "Classic inherited hot config reloaded");
+                    observe?.(ch.instanceName, "hot", "done");
                 }
             }
         }
         // warm_cap is fleet-owned; enforce the reloaded value immediately against
         // currently idle instances instead of waiting for a future state edge.
         this.enforceWarmCap();
+        // appliedFleetLevel is deliberately NOT updated here. It means "the
+        // fleet-level config this process started on"; a reconcile does not restart
+        // the process, so moving it would erase the fact that a restart is still
+        // owed and silence every later reminder.
         this.logger.info({ running: this.daemons.size, configured: Object.keys(newInstances).length }, "Reconcile complete");
+        return {};
+    }
+    /**
+     * `{channel, cold defaults}` as one comparable string — the part of the config
+     * a running fleet process cannot adopt without restarting.
+     */
+    fleetLevelSignature(config = this.fleetConfig) {
+        return fleetLevelSignature(config);
+    }
+    /**
+     * The config a reconcile is about to load, not the one held in memory.
+     *
+     * Settings mutates the in-memory object and writes the file; only the file
+     * goes back through defaults expansion. Forecasting from memory therefore
+     * misses every instance that a changed fleet default will restart — the user
+     * is told "restart AgEnD" and not told that five agents are about to go down.
+     */
+    nextFleetConfig() {
+        if (!this.configPath)
+            return this.fleetConfig;
+        try {
+            return loadFleetConfig(this.configPath);
+        }
+        catch (err) {
+            // An unparseable file is the reconcile's problem to report; the forecast
+            // falls back to what is running rather than failing the request.
+            this.logger.debug({ err }, "Apply plan fell back to the in-memory config");
+            return this.fleetConfig;
+        }
+    }
+    /**
+     * Does the config this process is running match the one a reconcile would
+     * load off disk?
+     *
+     * If not, every apply reports a fleet-level change that a restart cannot
+     * clear — restart, recompute, disagree again — a self-sustaining loop that
+     * the rate limit can only slow to three naggings an hour. Startup rewrites
+     * the file in three places before this point (slimFleetConfigAtStartup, the
+     * general auto-create, the general fixup), so the two can genuinely diverge.
+     *
+     * An offer to restart that cannot possibly succeed is worse than no offer, so
+     * the panel shows the mismatch instead of a button.
+     */
+    checkStartupSignatureConsistency() {
+        if (!this.configPath) {
+            this.fleetSignatureMismatch = null;
+            return;
+        }
+        let onDisk;
+        try {
+            onDisk = loadFleetConfig(this.configPath);
+        }
+        catch (err) {
+            this.logger.warn({ err }, "Could not re-read fleet.yaml to check the startup signature");
+            this.fleetSignatureMismatch = null;
+            return;
+        }
+        if (fleetLevelSignature(onDisk) === this.appliedFleetLevel) {
+            this.fleetSignatureMismatch = null;
+            return;
+        }
+        this.fleetSignatureMismatch = fleetLevelDifferences(this.fleetConfig, onDisk);
+        this.logger.warn({
+            keys: this.fleetSignatureMismatch,
+            configPath: this.configPath,
+        }, "fleet.yaml and the running configuration disagree on startup-only keys — every apply will ask for a restart that cannot clear it");
+    }
+    /**
+     * Is a live adapter already long-polling this bot token?
+     *
+     * Telegram's `getUpdates` has exactly one consumer: a second poller takes
+     * turns with the first and both miss messages. The setup wizard's "post in
+     * the group and I'll detect it" step is a second poller, so it has to know
+     * when the answer is "not against this token, not while I'm running".
+     */
+    isBotTokenInUse(token) {
+        if (!token)
+            return false;
+        const configured = this.fleetConfig?.channels
+            ?? (this.fleetConfig?.channel ? [this.fleetConfig.channel] : []);
+        for (const channel of configured) {
+            const envVar = channel?.bot_token_env;
+            if (!envVar)
+                continue;
+            // Compare the value, not the variable name: the same token can be
+            // reached through a differently named variable.
+            if (process.env[envVar] === token)
+                return true;
+        }
+        return false;
+    }
+    /** Non-null when startup found the running config and fleet.yaml disagreeing. */
+    fleetSignatureMismatchKeys() {
+        return this.fleetSignatureMismatch;
+    }
+    /**
+     * Restart AgEnD itself on behalf of a Settings apply.
+     *
+     * Deliberately a separate action from Apply: the panel is reachable from
+     * outside the LAN, and "restart the whole fleet" must never be something a
+     * single Apply click can carry along with it.
+     *
+     * The order below is the safety envelope, and the order matters:
+     * concurrency and consistency first (cheap, and a restart during a reconcile
+     * is the dangerous one), then the state checks, then the rate limit, then the
+     * audit notice — and only once all of that holds is the attempt written to
+     * disk and fsynced, before anything spawns.
+     */
+    async requestSettingsSelfRestart(jobId, key) {
+        // A retry after a lost response must not restart a second time. The key is
+        // recorded on the job, which survives the restart it triggers.
+        const already = this.applyJobs.findByRestartKey(key);
+        if (already)
+            return { ok: true, jobId: already.id, reused: true };
+        if (this.reconcileInFlight || this.activeApplyJobId) {
+            return { ok: false, status: 409, error: "a configuration reload is running — try again once it finishes" };
+        }
+        if (this.fleetSignatureMismatch) {
+            // Restarting cannot clear this, so offering it would be a loop.
+            return {
+                ok: false,
+                status: 409,
+                error: `fleet.yaml and the running configuration disagree on ${this.fleetSignatureMismatch.join(", ")} — check fleet.log before restarting`,
+            };
+        }
+        const job = this.applyJobs.get(jobId);
+        const row = job?.targets.find(item => item.target === APPLY_FLEET_TARGET);
+        if (!job || !row || row.status !== "restart-required") {
+            return { ok: false, status: 409, error: "that apply has no pending fleet-level change" };
+        }
+        // The row alone is not enough: an old job keeps its row for the whole
+        // retention window, so a change made and then reverted would still leave a
+        // job that looks restartable. The live signature is the authority — and it
+        // must be the same view of the config the plan uses.
+        if (this.appliedFleetLevel === null || this.appliedFleetLevel === this.fleetLevelSignature(this.nextFleetConfig())) {
+            return { ok: false, status: 409, error: "no fleet-level change is pending any more" };
+        }
+        const allowance = checkSelfRestartAllowance(this.dataDir);
+        if (!allowance.allowed) {
+            if (allowance.reason === "unreadable") {
+                // Fail closed: with the limit's own state in doubt, "no attempts yet"
+                // is the one reading that must not be assumed.
+                return {
+                    ok: false,
+                    status: 503,
+                    error: "the restart rate-limit file cannot be read — remove self-restart.json from the data dir on the host, or run `agend restart` there",
+                };
+            }
+            return {
+                ok: false,
+                status: 429,
+                error: allowance.reason === "too-soon"
+                    ? "AgEnD was restarted from Settings very recently"
+                    : "too many Settings-triggered restarts in the last hour",
+                retryAfterSeconds: allowance.retryAfterSeconds,
+            };
+        }
+        // Recorded before the notice, not after. If recording keeps failing (a
+        // read-only data dir), posting first would let whoever holds the token spam
+        // the channel with "restarting…" notices for restarts that never happen.
+        // The cost is that a failed announcement still spends an attempt, which is
+        // the right way round for a rate limit.
+        if (!recordSelfRestartAttempt(this.dataDir)) {
+            this.logger.error("Self-restart attempt could not be recorded — refusing to restart unmetered");
+            return { ok: false, status: 503, error: "could not record the restart attempt" };
+        }
+        // Out-of-band notice before the restart, so a panel restart is visible where
+        // the admins are. Refusing when it cannot be posted is the same rule as
+        // refusing when the progress marker cannot be written: no untraceable
+        // restarts.
+        const notice = await this.postSelfRestartNotice();
+        if (!notice) {
+            return {
+                ok: false,
+                status: 409,
+                error: "no chat channel is available to announce the restart — run `agend restart` on the host instead",
+            };
+        }
+        // Consume the row: it moves to running, which is also what lets the next
+        // process settle it (settleAfterRestart only touches non-terminal rows).
+        this.applyJobs.update(jobId, current => {
+            const target = current.targets.find(item => item.target === APPLY_FLEET_TARGET);
+            if (target)
+                target.status = "running";
+            current.restart_key = key;
+            current.deadlineMs = SELF_RESTART_DEADLINE_MS;
+        });
+        this.emitSseEvent("apply_progress", viewOf(this.applyJobs.get(jobId)));
+        const launched = await this.requestFullRestart(notice.adapter, notice.chatId, notice.threadId, notice.messageId)
+            .catch(err => {
+            this.logger.error({ err }, "Settings-triggered self restart failed to launch");
+            return false;
+        });
+        if (!launched) {
+            this.applyJobs.setTargetStatus(jobId, APPLY_FLEET_TARGET, "failed", "restart could not be launched");
+            this.applyJobs.finish(jobId, "restart could not be launched");
+            this.emitSseEvent("apply_progress", viewOf(this.applyJobs.get(jobId)));
+            return { ok: false, status: 409, error: "the restart could not be launched — see fleet.log" };
+        }
+        return { ok: true, jobId };
+    }
+    /** The audit notice, and the message the restart progress will edit. */
+    async postSelfRestartNotice() {
+        const groupId = this.fleetConfig?.channel?.group_id;
+        const adapter = this.adapter;
+        if (!groupId || !adapter)
+            return null;
+        const generalName = this.findGeneralInstance();
+        const rawThreadId = generalName ? this.fleetConfig?.instances[generalName]?.topic_id : undefined;
+        const threadId = rawThreadId != null ? String(rawThreadId) : undefined;
+        try {
+            const sent = await adapter.sendText(String(groupId), t("restart.settings_triggered"), { threadId });
+            if (!sent?.messageId)
+                return null;
+            return { adapter, chatId: sent.chatId, threadId: sent.threadId, messageId: sent.messageId };
+        }
+        catch (err) {
+            this.logger.error({ err }, "Could not announce the Settings-triggered restart — refusing to restart silently");
+            return null;
+        }
+    }
+    /** Jobs outlive this process on purpose; see apply-job.ts. */
+    get applyJobs() {
+        return (this.applyJobStoreCache ??= new ApplyJobStore(this.dataDir, Date.now, this.logger));
+    }
+    /**
+     * What a reconcile is about to do, per target.
+     *
+     * A forecast, not the record: it is built from the same `classifyInstanceChange`
+     * the reconcile decides with, but the rows that end up in the job are the ones
+     * the reconcile reports as it works. A target the forecast missed (a Classic
+     * instance inheriting a changed default) is added when it is first touched.
+     */
+    planConfigApply() {
+        const rows = [];
+        const nextConfig = this.nextFleetConfig();
+        const next = nextConfig?.instances ?? {};
+        const classicNames = new Set(this.classicChannels?.getAll().map(ch => ch.instanceName) ?? []);
+        for (const [name, config] of Object.entries(next)) {
+            const daemon = this.daemons.get(name);
+            if (!daemon) {
+                rows.push({ target: name, kind: "restart" });
+                continue;
+            }
+            const runtime = daemon.getConfigSnapshot?.();
+            if (!runtime)
+                continue;
+            const change = classifyInstanceChange(runtime, config);
+            if (change !== "none")
+                rows.push({ target: name, kind: change === "restart" ? "restart" : "hot" });
+        }
+        for (const name of this.daemons.keys()) {
+            if (!(name in next) && !classicNames.has(name))
+                rows.push({ target: name, kind: "restart" });
+        }
+        if (this.appliedFleetLevel !== null && this.appliedFleetLevel !== this.fleetLevelSignature(nextConfig)) {
+            rows.push({ target: APPLY_FLEET_TARGET, kind: "restart" });
+        }
+        return rows;
+    }
+    /**
+     * Start (or re-join) a Settings apply.
+     *
+     * The key is the client's. Handing back the existing job for a repeated key is
+     * the whole point: the retry after a lost response must not apply everything a
+     * second time.
+     */
+    startSettingsApply(key) {
+        // The key is checked first on purpose: a retry of the apply that is running
+        // right now must get its own job back, not "busy".
+        const existing = this.applyJobs.findByKey(key);
+        if (existing)
+            return { job: existing, reused: true };
+        if (this.reconcileInFlight || this.activeApplyJobId) {
+            return { busy: this.activeApplyJobId ? this.applyJobs.get(this.activeApplyJobId) : null };
+        }
+        const job = this.applyJobs.create(key, this.planConfigApply());
+        // Reserved synchronously: the work starts a microtask later, and a second
+        // request arriving in that gap must see the slot taken.
+        this.activeApplyJobId = job.id;
+        // Start after the caller has its answer, so the first thing the page renders
+        // is the whole plan with every row still pending — not a job the reconcile
+        // has already half-finished synchronously.
+        queueMicrotask(() => void this.runSettingsApply(job.id));
+        return { job, reused: false };
+    }
+    async runSettingsApply(jobId) {
+        const emit = () => {
+            const job = this.applyJobs.get(jobId);
+            // An accelerator only: these frames carry no event id, so a client that
+            // reconnects cannot ask for what it missed. GET is the authority.
+            if (job)
+                this.emitSseEvent("apply_progress", viewOf(job));
+        };
+        // Passed in rather than parked on `this`: a shared field would let a second
+        // reconcile redirect this job's reporting into another job's rows.
+        const observer = (target, kind, status, error) => {
+            this.applyJobs.update(jobId, job => {
+                let row = job.targets.find(item => item.target === target);
+                if (!row) {
+                    row = { target, kind, status: "pending" };
+                    job.targets.push(row);
+                }
+                row.kind = kind;
+                row.status = status;
+                if (error)
+                    row.error = error;
+            });
+            emit();
+        };
+        emit();
+        try {
+            const started = this.startExclusiveReconcile(observer);
+            if (!started) {
+                // The slot was reserved before the microtask, so this means a SIGHUP
+                // reconcile started in between. Report it instead of applying twice.
+                this.applyJobs.finish(jobId, "a config reload was already running");
+                return;
+            }
+            const outcome = await started;
+            this.applyJobs.finish(jobId, outcome.rejected);
+        }
+        catch (err) {
+            this.applyJobs.finish(jobId, err instanceof Error ? err.message : String(err));
+        }
+        finally {
+            this.activeApplyJobId = null;
+            emit();
+        }
     }
     async restartInstances() {
         if (!this.configPath) {
@@ -10972,6 +11464,12 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             this.initializeWebAuthTokens();
         this.healthServer = createServer((req, res) => {
             res.setHeader("Content-Type", "application/json");
+            // No Referer to a tunnel host, an upstream proxy, or any page linked from
+            // the panel — the dashboard URL is itself a credential-bearing address.
+            res.setHeader("Referrer-Policy", "no-referrer");
+            // Authorization now depends on a cookie, so a shared cache (a tunnel, a
+            // corporate proxy) must not serve one visitor's response to another.
+            res.setHeader("Vary", "Cookie");
             const requestPath = new URL(req.url ?? "/", `http://localhost:${port}`).pathname;
             // Browsers request this automatically and AgEnD does not ship an icon.
             // It is neither user data nor an API route, so do not turn the harmless
@@ -10997,15 +11495,25 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 // like the other /view data routes (usage-api.ts rejects non-GET).
             }
             else {
-                // All other endpoints require a valid token (query ?token= or X-Agend-Token header).
+                // All other endpoints require a session cookie or an X-Agend-Token
+                // header; a `?token=` in the URL is only redeemed for a cookie on a GET.
                 // /ui/* will also re-check in web-api.ts, which is harmless.
                 const parsedUrl = new URL(req.url ?? "/", `http://localhost:${port}`);
-                const headerToken = req.headers["x-agend-token"];
-                const providedToken = parsedUrl.searchParams.get("token")
-                    ?? (typeof headerToken === "string" ? headerToken : null);
-                if (!this.webToken || providedToken !== this.webToken) {
-                    res.writeHead(401);
-                    res.end(JSON.stringify({ error: WEB_TOKEN_INVALID_MESSAGE }));
+                const decision = decideWebGate(req, parsedUrl, this.webToken);
+                if (decision.kind === "reject") {
+                    res.writeHead(decision.status);
+                    res.end(JSON.stringify({ error: decision.message }));
+                    return;
+                }
+                if (decision.kind === "exchange") {
+                    res.setHeader("Set-Cookie", decision.setCookie);
+                    res.setHeader("Location", decision.location);
+                    // A cached redirect would replay a Set-Cookie for a rotated token.
+                    res.setHeader("Cache-Control", "no-store");
+                    res.writeHead(302);
+                    // Browsers follow the Location; a script that does not gets told why
+                    // its URL token stopped being echoed back as data.
+                    res.end(JSON.stringify({ redirect: decision.location }));
                     return;
                 }
             }
@@ -11243,8 +11751,11 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             this.logger.info({ port }, afterTakeover
                 ? "Health endpoint listening (after takeover)"
                 : "Health endpoint listening");
-            this.logger.info({ url: `http://localhost:${port}/ui?token=${this.webToken}` }, "Web UI available");
-            this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
+            // Never the token: fleet.log is readable by anything that can read the
+            // data dir, is copied into bug reports, and is tailed in shared terminals.
+            // `/dashboard` and `agend web` are the ways to get an authorized link.
+            this.logger.info({ url: `http://localhost:${port}/ui` }, "Web UI available (open it with /dashboard or `agend web`)");
+            this.logger.info({ url: `http://localhost:${port}/view` }, "Web View available");
         };
         this.healthServer.on("error", (err) => {
             this.healthServerListening = false;
@@ -11261,8 +11772,25 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                     if (existsSync(pidPath)) {
                         const oldPid = parseInt(readFileSync(pidPath, "utf-8").trim(), 10);
                         if (oldPid && oldPid !== process.pid) {
-                            process.kill(oldPid, "SIGTERM");
-                            this.logger.info({ oldPid }, "Killed old fleet process");
+                            // fleet.pid is a claim, not proof. A stale or wrong entry points
+                            // at whatever now holds that pid, and this used to SIGTERM it —
+                            // an unrelated process killed because a port was busy. Confirm
+                            // the target really is an AgEnD fleet, and when that cannot be
+                            // confirmed, do not signal: not killing costs a dashboard, and
+                            // killing costs somebody else's process.
+                            const commandLine = readProcessCommandLine(oldPid);
+                            if (isFleetStartCommandLine(commandLine)) {
+                                process.kill(oldPid, "SIGTERM");
+                                this.logger.info({ oldPid }, "Killed old fleet process");
+                            }
+                            else {
+                                this.logger.warn({
+                                    oldPid,
+                                    // Truncated: this is an unrelated process's command line, and
+                                    // fleet.log is copied into bug reports.
+                                    commandLine: commandLine ? `${commandLine.slice(0, 60)}${commandLine.length > 60 ? "…" : ""}` : "(unreadable)",
+                                }, "fleet.pid does not name an AgEnD fleet process — not signalling it");
+                            }
                         }
                     }
                 }