@songsid/agend 2.1.5-beta.21 → 2.1.5-beta.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channel/adapters/telegram.d.ts +16 -0
- package/dist/channel/adapters/telegram.js +47 -5
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/types.d.ts +1 -0
- package/dist/classic-channel-manager.d.ts +23 -3
- package/dist/classic-channel-manager.js +35 -0
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/config-validator.js +15 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/daemon.js +8 -1
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-manager.d.ts +42 -1
- package/dist/fleet-manager.js +259 -54
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +26 -2
- package/dist/locale.js +2 -2
- package/dist/locale.js.map +1 -1
- package/dist/settings-api.d.ts +1 -1
- package/dist/settings-api.js +38 -6
- package/dist/settings-api.js.map +1 -1
- package/dist/types.d.ts +7 -0
- package/dist/ui/dashboard.html +7 -5
- package/dist/ui/settings.html +93 -15
- package/dist/ui/view.html +8 -7
- package/package.json +1 -1
package/dist/fleet-manager.d.ts
CHANGED
|
@@ -139,6 +139,16 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
|
|
|
139
139
|
private topicCleanupInFlight;
|
|
140
140
|
private topicCleanupGeneration;
|
|
141
141
|
private topicProbeWarnings;
|
|
142
|
+
/**
|
|
143
|
+
* Consecutive unknown probe results per route (or per adapter for outage
|
|
144
|
+
* class reasons). A single transient never reaches the operator; only a
|
|
145
|
+
* streak of TOPIC_PROBE_UNKNOWN_ESCALATION does.
|
|
146
|
+
*/
|
|
147
|
+
private topicProbeUnknownStreak;
|
|
148
|
+
/** Unknown results in a row before the operator is told. 3 × 5 min poller = 15 min. */
|
|
149
|
+
static readonly TOPIC_PROBE_UNKNOWN_ESCALATION = 3;
|
|
150
|
+
/** Reasons that describe the adapter, not one topic — counted once per adapter. */
|
|
151
|
+
private static readonly TOPIC_PROBE_ADAPTER_SCOPED_REASONS;
|
|
142
152
|
logger: Logger;
|
|
143
153
|
private topicCommands;
|
|
144
154
|
sessionRegistry: Map<string, string>;
|
|
@@ -437,8 +447,14 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
|
|
|
437
447
|
/** Fleet admin is an explicit config allowlist entry, not merely an open/paired user. */
|
|
438
448
|
isFleetAdmin(userId: string, adapterId?: string): boolean;
|
|
439
449
|
changeInstancePauseState(name: string, action: "pause" | "wake"): Promise<"paused" | "awake" | "not_idle">;
|
|
450
|
+
/** Deliver an already-resolved hot snapshot without depending on IPC timing. */
|
|
451
|
+
private applyHotConfigUpdate;
|
|
452
|
+
private classicBehaviorUpdate;
|
|
440
453
|
/** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
|
|
441
|
-
restartClassicInstanceFromSettings(instanceName: string): Promise<void>;
|
|
454
|
+
restartClassicInstanceFromSettings(instanceName: string, changedFields?: string[]): Promise<void>;
|
|
455
|
+
/** Reload classicBot.yaml once. Kept callable so the periodic production
|
|
456
|
+
* path is covered without relying on fake timers around startAll(). */
|
|
457
|
+
private reloadClassicConfigFromDisk;
|
|
442
458
|
startInstance(name: string, config: InstanceConfig, topicMode: boolean, kind?: "fleet-topic" | "classic",
|
|
443
459
|
/**
|
|
444
460
|
* Explicit starts (CLI/API) may resume a paused or failed daemon. Startup
|
|
@@ -770,7 +786,32 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
|
|
|
770
786
|
private scheduleTopicCleanup;
|
|
771
787
|
private confirmedProbeFence;
|
|
772
788
|
private sameProbeFence;
|
|
789
|
+
private topicProbeStreakKey;
|
|
790
|
+
/** A definite answer (present or missing) ends the unknown streak for that route and its adapter. */
|
|
791
|
+
private clearTopicProbeUnknownStreak;
|
|
792
|
+
/**
|
|
793
|
+
* Record one unknown probe result. Nothing here can touch quarantine or
|
|
794
|
+
* removal: unknown is always retained data. The only question is whether
|
|
795
|
+
* the operator hears about it, and a single transient (one flaky HTTP call
|
|
796
|
+
* out of dozens per pass) must not — only a streak does.
|
|
797
|
+
*
|
|
798
|
+
* Used directly for single-route events (channelDelete, the pre-action
|
|
799
|
+
* fence). The periodic scan goes through a TopicProbePass instead, so one
|
|
800
|
+
* pass over N routes of a dead adapter counts as ONE check, not N.
|
|
801
|
+
*/
|
|
773
802
|
private warnTopicProbeUnknown;
|
|
803
|
+
/** One scan's worth of probe outcomes, applied to the streaks after the loop. */
|
|
804
|
+
private newTopicProbePass;
|
|
805
|
+
private passTopicProbeUnknown;
|
|
806
|
+
private passTopicProbeDefinite;
|
|
807
|
+
/**
|
|
808
|
+
* Apply a pass: a definite answer resets its keys, and an adapter that
|
|
809
|
+
* answered for any route this pass is evidently alive, so an unknown for the
|
|
810
|
+
* same adapter key in the same pass does not count — regardless of the order
|
|
811
|
+
* the routes happened to be probed in.
|
|
812
|
+
*/
|
|
813
|
+
private applyTopicProbePass;
|
|
814
|
+
private noteTopicProbeUnknown;
|
|
774
815
|
/** One fixed-snapshot topology pass. Automatic evidence can only quarantine. */
|
|
775
816
|
private runTopicCleanup;
|
|
776
817
|
/**
|
package/dist/fleet-manager.js
CHANGED
|
@@ -181,6 +181,7 @@ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
|
|
|
181
181
|
const IGNORED_REACTION_EMOJIS = new Set(["📷"]);
|
|
182
182
|
const HOT_INSTANCE_CONFIG_KEYS = new Set([
|
|
183
183
|
"tool_progress",
|
|
184
|
+
"reply_completion_guard",
|
|
184
185
|
"mcp_proxy_reply",
|
|
185
186
|
"auto_pause_after",
|
|
186
187
|
"warm_cap",
|
|
@@ -321,6 +322,29 @@ export class FleetManager {
|
|
|
321
322
|
topicCleanupInFlight = null;
|
|
322
323
|
topicCleanupGeneration = 0;
|
|
323
324
|
topicProbeWarnings = new Map();
|
|
325
|
+
/**
|
|
326
|
+
* Consecutive unknown probe results per route (or per adapter for outage
|
|
327
|
+
* class reasons). A single transient never reaches the operator; only a
|
|
328
|
+
* streak of TOPIC_PROBE_UNKNOWN_ESCALATION does.
|
|
329
|
+
*/
|
|
330
|
+
topicProbeUnknownStreak = new Map();
|
|
331
|
+
/** Unknown results in a row before the operator is told. 3 × 5 min poller = 15 min. */
|
|
332
|
+
static TOPIC_PROBE_UNKNOWN_ESCALATION = 3;
|
|
333
|
+
/** Reasons that describe the adapter, not one topic — counted once per adapter. */
|
|
334
|
+
static TOPIC_PROBE_ADAPTER_SCOPED_REASONS = new Set([
|
|
335
|
+
"owner-adapter-unavailable",
|
|
336
|
+
"owner-adapter-not-ready",
|
|
337
|
+
"owner-adapter-generation-changed",
|
|
338
|
+
"owner-adapter-changed-before-action",
|
|
339
|
+
"adapter-not-ready",
|
|
340
|
+
"adapter-not-initialized",
|
|
341
|
+
"adapter-generation-changed",
|
|
342
|
+
"topic-close-from-unready-adapter",
|
|
343
|
+
"topic-close-generation-changed",
|
|
344
|
+
// Telegram probe: the transport or Telegram itself is down, not one topic.
|
|
345
|
+
"transport-failed",
|
|
346
|
+
"provider-unavailable",
|
|
347
|
+
]);
|
|
324
348
|
logger = createLogger("info");
|
|
325
349
|
topicCommands;
|
|
326
350
|
// sessionName → instanceName mapping for external sessions
|
|
@@ -1482,8 +1506,27 @@ export class FleetManager {
|
|
|
1482
1506
|
await this.lifecycle.pause(name);
|
|
1483
1507
|
return this.lifecycle.isPaused(name) ? "paused" : "not_idle";
|
|
1484
1508
|
}
|
|
1509
|
+
/** Deliver an already-resolved hot snapshot without depending on IPC timing. */
|
|
1510
|
+
applyHotConfigUpdate(instanceName, update) {
|
|
1511
|
+
const daemon = this.daemons.get(instanceName);
|
|
1512
|
+
if (!daemon)
|
|
1513
|
+
return false;
|
|
1514
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1515
|
+
const sent = ipc?.connected === true && ipc.send({ type: "config_update", config: update });
|
|
1516
|
+
if (!sent) {
|
|
1517
|
+
daemon.applyConfigUpdate(update);
|
|
1518
|
+
this.logger.warn({ name: instanceName }, "Config-update IPC unavailable — applied hot config in-process");
|
|
1519
|
+
}
|
|
1520
|
+
return true;
|
|
1521
|
+
}
|
|
1522
|
+
classicBehaviorUpdate(instanceName) {
|
|
1523
|
+
return {
|
|
1524
|
+
tool_progress: this.classicChannels?.getToolProgressByInstance(instanceName, this.fleetConfig?.defaults?.tool_progress) ?? "off",
|
|
1525
|
+
reply_completion_guard: this.classicChannels?.getReplyCompletionGuardByInstance(instanceName, this.fleetConfig?.defaults?.reply_completion_guard) ?? true,
|
|
1526
|
+
};
|
|
1527
|
+
}
|
|
1485
1528
|
/** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
|
|
1486
|
-
async restartClassicInstanceFromSettings(instanceName) {
|
|
1529
|
+
async restartClassicInstanceFromSettings(instanceName, changedFields = []) {
|
|
1487
1530
|
if (!this.classicChannels)
|
|
1488
1531
|
throw new Error("Classic channel manager not initialized");
|
|
1489
1532
|
const wasRunning = this.daemons.has(instanceName);
|
|
@@ -1495,10 +1538,77 @@ export class FleetManager {
|
|
|
1495
1538
|
throw new Error("Classic channel not found after reload");
|
|
1496
1539
|
if (!wasRunning)
|
|
1497
1540
|
return;
|
|
1541
|
+
const hotOnly = changedFields.length > 0
|
|
1542
|
+
&& changedFields.every(field => field === "tool_progress" || field === "reply_completion_guard");
|
|
1543
|
+
if (hotOnly) {
|
|
1544
|
+
this.applyHotConfigUpdate(instanceName, this.classicBehaviorUpdate(instanceName));
|
|
1545
|
+
this.logger.info({ instanceName, fields: changedFields }, "Classic instance hot config reloaded");
|
|
1546
|
+
return;
|
|
1547
|
+
}
|
|
1498
1548
|
await this.stopInstance(instanceName);
|
|
1499
1549
|
await new Promise(resolve => setTimeout(resolve, 250));
|
|
1500
1550
|
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1501
1551
|
}
|
|
1552
|
+
/** Reload classicBot.yaml once. Kept callable so the periodic production
|
|
1553
|
+
* path is covered without relying on fake timers around startAll(). */
|
|
1554
|
+
async reloadClassicConfigFromDisk() {
|
|
1555
|
+
try {
|
|
1556
|
+
if (!this.classicChannels)
|
|
1557
|
+
return;
|
|
1558
|
+
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
1559
|
+
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
1560
|
+
const oldBackends = new Map();
|
|
1561
|
+
const oldModels = new Map();
|
|
1562
|
+
const oldAutoPause = new Map();
|
|
1563
|
+
const oldToolProgress = new Map();
|
|
1564
|
+
const oldReplyGuard = new Map();
|
|
1565
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
1566
|
+
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
1567
|
+
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
1568
|
+
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1569
|
+
oldToolProgress.set(ch.instanceName, this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress));
|
|
1570
|
+
oldReplyGuard.set(ch.instanceName, this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard));
|
|
1571
|
+
}
|
|
1572
|
+
if (!this.classicChannels.checkReload())
|
|
1573
|
+
return;
|
|
1574
|
+
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
1575
|
+
// throttle keeps a repeated report from flooding the topic.
|
|
1576
|
+
this.reportClassicUnrecoverableIds();
|
|
1577
|
+
this.reregisterClassicChannels();
|
|
1578
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
1579
|
+
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
1580
|
+
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
1581
|
+
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
1582
|
+
const newToolProgress = this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress);
|
|
1583
|
+
const newReplyGuard = this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard);
|
|
1584
|
+
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
1585
|
+
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
1586
|
+
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
1587
|
+
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
1588
|
+
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
1589
|
+
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
1590
|
+
// Small delay to let tmux window clean up
|
|
1591
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
1592
|
+
// The manager already holds the new backend/model/auto-pause; the
|
|
1593
|
+
// unattended helper reads them from it and schedules the delayed
|
|
1594
|
+
// retry on failure like every other unattended start.
|
|
1595
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
1596
|
+
}
|
|
1597
|
+
else if (this.daemons.has(ch.instanceName)
|
|
1598
|
+
&& (oldToolProgress.get(ch.instanceName) !== newToolProgress
|
|
1599
|
+
|| oldReplyGuard.get(ch.instanceName) !== newReplyGuard)) {
|
|
1600
|
+
this.applyHotConfigUpdate(ch.instanceName, {
|
|
1601
|
+
tool_progress: newToolProgress,
|
|
1602
|
+
reply_completion_guard: newReplyGuard,
|
|
1603
|
+
});
|
|
1604
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic instance hot config reloaded");
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
catch (err) {
|
|
1609
|
+
this.logger.warn({ err }, "classicBot.yaml reload error");
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1502
1612
|
async startInstance(name, config, topicMode, kind = "fleet-topic",
|
|
1503
1613
|
/**
|
|
1504
1614
|
* Explicit starts (CLI/API) may resume a paused or failed daemon. Startup
|
|
@@ -2356,48 +2466,8 @@ export class FleetManager {
|
|
|
2356
2466
|
this.instanceWorldBinding.set(ch.instanceName, ch.adapterId);
|
|
2357
2467
|
}
|
|
2358
2468
|
// Poll classicBot.yaml for external changes every 30s
|
|
2359
|
-
this.classicReloadTimer = setInterval(
|
|
2360
|
-
|
|
2361
|
-
if (!this.classicChannels)
|
|
2362
|
-
return;
|
|
2363
|
-
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
2364
|
-
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
2365
|
-
const oldBackends = new Map();
|
|
2366
|
-
const oldModels = new Map();
|
|
2367
|
-
const oldAutoPause = new Map();
|
|
2368
|
-
for (const ch of this.classicChannels.getAll()) {
|
|
2369
|
-
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
2370
|
-
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
2371
|
-
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
2372
|
-
}
|
|
2373
|
-
if (!this.classicChannels.checkReload())
|
|
2374
|
-
return;
|
|
2375
|
-
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
2376
|
-
// throttle keeps a repeated report from flooding the topic.
|
|
2377
|
-
this.reportClassicUnrecoverableIds();
|
|
2378
|
-
this.reregisterClassicChannels();
|
|
2379
|
-
for (const ch of this.classicChannels.getAll()) {
|
|
2380
|
-
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
2381
|
-
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
2382
|
-
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
2383
|
-
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
2384
|
-
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
2385
|
-
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
2386
|
-
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
2387
|
-
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
2388
|
-
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
2389
|
-
// Small delay to let tmux window clean up
|
|
2390
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
2391
|
-
// The manager already holds the new backend/model/auto-pause; the
|
|
2392
|
-
// unattended helper reads them from it and schedules the delayed
|
|
2393
|
-
// retry on failure like every other unattended start.
|
|
2394
|
-
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
2395
|
-
}
|
|
2396
|
-
}
|
|
2397
|
-
}
|
|
2398
|
-
catch (err) {
|
|
2399
|
-
this.logger.warn({ err }, "classicBot.yaml reload error");
|
|
2400
|
-
}
|
|
2469
|
+
this.classicReloadTimer = setInterval(() => {
|
|
2470
|
+
void this.reloadClassicConfigFromDisk();
|
|
2401
2471
|
}, 30_000);
|
|
2402
2472
|
const costGuardConfig = {
|
|
2403
2473
|
...DEFAULT_COST_GUARD,
|
|
@@ -5488,15 +5558,74 @@ export class FleetManager {
|
|
|
5488
5558
|
return false;
|
|
5489
5559
|
return result?.generation === undefined || result.generation === after.generation;
|
|
5490
5560
|
}
|
|
5491
|
-
|
|
5561
|
+
topicProbeStreakKey(threadId, adapterId, reason) {
|
|
5562
|
+
return FleetManager.TOPIC_PROBE_ADAPTER_SCOPED_REASONS.has(reason)
|
|
5563
|
+
? `adapter:${adapterId ?? "unbound"}`
|
|
5564
|
+
: `thread:${threadId}`;
|
|
5565
|
+
}
|
|
5566
|
+
/** A definite answer (present or missing) ends the unknown streak for that route and its adapter. */
|
|
5567
|
+
clearTopicProbeUnknownStreak(threadId, adapterId) {
|
|
5568
|
+
this.topicProbeUnknownStreak.delete(`thread:${threadId}`);
|
|
5569
|
+
this.topicProbeUnknownStreak.delete(`adapter:${adapterId ?? "unbound"}`);
|
|
5570
|
+
}
|
|
5571
|
+
/**
|
|
5572
|
+
* Record one unknown probe result. Nothing here can touch quarantine or
|
|
5573
|
+
* removal: unknown is always retained data. The only question is whether
|
|
5574
|
+
* the operator hears about it, and a single transient (one flaky HTTP call
|
|
5575
|
+
* out of dozens per pass) must not — only a streak does.
|
|
5576
|
+
*
|
|
5577
|
+
* Used directly for single-route events (channelDelete, the pre-action
|
|
5578
|
+
* fence). The periodic scan goes through a TopicProbePass instead, so one
|
|
5579
|
+
* pass over N routes of a dead adapter counts as ONE check, not N.
|
|
5580
|
+
*/
|
|
5581
|
+
warnTopicProbeUnknown(instanceName, threadId, adapterId, reason, detail) {
|
|
5582
|
+
this.noteTopicProbeUnknown(this.topicProbeStreakKey(threadId, adapterId, reason), { instanceName, threadId, adapterId, reason, detail });
|
|
5583
|
+
}
|
|
5584
|
+
/** One scan's worth of probe outcomes, applied to the streaks after the loop. */
|
|
5585
|
+
newTopicProbePass() {
|
|
5586
|
+
return { unknown: new Map(), definite: new Set() };
|
|
5587
|
+
}
|
|
5588
|
+
passTopicProbeUnknown(pass, instanceName, threadId, adapterId, reason, detail) {
|
|
5589
|
+
const key = this.topicProbeStreakKey(threadId, adapterId, reason);
|
|
5590
|
+
// First unknown per key per pass wins; the rest of the routes on a dead
|
|
5591
|
+
// adapter are the same observation, not additional checks.
|
|
5592
|
+
if (!pass.unknown.has(key))
|
|
5593
|
+
pass.unknown.set(key, { instanceName, threadId, adapterId, reason, detail });
|
|
5594
|
+
}
|
|
5595
|
+
passTopicProbeDefinite(pass, threadId, adapterId) {
|
|
5596
|
+
pass.definite.add(`thread:${threadId}`);
|
|
5597
|
+
pass.definite.add(`adapter:${adapterId ?? "unbound"}`);
|
|
5598
|
+
}
|
|
5599
|
+
/**
|
|
5600
|
+
* Apply a pass: a definite answer resets its keys, and an adapter that
|
|
5601
|
+
* answered for any route this pass is evidently alive, so an unknown for the
|
|
5602
|
+
* same adapter key in the same pass does not count — regardless of the order
|
|
5603
|
+
* the routes happened to be probed in.
|
|
5604
|
+
*/
|
|
5605
|
+
applyTopicProbePass(pass) {
|
|
5606
|
+
for (const key of pass.definite)
|
|
5607
|
+
this.topicProbeUnknownStreak.delete(key);
|
|
5608
|
+
for (const [key, ctx] of pass.unknown) {
|
|
5609
|
+
if (pass.definite.has(key))
|
|
5610
|
+
continue;
|
|
5611
|
+
this.noteTopicProbeUnknown(key, ctx);
|
|
5612
|
+
}
|
|
5613
|
+
}
|
|
5614
|
+
noteTopicProbeUnknown(streakKey, { instanceName, threadId, adapterId, reason, detail }) {
|
|
5615
|
+
const streak = (this.topicProbeUnknownStreak.get(streakKey) ?? 0) + 1;
|
|
5616
|
+
this.topicProbeUnknownStreak.set(streakKey, streak);
|
|
5617
|
+
if (streak < FleetManager.TOPIC_PROBE_UNKNOWN_ESCALATION) {
|
|
5618
|
+
this.logger.debug({ instanceName, threadId, adapterId, reason, detail, streak }, "Topic presence not confirmed this pass — transient, retaining instance and all data");
|
|
5619
|
+
return;
|
|
5620
|
+
}
|
|
5492
5621
|
const key = `${adapterId ?? "unbound"}:${reason}`;
|
|
5493
5622
|
const now = Date.now();
|
|
5494
5623
|
const last = this.topicProbeWarnings.get(key) ?? 0;
|
|
5495
5624
|
if (now - last < FleetManager.FLEET_ERROR_THROTTLE_MS)
|
|
5496
5625
|
return;
|
|
5497
5626
|
this.topicProbeWarnings.set(key, now);
|
|
5498
|
-
this.logger.error({ instanceName, threadId, adapterId, reason }, "Topic presence could not be confirmed — retaining instance and all data");
|
|
5499
|
-
this.notifyFleetError(t("fleet.topic_probe_unknown", instanceName, adapterId ?? "unbound"));
|
|
5627
|
+
this.logger.error({ instanceName, threadId, adapterId, reason, detail, streak }, "Topic presence could not be confirmed repeatedly — retaining instance and all data");
|
|
5628
|
+
this.notifyFleetError(t("fleet.topic_probe_unknown", instanceName, adapterId ?? "unbound", streak));
|
|
5500
5629
|
}
|
|
5501
5630
|
/** One fixed-snapshot topology pass. Automatic evidence can only quarantine. */
|
|
5502
5631
|
async runTopicCleanup(generation) {
|
|
@@ -5504,6 +5633,7 @@ export class FleetManager {
|
|
|
5504
5633
|
return;
|
|
5505
5634
|
const snapshot = [...this.routing.entries()].filter(([, target]) => isProbeableRouteTarget(target));
|
|
5506
5635
|
const missing = [];
|
|
5636
|
+
const pass = this.newTopicProbePass();
|
|
5507
5637
|
for (const [threadId, target] of snapshot) {
|
|
5508
5638
|
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5509
5639
|
return;
|
|
@@ -5513,12 +5643,12 @@ export class FleetManager {
|
|
|
5513
5643
|
const adapterId = this.getInstanceAdapterId(target.name);
|
|
5514
5644
|
const adapter = adapterId ? this.adapters.get(adapterId) : undefined;
|
|
5515
5645
|
if (!adapterId || !adapter?.probeTopicPresence) {
|
|
5516
|
-
this.
|
|
5646
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-unavailable");
|
|
5517
5647
|
continue;
|
|
5518
5648
|
}
|
|
5519
5649
|
const before = this.confirmedProbeFence(adapterId, adapter);
|
|
5520
5650
|
if (!before) {
|
|
5521
|
-
this.
|
|
5651
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-not-ready");
|
|
5522
5652
|
continue;
|
|
5523
5653
|
}
|
|
5524
5654
|
let result;
|
|
@@ -5531,17 +5661,24 @@ export class FleetManager {
|
|
|
5531
5661
|
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5532
5662
|
return;
|
|
5533
5663
|
if (!this.sameProbeFence(adapterId, adapter, before, result)) {
|
|
5534
|
-
this.
|
|
5664
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-generation-changed");
|
|
5535
5665
|
continue;
|
|
5536
5666
|
}
|
|
5537
5667
|
if (result.status === "unknown") {
|
|
5538
|
-
this.
|
|
5668
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, result.reason, result.detail);
|
|
5539
5669
|
}
|
|
5540
|
-
else
|
|
5541
|
-
|
|
5670
|
+
else {
|
|
5671
|
+
this.passTopicProbeDefinite(pass, threadId, adapterId);
|
|
5672
|
+
if (result.status === "missing") {
|
|
5673
|
+
missing.push({ threadId, target, adapterId, adapter, generation: result.generation });
|
|
5674
|
+
}
|
|
5542
5675
|
}
|
|
5543
5676
|
}
|
|
5544
|
-
if (generation !== this.topicCleanupGeneration || this.shuttingDown
|
|
5677
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5678
|
+
return;
|
|
5679
|
+
// One pass, one check: streaks move by at most one per key here.
|
|
5680
|
+
this.applyTopicProbePass(pass);
|
|
5681
|
+
if (missing.length === 0)
|
|
5545
5682
|
return;
|
|
5546
5683
|
if (missing.length > 1) {
|
|
5547
5684
|
this.logger.error({ missing: missing.map(item => ({ instanceName: item.target.name, threadId: item.threadId, adapterId: item.adapterId })) }, "Multiple topics appeared missing in one pass — treating topology evidence as untrusted and retaining all data");
|
|
@@ -5595,8 +5732,15 @@ export class FleetManager {
|
|
|
5595
5732
|
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-generation-changed");
|
|
5596
5733
|
return;
|
|
5597
5734
|
}
|
|
5735
|
+
if (result.status === "unknown") {
|
|
5736
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, result.reason, result.detail);
|
|
5737
|
+
return;
|
|
5738
|
+
}
|
|
5739
|
+
this.clearTopicProbeUnknownStreak(threadId, adapterId);
|
|
5598
5740
|
if (result.status !== "missing") {
|
|
5599
|
-
|
|
5741
|
+
// The gateway said deleted, REST says present: a definite answer, so it
|
|
5742
|
+
// is not an unknown streak — but it is worth one debug line.
|
|
5743
|
+
this.logger.debug({ instanceName: target.name, threadId, adapterId }, "channelDelete hint contradicted by REST — topic present, nothing to do");
|
|
5600
5744
|
return;
|
|
5601
5745
|
}
|
|
5602
5746
|
this.topicCommands.handleTopicDeleted(threadId, {
|
|
@@ -9999,11 +10143,19 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9999
10143
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
10000
10144
|
ensureWorkspaceGit(workDir);
|
|
10001
10145
|
const classicIdentity = this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName);
|
|
10146
|
+
const toolProgress = classicIdentity
|
|
10147
|
+
? this.classicChannels?.getToolProgress(classicIdentity.channelId, classicIdentity.adapterId, this.fleetConfig?.defaults?.tool_progress)
|
|
10148
|
+
: this.fleetConfig?.defaults?.tool_progress;
|
|
10149
|
+
const replyCompletionGuard = classicIdentity
|
|
10150
|
+
? this.classicChannels?.getReplyCompletionGuard(classicIdentity.channelId, classicIdentity.adapterId, this.fleetConfig?.defaults?.reply_completion_guard)
|
|
10151
|
+
: this.fleetConfig?.defaults?.reply_completion_guard;
|
|
10002
10152
|
const config = {
|
|
10003
10153
|
...DEFAULT_INSTANCE_CONFIG,
|
|
10004
10154
|
...this.fleetConfig?.defaults,
|
|
10005
10155
|
working_directory: workDir,
|
|
10006
10156
|
lightweight: true,
|
|
10157
|
+
tool_progress: toolProgress ?? "off",
|
|
10158
|
+
reply_completion_guard: replyCompletionGuard ?? true,
|
|
10007
10159
|
...(backend ? { backend } : {}),
|
|
10008
10160
|
...(model ? { model } : {}),
|
|
10009
10161
|
...(classicIdentity?.displayName ? { display_name: classicIdentity.displayName } : {}),
|
|
@@ -10396,6 +10548,30 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10396
10548
|
this.notifyFleetError(t("fleet.reload_rejected", why));
|
|
10397
10549
|
return;
|
|
10398
10550
|
}
|
|
10551
|
+
// Classic behavior settings share the fleet defaults but are not entries
|
|
10552
|
+
// in fleet.yaml. Snapshot the old effective chain before reloading the
|
|
10553
|
+
// Classic file so SIGHUP can hot-apply either source without waiting for
|
|
10554
|
+
// the 30-second Classic poller.
|
|
10555
|
+
const oldClassicBehavior = new Map();
|
|
10556
|
+
if (this.classicChannels) {
|
|
10557
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
10558
|
+
const runtimeConfig = this.daemons.get(ch.instanceName)?.getConfigSnapshot?.();
|
|
10559
|
+
oldClassicBehavior.set(ch.instanceName, {
|
|
10560
|
+
backend: this.classicChannels.getBackend(ch.channelId, ch.adapterId, oldConfig?.defaults?.backend),
|
|
10561
|
+
model: this.classicChannels.getModel(ch.channelId, ch.adapterId, oldConfig?.defaults?.model),
|
|
10562
|
+
autoPauseAfter: this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, oldConfig?.defaults?.auto_pause_after),
|
|
10563
|
+
// Settings mutates FleetManager's in-memory defaults before SIGHUP.
|
|
10564
|
+
// The live daemon is therefore the authority for the previous hot
|
|
10565
|
+
// values, exactly as in the fleet-topic reconciliation below.
|
|
10566
|
+
toolProgress: runtimeConfig?.tool_progress
|
|
10567
|
+
?? this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, oldConfig?.defaults?.tool_progress),
|
|
10568
|
+
replyCompletionGuard: runtimeConfig?.reply_completion_guard
|
|
10569
|
+
?? this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, oldConfig?.defaults?.reply_completion_guard),
|
|
10570
|
+
});
|
|
10571
|
+
}
|
|
10572
|
+
if (this.classicChannels.checkReload())
|
|
10573
|
+
this.reportClassicUnrecoverableIds();
|
|
10574
|
+
}
|
|
10399
10575
|
this.routing.rebuild(this.fleetConfig);
|
|
10400
10576
|
this.reregisterClassicChannels();
|
|
10401
10577
|
this.scheduler?.reload();
|
|
@@ -10460,6 +10636,35 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10460
10636
|
}
|
|
10461
10637
|
}
|
|
10462
10638
|
}
|
|
10639
|
+
// A Classic channel inherits fleet defaults beneath its own two levels.
|
|
10640
|
+
// Recompute that complete chain on SIGHUP. Only the two behavior switches
|
|
10641
|
+
// are hot; changes to backend/model/auto-pause retain the existing restart
|
|
10642
|
+
// semantics.
|
|
10643
|
+
if (this.classicChannels) {
|
|
10644
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
10645
|
+
const old = oldClassicBehavior.get(ch.instanceName);
|
|
10646
|
+
if (!old || !this.daemons.has(ch.instanceName))
|
|
10647
|
+
continue;
|
|
10648
|
+
const backend = this.classicChannels.getBackend(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.backend);
|
|
10649
|
+
const model = this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model);
|
|
10650
|
+
const autoPauseAfter = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
10651
|
+
if (old.backend !== backend || old.model !== model || old.autoPauseAfter !== autoPauseAfter) {
|
|
10652
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic cold config changed — restarting");
|
|
10653
|
+
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
10654
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after fleet reload");
|
|
10655
|
+
continue;
|
|
10656
|
+
}
|
|
10657
|
+
const toolProgress = this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress);
|
|
10658
|
+
const replyCompletionGuard = this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard);
|
|
10659
|
+
if (old.toolProgress !== toolProgress || old.replyCompletionGuard !== replyCompletionGuard) {
|
|
10660
|
+
this.applyHotConfigUpdate(ch.instanceName, {
|
|
10661
|
+
tool_progress: toolProgress,
|
|
10662
|
+
reply_completion_guard: replyCompletionGuard,
|
|
10663
|
+
});
|
|
10664
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic inherited hot config reloaded");
|
|
10665
|
+
}
|
|
10666
|
+
}
|
|
10667
|
+
}
|
|
10463
10668
|
// warm_cap is fleet-owned; enforce the reloaded value immediately against
|
|
10464
10669
|
// currently idle instances instead of waiting for a future state edge.
|
|
10465
10670
|
this.enforceWarmCap();
|