@songsid/agend 2.1.5-beta.21 → 2.1.5-beta.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channel/adapters/telegram.d.ts +40 -1
- package/dist/channel/adapters/telegram.js +110 -14
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/factory.js +6 -1
- package/dist/channel/factory.js.map +1 -1
- package/dist/channel/types.d.ts +11 -0
- package/dist/classic-channel-manager.d.ts +23 -3
- package/dist/classic-channel-manager.js +35 -0
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/config-validator.js +23 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/daemon.js +8 -1
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-manager.d.ts +42 -1
- package/dist/fleet-manager.js +270 -54
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +26 -2
- package/dist/locale.js +2 -2
- package/dist/locale.js.map +1 -1
- package/dist/settings-api.d.ts +1 -1
- package/dist/settings-api.js +38 -6
- package/dist/settings-api.js.map +1 -1
- package/dist/types.d.ts +7 -0
- package/dist/ui/dashboard.html +7 -5
- package/dist/ui/settings.html +93 -15
- package/dist/ui/view.html +8 -7
- package/package.json +1 -1
package/dist/fleet-manager.d.ts
CHANGED
|
@@ -139,6 +139,16 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
|
|
|
139
139
|
private topicCleanupInFlight;
|
|
140
140
|
private topicCleanupGeneration;
|
|
141
141
|
private topicProbeWarnings;
|
|
142
|
+
/**
|
|
143
|
+
* Consecutive unknown probe results per route (or per adapter for outage
|
|
144
|
+
* class reasons). A single transient never reaches the operator; only a
|
|
145
|
+
* streak of TOPIC_PROBE_UNKNOWN_ESCALATION does.
|
|
146
|
+
*/
|
|
147
|
+
private topicProbeUnknownStreak;
|
|
148
|
+
/** Unknown results in a row before the operator is told. 3 × 5 min poller = 15 min. */
|
|
149
|
+
static readonly TOPIC_PROBE_UNKNOWN_ESCALATION = 3;
|
|
150
|
+
/** Reasons that describe the adapter, not one topic — counted once per adapter. */
|
|
151
|
+
private static readonly TOPIC_PROBE_ADAPTER_SCOPED_REASONS;
|
|
142
152
|
logger: Logger;
|
|
143
153
|
private topicCommands;
|
|
144
154
|
sessionRegistry: Map<string, string>;
|
|
@@ -437,8 +447,14 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
|
|
|
437
447
|
/** Fleet admin is an explicit config allowlist entry, not merely an open/paired user. */
|
|
438
448
|
isFleetAdmin(userId: string, adapterId?: string): boolean;
|
|
439
449
|
changeInstancePauseState(name: string, action: "pause" | "wake"): Promise<"paused" | "awake" | "not_idle">;
|
|
450
|
+
/** Deliver an already-resolved hot snapshot without depending on IPC timing. */
|
|
451
|
+
private applyHotConfigUpdate;
|
|
452
|
+
private classicBehaviorUpdate;
|
|
440
453
|
/** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
|
|
441
|
-
restartClassicInstanceFromSettings(instanceName: string): Promise<void>;
|
|
454
|
+
restartClassicInstanceFromSettings(instanceName: string, changedFields?: string[]): Promise<void>;
|
|
455
|
+
/** Reload classicBot.yaml once. Kept callable so the periodic production
|
|
456
|
+
* path is covered without relying on fake timers around startAll(). */
|
|
457
|
+
private reloadClassicConfigFromDisk;
|
|
442
458
|
startInstance(name: string, config: InstanceConfig, topicMode: boolean, kind?: "fleet-topic" | "classic",
|
|
443
459
|
/**
|
|
444
460
|
* Explicit starts (CLI/API) may resume a paused or failed daemon. Startup
|
|
@@ -770,7 +786,32 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
|
|
|
770
786
|
private scheduleTopicCleanup;
|
|
771
787
|
private confirmedProbeFence;
|
|
772
788
|
private sameProbeFence;
|
|
789
|
+
private topicProbeStreakKey;
|
|
790
|
+
/** A definite answer (present or missing) ends the unknown streak for that route and its adapter. */
|
|
791
|
+
private clearTopicProbeUnknownStreak;
|
|
792
|
+
/**
|
|
793
|
+
* Record one unknown probe result. Nothing here can touch quarantine or
|
|
794
|
+
* removal: unknown is always retained data. The only question is whether
|
|
795
|
+
* the operator hears about it, and a single transient (one flaky HTTP call
|
|
796
|
+
* out of dozens per pass) must not — only a streak does.
|
|
797
|
+
*
|
|
798
|
+
* Used directly for single-route events (channelDelete, the pre-action
|
|
799
|
+
* fence). The periodic scan goes through a TopicProbePass instead, so one
|
|
800
|
+
* pass over N routes of a dead adapter counts as ONE check, not N.
|
|
801
|
+
*/
|
|
773
802
|
private warnTopicProbeUnknown;
|
|
803
|
+
/** One scan's worth of probe outcomes, applied to the streaks after the loop. */
|
|
804
|
+
private newTopicProbePass;
|
|
805
|
+
private passTopicProbeUnknown;
|
|
806
|
+
private passTopicProbeDefinite;
|
|
807
|
+
/**
|
|
808
|
+
* Apply a pass: a definite answer resets its keys, and an adapter that
|
|
809
|
+
* answered for any route this pass is evidently alive, so an unknown for the
|
|
810
|
+
* same adapter key in the same pass does not count — regardless of the order
|
|
811
|
+
* the routes happened to be probed in.
|
|
812
|
+
*/
|
|
813
|
+
private applyTopicProbePass;
|
|
814
|
+
private noteTopicProbeUnknown;
|
|
774
815
|
/** One fixed-snapshot topology pass. Automatic evidence can only quarantine. */
|
|
775
816
|
private runTopicCleanup;
|
|
776
817
|
/**
|
package/dist/fleet-manager.js
CHANGED
|
@@ -181,6 +181,7 @@ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
|
|
|
181
181
|
const IGNORED_REACTION_EMOJIS = new Set(["📷"]);
|
|
182
182
|
const HOT_INSTANCE_CONFIG_KEYS = new Set([
|
|
183
183
|
"tool_progress",
|
|
184
|
+
"reply_completion_guard",
|
|
184
185
|
"mcp_proxy_reply",
|
|
185
186
|
"auto_pause_after",
|
|
186
187
|
"warm_cap",
|
|
@@ -321,6 +322,29 @@ export class FleetManager {
|
|
|
321
322
|
topicCleanupInFlight = null;
|
|
322
323
|
topicCleanupGeneration = 0;
|
|
323
324
|
topicProbeWarnings = new Map();
|
|
325
|
+
/**
|
|
326
|
+
* Consecutive unknown probe results per route (or per adapter for outage
|
|
327
|
+
* class reasons). A single transient never reaches the operator; only a
|
|
328
|
+
* streak of TOPIC_PROBE_UNKNOWN_ESCALATION does.
|
|
329
|
+
*/
|
|
330
|
+
topicProbeUnknownStreak = new Map();
|
|
331
|
+
/** Unknown results in a row before the operator is told. 3 × 5 min poller = 15 min. */
|
|
332
|
+
static TOPIC_PROBE_UNKNOWN_ESCALATION = 3;
|
|
333
|
+
/** Reasons that describe the adapter, not one topic — counted once per adapter. */
|
|
334
|
+
static TOPIC_PROBE_ADAPTER_SCOPED_REASONS = new Set([
|
|
335
|
+
"owner-adapter-unavailable",
|
|
336
|
+
"owner-adapter-not-ready",
|
|
337
|
+
"owner-adapter-generation-changed",
|
|
338
|
+
"owner-adapter-changed-before-action",
|
|
339
|
+
"adapter-not-ready",
|
|
340
|
+
"adapter-not-initialized",
|
|
341
|
+
"adapter-generation-changed",
|
|
342
|
+
"topic-close-from-unready-adapter",
|
|
343
|
+
"topic-close-generation-changed",
|
|
344
|
+
// Telegram probe: the transport or Telegram itself is down, not one topic.
|
|
345
|
+
"transport-failed",
|
|
346
|
+
"provider-unavailable",
|
|
347
|
+
]);
|
|
324
348
|
logger = createLogger("info");
|
|
325
349
|
topicCommands;
|
|
326
350
|
// sessionName → instanceName mapping for external sessions
|
|
@@ -1482,8 +1506,27 @@ export class FleetManager {
|
|
|
1482
1506
|
await this.lifecycle.pause(name);
|
|
1483
1507
|
return this.lifecycle.isPaused(name) ? "paused" : "not_idle";
|
|
1484
1508
|
}
|
|
1509
|
+
/** Deliver an already-resolved hot snapshot without depending on IPC timing. */
|
|
1510
|
+
applyHotConfigUpdate(instanceName, update) {
|
|
1511
|
+
const daemon = this.daemons.get(instanceName);
|
|
1512
|
+
if (!daemon)
|
|
1513
|
+
return false;
|
|
1514
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1515
|
+
const sent = ipc?.connected === true && ipc.send({ type: "config_update", config: update });
|
|
1516
|
+
if (!sent) {
|
|
1517
|
+
daemon.applyConfigUpdate(update);
|
|
1518
|
+
this.logger.warn({ name: instanceName }, "Config-update IPC unavailable — applied hot config in-process");
|
|
1519
|
+
}
|
|
1520
|
+
return true;
|
|
1521
|
+
}
|
|
1522
|
+
classicBehaviorUpdate(instanceName) {
|
|
1523
|
+
return {
|
|
1524
|
+
tool_progress: this.classicChannels?.getToolProgressByInstance(instanceName, this.fleetConfig?.defaults?.tool_progress) ?? "off",
|
|
1525
|
+
reply_completion_guard: this.classicChannels?.getReplyCompletionGuardByInstance(instanceName, this.fleetConfig?.defaults?.reply_completion_guard) ?? true,
|
|
1526
|
+
};
|
|
1527
|
+
}
|
|
1485
1528
|
/** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
|
|
1486
|
-
async restartClassicInstanceFromSettings(instanceName) {
|
|
1529
|
+
async restartClassicInstanceFromSettings(instanceName, changedFields = []) {
|
|
1487
1530
|
if (!this.classicChannels)
|
|
1488
1531
|
throw new Error("Classic channel manager not initialized");
|
|
1489
1532
|
const wasRunning = this.daemons.has(instanceName);
|
|
@@ -1495,10 +1538,77 @@ export class FleetManager {
|
|
|
1495
1538
|
throw new Error("Classic channel not found after reload");
|
|
1496
1539
|
if (!wasRunning)
|
|
1497
1540
|
return;
|
|
1541
|
+
const hotOnly = changedFields.length > 0
|
|
1542
|
+
&& changedFields.every(field => field === "tool_progress" || field === "reply_completion_guard");
|
|
1543
|
+
if (hotOnly) {
|
|
1544
|
+
this.applyHotConfigUpdate(instanceName, this.classicBehaviorUpdate(instanceName));
|
|
1545
|
+
this.logger.info({ instanceName, fields: changedFields }, "Classic instance hot config reloaded");
|
|
1546
|
+
return;
|
|
1547
|
+
}
|
|
1498
1548
|
await this.stopInstance(instanceName);
|
|
1499
1549
|
await new Promise(resolve => setTimeout(resolve, 250));
|
|
1500
1550
|
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1501
1551
|
}
|
|
1552
|
+
/** Reload classicBot.yaml once. Kept callable so the periodic production
|
|
1553
|
+
* path is covered without relying on fake timers around startAll(). */
|
|
1554
|
+
async reloadClassicConfigFromDisk() {
|
|
1555
|
+
try {
|
|
1556
|
+
if (!this.classicChannels)
|
|
1557
|
+
return;
|
|
1558
|
+
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
1559
|
+
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
1560
|
+
const oldBackends = new Map();
|
|
1561
|
+
const oldModels = new Map();
|
|
1562
|
+
const oldAutoPause = new Map();
|
|
1563
|
+
const oldToolProgress = new Map();
|
|
1564
|
+
const oldReplyGuard = new Map();
|
|
1565
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
1566
|
+
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
1567
|
+
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
1568
|
+
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1569
|
+
oldToolProgress.set(ch.instanceName, this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress));
|
|
1570
|
+
oldReplyGuard.set(ch.instanceName, this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard));
|
|
1571
|
+
}
|
|
1572
|
+
if (!this.classicChannels.checkReload())
|
|
1573
|
+
return;
|
|
1574
|
+
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
1575
|
+
// throttle keeps a repeated report from flooding the topic.
|
|
1576
|
+
this.reportClassicUnrecoverableIds();
|
|
1577
|
+
this.reregisterClassicChannels();
|
|
1578
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
1579
|
+
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
1580
|
+
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
1581
|
+
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
1582
|
+
const newToolProgress = this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress);
|
|
1583
|
+
const newReplyGuard = this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard);
|
|
1584
|
+
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
1585
|
+
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
1586
|
+
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
1587
|
+
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
1588
|
+
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
1589
|
+
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
1590
|
+
// Small delay to let tmux window clean up
|
|
1591
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
1592
|
+
// The manager already holds the new backend/model/auto-pause; the
|
|
1593
|
+
// unattended helper reads them from it and schedules the delayed
|
|
1594
|
+
// retry on failure like every other unattended start.
|
|
1595
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
1596
|
+
}
|
|
1597
|
+
else if (this.daemons.has(ch.instanceName)
|
|
1598
|
+
&& (oldToolProgress.get(ch.instanceName) !== newToolProgress
|
|
1599
|
+
|| oldReplyGuard.get(ch.instanceName) !== newReplyGuard)) {
|
|
1600
|
+
this.applyHotConfigUpdate(ch.instanceName, {
|
|
1601
|
+
tool_progress: newToolProgress,
|
|
1602
|
+
reply_completion_guard: newReplyGuard,
|
|
1603
|
+
});
|
|
1604
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic instance hot config reloaded");
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
catch (err) {
|
|
1609
|
+
this.logger.warn({ err }, "classicBot.yaml reload error");
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1502
1612
|
async startInstance(name, config, topicMode, kind = "fleet-topic",
|
|
1503
1613
|
/**
|
|
1504
1614
|
* Explicit starts (CLI/API) may resume a paused or failed daemon. Startup
|
|
@@ -2356,48 +2466,8 @@ export class FleetManager {
|
|
|
2356
2466
|
this.instanceWorldBinding.set(ch.instanceName, ch.adapterId);
|
|
2357
2467
|
}
|
|
2358
2468
|
// Poll classicBot.yaml for external changes every 30s
|
|
2359
|
-
this.classicReloadTimer = setInterval(
|
|
2360
|
-
|
|
2361
|
-
if (!this.classicChannels)
|
|
2362
|
-
return;
|
|
2363
|
-
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
2364
|
-
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
2365
|
-
const oldBackends = new Map();
|
|
2366
|
-
const oldModels = new Map();
|
|
2367
|
-
const oldAutoPause = new Map();
|
|
2368
|
-
for (const ch of this.classicChannels.getAll()) {
|
|
2369
|
-
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
2370
|
-
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
2371
|
-
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
2372
|
-
}
|
|
2373
|
-
if (!this.classicChannels.checkReload())
|
|
2374
|
-
return;
|
|
2375
|
-
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
2376
|
-
// throttle keeps a repeated report from flooding the topic.
|
|
2377
|
-
this.reportClassicUnrecoverableIds();
|
|
2378
|
-
this.reregisterClassicChannels();
|
|
2379
|
-
for (const ch of this.classicChannels.getAll()) {
|
|
2380
|
-
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
2381
|
-
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
2382
|
-
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
2383
|
-
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
2384
|
-
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
2385
|
-
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
2386
|
-
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
2387
|
-
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
2388
|
-
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
2389
|
-
// Small delay to let tmux window clean up
|
|
2390
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
2391
|
-
// The manager already holds the new backend/model/auto-pause; the
|
|
2392
|
-
// unattended helper reads them from it and schedules the delayed
|
|
2393
|
-
// retry on failure like every other unattended start.
|
|
2394
|
-
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
2395
|
-
}
|
|
2396
|
-
}
|
|
2397
|
-
}
|
|
2398
|
-
catch (err) {
|
|
2399
|
-
this.logger.warn({ err }, "classicBot.yaml reload error");
|
|
2400
|
-
}
|
|
2469
|
+
this.classicReloadTimer = setInterval(() => {
|
|
2470
|
+
void this.reloadClassicConfigFromDisk();
|
|
2401
2471
|
}, 30_000);
|
|
2402
2472
|
const costGuardConfig = {
|
|
2403
2473
|
...DEFAULT_COST_GUARD,
|
|
@@ -5488,15 +5558,74 @@ export class FleetManager {
|
|
|
5488
5558
|
return false;
|
|
5489
5559
|
return result?.generation === undefined || result.generation === after.generation;
|
|
5490
5560
|
}
|
|
5491
|
-
|
|
5561
|
+
topicProbeStreakKey(threadId, adapterId, reason) {
|
|
5562
|
+
return FleetManager.TOPIC_PROBE_ADAPTER_SCOPED_REASONS.has(reason)
|
|
5563
|
+
? `adapter:${adapterId ?? "unbound"}`
|
|
5564
|
+
: `thread:${threadId}`;
|
|
5565
|
+
}
|
|
5566
|
+
/** A definite answer (present or missing) ends the unknown streak for that route and its adapter. */
|
|
5567
|
+
clearTopicProbeUnknownStreak(threadId, adapterId) {
|
|
5568
|
+
this.topicProbeUnknownStreak.delete(`thread:${threadId}`);
|
|
5569
|
+
this.topicProbeUnknownStreak.delete(`adapter:${adapterId ?? "unbound"}`);
|
|
5570
|
+
}
|
|
5571
|
+
/**
|
|
5572
|
+
* Record one unknown probe result. Nothing here can touch quarantine or
|
|
5573
|
+
* removal: unknown is always retained data. The only question is whether
|
|
5574
|
+
* the operator hears about it, and a single transient (one flaky HTTP call
|
|
5575
|
+
* out of dozens per pass) must not — only a streak does.
|
|
5576
|
+
*
|
|
5577
|
+
* Used directly for single-route events (channelDelete, the pre-action
|
|
5578
|
+
* fence). The periodic scan goes through a TopicProbePass instead, so one
|
|
5579
|
+
* pass over N routes of a dead adapter counts as ONE check, not N.
|
|
5580
|
+
*/
|
|
5581
|
+
warnTopicProbeUnknown(instanceName, threadId, adapterId, reason, detail) {
|
|
5582
|
+
this.noteTopicProbeUnknown(this.topicProbeStreakKey(threadId, adapterId, reason), { instanceName, threadId, adapterId, reason, detail });
|
|
5583
|
+
}
|
|
5584
|
+
/** One scan's worth of probe outcomes, applied to the streaks after the loop. */
|
|
5585
|
+
newTopicProbePass() {
|
|
5586
|
+
return { unknown: new Map(), definite: new Set() };
|
|
5587
|
+
}
|
|
5588
|
+
passTopicProbeUnknown(pass, instanceName, threadId, adapterId, reason, detail) {
|
|
5589
|
+
const key = this.topicProbeStreakKey(threadId, adapterId, reason);
|
|
5590
|
+
// First unknown per key per pass wins; the rest of the routes on a dead
|
|
5591
|
+
// adapter are the same observation, not additional checks.
|
|
5592
|
+
if (!pass.unknown.has(key))
|
|
5593
|
+
pass.unknown.set(key, { instanceName, threadId, adapterId, reason, detail });
|
|
5594
|
+
}
|
|
5595
|
+
passTopicProbeDefinite(pass, threadId, adapterId) {
|
|
5596
|
+
pass.definite.add(`thread:${threadId}`);
|
|
5597
|
+
pass.definite.add(`adapter:${adapterId ?? "unbound"}`);
|
|
5598
|
+
}
|
|
5599
|
+
/**
|
|
5600
|
+
* Apply a pass: a definite answer resets its keys, and an adapter that
|
|
5601
|
+
* answered for any route this pass is evidently alive, so an unknown for the
|
|
5602
|
+
* same adapter key in the same pass does not count — regardless of the order
|
|
5603
|
+
* the routes happened to be probed in.
|
|
5604
|
+
*/
|
|
5605
|
+
applyTopicProbePass(pass) {
|
|
5606
|
+
for (const key of pass.definite)
|
|
5607
|
+
this.topicProbeUnknownStreak.delete(key);
|
|
5608
|
+
for (const [key, ctx] of pass.unknown) {
|
|
5609
|
+
if (pass.definite.has(key))
|
|
5610
|
+
continue;
|
|
5611
|
+
this.noteTopicProbeUnknown(key, ctx);
|
|
5612
|
+
}
|
|
5613
|
+
}
|
|
5614
|
+
noteTopicProbeUnknown(streakKey, { instanceName, threadId, adapterId, reason, detail }) {
|
|
5615
|
+
const streak = (this.topicProbeUnknownStreak.get(streakKey) ?? 0) + 1;
|
|
5616
|
+
this.topicProbeUnknownStreak.set(streakKey, streak);
|
|
5617
|
+
if (streak < FleetManager.TOPIC_PROBE_UNKNOWN_ESCALATION) {
|
|
5618
|
+
this.logger.debug({ instanceName, threadId, adapterId, reason, detail, streak }, "Topic presence not confirmed this pass — transient, retaining instance and all data");
|
|
5619
|
+
return;
|
|
5620
|
+
}
|
|
5492
5621
|
const key = `${adapterId ?? "unbound"}:${reason}`;
|
|
5493
5622
|
const now = Date.now();
|
|
5494
5623
|
const last = this.topicProbeWarnings.get(key) ?? 0;
|
|
5495
5624
|
if (now - last < FleetManager.FLEET_ERROR_THROTTLE_MS)
|
|
5496
5625
|
return;
|
|
5497
5626
|
this.topicProbeWarnings.set(key, now);
|
|
5498
|
-
this.logger.error({ instanceName, threadId, adapterId, reason }, "Topic presence could not be confirmed — retaining instance and all data");
|
|
5499
|
-
this.notifyFleetError(t("fleet.topic_probe_unknown", instanceName, adapterId ?? "unbound"));
|
|
5627
|
+
this.logger.error({ instanceName, threadId, adapterId, reason, detail, streak }, "Topic presence could not be confirmed repeatedly — retaining instance and all data");
|
|
5628
|
+
this.notifyFleetError(t("fleet.topic_probe_unknown", instanceName, adapterId ?? "unbound", streak));
|
|
5500
5629
|
}
|
|
5501
5630
|
/** One fixed-snapshot topology pass. Automatic evidence can only quarantine. */
|
|
5502
5631
|
async runTopicCleanup(generation) {
|
|
@@ -5504,6 +5633,8 @@ export class FleetManager {
|
|
|
5504
5633
|
return;
|
|
5505
5634
|
const snapshot = [...this.routing.entries()].filter(([, target]) => isProbeableRouteTarget(target));
|
|
5506
5635
|
const missing = [];
|
|
5636
|
+
const pass = this.newTopicProbePass();
|
|
5637
|
+
const skippedOnDemand = new Set();
|
|
5507
5638
|
for (const [threadId, target] of snapshot) {
|
|
5508
5639
|
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5509
5640
|
return;
|
|
@@ -5513,12 +5644,22 @@ export class FleetManager {
|
|
|
5513
5644
|
const adapterId = this.getInstanceAdapterId(target.name);
|
|
5514
5645
|
const adapter = adapterId ? this.adapters.get(adapterId) : undefined;
|
|
5515
5646
|
if (!adapterId || !adapter?.probeTopicPresence) {
|
|
5516
|
-
this.
|
|
5647
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-unavailable");
|
|
5648
|
+
continue;
|
|
5649
|
+
}
|
|
5650
|
+
// An on-demand adapter's probe is reserved for confirming a delivery
|
|
5651
|
+
// failure hint (handleProviderTopicClosed); the periodic scan leaves its
|
|
5652
|
+
// routes alone — no probe, no unknown, no streak, no notice.
|
|
5653
|
+
if (adapter.topicProbePolicy?.() === "on-demand") {
|
|
5654
|
+
if (!skippedOnDemand.has(adapterId)) {
|
|
5655
|
+
skippedOnDemand.add(adapterId);
|
|
5656
|
+
this.logger.debug({ adapterId }, "Topic scan skipping on-demand adapter — presence is confirmed only on delivery failure");
|
|
5657
|
+
}
|
|
5517
5658
|
continue;
|
|
5518
5659
|
}
|
|
5519
5660
|
const before = this.confirmedProbeFence(adapterId, adapter);
|
|
5520
5661
|
if (!before) {
|
|
5521
|
-
this.
|
|
5662
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-not-ready");
|
|
5522
5663
|
continue;
|
|
5523
5664
|
}
|
|
5524
5665
|
let result;
|
|
@@ -5531,17 +5672,24 @@ export class FleetManager {
|
|
|
5531
5672
|
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5532
5673
|
return;
|
|
5533
5674
|
if (!this.sameProbeFence(adapterId, adapter, before, result)) {
|
|
5534
|
-
this.
|
|
5675
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-generation-changed");
|
|
5535
5676
|
continue;
|
|
5536
5677
|
}
|
|
5537
5678
|
if (result.status === "unknown") {
|
|
5538
|
-
this.
|
|
5679
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, result.reason, result.detail);
|
|
5539
5680
|
}
|
|
5540
|
-
else
|
|
5541
|
-
|
|
5681
|
+
else {
|
|
5682
|
+
this.passTopicProbeDefinite(pass, threadId, adapterId);
|
|
5683
|
+
if (result.status === "missing") {
|
|
5684
|
+
missing.push({ threadId, target, adapterId, adapter, generation: result.generation });
|
|
5685
|
+
}
|
|
5542
5686
|
}
|
|
5543
5687
|
}
|
|
5544
|
-
if (generation !== this.topicCleanupGeneration || this.shuttingDown
|
|
5688
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5689
|
+
return;
|
|
5690
|
+
// One pass, one check: streaks move by at most one per key here.
|
|
5691
|
+
this.applyTopicProbePass(pass);
|
|
5692
|
+
if (missing.length === 0)
|
|
5545
5693
|
return;
|
|
5546
5694
|
if (missing.length > 1) {
|
|
5547
5695
|
this.logger.error({ missing: missing.map(item => ({ instanceName: item.target.name, threadId: item.threadId, adapterId: item.adapterId })) }, "Multiple topics appeared missing in one pass — treating topology evidence as untrusted and retaining all data");
|
|
@@ -5595,8 +5743,15 @@ export class FleetManager {
|
|
|
5595
5743
|
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-generation-changed");
|
|
5596
5744
|
return;
|
|
5597
5745
|
}
|
|
5746
|
+
if (result.status === "unknown") {
|
|
5747
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, result.reason, result.detail);
|
|
5748
|
+
return;
|
|
5749
|
+
}
|
|
5750
|
+
this.clearTopicProbeUnknownStreak(threadId, adapterId);
|
|
5598
5751
|
if (result.status !== "missing") {
|
|
5599
|
-
|
|
5752
|
+
// The gateway said deleted, REST says present: a definite answer, so it
|
|
5753
|
+
// is not an unknown streak — but it is worth one debug line.
|
|
5754
|
+
this.logger.debug({ instanceName: target.name, threadId, adapterId }, "channelDelete hint contradicted by REST — topic present, nothing to do");
|
|
5600
5755
|
return;
|
|
5601
5756
|
}
|
|
5602
5757
|
this.topicCommands.handleTopicDeleted(threadId, {
|
|
@@ -9999,11 +10154,19 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9999
10154
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
10000
10155
|
ensureWorkspaceGit(workDir);
|
|
10001
10156
|
const classicIdentity = this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName);
|
|
10157
|
+
const toolProgress = classicIdentity
|
|
10158
|
+
? this.classicChannels?.getToolProgress(classicIdentity.channelId, classicIdentity.adapterId, this.fleetConfig?.defaults?.tool_progress)
|
|
10159
|
+
: this.fleetConfig?.defaults?.tool_progress;
|
|
10160
|
+
const replyCompletionGuard = classicIdentity
|
|
10161
|
+
? this.classicChannels?.getReplyCompletionGuard(classicIdentity.channelId, classicIdentity.adapterId, this.fleetConfig?.defaults?.reply_completion_guard)
|
|
10162
|
+
: this.fleetConfig?.defaults?.reply_completion_guard;
|
|
10002
10163
|
const config = {
|
|
10003
10164
|
...DEFAULT_INSTANCE_CONFIG,
|
|
10004
10165
|
...this.fleetConfig?.defaults,
|
|
10005
10166
|
working_directory: workDir,
|
|
10006
10167
|
lightweight: true,
|
|
10168
|
+
tool_progress: toolProgress ?? "off",
|
|
10169
|
+
reply_completion_guard: replyCompletionGuard ?? true,
|
|
10007
10170
|
...(backend ? { backend } : {}),
|
|
10008
10171
|
...(model ? { model } : {}),
|
|
10009
10172
|
...(classicIdentity?.displayName ? { display_name: classicIdentity.displayName } : {}),
|
|
@@ -10396,6 +10559,30 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10396
10559
|
this.notifyFleetError(t("fleet.reload_rejected", why));
|
|
10397
10560
|
return;
|
|
10398
10561
|
}
|
|
10562
|
+
// Classic behavior settings share the fleet defaults but are not entries
|
|
10563
|
+
// in fleet.yaml. Snapshot the old effective chain before reloading the
|
|
10564
|
+
// Classic file so SIGHUP can hot-apply either source without waiting for
|
|
10565
|
+
// the 30-second Classic poller.
|
|
10566
|
+
const oldClassicBehavior = new Map();
|
|
10567
|
+
if (this.classicChannels) {
|
|
10568
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
10569
|
+
const runtimeConfig = this.daemons.get(ch.instanceName)?.getConfigSnapshot?.();
|
|
10570
|
+
oldClassicBehavior.set(ch.instanceName, {
|
|
10571
|
+
backend: this.classicChannels.getBackend(ch.channelId, ch.adapterId, oldConfig?.defaults?.backend),
|
|
10572
|
+
model: this.classicChannels.getModel(ch.channelId, ch.adapterId, oldConfig?.defaults?.model),
|
|
10573
|
+
autoPauseAfter: this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, oldConfig?.defaults?.auto_pause_after),
|
|
10574
|
+
// Settings mutates FleetManager's in-memory defaults before SIGHUP.
|
|
10575
|
+
// The live daemon is therefore the authority for the previous hot
|
|
10576
|
+
// values, exactly as in the fleet-topic reconciliation below.
|
|
10577
|
+
toolProgress: runtimeConfig?.tool_progress
|
|
10578
|
+
?? this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, oldConfig?.defaults?.tool_progress),
|
|
10579
|
+
replyCompletionGuard: runtimeConfig?.reply_completion_guard
|
|
10580
|
+
?? this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, oldConfig?.defaults?.reply_completion_guard),
|
|
10581
|
+
});
|
|
10582
|
+
}
|
|
10583
|
+
if (this.classicChannels.checkReload())
|
|
10584
|
+
this.reportClassicUnrecoverableIds();
|
|
10585
|
+
}
|
|
10399
10586
|
this.routing.rebuild(this.fleetConfig);
|
|
10400
10587
|
this.reregisterClassicChannels();
|
|
10401
10588
|
this.scheduler?.reload();
|
|
@@ -10460,6 +10647,35 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10460
10647
|
}
|
|
10461
10648
|
}
|
|
10462
10649
|
}
|
|
10650
|
+
// A Classic channel inherits fleet defaults beneath its own two levels.
|
|
10651
|
+
// Recompute that complete chain on SIGHUP. Only the two behavior switches
|
|
10652
|
+
// are hot; changes to backend/model/auto-pause retain the existing restart
|
|
10653
|
+
// semantics.
|
|
10654
|
+
if (this.classicChannels) {
|
|
10655
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
10656
|
+
const old = oldClassicBehavior.get(ch.instanceName);
|
|
10657
|
+
if (!old || !this.daemons.has(ch.instanceName))
|
|
10658
|
+
continue;
|
|
10659
|
+
const backend = this.classicChannels.getBackend(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.backend);
|
|
10660
|
+
const model = this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model);
|
|
10661
|
+
const autoPauseAfter = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
10662
|
+
if (old.backend !== backend || old.model !== model || old.autoPauseAfter !== autoPauseAfter) {
|
|
10663
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic cold config changed — restarting");
|
|
10664
|
+
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
10665
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after fleet reload");
|
|
10666
|
+
continue;
|
|
10667
|
+
}
|
|
10668
|
+
const toolProgress = this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress);
|
|
10669
|
+
const replyCompletionGuard = this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard);
|
|
10670
|
+
if (old.toolProgress !== toolProgress || old.replyCompletionGuard !== replyCompletionGuard) {
|
|
10671
|
+
this.applyHotConfigUpdate(ch.instanceName, {
|
|
10672
|
+
tool_progress: toolProgress,
|
|
10673
|
+
reply_completion_guard: replyCompletionGuard,
|
|
10674
|
+
});
|
|
10675
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic inherited hot config reloaded");
|
|
10676
|
+
}
|
|
10677
|
+
}
|
|
10678
|
+
}
|
|
10463
10679
|
// warm_cap is fleet-owned; enforce the reloaded value immediately against
|
|
10464
10680
|
// currently idle instances instead of waiting for a future state edge.
|
|
10465
10681
|
this.enforceWarmCap();
|