@songsid/agend 2.1.0-beta.36 → 2.1.0-beta.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channel/mcp-tools.js +1 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/daemon.d.ts +7 -0
- package/dist/daemon.js +105 -70
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-manager.js +4 -1
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-system-prompt.js +1 -1
- package/dist/outbound-schemas.d.ts +3 -1
- package/dist/outbound-schemas.js +12 -2
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/scheduler/db.d.ts +8 -0
- package/dist/scheduler/db.js +66 -5
- package/dist/scheduler/db.js.map +1 -1
- package/dist/scheduler/db.test.js +50 -0
- package/dist/scheduler/db.test.js.map +1 -1
- package/dist/scheduler/scheduler.d.ts +6 -0
- package/dist/scheduler/scheduler.js +99 -16
- package/dist/scheduler/scheduler.js.map +1 -1
- package/dist/scheduler/scheduler.test.js +131 -1
- package/dist/scheduler/scheduler.test.js.map +1 -1
- package/dist/scheduler/types.d.ts +7 -3
- package/dist/scheduler/types.js.map +1 -1
- package/package.json +1 -1
|
@@ -32,7 +32,7 @@ const DEFS = [
|
|
|
32
32
|
["validate_config", schemas.ValidateConfigArgs,
|
|
33
33
|
"Validate the fleet's config. Reads fleet.yaml + classicBot.yaml and returns { fleet, classic } validation results, each with errors[] and warnings[]."],
|
|
34
34
|
["create_schedule", schemas.CreateScheduleArgs,
|
|
35
|
-
"Create a cron-
|
|
35
|
+
"Create a recurring cron schedule or a one-shot ISO datetime schedule. When triggered, sends a message to the target instance."],
|
|
36
36
|
["list_schedules", schemas.ListSchedulesArgs,
|
|
37
37
|
"List all schedules. Optionally filter by target instance."],
|
|
38
38
|
["update_schedule", schemas.UpdateScheduleArgs,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mcp-tools.js","sourceRoot":"","sources":["../../src/channel/mcp-tools.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,CAAC,EAAgB,MAAM,KAAK,CAAC;AACtC,OAAO,KAAK,OAAO,MAAM,wBAAwB,CAAC;AAWlD,4DAA4D;AAC5D,SAAS,iBAAiB,CAAC,MAAe;IACxC,MAAM,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAA4B,CAAC;IAC9D,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC;IAChD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,IAAwB,CAAC;AAClC,CAAC;AAQD;;;GAGG;AACH,MAAM,IAAI,GAAqC;IAC7C,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS;QACzB,4FAA4F,CAAC;IAC/F,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS;QACzB,6CAA6C,CAAC;IAChD,CAAC,cAAc,EAAE,OAAO,CAAC,eAAe;QACtC,8EAA8E,CAAC;IACjF,CAAC,qBAAqB,EAAE,OAAO,CAAC,sBAAsB;QACpD,+FAA+F,CAAC;IAClG,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,uJAAuJ,CAAC;IAC1J,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,
|
|
1
|
+
{"version":3,"file":"mcp-tools.js","sourceRoot":"","sources":["../../src/channel/mcp-tools.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,CAAC,EAAgB,MAAM,KAAK,CAAC;AACtC,OAAO,KAAK,OAAO,MAAM,wBAAwB,CAAC;AAWlD,4DAA4D;AAC5D,SAAS,iBAAiB,CAAC,MAAe;IACxC,MAAM,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAA4B,CAAC;IAC9D,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC;IAChD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,IAAwB,CAAC;AAClC,CAAC;AAQD;;;GAGG;AACH,MAAM,IAAI,GAAqC;IAC7C,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS;QACzB,4FAA4F,CAAC;IAC/F,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS;QACzB,6CAA6C,CAAC;IAChD,CAAC,cAAc,EAAE,OAAO,CAAC,eAAe;QACtC,8EAA8E,CAAC;IACjF,CAAC,qBAAqB,EAAE,OAAO,CAAC,sBAAsB;QACpD,+FAA+F,CAAC;IAClG,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,uJAAuJ,CAAC;IAC1J,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,+HAA+H,CAAC;IAClI,CAAC,gBAAgB,EAAE,OAAO,CAAC,iBAAiB;QAC1C,2DAA2D,CAAC;IAC9D,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,sEAAsE,CAAC;IACzE,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,0BAA0B,CAAC;IAC7B,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa;QAC5B,iKAAiK,CAAC;IACpK,CAAC,eAAe,EAAE,OAAO,CAAC,gBAAgB;QACxC,yOAAyO,CAAC;IAC5O,CAAC,gBAAgB,EAAE,OAAO,CAAC,iBAAiB;QAC1C,8HAA8H,CAAC;IACjI,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,yCAAyC,CAAC;IAC5C,CAAC,WAAW,EAAE,OAAO,CAAC,aAAa;QACjC,8FAA8F,CAAC;IACjG,CAAC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC7C,2EAA2E,CAAC;IAC9E,CAAC,qBAAqB,EAAE,OAAO,CAAC,sBAAsB;QACpD,sIAAsI,CAAC;IACzI,CAAC,eAAe,EAAE,OAAO,CAAC,gBAAgB;QACxC,sJAAsJ,CAAC;IACzJ,CAAC,eAAe,EAAE,OAAO,CAAC,gBAAgB;QACxC,yIAAyI,CAAC;IAC5I,CAAC,aAAa,EAAE,OAAO,CAAC,cAAc;QACpC,6FAA6F,CAAC;IAChG,CAAC,aAAa,EAAE,OAAO,CAAC,cAAc;QACpC,wDAAwD,CAAC;IAC3D,CAAC,YAAY,EAAE,OAAO,CAAC,aAAa;QAClC,uDAAuD,CAAC;IAC1D,CAAC,aAAa,EAAE,OAAO,CAAC,cAAc;QACpC,sEAAsE,CAAC;IACzE,CAAC,mBAAmB,EAAE,OAAO,CAAC,oBAAoB;QAChD,wHAAwH,CAAC;IAC3H,CAAC,gBAAgB,EAAE,OAAO,CAAC,iBAAiB;QAC1C,qEAAqE,CAAC;IACxE,CAAC,gBAAgB,EAAE,OAAO,CAAC,iBAAiB;QAC1C,mCAAmC,CAAC;IACtC,CAAC,kBAAkB,EAAE,OAAO,CAAC,mBAAmB;QAC9C,6EAA6E,CAAC;IAChF,CAAC,gBAAgB,EAAE,OAAO,CAAC,iBAAiB;QAC1C,qFAAqF,CAAC;IACxF,CAAC,eAAe,EAAE,OAAO,CAAC,gBAAgB;QACxC,qCAAqC,CAAC;IACxC,CAAC,eAAe,EAAE,OAAO,CAAC,gBAAgB;QACxC,kDAAkD,CAAC;IACrD,CAAC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC7C,qEAAqE,CAAC;IACxE,CAAC,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;QAC/C,iEAAiE,CAAC;IACpE,CAAC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC7C,iHAAiH,CAAC;IACpH,CAAC,wBAAwB,EAAE,OAAO,CAAC,wBAAwB;QACzD,yHAAyH,CAAC;IAC5H,CAAC,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;QACvD,qFAAqF,CAAC;IACxF,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,uKAAuK,CAAC;IAC1K,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,0EAA0E,CAAC;IAC7E,CAAC,kBAAkB,EAAE,OAAO,CAAC,mBAAmB;QAC9C,2OAA2O,CAAC;IAC9O,CAAC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC7C,yHAAyH,CAAC;IAC5H,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,oIAAoI,CAAC;IACvI,CAAC,eAAe,EAAE,OAAO,CAAC,gBAAgB;QACxC,yIAAyI,CAAC;IAC5I,CAAC,cAAc,EAAE,OAAO,CAAC,eAAe;QACtC,gDAAgD,CAAC;IACnD,CAAC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;QAC5C,qFAAqF,CAAC;IACxF,CAAC,qBAAqB,EAAE,OAAO,CAAC,sBAAsB;QACpD,6EAA6E,CAAC;IAChF,CAAC,kBAAkB,EAAE,OAAO,CAAC,mBAAmB;QAC9C,mEAAmE,CAAC;CACvE,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAc,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC;IACzE,IAAI;IACJ,WAAW;IACX,WAAW,EAAE,iBAAiB,CAAC,MAAM,CAAC;CACvC,CAAC,CAAC,CAAC;AAEJ,sEAAsE;AACtE,MAAM,CAAC,MAAM,SAAS,GAA6B;IACjD,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5B,QAAQ,EAAE;QACR,OAAO,EAAE,OAAO,EAAE,cAAc;QAChC,kBAAkB,EAAE,WAAW,EAAE,gBAAgB,EAAE,mBAAmB;QACtE,gBAAgB,EAAE,eAAe,EAAE,MAAM,EAAE,kBAAkB,EAAE,iBAAiB;QAChF,iBAAiB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,kBAAkB;KAC/E;IACD,OAAO,EAAE,CAAC,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,qBAAqB,CAAC;CAChF,CAAC"}
|
package/dist/daemon.d.ts
CHANGED
|
@@ -157,6 +157,8 @@ export declare class Daemon extends EventEmitter {
|
|
|
157
157
|
private runtimeMonitorsFrozen;
|
|
158
158
|
private errorWaitingForRecovery;
|
|
159
159
|
private errorDetectedAt;
|
|
160
|
+
private errorRecoveryDeadlineAt;
|
|
161
|
+
private activeErrorPatternKey;
|
|
160
162
|
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
161
163
|
get isErrorState(): boolean;
|
|
162
164
|
get isPaused(): boolean;
|
|
@@ -170,8 +172,11 @@ export declare class Daemon extends EventEmitter {
|
|
|
170
172
|
private static FAILOVER_COOLDOWN_MS;
|
|
171
173
|
private lastErrorNotifiedAt;
|
|
172
174
|
private static ERROR_COOLDOWN_MS;
|
|
175
|
+
private static ERROR_RECOVERY_TIMEOUT_MS;
|
|
173
176
|
private lastErrorCount;
|
|
174
177
|
private lastDetectedErrorType;
|
|
178
|
+
private static errorPatternKey;
|
|
179
|
+
private clearErrorRecoveryGate;
|
|
175
180
|
constructor(name: string, config: InstanceConfig, instanceDir: string, topicMode?: boolean, backend?: CliBackend | undefined, controlClient?: TmuxControlClient | undefined, rootLogger?: Logger);
|
|
176
181
|
start(): Promise<void>;
|
|
177
182
|
private startHealthCheck;
|
|
@@ -191,6 +196,8 @@ export declare class Daemon extends EventEmitter {
|
|
|
191
196
|
* (ready pattern visible), it goes back to monitoring for new errors.
|
|
192
197
|
*/
|
|
193
198
|
private startErrorMonitor;
|
|
199
|
+
/** Evaluate one pane snapshot. Kept synchronous so state-machine edges are unit-testable. */
|
|
200
|
+
private evaluateErrorPatterns;
|
|
194
201
|
/**
|
|
195
202
|
* Interrupt the CLI's current generation (cancel button / `/cancel`).
|
|
196
203
|
* Direct tmux key event (not a paste) so it registers as the interrupt key.
|
package/dist/daemon.js
CHANGED
|
@@ -331,6 +331,8 @@ export class Daemon extends EventEmitter {
|
|
|
331
331
|
runtimeMonitorsFrozen = false;
|
|
332
332
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
333
333
|
errorDetectedAt = 0;
|
|
334
|
+
errorRecoveryDeadlineAt = 0;
|
|
335
|
+
activeErrorPatternKey = null;
|
|
334
336
|
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
335
337
|
get isErrorState() {
|
|
336
338
|
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || Daemon.tmuxServerPaused;
|
|
@@ -348,9 +350,10 @@ export class Daemon extends EventEmitter {
|
|
|
348
350
|
}
|
|
349
351
|
lastFailoverAt = 0; // cooldown: prevent repeated failover triggers
|
|
350
352
|
static FAILOVER_COOLDOWN_MS = 5 * 60_000; // 5 minutes
|
|
351
|
-
lastErrorNotifiedAt = new Map(); // per-
|
|
353
|
+
lastErrorNotifiedAt = new Map(); // per-pattern cooldown for all actions
|
|
352
354
|
static ERROR_COOLDOWN_MS = 5 * 60_000;
|
|
353
|
-
|
|
355
|
+
static ERROR_RECOVERY_TIMEOUT_MS = 5 * 60_000;
|
|
356
|
+
// Count-based dedup: per error pattern, the number of occurrences already
|
|
354
357
|
// accounted for. A scan counts occurrences across the WHOLE pane; count > this
|
|
355
358
|
// baseline means a NEW error appeared. On recovery we absorb the current count
|
|
356
359
|
// (not reset to 0) so the just-handled error doesn't re-trigger, while a later
|
|
@@ -359,6 +362,15 @@ export class Daemon extends EventEmitter {
|
|
|
359
362
|
// still registers as new (prevents the old hash-dedup's permanent suppression).
|
|
360
363
|
lastErrorCount = new Map();
|
|
361
364
|
lastDetectedErrorType = null;
|
|
365
|
+
static errorPatternKey(ep) {
|
|
366
|
+
return `${ep.type}:${ep.pattern.source}`;
|
|
367
|
+
}
|
|
368
|
+
clearErrorRecoveryGate() {
|
|
369
|
+
this.errorWaitingForRecovery = false;
|
|
370
|
+
this.errorDetectedAt = 0;
|
|
371
|
+
this.errorRecoveryDeadlineAt = 0;
|
|
372
|
+
this.activeErrorPatternKey = null;
|
|
373
|
+
}
|
|
362
374
|
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger) {
|
|
363
375
|
super();
|
|
364
376
|
this.name = name;
|
|
@@ -939,6 +951,10 @@ export class Daemon extends EventEmitter {
|
|
|
939
951
|
return;
|
|
940
952
|
this.processStatus = status;
|
|
941
953
|
if (status === "running") {
|
|
954
|
+
// A successful crash respawn is a new pane generation. It is ready by
|
|
955
|
+
// this point, so an error gate inherited from the dead pane must not keep
|
|
956
|
+
// the new process from being monitored.
|
|
957
|
+
this.clearErrorRecoveryGate();
|
|
942
958
|
this.startInstanceStateMonitor();
|
|
943
959
|
}
|
|
944
960
|
else {
|
|
@@ -983,13 +999,6 @@ export class Daemon extends EventEmitter {
|
|
|
983
999
|
if (!alive)
|
|
984
1000
|
return;
|
|
985
1001
|
const pane = await this.tmux.capturePane();
|
|
986
|
-
// Count occurrences of a pattern across the WHOLE pane (not just the text
|
|
987
|
-
// after the last ready prompt — a fast recovery can put "ready" AFTER the
|
|
988
|
-
// error line, which the old scanText approach missed).
|
|
989
|
-
const countMatches = (pattern) => {
|
|
990
|
-
const g = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g");
|
|
991
|
-
return (pane.match(g) || []).length;
|
|
992
|
-
};
|
|
993
1002
|
// Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch)
|
|
994
1003
|
for (const dialog of dialogs) {
|
|
995
1004
|
if (!dialog.pattern.test(pane))
|
|
@@ -1007,73 +1016,96 @@ export class Daemon extends EventEmitter {
|
|
|
1007
1016
|
}
|
|
1008
1017
|
return; // Dialog dismissed, skip error checks this cycle
|
|
1009
1018
|
}
|
|
1010
|
-
|
|
1011
|
-
if (this.errorWaitingForRecovery) {
|
|
1012
|
-
if (readyPattern.test(pane)) {
|
|
1013
|
-
const downtime = Math.round((Date.now() - this.errorDetectedAt) / 1000);
|
|
1014
|
-
// Absorb the current count of the just-handled error type so it isn't
|
|
1015
|
-
// re-triggered; a later NEW occurrence pushes the count higher again.
|
|
1016
|
-
if (this.lastDetectedErrorType) {
|
|
1017
|
-
const ep = patterns.find(p => p.type === this.lastDetectedErrorType);
|
|
1018
|
-
if (ep)
|
|
1019
|
-
this.lastErrorCount.set(ep.type, countMatches(ep.pattern));
|
|
1020
|
-
}
|
|
1021
|
-
this.errorWaitingForRecovery = false;
|
|
1022
|
-
this.errorDetectedAt = 0;
|
|
1023
|
-
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
1024
|
-
this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
|
|
1025
|
-
}
|
|
1026
|
-
return; // Don't check for errors while waiting for recovery
|
|
1027
|
-
}
|
|
1028
|
-
// State: monitoring — count-based new-error detection over the full pane
|
|
1029
|
-
for (const ep of patterns) {
|
|
1030
|
-
const count = countMatches(ep.pattern);
|
|
1031
|
-
const seen = this.lastErrorCount.get(ep.type) ?? 0;
|
|
1032
|
-
if (count <= seen) {
|
|
1033
|
-
// Occurrences scrolled out of the capture buffer → lower the baseline
|
|
1034
|
-
// so a future re-occurrence still counts as new (no permanent suppress).
|
|
1035
|
-
if (count < seen)
|
|
1036
|
-
this.lastErrorCount.set(ep.type, count);
|
|
1037
|
-
continue;
|
|
1038
|
-
}
|
|
1039
|
-
// count > seen → a NEW occurrence of this error appeared.
|
|
1040
|
-
// Cooldown: 2nd-layer guard so the same type isn't re-notified within
|
|
1041
|
-
// the window. Leave the count unconsumed so it fires once cooldown ends.
|
|
1042
|
-
if (!ep.skipCooldown) {
|
|
1043
|
-
const lastNotified = this.lastErrorNotifiedAt.get(ep.type) ?? 0;
|
|
1044
|
-
if (Date.now() - lastNotified < Daemon.ERROR_COOLDOWN_MS) {
|
|
1045
|
-
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (cooldown active)");
|
|
1046
|
-
break;
|
|
1047
|
-
}
|
|
1048
|
-
}
|
|
1049
|
-
if (ep.action === "failover" && Date.now() - this.lastFailoverAt < Daemon.FAILOVER_COOLDOWN_MS) {
|
|
1050
|
-
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (failover cooldown active)");
|
|
1051
|
-
break;
|
|
1052
|
-
}
|
|
1053
|
-
this.lastErrorCount.set(ep.type, count);
|
|
1054
|
-
// skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
|
|
1055
|
-
// back at its prompt immediately). Its ready-pattern only matches the
|
|
1056
|
-
// startup banner, so entering "waiting for recovery" would never clear
|
|
1057
|
-
// and would block ALL future error detection. Just absorb the baseline
|
|
1058
|
-
// (done above) and keep monitoring so the next occurrence still fires.
|
|
1059
|
-
if (!ep.skipRecoveryWait) {
|
|
1060
|
-
this.errorWaitingForRecovery = true;
|
|
1061
|
-
this.errorDetectedAt = Date.now();
|
|
1062
|
-
this.lastDetectedErrorType = ep.type;
|
|
1063
|
-
}
|
|
1064
|
-
this.lastErrorNotifiedAt.set(ep.type, Date.now());
|
|
1065
|
-
if (ep.action === "failover")
|
|
1066
|
-
this.lastFailoverAt = Date.now();
|
|
1067
|
-
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${ep.message}`);
|
|
1068
|
-
this.emit("pty_error", { name: this.name, ...ep });
|
|
1069
|
-
break; // Only handle first new error per scan
|
|
1070
|
-
}
|
|
1019
|
+
this.evaluateErrorPatterns(pane, patterns, readyPattern);
|
|
1071
1020
|
}
|
|
1072
1021
|
catch {
|
|
1073
1022
|
// capturePane can fail if window is transitioning — ignore
|
|
1074
1023
|
}
|
|
1075
1024
|
}, 5_000); // Check every 5 seconds (runtime dialogs need fast response)
|
|
1076
1025
|
}
|
|
1026
|
+
/** Evaluate one pane snapshot. Kept synchronous so state-machine edges are unit-testable. */
|
|
1027
|
+
evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now()) {
|
|
1028
|
+
// Count occurrences across the WHOLE pane (not just text after the last
|
|
1029
|
+
// ready prompt). Clone with `g` so stateful backend regexes cannot leak
|
|
1030
|
+
// lastIndex between monitor cycles.
|
|
1031
|
+
const countMatches = (pattern) => {
|
|
1032
|
+
const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
|
|
1033
|
+
return (pane.match(new RegExp(pattern.source, flags)) || []).length;
|
|
1034
|
+
};
|
|
1035
|
+
// State: waiting for recovery. A missing/outdated ready pattern must not
|
|
1036
|
+
// suppress every future error forever, so the gate has a hard deadline.
|
|
1037
|
+
if (this.errorWaitingForRecovery) {
|
|
1038
|
+
if (readyPattern.test(pane)) {
|
|
1039
|
+
const downtime = Math.round((now - this.errorDetectedAt) / 1000);
|
|
1040
|
+
const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
|
|
1041
|
+
if (active) {
|
|
1042
|
+
this.lastErrorCount.set(Daemon.errorPatternKey(active), countMatches(active.pattern));
|
|
1043
|
+
}
|
|
1044
|
+
this.clearErrorRecoveryGate();
|
|
1045
|
+
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
1046
|
+
this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
|
|
1047
|
+
return;
|
|
1048
|
+
}
|
|
1049
|
+
if (now <= this.errorRecoveryDeadlineAt)
|
|
1050
|
+
return;
|
|
1051
|
+
// Re-arm the active occurrence as a reminder. Without lowering its
|
|
1052
|
+
// baseline, clearing the gate alone would still leave count === seen and
|
|
1053
|
+
// could never satisfy the promised post-timeout re-notification.
|
|
1054
|
+
const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
|
|
1055
|
+
if (active) {
|
|
1056
|
+
const key = Daemon.errorPatternKey(active);
|
|
1057
|
+
const count = countMatches(active.pattern);
|
|
1058
|
+
if (count > 0)
|
|
1059
|
+
this.lastErrorCount.set(key, count - 1);
|
|
1060
|
+
}
|
|
1061
|
+
this.logger.warn({ errorType: this.lastDetectedErrorType }, "PTY error recovery deadline expired — resuming error scan");
|
|
1062
|
+
this.clearErrorRecoveryGate();
|
|
1063
|
+
}
|
|
1064
|
+
// State: monitoring — count-based new-error detection over the full pane.
|
|
1065
|
+
for (const ep of patterns) {
|
|
1066
|
+
const key = Daemon.errorPatternKey(ep);
|
|
1067
|
+
const count = countMatches(ep.pattern);
|
|
1068
|
+
const seen = this.lastErrorCount.get(key) ?? 0;
|
|
1069
|
+
if (count <= seen) {
|
|
1070
|
+
// Occurrences scrolled out of the capture buffer → lower the baseline
|
|
1071
|
+
// so a future re-occurrence still counts as new (no permanent suppress).
|
|
1072
|
+
if (count < seen)
|
|
1073
|
+
this.lastErrorCount.set(key, count);
|
|
1074
|
+
continue;
|
|
1075
|
+
}
|
|
1076
|
+
// count > seen → a NEW occurrence of this exact pattern appeared.
|
|
1077
|
+
// Leave a cooldown-suppressed count unconsumed so it can fire once the
|
|
1078
|
+
// cooldown expires, but continue scanning unrelated patterns this cycle.
|
|
1079
|
+
if (!ep.skipCooldown) {
|
|
1080
|
+
const lastNotified = this.lastErrorNotifiedAt.get(key) ?? 0;
|
|
1081
|
+
if (now - lastNotified < Daemon.ERROR_COOLDOWN_MS) {
|
|
1082
|
+
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (cooldown active)");
|
|
1083
|
+
continue;
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
if (ep.action === "failover" && now - this.lastFailoverAt < Daemon.FAILOVER_COOLDOWN_MS) {
|
|
1087
|
+
this.logger.debug({ errorType: ep.type }, "PTY error suppressed (failover cooldown active)");
|
|
1088
|
+
continue;
|
|
1089
|
+
}
|
|
1090
|
+
this.lastErrorCount.set(key, count);
|
|
1091
|
+
// skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
|
|
1092
|
+
// back at its prompt immediately). Keep monitoring so the next occurrence
|
|
1093
|
+
// can fire without waiting for a possibly startup-only ready pattern.
|
|
1094
|
+
if (!ep.skipRecoveryWait) {
|
|
1095
|
+
this.errorWaitingForRecovery = true;
|
|
1096
|
+
this.errorDetectedAt = now;
|
|
1097
|
+
this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
|
|
1098
|
+
this.activeErrorPatternKey = key;
|
|
1099
|
+
this.lastDetectedErrorType = ep.type;
|
|
1100
|
+
}
|
|
1101
|
+
this.lastErrorNotifiedAt.set(key, now);
|
|
1102
|
+
if (ep.action === "failover")
|
|
1103
|
+
this.lastFailoverAt = now;
|
|
1104
|
+
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${ep.message}`);
|
|
1105
|
+
this.emit("pty_error", { name: this.name, ...ep });
|
|
1106
|
+
break; // Only handle first unsuppressed new error per scan
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1077
1109
|
/**
|
|
1078
1110
|
* Interrupt the CLI's current generation (cancel button / `/cancel`).
|
|
1079
1111
|
* Direct tmux key event (not a paste) so it registers as the interrupt key.
|
|
@@ -1287,6 +1319,9 @@ export class Daemon extends EventEmitter {
|
|
|
1287
1319
|
this.pauseWakeState = "active";
|
|
1288
1320
|
this.healthCheckPaused = false;
|
|
1289
1321
|
this.pauseRequested = false;
|
|
1322
|
+
// trySpawn resolved only after the new CLI reached its ready prompt.
|
|
1323
|
+
// Discard any recovery gate retained while monitors were frozen.
|
|
1324
|
+
this.clearErrorRecoveryGate();
|
|
1290
1325
|
clearPausedMarker(this.instanceDir);
|
|
1291
1326
|
this.transcriptMonitor?.resetOffset();
|
|
1292
1327
|
this.resumeRuntimeMonitors();
|