@hueyexe/opencode-ensemble 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -3
- package/dist/index.js +221 -16
- package/package.json +1 -1
- package/src/hooks.ts +8 -1
- package/src/index.ts +47 -7
- package/src/notify.ts +41 -0
- package/src/recovery.ts +19 -1
- package/src/tools/team-broadcast.ts +4 -1
- package/src/tools/team-cleanup.ts +1 -1
- package/src/tools/team-create.ts +1 -1
- package/src/tools/team-shutdown.ts +14 -2
- package/src/tools/team-spawn.ts +25 -10
- package/src/tools/team-status.ts +23 -0
- package/src/watchdog.ts +92 -0
package/README.md
CHANGED
|
@@ -151,10 +151,11 @@ The plugin works within OpenCode's existing TUI. There's no custom team panel (t
|
|
|
151
151
|
|
|
152
152
|
What you do get:
|
|
153
153
|
|
|
154
|
-
- **Toast notifications** when teammates spawn, finish, error, or
|
|
154
|
+
- **Toast notifications** when teammates spawn, finish, error, shut down, or get rate-limited
|
|
155
|
+
- **Working progress toasts** showing who's still active after every status change (e.g. "Working: alice, bob (2/3)")
|
|
155
156
|
- **Rich tool titles** in the sidebar (e.g. "Spawned alice (build)", "Message -> bob", "Task board (3 tasks)")
|
|
156
157
|
- **Session switching** via `team_view` to see any teammate's full chat log
|
|
157
|
-
- **Status checks** via `team_status` for a snapshot of the whole team
|
|
158
|
+
- **Status checks** via `team_status` for a snapshot of the whole team (also fires a toast)
|
|
158
159
|
|
|
159
160
|
Teammate messages arrive in the lead's session as `[Team message from alice]: ...` blocks. They look like user messages in the TUI because that's how `promptAsync` delivery works. The content is clearly labeled with the sender's name.
|
|
160
161
|
|
|
@@ -163,7 +164,10 @@ Teammate messages arrive in the lead's session as `[Team message from alice]: ..
|
|
|
163
164
|
- **SQLite** (`bun:sqlite`, WAL mode) for teams, members, tasks, and messages
|
|
164
165
|
- **promptAsync** for message delivery: injects a message and starts the prompt loop in one call
|
|
165
166
|
- **Sub-agent isolation**: teammates' sub-agents can't use team tools (parent chain tracking, max depth 10)
|
|
166
|
-
- **Crash recovery**: stale busy members marked as errored on restart, undelivered messages redelivered
|
|
167
|
+
- **Crash recovery**: stale busy members marked as errored on restart, orphaned sessions aborted, undelivered messages redelivered
|
|
168
|
+
- **Spawn rollback**: if the initial prompt fails, the member is cleaned up and the session aborted
|
|
169
|
+
- **Timeout watchdog**: teammates stuck busy beyond the TTL are automatically timed out and aborted
|
|
170
|
+
- **Shutdown resilience**: abort failures handled gracefully, busy-after-shutdown triggers re-abort
|
|
167
171
|
- **Rate limiting**: token bucket (configurable via `OPENCODE_ENSEMBLE_RATE_LIMIT`, default 10 tokens/sec)
|
|
168
172
|
|
|
169
173
|
## Configuration
|
|
@@ -174,6 +178,12 @@ OPENCODE_ENSEMBLE_RATE_LIMIT=20
|
|
|
174
178
|
|
|
175
179
|
# Disable rate limiting
|
|
176
180
|
OPENCODE_ENSEMBLE_RATE_LIMIT=0
|
|
181
|
+
|
|
182
|
+
# Adjust teammate timeout (default: 1800000ms = 30 minutes)
|
|
183
|
+
OPENCODE_ENSEMBLE_TIMEOUT=3600000
|
|
184
|
+
|
|
185
|
+
# Disable timeout watchdog
|
|
186
|
+
OPENCODE_ENSEMBLE_TIMEOUT=0
|
|
177
187
|
```
|
|
178
188
|
|
|
179
189
|
## Development
|
package/dist/index.js
CHANGED
|
@@ -12476,17 +12476,59 @@ function broadcastMessage(db, input) {
|
|
|
12476
12476
|
db.run("INSERT INTO team_message (id, team_id, from_name, to_name, content, delivered, time_created) VALUES (?, ?, ?, NULL, ?, 0, ?)", [id, input.teamId, input.from, input.content, Date.now()]);
|
|
12477
12477
|
return id;
|
|
12478
12478
|
}
|
|
12479
|
+
function getUndeliveredMessages(db, teamId) {
|
|
12480
|
+
return db.query("SELECT * FROM team_message WHERE team_id = ? AND delivered = 0 ORDER BY time_created ASC").all(teamId);
|
|
12481
|
+
}
|
|
12479
12482
|
function markDelivered(db, messageId) {
|
|
12480
12483
|
db.run("UPDATE team_message SET delivered = 1 WHERE id = ?", [messageId]);
|
|
12481
12484
|
}
|
|
12482
12485
|
|
|
12483
12486
|
// src/recovery.ts
|
|
12484
|
-
function recoverStaleMembers(db) {
|
|
12487
|
+
async function recoverStaleMembers(db, client) {
|
|
12488
|
+
const stale = db.query(`SELECT tm.session_id FROM team_member tm
|
|
12489
|
+
JOIN team t ON tm.team_id = t.id
|
|
12490
|
+
WHERE tm.status = 'busy' AND t.status = 'active'`).all();
|
|
12485
12491
|
const result = db.run(`UPDATE team_member SET status = 'error', execution_status = 'idle', time_updated = ?
|
|
12486
12492
|
WHERE status = 'busy'
|
|
12487
12493
|
AND team_id IN (SELECT id FROM team WHERE status = 'active')`, [Date.now()]);
|
|
12494
|
+
if (client) {
|
|
12495
|
+
for (const member of stale) {
|
|
12496
|
+
try {
|
|
12497
|
+
await client.session.abort({ path: { id: member.session_id } });
|
|
12498
|
+
} catch {}
|
|
12499
|
+
}
|
|
12500
|
+
}
|
|
12488
12501
|
return { interrupted: result.changes };
|
|
12489
12502
|
}
|
|
12503
|
+
async function recoverUndeliveredMessages(db, client, registry2) {
|
|
12504
|
+
const teams = db.query("SELECT id, lead_session_id FROM team WHERE status = 'active'").all();
|
|
12505
|
+
let redelivered = 0;
|
|
12506
|
+
for (const team of teams) {
|
|
12507
|
+
const messages = getUndeliveredMessages(db, team.id);
|
|
12508
|
+
for (const msg of messages) {
|
|
12509
|
+
let recipientSessionId;
|
|
12510
|
+
if (msg.to_name === "lead") {
|
|
12511
|
+
recipientSessionId = team.lead_session_id;
|
|
12512
|
+
} else if (msg.to_name) {
|
|
12513
|
+
const entry = registry2.getByName(team.id, msg.to_name);
|
|
12514
|
+
recipientSessionId = entry?.sessionId;
|
|
12515
|
+
} else {
|
|
12516
|
+
continue;
|
|
12517
|
+
}
|
|
12518
|
+
if (!recipientSessionId)
|
|
12519
|
+
continue;
|
|
12520
|
+
try {
|
|
12521
|
+
await client.session.promptAsync({
|
|
12522
|
+
path: { id: recipientSessionId },
|
|
12523
|
+
body: { parts: [{ type: "text", text: `[Recovered team message from ${msg.from_name}]: ${msg.content}` }] }
|
|
12524
|
+
});
|
|
12525
|
+
markDelivered(db, msg.id);
|
|
12526
|
+
redelivered++;
|
|
12527
|
+
} catch {}
|
|
12528
|
+
}
|
|
12529
|
+
}
|
|
12530
|
+
return { redelivered };
|
|
12531
|
+
}
|
|
12490
12532
|
|
|
12491
12533
|
// src/state.ts
|
|
12492
12534
|
class MemberRegistry {
|
|
@@ -12578,6 +12620,11 @@ function handleSessionStatusEvent(db, registry2, sessionId, status) {
|
|
|
12578
12620
|
db.run("UPDATE team_member SET status = 'busy', execution_status = 'running', time_updated = ? WHERE team_id = ? AND name = ?", [Date.now(), entry.teamId, entry.memberName]);
|
|
12579
12621
|
return { memberName: entry.memberName, teamId: entry.teamId, from: member.status, to: "busy" };
|
|
12580
12622
|
}
|
|
12623
|
+
if (member.status === "shutdown_requested") {
|
|
12624
|
+
return { memberName: entry.memberName, teamId: entry.teamId, from: "shutdown_requested", to: "busy_while_shutdown" };
|
|
12625
|
+
}
|
|
12626
|
+
} else if (status === "retry") {
|
|
12627
|
+
return { memberName: entry.memberName, teamId: entry.teamId, from: member.status, to: "retry" };
|
|
12581
12628
|
}
|
|
12582
12629
|
return;
|
|
12583
12630
|
}
|
|
@@ -12633,6 +12680,30 @@ async function notifyTeamEvent(client, type, data) {
|
|
|
12633
12680
|
});
|
|
12634
12681
|
} catch {}
|
|
12635
12682
|
}
|
|
12683
|
+
async function notifyWorkingProgress(client, db, teamId) {
|
|
12684
|
+
const members = db.query("SELECT name, status FROM team_member WHERE team_id = ? ORDER BY time_created ASC").all(teamId);
|
|
12685
|
+
if (members.length === 0)
|
|
12686
|
+
return;
|
|
12687
|
+
const busy = members.filter((m) => m.status === "busy");
|
|
12688
|
+
try {
|
|
12689
|
+
if (busy.length === 0) {
|
|
12690
|
+
await client.tui.showToast({
|
|
12691
|
+
title: "Team",
|
|
12692
|
+
message: "All teammates finished",
|
|
12693
|
+
variant: "success",
|
|
12694
|
+
duration: 4000
|
|
12695
|
+
});
|
|
12696
|
+
} else {
|
|
12697
|
+
const names = busy.map((m) => m.name).join(", ");
|
|
12698
|
+
await client.tui.showToast({
|
|
12699
|
+
title: "Team",
|
|
12700
|
+
message: `Working: ${names} (${busy.length}/${members.length})`,
|
|
12701
|
+
variant: "info",
|
|
12702
|
+
duration: 5000
|
|
12703
|
+
});
|
|
12704
|
+
}
|
|
12705
|
+
} catch {}
|
|
12706
|
+
}
|
|
12636
12707
|
|
|
12637
12708
|
// src/types.ts
|
|
12638
12709
|
function findTeamBySession(db, registry2, sessionId) {
|
|
@@ -12670,7 +12741,7 @@ async function executeTeamCreate(deps, args, sessionId) {
|
|
|
12670
12741
|
const id = generateId("team");
|
|
12671
12742
|
const now = Date.now();
|
|
12672
12743
|
deps.db.run("INSERT INTO team (id, name, lead_session_id, status, delegate, time_created, time_updated) VALUES (?, ?, ?, 'active', 0, ?, ?)", [id, args.name, sessionId, now, now]);
|
|
12673
|
-
return `Team "${args.name}" created (ID: ${id}). You are the lead. Use team_spawn to add teammates.`;
|
|
12744
|
+
return `Team "${args.name}" created (ID: ${id}). You are the lead. Use team_spawn to add teammates. Teammates will message you when done \u2014 do not poll team_status repeatedly.`;
|
|
12674
12745
|
}
|
|
12675
12746
|
|
|
12676
12747
|
// src/tools/team-spawn.ts
|
|
@@ -12709,8 +12780,9 @@ async function executeTeamSpawn(deps, args, sessionId) {
|
|
|
12709
12780
|
"- team_claim: claim a pending task from the shared board",
|
|
12710
12781
|
"",
|
|
12711
12782
|
"When you finish your task:",
|
|
12712
|
-
"1.
|
|
12713
|
-
"2.
|
|
12783
|
+
"1. If you claimed a task, mark it complete using team_tasks_complete.",
|
|
12784
|
+
"2. Send ONE message to the lead with your findings using team_message.",
|
|
12785
|
+
"3. STOP. Do not send follow-up confirmations, status updates, or 'standing by' messages.",
|
|
12714
12786
|
"",
|
|
12715
12787
|
"If you are blocked, send ONE message to the lead describing the specific blocker.",
|
|
12716
12788
|
"",
|
|
@@ -12718,13 +12790,26 @@ async function executeTeamSpawn(deps, args, sessionId) {
|
|
|
12718
12790
|
"",
|
|
12719
12791
|
"Your task:",
|
|
12720
12792
|
args.prompt
|
|
12721
|
-
]
|
|
12793
|
+
];
|
|
12794
|
+
if (args.claim_task) {
|
|
12795
|
+
context.push("", `You have been assigned task ${args.claim_task}. Mark it complete when done.`);
|
|
12796
|
+
}
|
|
12797
|
+
const contextStr = context.join(`
|
|
12722
12798
|
`);
|
|
12723
|
-
|
|
12724
|
-
|
|
12725
|
-
|
|
12726
|
-
|
|
12727
|
-
|
|
12799
|
+
try {
|
|
12800
|
+
await deps.client.session.promptAsync({
|
|
12801
|
+
path: { id: childSessionId },
|
|
12802
|
+
body: { parts: [{ type: "text", text: contextStr }] }
|
|
12803
|
+
});
|
|
12804
|
+
} catch (err) {
|
|
12805
|
+
deps.db.run("DELETE FROM team_member WHERE team_id = ? AND name = ?", [teamInfo.teamId, args.name]);
|
|
12806
|
+
deps.registry.unregister(childSessionId);
|
|
12807
|
+
try {
|
|
12808
|
+
await deps.client.session.abort({ path: { id: childSessionId } });
|
|
12809
|
+
} catch {}
|
|
12810
|
+
throw new Error(`Failed to send initial prompt to teammate "${args.name}": ${err instanceof Error ? err.message : String(err)}`);
|
|
12811
|
+
}
|
|
12812
|
+
return `Teammate "${args.name}" spawned (session: ${childSessionId}, agent: ${args.agent}). Working on: ${args.prompt.slice(0, 100)}${args.prompt.length > 100 ? "..." : ""} Teammates will message you when done. Do not poll team_status to check.`;
|
|
12728
12813
|
}
|
|
12729
12814
|
|
|
12730
12815
|
// src/tools/team-message.ts
|
|
@@ -12783,7 +12868,9 @@ async function executeTeamBroadcast(deps, args, sessionId) {
|
|
|
12783
12868
|
delivered++;
|
|
12784
12869
|
} catch {}
|
|
12785
12870
|
}
|
|
12786
|
-
|
|
12871
|
+
if (delivered > 0) {
|
|
12872
|
+
markDelivered(deps.db, msgId);
|
|
12873
|
+
}
|
|
12787
12874
|
return `Broadcast sent to ${delivered} recipient${delivered !== 1 ? "s" : ""}.`;
|
|
12788
12875
|
}
|
|
12789
12876
|
|
|
@@ -12800,7 +12887,18 @@ async function executeTeamShutdown(deps, args, sessionId) {
|
|
|
12800
12887
|
if (member.status === "shutdown")
|
|
12801
12888
|
throw new Error(`Teammate "${args.member}" is already shut down`);
|
|
12802
12889
|
deps.db.run("UPDATE team_member SET status = 'shutdown_requested', time_updated = ? WHERE team_id = ? AND name = ?", [Date.now(), teamInfo.teamId, args.member]);
|
|
12803
|
-
|
|
12890
|
+
try {
|
|
12891
|
+
await deps.client.session.abort({ path: { id: member.session_id } });
|
|
12892
|
+
} catch {
|
|
12893
|
+
try {
|
|
12894
|
+
await deps.client.tui.showToast({
|
|
12895
|
+
title: "Team",
|
|
12896
|
+
message: `Failed to abort ${args.member} session \u2014 will rely on event hook`,
|
|
12897
|
+
variant: "warning",
|
|
12898
|
+
duration: 4000
|
|
12899
|
+
});
|
|
12900
|
+
} catch {}
|
|
12901
|
+
}
|
|
12804
12902
|
try {
|
|
12805
12903
|
const statuses = await deps.client.session.status();
|
|
12806
12904
|
const sessionStatus = statuses.data?.[member.session_id];
|
|
@@ -12820,7 +12918,7 @@ async function executeTeamCleanup(deps, args, sessionId) {
|
|
|
12820
12918
|
if (teamInfo.role !== "lead")
|
|
12821
12919
|
throw new Error("Only the team lead can clean up the team.");
|
|
12822
12920
|
const members = deps.db.query("SELECT name, session_id, status FROM team_member WHERE team_id = ?").all(teamInfo.teamId);
|
|
12823
|
-
const active = members.filter((m) => m.status !== "shutdown");
|
|
12921
|
+
const active = members.filter((m) => m.status !== "shutdown" && m.status !== "shutdown_requested" && m.status !== "error");
|
|
12824
12922
|
if (active.length > 0 && !args.force) {
|
|
12825
12923
|
const names = active.map((m) => m.name).join(", ");
|
|
12826
12924
|
throw new Error(`Cannot clean up team "${teamInfo.teamName}": ${active.length} member(s) still active: ${names}. Use team_shutdown on each member first, or call team_cleanup with force: true to abort them immediately.`);
|
|
@@ -12982,6 +13080,25 @@ async function executeTeamStatus(deps, sessionId) {
|
|
|
12982
13080
|
lines.push("");
|
|
12983
13081
|
lines.push(`Tasks: ${tasks.length} total (${parts.join(", ")})`);
|
|
12984
13082
|
}
|
|
13083
|
+
if (members.length > 0) {
|
|
13084
|
+
const memberParts = members.map((m) => {
|
|
13085
|
+
const label = m.status === "busy" ? "working" : m.status === "ready" ? "idle" : m.status;
|
|
13086
|
+
return `${m.name} [${label}]`;
|
|
13087
|
+
});
|
|
13088
|
+
let toastMsg = memberParts.join(", ");
|
|
13089
|
+
if (tasks.length > 0) {
|
|
13090
|
+
const completed = tasks.filter((t) => t.status === "completed").length;
|
|
13091
|
+
toastMsg += ` | Tasks: ${completed}/${tasks.length} done`;
|
|
13092
|
+
}
|
|
13093
|
+
try {
|
|
13094
|
+
await deps.client.tui.showToast({
|
|
13095
|
+
title: "Team",
|
|
13096
|
+
message: toastMsg,
|
|
13097
|
+
variant: "info",
|
|
13098
|
+
duration: 4000
|
|
13099
|
+
});
|
|
13100
|
+
} catch {}
|
|
13101
|
+
}
|
|
12985
13102
|
return lines.join(`
|
|
12986
13103
|
`);
|
|
12987
13104
|
}
|
|
@@ -13061,25 +13178,86 @@ class TokenBucket {
|
|
|
13061
13178
|
}
|
|
13062
13179
|
}
|
|
13063
13180
|
|
|
13181
|
+
// src/watchdog.ts
|
|
13182
|
+
class Watchdog {
|
|
13183
|
+
db;
|
|
13184
|
+
client;
|
|
13185
|
+
registry;
|
|
13186
|
+
ttlMs;
|
|
13187
|
+
checkIntervalMs;
|
|
13188
|
+
timer;
|
|
13189
|
+
constructor(opts) {
|
|
13190
|
+
this.db = opts.db;
|
|
13191
|
+
this.client = opts.client;
|
|
13192
|
+
this.registry = opts.registry;
|
|
13193
|
+
this.ttlMs = opts.ttlMs;
|
|
13194
|
+
this.checkIntervalMs = opts.checkIntervalMs ?? 60000;
|
|
13195
|
+
}
|
|
13196
|
+
async check() {
|
|
13197
|
+
if (this.ttlMs === 0)
|
|
13198
|
+
return;
|
|
13199
|
+
const cutoff = Date.now() - this.ttlMs;
|
|
13200
|
+
const stale = this.db.query(`SELECT tm.team_id, tm.name, tm.session_id
|
|
13201
|
+
FROM team_member tm
|
|
13202
|
+
JOIN team t ON tm.team_id = t.id
|
|
13203
|
+
WHERE t.status = 'active'
|
|
13204
|
+
AND tm.status = 'busy'
|
|
13205
|
+
AND tm.time_updated < ?`).all(cutoff);
|
|
13206
|
+
for (const member of stale) {
|
|
13207
|
+
this.db.run("UPDATE team_member SET status = 'error', execution_status = 'timed_out', time_updated = ? WHERE team_id = ? AND name = ?", [Date.now(), member.team_id, member.name]);
|
|
13208
|
+
try {
|
|
13209
|
+
await this.client.session.abort({ path: { id: member.session_id } });
|
|
13210
|
+
} catch {}
|
|
13211
|
+
try {
|
|
13212
|
+
await this.client.tui.showToast({
|
|
13213
|
+
title: "Team",
|
|
13214
|
+
message: `${member.name} timed out`,
|
|
13215
|
+
variant: "warning",
|
|
13216
|
+
duration: 5000
|
|
13217
|
+
});
|
|
13218
|
+
} catch {}
|
|
13219
|
+
}
|
|
13220
|
+
}
|
|
13221
|
+
start() {
|
|
13222
|
+
if (this.timer)
|
|
13223
|
+
return;
|
|
13224
|
+
if (this.ttlMs === 0)
|
|
13225
|
+
return;
|
|
13226
|
+
this.timer = setInterval(() => this.check(), this.checkIntervalMs);
|
|
13227
|
+
}
|
|
13228
|
+
stop() {
|
|
13229
|
+
if (this.timer) {
|
|
13230
|
+
clearInterval(this.timer);
|
|
13231
|
+
this.timer = undefined;
|
|
13232
|
+
}
|
|
13233
|
+
}
|
|
13234
|
+
isRunning() {
|
|
13235
|
+
return this.timer !== undefined;
|
|
13236
|
+
}
|
|
13237
|
+
}
|
|
13238
|
+
|
|
13064
13239
|
// src/index.ts
|
|
13065
13240
|
var DEFAULT_RATE_LIMIT_CAPACITY = 10;
|
|
13066
13241
|
var DEFAULT_RATE_LIMIT_REFILL = 2;
|
|
13067
13242
|
var DEFAULT_RATE_LIMIT_INTERVAL_MS = 1000;
|
|
13243
|
+
var DEFAULT_WATCHDOG_TTL_MS = 30 * 60 * 1000;
|
|
13244
|
+
var DEFAULT_WATCHDOG_CHECK_MS = 60 * 1000;
|
|
13068
13245
|
var plugin = async (input) => {
|
|
13069
13246
|
const dbPath = getDbPath();
|
|
13070
13247
|
mkdirSync(path2.dirname(dbPath), { recursive: true });
|
|
13071
13248
|
const db = createDb(dbPath);
|
|
13072
13249
|
const registry2 = new MemberRegistry;
|
|
13073
13250
|
const tracker = new DescendantTracker;
|
|
13074
|
-
const
|
|
13251
|
+
const client = input.client;
|
|
13252
|
+
const deps = { db, registry: registry2, tracker, client };
|
|
13253
|
+
const recovery = await recoverStaleMembers(db, client);
|
|
13075
13254
|
if (recovery.interrupted > 0) {
|
|
13076
13255
|
const members = db.query("SELECT tm.team_id, tm.name, tm.session_id FROM team_member tm JOIN team t ON tm.team_id = t.id WHERE t.status = 'active'").all();
|
|
13077
13256
|
for (const m of members) {
|
|
13078
13257
|
registry2.register(m.team_id, m.name, m.session_id);
|
|
13079
13258
|
}
|
|
13080
13259
|
}
|
|
13081
|
-
|
|
13082
|
-
const deps = { db, registry: registry2, tracker, client };
|
|
13260
|
+
recoverUndeliveredMessages(db, client, registry2).catch(() => {});
|
|
13083
13261
|
const rateLimitEnv = process.env.OPENCODE_ENSEMBLE_RATE_LIMIT;
|
|
13084
13262
|
const rateLimitCapacity = rateLimitEnv === "0" ? 0 : parseInt(rateLimitEnv ?? "", 10) || DEFAULT_RATE_LIMIT_CAPACITY;
|
|
13085
13263
|
const rateLimiter = new TokenBucket({
|
|
@@ -13087,6 +13265,16 @@ var plugin = async (input) => {
|
|
|
13087
13265
|
refillRate: DEFAULT_RATE_LIMIT_REFILL,
|
|
13088
13266
|
refillIntervalMs: DEFAULT_RATE_LIMIT_INTERVAL_MS
|
|
13089
13267
|
});
|
|
13268
|
+
const timeoutEnv = process.env.OPENCODE_ENSEMBLE_TIMEOUT;
|
|
13269
|
+
const watchdogTtl = timeoutEnv === "0" ? 0 : parseInt(timeoutEnv ?? "", 10) || DEFAULT_WATCHDOG_TTL_MS;
|
|
13270
|
+
const watchdog = new Watchdog({
|
|
13271
|
+
db,
|
|
13272
|
+
client,
|
|
13273
|
+
registry: registry2,
|
|
13274
|
+
ttlMs: watchdogTtl,
|
|
13275
|
+
checkIntervalMs: DEFAULT_WATCHDOG_CHECK_MS
|
|
13276
|
+
});
|
|
13277
|
+
watchdog.start();
|
|
13090
13278
|
return {
|
|
13091
13279
|
async event({ event }) {
|
|
13092
13280
|
if (event.type === "session.status") {
|
|
@@ -13100,7 +13288,24 @@ var plugin = async (input) => {
|
|
|
13100
13288
|
notifyTeamEvent(client, "completed", { memberName: transition.memberName });
|
|
13101
13289
|
} else if (transition.to === "error") {
|
|
13102
13290
|
notifyTeamEvent(client, "error", { memberName: transition.memberName });
|
|
13291
|
+
} else if (transition.to === "retry") {
|
|
13292
|
+
try {
|
|
13293
|
+
await client.tui.showToast({
|
|
13294
|
+
title: "Team",
|
|
13295
|
+
message: `${transition.memberName} is being rate-limited`,
|
|
13296
|
+
variant: "warning",
|
|
13297
|
+
duration: 3000
|
|
13298
|
+
});
|
|
13299
|
+
} catch {}
|
|
13300
|
+
} else if (transition.to === "busy_while_shutdown") {
|
|
13301
|
+
const entry = registry2.getBySession(sessionID);
|
|
13302
|
+
if (entry) {
|
|
13303
|
+
try {
|
|
13304
|
+
await client.session.abort({ path: { id: sessionID } });
|
|
13305
|
+
} catch {}
|
|
13306
|
+
}
|
|
13103
13307
|
}
|
|
13308
|
+
await notifyWorkingProgress(client, db, transition.teamId);
|
|
13104
13309
|
}
|
|
13105
13310
|
}
|
|
13106
13311
|
if (event.type === "session.created") {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hueyexe/opencode-ensemble",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Agent teams for OpenCode — parallel agents with peer-to-peer communication, shared tasks, and coordinated execution",
|
|
5
5
|
"module": "src/index.ts",
|
|
6
6
|
"main": "dist/index.js",
|
package/src/hooks.ts
CHANGED
|
@@ -50,6 +50,13 @@ export function handleSessionStatusEvent(
|
|
|
50
50
|
)
|
|
51
51
|
return { memberName: entry.memberName, teamId: entry.teamId, from: member.status, to: "busy" }
|
|
52
52
|
}
|
|
53
|
+
// Session went busy while shutdown was requested — signal for re-abort
|
|
54
|
+
if (member.status === "shutdown_requested") {
|
|
55
|
+
return { memberName: entry.memberName, teamId: entry.teamId, from: "shutdown_requested", to: "busy_while_shutdown" }
|
|
56
|
+
}
|
|
57
|
+
} else if (status === "retry") {
|
|
58
|
+
// Session is being rate-limited — signal for toast but don't change state
|
|
59
|
+
return { memberName: entry.memberName, teamId: entry.teamId, from: member.status, to: "retry" }
|
|
53
60
|
}
|
|
54
61
|
return undefined
|
|
55
62
|
}
|
|
@@ -71,7 +78,7 @@ export function handleSessionCreatedEvent(
|
|
|
71
78
|
/**
|
|
72
79
|
* Check whether a tool call should be blocked for sub-agent isolation.
|
|
73
80
|
* Throws if the tool is a team tool and the session is a descendant of a team member.
|
|
74
|
-
* OQ-11:
|
|
81
|
+
* OQ-11: confirmed — throwing inside tool.execute.before fails the tool call gracefully (verified in live testing).
|
|
75
82
|
*/
|
|
76
83
|
export function checkToolIsolation(
|
|
77
84
|
registry: MemberRegistry,
|
package/src/index.ts
CHANGED
|
@@ -3,10 +3,10 @@ import { tool } from "@opencode-ai/plugin"
|
|
|
3
3
|
import path from "path"
|
|
4
4
|
import { mkdirSync } from "node:fs"
|
|
5
5
|
import { createDb, getDbPath } from "./db"
|
|
6
|
-
import { recoverStaleMembers } from "./recovery"
|
|
6
|
+
import { recoverStaleMembers, recoverUndeliveredMessages } from "./recovery"
|
|
7
7
|
import { MemberRegistry, DescendantTracker } from "./state"
|
|
8
8
|
import { handleSessionStatusEvent, handleSessionCreatedEvent, checkToolIsolation } from "./hooks"
|
|
9
|
-
import { notifyTeamEvent } from "./notify"
|
|
9
|
+
import { notifyTeamEvent, notifyWorkingProgress } from "./notify"
|
|
10
10
|
import { executeTeamCreate } from "./tools/team-create"
|
|
11
11
|
import { executeTeamSpawn } from "./tools/team-spawn"
|
|
12
12
|
import { executeTeamMessage } from "./tools/team-message"
|
|
@@ -22,10 +22,13 @@ import { executeTeamStatus } from "./tools/team-status"
|
|
|
22
22
|
import { executeTeamView } from "./tools/team-view"
|
|
23
23
|
import type { ToolDeps, PluginClient } from "./types"
|
|
24
24
|
import { TokenBucket } from "./rate-limit"
|
|
25
|
+
import { Watchdog } from "./watchdog"
|
|
25
26
|
|
|
26
27
|
const DEFAULT_RATE_LIMIT_CAPACITY = 10
|
|
27
28
|
const DEFAULT_RATE_LIMIT_REFILL = 2
|
|
28
29
|
const DEFAULT_RATE_LIMIT_INTERVAL_MS = 1000
|
|
30
|
+
const DEFAULT_WATCHDOG_TTL_MS = 30 * 60 * 1000 // 30 minutes
|
|
31
|
+
const DEFAULT_WATCHDOG_CHECK_MS = 60 * 1000 // 60 seconds
|
|
29
32
|
|
|
30
33
|
/**
|
|
31
34
|
* opencode-ensemble plugin entry point.
|
|
@@ -42,8 +45,12 @@ const plugin: Plugin = async (input) => {
|
|
|
42
45
|
const registry = new MemberRegistry()
|
|
43
46
|
const tracker = new DescendantTracker()
|
|
44
47
|
|
|
45
|
-
//
|
|
46
|
-
const
|
|
48
|
+
// Build shared tool dependencies
|
|
49
|
+
const client = input.client as unknown as PluginClient
|
|
50
|
+
const deps: ToolDeps = { db, registry, tracker, client }
|
|
51
|
+
|
|
52
|
+
// Run recovery on init — mark stale busy members as error + abort orphaned sessions
|
|
53
|
+
const recovery = await recoverStaleMembers(db, client)
|
|
47
54
|
if (recovery.interrupted > 0) {
|
|
48
55
|
// Rebuild registry from DB for recovered members
|
|
49
56
|
const members = db.query(
|
|
@@ -54,9 +61,10 @@ const plugin: Plugin = async (input) => {
|
|
|
54
61
|
}
|
|
55
62
|
}
|
|
56
63
|
|
|
57
|
-
//
|
|
58
|
-
|
|
59
|
-
|
|
64
|
+
// Redeliver undelivered messages from previous sessions
|
|
65
|
+
recoverUndeliveredMessages(db, client, registry).catch(() => {
|
|
66
|
+
// Best effort — don't block plugin init
|
|
67
|
+
})
|
|
60
68
|
|
|
61
69
|
// Initialize rate limiter
|
|
62
70
|
// OPENCODE_ENSEMBLE_RATE_LIMIT=0 disables it entirely
|
|
@@ -68,6 +76,17 @@ const plugin: Plugin = async (input) => {
|
|
|
68
76
|
refillIntervalMs: DEFAULT_RATE_LIMIT_INTERVAL_MS,
|
|
69
77
|
})
|
|
70
78
|
|
|
79
|
+
// Initialize watchdog for teammate timeouts
|
|
80
|
+
// OPENCODE_ENSEMBLE_TIMEOUT=0 disables it entirely
|
|
81
|
+
const timeoutEnv = process.env.OPENCODE_ENSEMBLE_TIMEOUT
|
|
82
|
+
const watchdogTtl = timeoutEnv === "0" ? 0 : (parseInt(timeoutEnv ?? "", 10) || DEFAULT_WATCHDOG_TTL_MS)
|
|
83
|
+
const watchdog = new Watchdog({
|
|
84
|
+
db, client, registry,
|
|
85
|
+
ttlMs: watchdogTtl,
|
|
86
|
+
checkIntervalMs: DEFAULT_WATCHDOG_CHECK_MS,
|
|
87
|
+
})
|
|
88
|
+
watchdog.start()
|
|
89
|
+
|
|
71
90
|
return {
|
|
72
91
|
// Event hook — drives state machine transitions + descendant tracking + toasts
|
|
73
92
|
async event({ event }) {
|
|
@@ -84,7 +103,28 @@ const plugin: Plugin = async (input) => {
|
|
|
84
103
|
notifyTeamEvent(client, "completed", { memberName: transition.memberName })
|
|
85
104
|
} else if (transition.to === "error") {
|
|
86
105
|
notifyTeamEvent(client, "error", { memberName: transition.memberName })
|
|
106
|
+
} else if (transition.to === "retry") {
|
|
107
|
+
// Teammate is being rate-limited — notify user
|
|
108
|
+
try {
|
|
109
|
+
await client.tui.showToast({
|
|
110
|
+
title: "Team",
|
|
111
|
+
message: `${transition.memberName} is being rate-limited`,
|
|
112
|
+
variant: "warning",
|
|
113
|
+
duration: 3000,
|
|
114
|
+
})
|
|
115
|
+
} catch { /* TUI may not be available */ }
|
|
116
|
+
} else if (transition.to === "busy_while_shutdown") {
|
|
117
|
+
// Session went busy after shutdown was requested — re-issue abort
|
|
118
|
+
const entry = registry.getBySession(sessionID)
|
|
119
|
+
if (entry) {
|
|
120
|
+
try {
|
|
121
|
+
await client.session.abort({ path: { id: sessionID } })
|
|
122
|
+
} catch { /* best effort */ }
|
|
123
|
+
}
|
|
87
124
|
}
|
|
125
|
+
|
|
126
|
+
// Show working progress after every transition so the user sees who's still active
|
|
127
|
+
await notifyWorkingProgress(client, db, transition.teamId)
|
|
88
128
|
}
|
|
89
129
|
}
|
|
90
130
|
|
package/src/notify.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite"
|
|
1
2
|
import type { PluginClient } from "./types"
|
|
2
3
|
|
|
3
4
|
type TeamEventType = "spawn" | "message" | "completed" | "error" | "shutdown"
|
|
@@ -52,3 +53,43 @@ export async function notifyTeamEvent(
|
|
|
52
53
|
// TUI may not be available — silently ignore
|
|
53
54
|
}
|
|
54
55
|
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Fire a toast showing which teammates are still working.
|
|
59
|
+
* Called on every session.status transition so the user sees live progress.
|
|
60
|
+
* Does nothing if the team has no members.
|
|
61
|
+
*/
|
|
62
|
+
export async function notifyWorkingProgress(
|
|
63
|
+
client: PluginClient,
|
|
64
|
+
db: Database,
|
|
65
|
+
teamId: string,
|
|
66
|
+
): Promise<void> {
|
|
67
|
+
const members = db.query(
|
|
68
|
+
"SELECT name, status FROM team_member WHERE team_id = ? ORDER BY time_created ASC"
|
|
69
|
+
).all(teamId) as Array<{ name: string; status: string }>
|
|
70
|
+
|
|
71
|
+
if (members.length === 0) return
|
|
72
|
+
|
|
73
|
+
const busy = members.filter(m => m.status === "busy")
|
|
74
|
+
|
|
75
|
+
try {
|
|
76
|
+
if (busy.length === 0) {
|
|
77
|
+
await client.tui.showToast({
|
|
78
|
+
title: "Team",
|
|
79
|
+
message: "All teammates finished",
|
|
80
|
+
variant: "success",
|
|
81
|
+
duration: 4000,
|
|
82
|
+
})
|
|
83
|
+
} else {
|
|
84
|
+
const names = busy.map(m => m.name).join(", ")
|
|
85
|
+
await client.tui.showToast({
|
|
86
|
+
title: "Team",
|
|
87
|
+
message: `Working: ${names} (${busy.length}/${members.length})`,
|
|
88
|
+
variant: "info",
|
|
89
|
+
duration: 5000,
|
|
90
|
+
})
|
|
91
|
+
}
|
|
92
|
+
} catch {
|
|
93
|
+
// TUI may not be available — silently ignore
|
|
94
|
+
}
|
|
95
|
+
}
|
package/src/recovery.ts
CHANGED
|
@@ -6,16 +6,34 @@ import { getUndeliveredMessages, markDelivered } from "./messaging"
|
|
|
6
6
|
/**
|
|
7
7
|
* Scan for team members stuck in 'busy' status (stale from a crash)
|
|
8
8
|
* and mark them as 'error' with execution_status 'idle'.
|
|
9
|
+
* Also aborts their orphaned sessions (best effort).
|
|
9
10
|
* Only processes members in active teams.
|
|
10
11
|
* Returns the count of interrupted members.
|
|
11
12
|
*/
|
|
12
|
-
export function recoverStaleMembers(db: Database): { interrupted: number } {
|
|
13
|
+
export async function recoverStaleMembers(db: Database, client?: PluginClient): Promise<{ interrupted: number }> {
|
|
14
|
+
// Find stale members before updating so we can abort their sessions
|
|
15
|
+
const stale = db.query(
|
|
16
|
+
`SELECT tm.session_id FROM team_member tm
|
|
17
|
+
JOIN team t ON tm.team_id = t.id
|
|
18
|
+
WHERE tm.status = 'busy' AND t.status = 'active'`
|
|
19
|
+
).all() as Array<{ session_id: string }>
|
|
20
|
+
|
|
13
21
|
const result = db.run(
|
|
14
22
|
`UPDATE team_member SET status = 'error', execution_status = 'idle', time_updated = ?
|
|
15
23
|
WHERE status = 'busy'
|
|
16
24
|
AND team_id IN (SELECT id FROM team WHERE status = 'active')`,
|
|
17
25
|
[Date.now()]
|
|
18
26
|
)
|
|
27
|
+
|
|
28
|
+
// Abort orphaned sessions (best effort)
|
|
29
|
+
if (client) {
|
|
30
|
+
for (const member of stale) {
|
|
31
|
+
try {
|
|
32
|
+
await client.session.abort({ path: { id: member.session_id } })
|
|
33
|
+
} catch { /* best effort */ }
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
19
37
|
return { interrupted: result.changes }
|
|
20
38
|
}
|
|
21
39
|
|
|
@@ -53,7 +53,10 @@ export async function executeTeamBroadcast(
|
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
// Only mark delivered if at least one recipient received it
|
|
57
|
+
if (delivered > 0) {
|
|
58
|
+
markDelivered(deps.db, msgId)
|
|
59
|
+
}
|
|
57
60
|
|
|
58
61
|
return `Broadcast sent to ${delivered} recipient${delivered !== 1 ? "s" : ""}.`
|
|
59
62
|
}
|
|
@@ -17,7 +17,7 @@ export async function executeTeamCleanup(
|
|
|
17
17
|
const members = deps.db.query("SELECT name, session_id, status FROM team_member WHERE team_id = ?")
|
|
18
18
|
.all(teamInfo.teamId) as Array<{ name: string; session_id: string; status: string }>
|
|
19
19
|
|
|
20
|
-
const active = members.filter(m => m.status !== "shutdown")
|
|
20
|
+
const active = members.filter(m => m.status !== "shutdown" && m.status !== "shutdown_requested" && m.status !== "error")
|
|
21
21
|
|
|
22
22
|
if (active.length > 0 && !args.force) {
|
|
23
23
|
const names = active.map(m => m.name).join(", ")
|
package/src/tools/team-create.ts
CHANGED
|
@@ -28,5 +28,5 @@ export async function executeTeamCreate(
|
|
|
28
28
|
[id, args.name, sessionId, now, now]
|
|
29
29
|
)
|
|
30
30
|
|
|
31
|
-
return `Team "${args.name}" created (ID: ${id}). You are the lead. Use team_spawn to add teammates.`
|
|
31
|
+
return `Team "${args.name}" created (ID: ${id}). You are the lead. Use team_spawn to add teammates. Teammates will message you when done — do not poll team_status repeatedly.`
|
|
32
32
|
}
|
|
@@ -27,8 +27,20 @@ export async function executeTeamShutdown(
|
|
|
27
27
|
[Date.now(), teamInfo.teamId, args.member]
|
|
28
28
|
)
|
|
29
29
|
|
|
30
|
-
// Call abort on the member's session
|
|
31
|
-
|
|
30
|
+
// Call abort on the member's session (best effort)
|
|
31
|
+
try {
|
|
32
|
+
await deps.client.session.abort({ path: { id: member.session_id } })
|
|
33
|
+
} catch {
|
|
34
|
+
// Abort failed — session may already be gone. Fire a warning toast.
|
|
35
|
+
try {
|
|
36
|
+
await deps.client.tui.showToast({
|
|
37
|
+
title: "Team",
|
|
38
|
+
message: `Failed to abort ${args.member} session — will rely on event hook`,
|
|
39
|
+
variant: "warning",
|
|
40
|
+
duration: 4000,
|
|
41
|
+
})
|
|
42
|
+
} catch { /* TUI may not be available */ }
|
|
43
|
+
}
|
|
32
44
|
|
|
33
45
|
// Fallback: poll session status after abort. If already idle, transition
|
|
34
46
|
// directly to shutdown. This handles the case where abort() on an
|
package/src/tools/team-spawn.ts
CHANGED
|
@@ -54,8 +54,9 @@ export async function executeTeamSpawn(
|
|
|
54
54
|
"- team_claim: claim a pending task from the shared board",
|
|
55
55
|
"",
|
|
56
56
|
"When you finish your task:",
|
|
57
|
-
"1.
|
|
58
|
-
"2.
|
|
57
|
+
"1. If you claimed a task, mark it complete using team_tasks_complete.",
|
|
58
|
+
"2. Send ONE message to the lead with your findings using team_message.",
|
|
59
|
+
"3. STOP. Do not send follow-up confirmations, status updates, or 'standing by' messages.",
|
|
59
60
|
"",
|
|
60
61
|
"If you are blocked, send ONE message to the lead describing the specific blocker.",
|
|
61
62
|
"",
|
|
@@ -63,15 +64,29 @@ export async function executeTeamSpawn(
|
|
|
63
64
|
"",
|
|
64
65
|
"Your task:",
|
|
65
66
|
args.prompt,
|
|
66
|
-
]
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
if (args.claim_task) {
|
|
70
|
+
context.push("", `You have been assigned task ${args.claim_task}. Mark it complete when done.`)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const contextStr = context.join("\n")
|
|
67
74
|
|
|
68
75
|
// Fire-and-forget: send prompt to teammate session
|
|
69
|
-
// OQ-1:
|
|
70
|
-
// OQ-10:
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
76
|
+
// OQ-1: confirmed — promptAsync queues when session is busy (verified in live testing)
|
|
77
|
+
// OQ-10: confirmed — fresh session accepts promptAsync without session.init() (verified in live testing)
|
|
78
|
+
try {
|
|
79
|
+
await deps.client.session.promptAsync({
|
|
80
|
+
path: { id: childSessionId },
|
|
81
|
+
body: { parts: [{ type: "text", text: contextStr }] },
|
|
82
|
+
})
|
|
83
|
+
} catch (err) {
|
|
84
|
+
// Rollback: clean up DB, registry, and abort the orphaned session
|
|
85
|
+
deps.db.run("DELETE FROM team_member WHERE team_id = ? AND name = ?", [teamInfo.teamId, args.name])
|
|
86
|
+
deps.registry.unregister(childSessionId)
|
|
87
|
+
try { await deps.client.session.abort({ path: { id: childSessionId } }) } catch { /* best effort */ }
|
|
88
|
+
throw new Error(`Failed to send initial prompt to teammate "${args.name}": ${err instanceof Error ? err.message : String(err)}`)
|
|
89
|
+
}
|
|
75
90
|
|
|
76
|
-
return `Teammate "${args.name}" spawned (session: ${childSessionId}, agent: ${args.agent}). Working on: ${args.prompt.slice(0, 100)}${args.prompt.length > 100 ? "..." : ""}
|
|
91
|
+
return `Teammate "${args.name}" spawned (session: ${childSessionId}, agent: ${args.agent}). Working on: ${args.prompt.slice(0, 100)}${args.prompt.length > 100 ? "..." : ""} Teammates will message you when done. Do not poll team_status to check.`
|
|
77
92
|
}
|
package/src/tools/team-status.ts
CHANGED
|
@@ -45,5 +45,28 @@ export async function executeTeamStatus(
|
|
|
45
45
|
lines.push(`Tasks: ${tasks.length} total (${parts.join(", ")})`)
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
// Fire a toast so the user (not just the model) sees the status summary
|
|
49
|
+
if (members.length > 0) {
|
|
50
|
+
const memberParts = members.map(m => {
|
|
51
|
+
const label = m.status === "busy" ? "working" : m.status === "ready" ? "idle" : m.status
|
|
52
|
+
return `${m.name} [${label}]`
|
|
53
|
+
})
|
|
54
|
+
let toastMsg = memberParts.join(", ")
|
|
55
|
+
if (tasks.length > 0) {
|
|
56
|
+
const completed = tasks.filter(t => t.status === "completed").length
|
|
57
|
+
toastMsg += ` | Tasks: ${completed}/${tasks.length} done`
|
|
58
|
+
}
|
|
59
|
+
try {
|
|
60
|
+
await deps.client.tui.showToast({
|
|
61
|
+
title: "Team",
|
|
62
|
+
message: toastMsg,
|
|
63
|
+
variant: "info",
|
|
64
|
+
duration: 4000,
|
|
65
|
+
})
|
|
66
|
+
} catch {
|
|
67
|
+
// TUI may not be available — silently ignore
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
48
71
|
return lines.join("\n")
|
|
49
72
|
}
|
package/src/watchdog.ts
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite"
|
|
2
|
+
import type { PluginClient } from "./types"
|
|
3
|
+
import type { MemberRegistry } from "./state"
|
|
4
|
+
|
|
5
|
+
interface WatchdogOpts {
|
|
6
|
+
db: Database
|
|
7
|
+
client: PluginClient
|
|
8
|
+
registry: MemberRegistry
|
|
9
|
+
/** Maximum time a member can stay busy before being timed out. 0 disables. */
|
|
10
|
+
ttlMs: number
|
|
11
|
+
/** How often to run the check. Defaults to 60s. */
|
|
12
|
+
checkIntervalMs?: number
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Periodic watchdog that times out teammates stuck in busy state.
|
|
17
|
+
* Transitions them to error/timed_out, aborts their session, and fires a toast.
|
|
18
|
+
*/
|
|
19
|
+
export class Watchdog {
|
|
20
|
+
private readonly db: Database
|
|
21
|
+
private readonly client: PluginClient
|
|
22
|
+
private readonly registry: MemberRegistry
|
|
23
|
+
private readonly ttlMs: number
|
|
24
|
+
private readonly checkIntervalMs: number
|
|
25
|
+
private timer: ReturnType<typeof setInterval> | undefined
|
|
26
|
+
|
|
27
|
+
constructor(opts: WatchdogOpts) {
|
|
28
|
+
this.db = opts.db
|
|
29
|
+
this.client = opts.client
|
|
30
|
+
this.registry = opts.registry
|
|
31
|
+
this.ttlMs = opts.ttlMs
|
|
32
|
+
this.checkIntervalMs = opts.checkIntervalMs ?? 60_000
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Run a single check for stale busy members. */
|
|
36
|
+
async check(): Promise<void> {
|
|
37
|
+
if (this.ttlMs === 0) return
|
|
38
|
+
|
|
39
|
+
const cutoff = Date.now() - this.ttlMs
|
|
40
|
+
const stale = this.db.query(
|
|
41
|
+
`SELECT tm.team_id, tm.name, tm.session_id
|
|
42
|
+
FROM team_member tm
|
|
43
|
+
JOIN team t ON tm.team_id = t.id
|
|
44
|
+
WHERE t.status = 'active'
|
|
45
|
+
AND tm.status = 'busy'
|
|
46
|
+
AND tm.time_updated < ?`
|
|
47
|
+
).all(cutoff) as Array<{ team_id: string; name: string; session_id: string }>
|
|
48
|
+
|
|
49
|
+
for (const member of stale) {
|
|
50
|
+
// Mark as timed out
|
|
51
|
+
this.db.run(
|
|
52
|
+
"UPDATE team_member SET status = 'error', execution_status = 'timed_out', time_updated = ? WHERE team_id = ? AND name = ?",
|
|
53
|
+
[Date.now(), member.team_id, member.name]
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
// Abort session (best effort)
|
|
57
|
+
try {
|
|
58
|
+
await this.client.session.abort({ path: { id: member.session_id } })
|
|
59
|
+
} catch { /* best effort */ }
|
|
60
|
+
|
|
61
|
+
// Notify
|
|
62
|
+
try {
|
|
63
|
+
await this.client.tui.showToast({
|
|
64
|
+
title: "Team",
|
|
65
|
+
message: `${member.name} timed out`,
|
|
66
|
+
variant: "warning",
|
|
67
|
+
duration: 5000,
|
|
68
|
+
})
|
|
69
|
+
} catch { /* TUI may not be available */ }
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Start the periodic check. */
|
|
74
|
+
start(): void {
|
|
75
|
+
if (this.timer) return
|
|
76
|
+
if (this.ttlMs === 0) return
|
|
77
|
+
this.timer = setInterval(() => this.check(), this.checkIntervalMs)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Stop the periodic check. */
|
|
81
|
+
stop(): void {
|
|
82
|
+
if (this.timer) {
|
|
83
|
+
clearInterval(this.timer)
|
|
84
|
+
this.timer = undefined
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Whether the watchdog is currently running. */
|
|
89
|
+
isRunning(): boolean {
|
|
90
|
+
return this.timer !== undefined
|
|
91
|
+
}
|
|
92
|
+
}
|