@hueyexe/opencode-ensemble 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -3
- package/dist/index.js +166 -10
- package/package.json +1 -1
- package/src/hooks.ts +8 -1
- package/src/index.ts +43 -6
- package/src/recovery.ts +19 -1
- package/src/tools/team-broadcast.ts +4 -1
- package/src/tools/team-shutdown.ts +14 -2
- package/src/tools/team-spawn.ts +14 -6
- package/src/watchdog.ts +92 -0
package/README.md
CHANGED
|
@@ -151,10 +151,11 @@ The plugin works within OpenCode's existing TUI. There's no custom team panel (t
|
|
|
151
151
|
|
|
152
152
|
What you do get:
|
|
153
153
|
|
|
154
|
-
- **Toast notifications** when teammates spawn, finish, error, or
|
|
154
|
+
- **Toast notifications** when teammates spawn, finish, error, shut down, or get rate-limited
|
|
155
|
+
- **Working progress toasts** showing who's still active after every status change (e.g. "Working: alice, bob (2/3)")
|
|
155
156
|
- **Rich tool titles** in the sidebar (e.g. "Spawned alice (build)", "Message -> bob", "Task board (3 tasks)")
|
|
156
157
|
- **Session switching** via `team_view` to see any teammate's full chat log
|
|
157
|
-
- **Status checks** via `team_status` for a snapshot of the whole team
|
|
158
|
+
- **Status checks** via `team_status` for a snapshot of the whole team (also fires a toast)
|
|
158
159
|
|
|
159
160
|
Teammate messages arrive in the lead's session as `[Team message from alice]: ...` blocks. They look like user messages in the TUI because that's how `promptAsync` delivery works. The content is clearly labeled with the sender's name.
|
|
160
161
|
|
|
@@ -163,7 +164,10 @@ Teammate messages arrive in the lead's session as `[Team message from alice]: ..
|
|
|
163
164
|
- **SQLite** (`bun:sqlite`, WAL mode) for teams, members, tasks, and messages
|
|
164
165
|
- **promptAsync** for message delivery: injects a message and starts the prompt loop in one call
|
|
165
166
|
- **Sub-agent isolation**: teammates' sub-agents can't use team tools (parent chain tracking, max depth 10)
|
|
166
|
-
- **Crash recovery**: stale busy members marked as errored on restart, undelivered messages redelivered
|
|
167
|
+
- **Crash recovery**: stale busy members marked as errored on restart, orphaned sessions aborted, undelivered messages redelivered
|
|
168
|
+
- **Spawn rollback**: if the initial prompt fails, the member is cleaned up and the session aborted
|
|
169
|
+
- **Timeout watchdog**: teammates stuck busy beyond the TTL are automatically timed out and aborted
|
|
170
|
+
- **Shutdown resilience**: abort failures handled gracefully, busy-after-shutdown triggers re-abort
|
|
167
171
|
- **Rate limiting**: token bucket (configurable via `OPENCODE_ENSEMBLE_RATE_LIMIT`, default 10 tokens/sec)
|
|
168
172
|
|
|
169
173
|
## Configuration
|
|
@@ -174,6 +178,12 @@ OPENCODE_ENSEMBLE_RATE_LIMIT=20
|
|
|
174
178
|
|
|
175
179
|
# Disable rate limiting
|
|
176
180
|
OPENCODE_ENSEMBLE_RATE_LIMIT=0
|
|
181
|
+
|
|
182
|
+
# Adjust teammate timeout (default: 1800000ms = 30 minutes)
|
|
183
|
+
OPENCODE_ENSEMBLE_TIMEOUT=3600000
|
|
184
|
+
|
|
185
|
+
# Disable timeout watchdog
|
|
186
|
+
OPENCODE_ENSEMBLE_TIMEOUT=0
|
|
177
187
|
```
|
|
178
188
|
|
|
179
189
|
## Development
|
package/dist/index.js
CHANGED
|
@@ -12476,17 +12476,59 @@ function broadcastMessage(db, input) {
|
|
|
12476
12476
|
db.run("INSERT INTO team_message (id, team_id, from_name, to_name, content, delivered, time_created) VALUES (?, ?, ?, NULL, ?, 0, ?)", [id, input.teamId, input.from, input.content, Date.now()]);
|
|
12477
12477
|
return id;
|
|
12478
12478
|
}
|
|
12479
|
+
function getUndeliveredMessages(db, teamId) {
|
|
12480
|
+
return db.query("SELECT * FROM team_message WHERE team_id = ? AND delivered = 0 ORDER BY time_created ASC").all(teamId);
|
|
12481
|
+
}
|
|
12479
12482
|
function markDelivered(db, messageId) {
|
|
12480
12483
|
db.run("UPDATE team_message SET delivered = 1 WHERE id = ?", [messageId]);
|
|
12481
12484
|
}
|
|
12482
12485
|
|
|
12483
12486
|
// src/recovery.ts
|
|
12484
|
-
function recoverStaleMembers(db) {
|
|
12487
|
+
async function recoverStaleMembers(db, client) {
|
|
12488
|
+
const stale = db.query(`SELECT tm.session_id FROM team_member tm
|
|
12489
|
+
JOIN team t ON tm.team_id = t.id
|
|
12490
|
+
WHERE tm.status = 'busy' AND t.status = 'active'`).all();
|
|
12485
12491
|
const result = db.run(`UPDATE team_member SET status = 'error', execution_status = 'idle', time_updated = ?
|
|
12486
12492
|
WHERE status = 'busy'
|
|
12487
12493
|
AND team_id IN (SELECT id FROM team WHERE status = 'active')`, [Date.now()]);
|
|
12494
|
+
if (client) {
|
|
12495
|
+
for (const member of stale) {
|
|
12496
|
+
try {
|
|
12497
|
+
await client.session.abort({ path: { id: member.session_id } });
|
|
12498
|
+
} catch {}
|
|
12499
|
+
}
|
|
12500
|
+
}
|
|
12488
12501
|
return { interrupted: result.changes };
|
|
12489
12502
|
}
|
|
12503
|
+
async function recoverUndeliveredMessages(db, client, registry2) {
|
|
12504
|
+
const teams = db.query("SELECT id, lead_session_id FROM team WHERE status = 'active'").all();
|
|
12505
|
+
let redelivered = 0;
|
|
12506
|
+
for (const team of teams) {
|
|
12507
|
+
const messages = getUndeliveredMessages(db, team.id);
|
|
12508
|
+
for (const msg of messages) {
|
|
12509
|
+
let recipientSessionId;
|
|
12510
|
+
if (msg.to_name === "lead") {
|
|
12511
|
+
recipientSessionId = team.lead_session_id;
|
|
12512
|
+
} else if (msg.to_name) {
|
|
12513
|
+
const entry = registry2.getByName(team.id, msg.to_name);
|
|
12514
|
+
recipientSessionId = entry?.sessionId;
|
|
12515
|
+
} else {
|
|
12516
|
+
continue;
|
|
12517
|
+
}
|
|
12518
|
+
if (!recipientSessionId)
|
|
12519
|
+
continue;
|
|
12520
|
+
try {
|
|
12521
|
+
await client.session.promptAsync({
|
|
12522
|
+
path: { id: recipientSessionId },
|
|
12523
|
+
body: { parts: [{ type: "text", text: `[Recovered team message from ${msg.from_name}]: ${msg.content}` }] }
|
|
12524
|
+
});
|
|
12525
|
+
markDelivered(db, msg.id);
|
|
12526
|
+
redelivered++;
|
|
12527
|
+
} catch {}
|
|
12528
|
+
}
|
|
12529
|
+
}
|
|
12530
|
+
return { redelivered };
|
|
12531
|
+
}
|
|
12490
12532
|
|
|
12491
12533
|
// src/state.ts
|
|
12492
12534
|
class MemberRegistry {
|
|
@@ -12578,6 +12620,11 @@ function handleSessionStatusEvent(db, registry2, sessionId, status) {
|
|
|
12578
12620
|
db.run("UPDATE team_member SET status = 'busy', execution_status = 'running', time_updated = ? WHERE team_id = ? AND name = ?", [Date.now(), entry.teamId, entry.memberName]);
|
|
12579
12621
|
return { memberName: entry.memberName, teamId: entry.teamId, from: member.status, to: "busy" };
|
|
12580
12622
|
}
|
|
12623
|
+
if (member.status === "shutdown_requested") {
|
|
12624
|
+
return { memberName: entry.memberName, teamId: entry.teamId, from: "shutdown_requested", to: "busy_while_shutdown" };
|
|
12625
|
+
}
|
|
12626
|
+
} else if (status === "retry") {
|
|
12627
|
+
return { memberName: entry.memberName, teamId: entry.teamId, from: member.status, to: "retry" };
|
|
12581
12628
|
}
|
|
12582
12629
|
return;
|
|
12583
12630
|
}
|
|
@@ -12749,10 +12796,19 @@ async function executeTeamSpawn(deps, args, sessionId) {
|
|
|
12749
12796
|
}
|
|
12750
12797
|
const contextStr = context.join(`
|
|
12751
12798
|
`);
|
|
12752
|
-
|
|
12753
|
-
|
|
12754
|
-
|
|
12755
|
-
|
|
12799
|
+
try {
|
|
12800
|
+
await deps.client.session.promptAsync({
|
|
12801
|
+
path: { id: childSessionId },
|
|
12802
|
+
body: { parts: [{ type: "text", text: contextStr }] }
|
|
12803
|
+
});
|
|
12804
|
+
} catch (err) {
|
|
12805
|
+
deps.db.run("DELETE FROM team_member WHERE team_id = ? AND name = ?", [teamInfo.teamId, args.name]);
|
|
12806
|
+
deps.registry.unregister(childSessionId);
|
|
12807
|
+
try {
|
|
12808
|
+
await deps.client.session.abort({ path: { id: childSessionId } });
|
|
12809
|
+
} catch {}
|
|
12810
|
+
throw new Error(`Failed to send initial prompt to teammate "${args.name}": ${err instanceof Error ? err.message : String(err)}`);
|
|
12811
|
+
}
|
|
12756
12812
|
return `Teammate "${args.name}" spawned (session: ${childSessionId}, agent: ${args.agent}). Working on: ${args.prompt.slice(0, 100)}${args.prompt.length > 100 ? "..." : ""} Teammates will message you when done. Do not poll team_status to check.`;
|
|
12757
12813
|
}
|
|
12758
12814
|
|
|
@@ -12812,7 +12868,9 @@ async function executeTeamBroadcast(deps, args, sessionId) {
|
|
|
12812
12868
|
delivered++;
|
|
12813
12869
|
} catch {}
|
|
12814
12870
|
}
|
|
12815
|
-
|
|
12871
|
+
if (delivered > 0) {
|
|
12872
|
+
markDelivered(deps.db, msgId);
|
|
12873
|
+
}
|
|
12816
12874
|
return `Broadcast sent to ${delivered} recipient${delivered !== 1 ? "s" : ""}.`;
|
|
12817
12875
|
}
|
|
12818
12876
|
|
|
@@ -12829,7 +12887,18 @@ async function executeTeamShutdown(deps, args, sessionId) {
|
|
|
12829
12887
|
if (member.status === "shutdown")
|
|
12830
12888
|
throw new Error(`Teammate "${args.member}" is already shut down`);
|
|
12831
12889
|
deps.db.run("UPDATE team_member SET status = 'shutdown_requested', time_updated = ? WHERE team_id = ? AND name = ?", [Date.now(), teamInfo.teamId, args.member]);
|
|
12832
|
-
|
|
12890
|
+
try {
|
|
12891
|
+
await deps.client.session.abort({ path: { id: member.session_id } });
|
|
12892
|
+
} catch {
|
|
12893
|
+
try {
|
|
12894
|
+
await deps.client.tui.showToast({
|
|
12895
|
+
title: "Team",
|
|
12896
|
+
message: `Failed to abort ${args.member} session \u2014 will rely on event hook`,
|
|
12897
|
+
variant: "warning",
|
|
12898
|
+
duration: 4000
|
|
12899
|
+
});
|
|
12900
|
+
} catch {}
|
|
12901
|
+
}
|
|
12833
12902
|
try {
|
|
12834
12903
|
const statuses = await deps.client.session.status();
|
|
12835
12904
|
const sessionStatus = statuses.data?.[member.session_id];
|
|
@@ -13109,25 +13178,86 @@ class TokenBucket {
|
|
|
13109
13178
|
}
|
|
13110
13179
|
}
|
|
13111
13180
|
|
|
13181
|
+
// src/watchdog.ts
|
|
13182
|
+
class Watchdog {
|
|
13183
|
+
db;
|
|
13184
|
+
client;
|
|
13185
|
+
registry;
|
|
13186
|
+
ttlMs;
|
|
13187
|
+
checkIntervalMs;
|
|
13188
|
+
timer;
|
|
13189
|
+
constructor(opts) {
|
|
13190
|
+
this.db = opts.db;
|
|
13191
|
+
this.client = opts.client;
|
|
13192
|
+
this.registry = opts.registry;
|
|
13193
|
+
this.ttlMs = opts.ttlMs;
|
|
13194
|
+
this.checkIntervalMs = opts.checkIntervalMs ?? 60000;
|
|
13195
|
+
}
|
|
13196
|
+
async check() {
|
|
13197
|
+
if (this.ttlMs === 0)
|
|
13198
|
+
return;
|
|
13199
|
+
const cutoff = Date.now() - this.ttlMs;
|
|
13200
|
+
const stale = this.db.query(`SELECT tm.team_id, tm.name, tm.session_id
|
|
13201
|
+
FROM team_member tm
|
|
13202
|
+
JOIN team t ON tm.team_id = t.id
|
|
13203
|
+
WHERE t.status = 'active'
|
|
13204
|
+
AND tm.status = 'busy'
|
|
13205
|
+
AND tm.time_updated < ?`).all(cutoff);
|
|
13206
|
+
for (const member of stale) {
|
|
13207
|
+
this.db.run("UPDATE team_member SET status = 'error', execution_status = 'timed_out', time_updated = ? WHERE team_id = ? AND name = ?", [Date.now(), member.team_id, member.name]);
|
|
13208
|
+
try {
|
|
13209
|
+
await this.client.session.abort({ path: { id: member.session_id } });
|
|
13210
|
+
} catch {}
|
|
13211
|
+
try {
|
|
13212
|
+
await this.client.tui.showToast({
|
|
13213
|
+
title: "Team",
|
|
13214
|
+
message: `${member.name} timed out`,
|
|
13215
|
+
variant: "warning",
|
|
13216
|
+
duration: 5000
|
|
13217
|
+
});
|
|
13218
|
+
} catch {}
|
|
13219
|
+
}
|
|
13220
|
+
}
|
|
13221
|
+
start() {
|
|
13222
|
+
if (this.timer)
|
|
13223
|
+
return;
|
|
13224
|
+
if (this.ttlMs === 0)
|
|
13225
|
+
return;
|
|
13226
|
+
this.timer = setInterval(() => this.check(), this.checkIntervalMs);
|
|
13227
|
+
}
|
|
13228
|
+
stop() {
|
|
13229
|
+
if (this.timer) {
|
|
13230
|
+
clearInterval(this.timer);
|
|
13231
|
+
this.timer = undefined;
|
|
13232
|
+
}
|
|
13233
|
+
}
|
|
13234
|
+
isRunning() {
|
|
13235
|
+
return this.timer !== undefined;
|
|
13236
|
+
}
|
|
13237
|
+
}
|
|
13238
|
+
|
|
13112
13239
|
// src/index.ts
|
|
13113
13240
|
var DEFAULT_RATE_LIMIT_CAPACITY = 10;
|
|
13114
13241
|
var DEFAULT_RATE_LIMIT_REFILL = 2;
|
|
13115
13242
|
var DEFAULT_RATE_LIMIT_INTERVAL_MS = 1000;
|
|
13243
|
+
var DEFAULT_WATCHDOG_TTL_MS = 30 * 60 * 1000;
|
|
13244
|
+
var DEFAULT_WATCHDOG_CHECK_MS = 60 * 1000;
|
|
13116
13245
|
var plugin = async (input) => {
|
|
13117
13246
|
const dbPath = getDbPath();
|
|
13118
13247
|
mkdirSync(path2.dirname(dbPath), { recursive: true });
|
|
13119
13248
|
const db = createDb(dbPath);
|
|
13120
13249
|
const registry2 = new MemberRegistry;
|
|
13121
13250
|
const tracker = new DescendantTracker;
|
|
13122
|
-
const
|
|
13251
|
+
const client = input.client;
|
|
13252
|
+
const deps = { db, registry: registry2, tracker, client };
|
|
13253
|
+
const recovery = await recoverStaleMembers(db, client);
|
|
13123
13254
|
if (recovery.interrupted > 0) {
|
|
13124
13255
|
const members = db.query("SELECT tm.team_id, tm.name, tm.session_id FROM team_member tm JOIN team t ON tm.team_id = t.id WHERE t.status = 'active'").all();
|
|
13125
13256
|
for (const m of members) {
|
|
13126
13257
|
registry2.register(m.team_id, m.name, m.session_id);
|
|
13127
13258
|
}
|
|
13128
13259
|
}
|
|
13129
|
-
|
|
13130
|
-
const deps = { db, registry: registry2, tracker, client };
|
|
13260
|
+
recoverUndeliveredMessages(db, client, registry2).catch(() => {});
|
|
13131
13261
|
const rateLimitEnv = process.env.OPENCODE_ENSEMBLE_RATE_LIMIT;
|
|
13132
13262
|
const rateLimitCapacity = rateLimitEnv === "0" ? 0 : parseInt(rateLimitEnv ?? "", 10) || DEFAULT_RATE_LIMIT_CAPACITY;
|
|
13133
13263
|
const rateLimiter = new TokenBucket({
|
|
@@ -13135,6 +13265,16 @@ var plugin = async (input) => {
|
|
|
13135
13265
|
refillRate: DEFAULT_RATE_LIMIT_REFILL,
|
|
13136
13266
|
refillIntervalMs: DEFAULT_RATE_LIMIT_INTERVAL_MS
|
|
13137
13267
|
});
|
|
13268
|
+
const timeoutEnv = process.env.OPENCODE_ENSEMBLE_TIMEOUT;
|
|
13269
|
+
const watchdogTtl = timeoutEnv === "0" ? 0 : parseInt(timeoutEnv ?? "", 10) || DEFAULT_WATCHDOG_TTL_MS;
|
|
13270
|
+
const watchdog = new Watchdog({
|
|
13271
|
+
db,
|
|
13272
|
+
client,
|
|
13273
|
+
registry: registry2,
|
|
13274
|
+
ttlMs: watchdogTtl,
|
|
13275
|
+
checkIntervalMs: DEFAULT_WATCHDOG_CHECK_MS
|
|
13276
|
+
});
|
|
13277
|
+
watchdog.start();
|
|
13138
13278
|
return {
|
|
13139
13279
|
async event({ event }) {
|
|
13140
13280
|
if (event.type === "session.status") {
|
|
@@ -13148,6 +13288,22 @@ var plugin = async (input) => {
|
|
|
13148
13288
|
notifyTeamEvent(client, "completed", { memberName: transition.memberName });
|
|
13149
13289
|
} else if (transition.to === "error") {
|
|
13150
13290
|
notifyTeamEvent(client, "error", { memberName: transition.memberName });
|
|
13291
|
+
} else if (transition.to === "retry") {
|
|
13292
|
+
try {
|
|
13293
|
+
await client.tui.showToast({
|
|
13294
|
+
title: "Team",
|
|
13295
|
+
message: `${transition.memberName} is being rate-limited`,
|
|
13296
|
+
variant: "warning",
|
|
13297
|
+
duration: 3000
|
|
13298
|
+
});
|
|
13299
|
+
} catch {}
|
|
13300
|
+
} else if (transition.to === "busy_while_shutdown") {
|
|
13301
|
+
const entry = registry2.getBySession(sessionID);
|
|
13302
|
+
if (entry) {
|
|
13303
|
+
try {
|
|
13304
|
+
await client.session.abort({ path: { id: sessionID } });
|
|
13305
|
+
} catch {}
|
|
13306
|
+
}
|
|
13151
13307
|
}
|
|
13152
13308
|
await notifyWorkingProgress(client, db, transition.teamId);
|
|
13153
13309
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hueyexe/opencode-ensemble",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Agent teams for OpenCode — parallel agents with peer-to-peer communication, shared tasks, and coordinated execution",
|
|
5
5
|
"module": "src/index.ts",
|
|
6
6
|
"main": "dist/index.js",
|
package/src/hooks.ts
CHANGED
|
@@ -50,6 +50,13 @@ export function handleSessionStatusEvent(
|
|
|
50
50
|
)
|
|
51
51
|
return { memberName: entry.memberName, teamId: entry.teamId, from: member.status, to: "busy" }
|
|
52
52
|
}
|
|
53
|
+
// Session went busy while shutdown was requested — signal for re-abort
|
|
54
|
+
if (member.status === "shutdown_requested") {
|
|
55
|
+
return { memberName: entry.memberName, teamId: entry.teamId, from: "shutdown_requested", to: "busy_while_shutdown" }
|
|
56
|
+
}
|
|
57
|
+
} else if (status === "retry") {
|
|
58
|
+
// Session is being rate-limited — signal for toast but don't change state
|
|
59
|
+
return { memberName: entry.memberName, teamId: entry.teamId, from: member.status, to: "retry" }
|
|
53
60
|
}
|
|
54
61
|
return undefined
|
|
55
62
|
}
|
|
@@ -71,7 +78,7 @@ export function handleSessionCreatedEvent(
|
|
|
71
78
|
/**
|
|
72
79
|
* Check whether a tool call should be blocked for sub-agent isolation.
|
|
73
80
|
* Throws if the tool is a team tool and the session is a descendant of a team member.
|
|
74
|
-
* OQ-11:
|
|
81
|
+
* OQ-11: confirmed — throwing inside tool.execute.before fails the tool call gracefully (verified in live testing).
|
|
75
82
|
*/
|
|
76
83
|
export function checkToolIsolation(
|
|
77
84
|
registry: MemberRegistry,
|
package/src/index.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { tool } from "@opencode-ai/plugin"
|
|
|
3
3
|
import path from "path"
|
|
4
4
|
import { mkdirSync } from "node:fs"
|
|
5
5
|
import { createDb, getDbPath } from "./db"
|
|
6
|
-
import { recoverStaleMembers } from "./recovery"
|
|
6
|
+
import { recoverStaleMembers, recoverUndeliveredMessages } from "./recovery"
|
|
7
7
|
import { MemberRegistry, DescendantTracker } from "./state"
|
|
8
8
|
import { handleSessionStatusEvent, handleSessionCreatedEvent, checkToolIsolation } from "./hooks"
|
|
9
9
|
import { notifyTeamEvent, notifyWorkingProgress } from "./notify"
|
|
@@ -22,10 +22,13 @@ import { executeTeamStatus } from "./tools/team-status"
|
|
|
22
22
|
import { executeTeamView } from "./tools/team-view"
|
|
23
23
|
import type { ToolDeps, PluginClient } from "./types"
|
|
24
24
|
import { TokenBucket } from "./rate-limit"
|
|
25
|
+
import { Watchdog } from "./watchdog"
|
|
25
26
|
|
|
26
27
|
const DEFAULT_RATE_LIMIT_CAPACITY = 10
|
|
27
28
|
const DEFAULT_RATE_LIMIT_REFILL = 2
|
|
28
29
|
const DEFAULT_RATE_LIMIT_INTERVAL_MS = 1000
|
|
30
|
+
const DEFAULT_WATCHDOG_TTL_MS = 30 * 60 * 1000 // 30 minutes
|
|
31
|
+
const DEFAULT_WATCHDOG_CHECK_MS = 60 * 1000 // 60 seconds
|
|
29
32
|
|
|
30
33
|
/**
|
|
31
34
|
* opencode-ensemble plugin entry point.
|
|
@@ -42,8 +45,12 @@ const plugin: Plugin = async (input) => {
|
|
|
42
45
|
const registry = new MemberRegistry()
|
|
43
46
|
const tracker = new DescendantTracker()
|
|
44
47
|
|
|
45
|
-
//
|
|
46
|
-
const
|
|
48
|
+
// Build shared tool dependencies
|
|
49
|
+
const client = input.client as unknown as PluginClient
|
|
50
|
+
const deps: ToolDeps = { db, registry, tracker, client }
|
|
51
|
+
|
|
52
|
+
// Run recovery on init — mark stale busy members as error + abort orphaned sessions
|
|
53
|
+
const recovery = await recoverStaleMembers(db, client)
|
|
47
54
|
if (recovery.interrupted > 0) {
|
|
48
55
|
// Rebuild registry from DB for recovered members
|
|
49
56
|
const members = db.query(
|
|
@@ -54,9 +61,10 @@ const plugin: Plugin = async (input) => {
|
|
|
54
61
|
}
|
|
55
62
|
}
|
|
56
63
|
|
|
57
|
-
//
|
|
58
|
-
|
|
59
|
-
|
|
64
|
+
// Redeliver undelivered messages from previous sessions
|
|
65
|
+
recoverUndeliveredMessages(db, client, registry).catch(() => {
|
|
66
|
+
// Best effort — don't block plugin init
|
|
67
|
+
})
|
|
60
68
|
|
|
61
69
|
// Initialize rate limiter
|
|
62
70
|
// OPENCODE_ENSEMBLE_RATE_LIMIT=0 disables it entirely
|
|
@@ -68,6 +76,17 @@ const plugin: Plugin = async (input) => {
|
|
|
68
76
|
refillIntervalMs: DEFAULT_RATE_LIMIT_INTERVAL_MS,
|
|
69
77
|
})
|
|
70
78
|
|
|
79
|
+
// Initialize watchdog for teammate timeouts
|
|
80
|
+
// OPENCODE_ENSEMBLE_TIMEOUT=0 disables it entirely
|
|
81
|
+
const timeoutEnv = process.env.OPENCODE_ENSEMBLE_TIMEOUT
|
|
82
|
+
const watchdogTtl = timeoutEnv === "0" ? 0 : (parseInt(timeoutEnv ?? "", 10) || DEFAULT_WATCHDOG_TTL_MS)
|
|
83
|
+
const watchdog = new Watchdog({
|
|
84
|
+
db, client, registry,
|
|
85
|
+
ttlMs: watchdogTtl,
|
|
86
|
+
checkIntervalMs: DEFAULT_WATCHDOG_CHECK_MS,
|
|
87
|
+
})
|
|
88
|
+
watchdog.start()
|
|
89
|
+
|
|
71
90
|
return {
|
|
72
91
|
// Event hook — drives state machine transitions + descendant tracking + toasts
|
|
73
92
|
async event({ event }) {
|
|
@@ -84,6 +103,24 @@ const plugin: Plugin = async (input) => {
|
|
|
84
103
|
notifyTeamEvent(client, "completed", { memberName: transition.memberName })
|
|
85
104
|
} else if (transition.to === "error") {
|
|
86
105
|
notifyTeamEvent(client, "error", { memberName: transition.memberName })
|
|
106
|
+
} else if (transition.to === "retry") {
|
|
107
|
+
// Teammate is being rate-limited — notify user
|
|
108
|
+
try {
|
|
109
|
+
await client.tui.showToast({
|
|
110
|
+
title: "Team",
|
|
111
|
+
message: `${transition.memberName} is being rate-limited`,
|
|
112
|
+
variant: "warning",
|
|
113
|
+
duration: 3000,
|
|
114
|
+
})
|
|
115
|
+
} catch { /* TUI may not be available */ }
|
|
116
|
+
} else if (transition.to === "busy_while_shutdown") {
|
|
117
|
+
// Session went busy after shutdown was requested — re-issue abort
|
|
118
|
+
const entry = registry.getBySession(sessionID)
|
|
119
|
+
if (entry) {
|
|
120
|
+
try {
|
|
121
|
+
await client.session.abort({ path: { id: sessionID } })
|
|
122
|
+
} catch { /* best effort */ }
|
|
123
|
+
}
|
|
87
124
|
}
|
|
88
125
|
|
|
89
126
|
// Show working progress after every transition so the user sees who's still active
|
package/src/recovery.ts
CHANGED
|
@@ -6,16 +6,34 @@ import { getUndeliveredMessages, markDelivered } from "./messaging"
|
|
|
6
6
|
/**
|
|
7
7
|
* Scan for team members stuck in 'busy' status (stale from a crash)
|
|
8
8
|
* and mark them as 'error' with execution_status 'idle'.
|
|
9
|
+
* Also aborts their orphaned sessions (best effort).
|
|
9
10
|
* Only processes members in active teams.
|
|
10
11
|
* Returns the count of interrupted members.
|
|
11
12
|
*/
|
|
12
|
-
export function recoverStaleMembers(db: Database): { interrupted: number } {
|
|
13
|
+
export async function recoverStaleMembers(db: Database, client?: PluginClient): Promise<{ interrupted: number }> {
|
|
14
|
+
// Find stale members before updating so we can abort their sessions
|
|
15
|
+
const stale = db.query(
|
|
16
|
+
`SELECT tm.session_id FROM team_member tm
|
|
17
|
+
JOIN team t ON tm.team_id = t.id
|
|
18
|
+
WHERE tm.status = 'busy' AND t.status = 'active'`
|
|
19
|
+
).all() as Array<{ session_id: string }>
|
|
20
|
+
|
|
13
21
|
const result = db.run(
|
|
14
22
|
`UPDATE team_member SET status = 'error', execution_status = 'idle', time_updated = ?
|
|
15
23
|
WHERE status = 'busy'
|
|
16
24
|
AND team_id IN (SELECT id FROM team WHERE status = 'active')`,
|
|
17
25
|
[Date.now()]
|
|
18
26
|
)
|
|
27
|
+
|
|
28
|
+
// Abort orphaned sessions (best effort)
|
|
29
|
+
if (client) {
|
|
30
|
+
for (const member of stale) {
|
|
31
|
+
try {
|
|
32
|
+
await client.session.abort({ path: { id: member.session_id } })
|
|
33
|
+
} catch { /* best effort */ }
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
19
37
|
return { interrupted: result.changes }
|
|
20
38
|
}
|
|
21
39
|
|
|
@@ -53,7 +53,10 @@ export async function executeTeamBroadcast(
|
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
// Only mark delivered if at least one recipient received it
|
|
57
|
+
if (delivered > 0) {
|
|
58
|
+
markDelivered(deps.db, msgId)
|
|
59
|
+
}
|
|
57
60
|
|
|
58
61
|
return `Broadcast sent to ${delivered} recipient${delivered !== 1 ? "s" : ""}.`
|
|
59
62
|
}
|
|
@@ -27,8 +27,20 @@ export async function executeTeamShutdown(
|
|
|
27
27
|
[Date.now(), teamInfo.teamId, args.member]
|
|
28
28
|
)
|
|
29
29
|
|
|
30
|
-
// Call abort on the member's session
|
|
31
|
-
|
|
30
|
+
// Call abort on the member's session (best effort)
|
|
31
|
+
try {
|
|
32
|
+
await deps.client.session.abort({ path: { id: member.session_id } })
|
|
33
|
+
} catch {
|
|
34
|
+
// Abort failed — session may already be gone. Fire a warning toast.
|
|
35
|
+
try {
|
|
36
|
+
await deps.client.tui.showToast({
|
|
37
|
+
title: "Team",
|
|
38
|
+
message: `Failed to abort ${args.member} session — will rely on event hook`,
|
|
39
|
+
variant: "warning",
|
|
40
|
+
duration: 4000,
|
|
41
|
+
})
|
|
42
|
+
} catch { /* TUI may not be available */ }
|
|
43
|
+
}
|
|
32
44
|
|
|
33
45
|
// Fallback: poll session status after abort. If already idle, transition
|
|
34
46
|
// directly to shutdown. This handles the case where abort() on an
|
package/src/tools/team-spawn.ts
CHANGED
|
@@ -73,12 +73,20 @@ export async function executeTeamSpawn(
|
|
|
73
73
|
const contextStr = context.join("\n")
|
|
74
74
|
|
|
75
75
|
// Fire-and-forget: send prompt to teammate session
|
|
76
|
-
// OQ-1:
|
|
77
|
-
// OQ-10:
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
76
|
+
// OQ-1: confirmed — promptAsync queues when session is busy (verified in live testing)
|
|
77
|
+
// OQ-10: confirmed — fresh session accepts promptAsync without session.init() (verified in live testing)
|
|
78
|
+
try {
|
|
79
|
+
await deps.client.session.promptAsync({
|
|
80
|
+
path: { id: childSessionId },
|
|
81
|
+
body: { parts: [{ type: "text", text: contextStr }] },
|
|
82
|
+
})
|
|
83
|
+
} catch (err) {
|
|
84
|
+
// Rollback: clean up DB, registry, and abort the orphaned session
|
|
85
|
+
deps.db.run("DELETE FROM team_member WHERE team_id = ? AND name = ?", [teamInfo.teamId, args.name])
|
|
86
|
+
deps.registry.unregister(childSessionId)
|
|
87
|
+
try { await deps.client.session.abort({ path: { id: childSessionId } }) } catch { /* best effort */ }
|
|
88
|
+
throw new Error(`Failed to send initial prompt to teammate "${args.name}": ${err instanceof Error ? err.message : String(err)}`)
|
|
89
|
+
}
|
|
82
90
|
|
|
83
91
|
return `Teammate "${args.name}" spawned (session: ${childSessionId}, agent: ${args.agent}). Working on: ${args.prompt.slice(0, 100)}${args.prompt.length > 100 ? "..." : ""} Teammates will message you when done. Do not poll team_status to check.`
|
|
84
92
|
}
|
package/src/watchdog.ts
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite"
|
|
2
|
+
import type { PluginClient } from "./types"
|
|
3
|
+
import type { MemberRegistry } from "./state"
|
|
4
|
+
|
|
5
|
+
interface WatchdogOpts {
|
|
6
|
+
db: Database
|
|
7
|
+
client: PluginClient
|
|
8
|
+
registry: MemberRegistry
|
|
9
|
+
/** Maximum time a member can stay busy before being timed out. 0 disables. */
|
|
10
|
+
ttlMs: number
|
|
11
|
+
/** How often to run the check. Defaults to 60s. */
|
|
12
|
+
checkIntervalMs?: number
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Periodic watchdog that times out teammates stuck in busy state.
|
|
17
|
+
* Transitions them to error/timed_out, aborts their session, and fires a toast.
|
|
18
|
+
*/
|
|
19
|
+
export class Watchdog {
|
|
20
|
+
private readonly db: Database
|
|
21
|
+
private readonly client: PluginClient
|
|
22
|
+
private readonly registry: MemberRegistry
|
|
23
|
+
private readonly ttlMs: number
|
|
24
|
+
private readonly checkIntervalMs: number
|
|
25
|
+
private timer: ReturnType<typeof setInterval> | undefined
|
|
26
|
+
|
|
27
|
+
constructor(opts: WatchdogOpts) {
|
|
28
|
+
this.db = opts.db
|
|
29
|
+
this.client = opts.client
|
|
30
|
+
this.registry = opts.registry
|
|
31
|
+
this.ttlMs = opts.ttlMs
|
|
32
|
+
this.checkIntervalMs = opts.checkIntervalMs ?? 60_000
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Run a single check for stale busy members. */
|
|
36
|
+
async check(): Promise<void> {
|
|
37
|
+
if (this.ttlMs === 0) return
|
|
38
|
+
|
|
39
|
+
const cutoff = Date.now() - this.ttlMs
|
|
40
|
+
const stale = this.db.query(
|
|
41
|
+
`SELECT tm.team_id, tm.name, tm.session_id
|
|
42
|
+
FROM team_member tm
|
|
43
|
+
JOIN team t ON tm.team_id = t.id
|
|
44
|
+
WHERE t.status = 'active'
|
|
45
|
+
AND tm.status = 'busy'
|
|
46
|
+
AND tm.time_updated < ?`
|
|
47
|
+
).all(cutoff) as Array<{ team_id: string; name: string; session_id: string }>
|
|
48
|
+
|
|
49
|
+
for (const member of stale) {
|
|
50
|
+
// Mark as timed out
|
|
51
|
+
this.db.run(
|
|
52
|
+
"UPDATE team_member SET status = 'error', execution_status = 'timed_out', time_updated = ? WHERE team_id = ? AND name = ?",
|
|
53
|
+
[Date.now(), member.team_id, member.name]
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
// Abort session (best effort)
|
|
57
|
+
try {
|
|
58
|
+
await this.client.session.abort({ path: { id: member.session_id } })
|
|
59
|
+
} catch { /* best effort */ }
|
|
60
|
+
|
|
61
|
+
// Notify
|
|
62
|
+
try {
|
|
63
|
+
await this.client.tui.showToast({
|
|
64
|
+
title: "Team",
|
|
65
|
+
message: `${member.name} timed out`,
|
|
66
|
+
variant: "warning",
|
|
67
|
+
duration: 5000,
|
|
68
|
+
})
|
|
69
|
+
} catch { /* TUI may not be available */ }
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Start the periodic check. */
|
|
74
|
+
start(): void {
|
|
75
|
+
if (this.timer) return
|
|
76
|
+
if (this.ttlMs === 0) return
|
|
77
|
+
this.timer = setInterval(() => this.check(), this.checkIntervalMs)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Stop the periodic check. */
|
|
81
|
+
stop(): void {
|
|
82
|
+
if (this.timer) {
|
|
83
|
+
clearInterval(this.timer)
|
|
84
|
+
this.timer = undefined
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Whether the watchdog is currently running. */
|
|
89
|
+
isRunning(): boolean {
|
|
90
|
+
return this.timer !== undefined
|
|
91
|
+
}
|
|
92
|
+
}
|