@hueyexe/opencode-ensemble 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -8,11 +8,13 @@ import { wrapThrowingClient } from "./client"
8
8
  import { recoverStaleMembers, recoverUndeliveredMessages, recoverOrphanedWorktrees } from "./recovery"
9
9
  import { MemberRegistry, DescendantTracker } from "./state"
10
10
  import { isWorktreeInstance } from "./util"
11
- import { handleSessionStatusEvent, handleSessionCreatedEvent, checkToolIsolation } from "./hooks"
11
+ import { handleSessionStatusEvent, handleSessionCreatedEvent, checkToolIsolation, shouldNudgeIdleMember } from "./hooks"
12
12
  import { notifyTeamEvent, notifyWorkingProgress } from "./notify"
13
13
  import { buildLeadSystemPrompt, buildTeammateSystemPrompt, buildTeamCompactionContext } from "./system-prompt"
14
14
  import { log, initLog } from "./log"
15
15
  import { findTeamBySession } from "./types"
16
+ import { loadConfig } from "./config"
17
+ import { ProgressTracker } from "./progress"
16
18
  import { executeTeamCreate } from "./tools/team-create"
17
19
  import { executeTeamSpawn } from "./tools/team-spawn"
18
20
  import { executeTeamMessage } from "./tools/team-message"
@@ -30,10 +32,8 @@ import type { ToolDeps, } from "./types"
30
32
  import { TokenBucket } from "./rate-limit"
31
33
  import { Watchdog } from "./watchdog"
32
34
 
33
- const DEFAULT_RATE_LIMIT_CAPACITY = 10
34
35
  const DEFAULT_RATE_LIMIT_REFILL = 2
35
36
  const DEFAULT_RATE_LIMIT_INTERVAL_MS = 1000
36
- const DEFAULT_WATCHDOG_TTL_MS = 30 * 60 * 1000 // 30 minutes
37
37
  const DEFAULT_WATCHDOG_CHECK_MS = 60 * 1000 // 60 seconds
38
38
 
39
39
  /**
@@ -47,9 +47,14 @@ const plugin: Plugin = async (input) => {
47
47
  mkdirSync(path.dirname(dbPath), { recursive: true })
48
48
  const db = createDb(dbPath)
49
49
 
50
+ // Load plugin configuration (global → project → env vars)
51
+ const config = loadConfig(input.directory)
52
+
50
53
  // Initialize in-memory state
51
54
  const registry = new MemberRegistry()
52
55
  const tracker = new DescendantTracker()
56
+ const nudgedMembers = new Set<string>()
57
+ const progressTracker = new ProgressTracker()
53
58
 
54
59
  // Extract the working HeyAPI transport from the plugin-provided v1 client and pass it
55
60
  // to the v2 OpencodeClient. The plugin framework provides a v1 client which stores its
@@ -88,24 +93,22 @@ const plugin: Plugin = async (input) => {
88
93
  log(`init:skip-recovery (worktree instance: ${input.directory})`)
89
94
  }
90
95
 
91
- // Initialize rate limiter
92
- // OPENCODE_ENSEMBLE_RATE_LIMIT=0 disables it entirely
93
- const rateLimitEnv = process.env.OPENCODE_ENSEMBLE_RATE_LIMIT
94
- const rateLimitCapacity = rateLimitEnv === "0" ? 0 : (parseInt(rateLimitEnv ?? "", 10) || DEFAULT_RATE_LIMIT_CAPACITY)
96
+ // Initialize rate limiter — config value already accounts for env var override
95
97
  const rateLimiter = new TokenBucket({
96
- capacity: rateLimitCapacity,
98
+ capacity: config.rateLimitCapacity,
97
99
  refillRate: DEFAULT_RATE_LIMIT_REFILL,
98
100
  refillIntervalMs: DEFAULT_RATE_LIMIT_INTERVAL_MS,
99
101
  })
100
102
 
101
- // Initialize watchdog for teammate timeouts
102
- // OPENCODE_ENSEMBLE_TIMEOUT=0 disables it entirely
103
- const timeoutEnv = process.env.OPENCODE_ENSEMBLE_TIMEOUT
104
- const watchdogTtl = timeoutEnv === "0" ? 0 : (parseInt(timeoutEnv ?? "", 10) || DEFAULT_WATCHDOG_TTL_MS)
103
+ // Initialize watchdog — config value already accounts for env var override
105
104
  const watchdog = new Watchdog({
106
105
  db, client, registry,
107
- ttlMs: watchdogTtl,
106
+ ttlMs: config.timeoutMs,
108
107
  checkIntervalMs: DEFAULT_WATCHDOG_CHECK_MS,
108
+ progressTracker,
109
+ stallThresholdMs: config.stallThresholdMs,
110
+ stallMinSteps: config.stallMinSteps,
111
+ stallTokenThreshold: config.stallTokenThreshold,
109
112
  })
110
113
  watchdog.start()
111
114
 
@@ -123,6 +126,16 @@ const plugin: Plugin = async (input) => {
123
126
  notifyTeamEvent(client, "shutdown", { memberName: transition.memberName })
124
127
  } else if (transition.to === "ready" && transition.from === "busy") {
125
128
  notifyTeamEvent(client, "completed", { memberName: transition.memberName })
129
+ // Nudge teammate if they went idle without reporting to the lead (once only)
130
+ const nudgeKey = `${transition.teamId}:${transition.memberName}`
131
+ if (!nudgedMembers.has(nudgeKey) && shouldNudgeIdleMember(db, transition.teamId, transition.memberName)) {
132
+ nudgedMembers.add(nudgeKey)
133
+ log(`nudge:idle-without-report name=${transition.memberName}`)
134
+ client.session.promptAsync({
135
+ sessionID,
136
+ parts: [{ type: "text", text: "[System]: You completed your work but did not report results. Send your findings to the lead via team_message now." }],
137
+ }).catch(() => { /* best effort */ })
138
+ }
126
139
  } else if (transition.to === "error") {
127
140
  notifyTeamEvent(client, "error", { memberName: transition.memberName })
128
141
  } else if (transition.to === "retry") {
@@ -174,6 +187,16 @@ const plugin: Plugin = async (input) => {
174
187
  handleSessionCreatedEvent(tracker, info.id, info.parentID)
175
188
  }
176
189
  }
190
+
191
+ // Track per-step output tokens for stall detection
192
+ if (event.type === "message.part.updated") {
193
+ const part = (event.properties as { part?: { type?: string; sessionID?: string; tokens?: { output?: number } } }).part
194
+ if (part?.type === "step-finish" && part.sessionID && part.tokens?.output !== undefined) {
195
+ if (registry.getBySession(part.sessionID)) {
196
+ progressTracker.recordStep(part.sessionID, part.tokens.output)
197
+ }
198
+ }
199
+ }
177
200
  },
178
201
 
179
202
  // Sub-agent isolation + rate limiting hook
@@ -272,6 +295,8 @@ const plugin: Plugin = async (input) => {
272
295
  },
273
296
  async execute(args, ctx) {
274
297
  const result = await executeTeamMessage(deps, args, ctx.sessionID)
298
+ // Track message activity for stall detection
299
+ progressTracker.recordMessage(ctx.sessionID)
275
300
  ctx.metadata({ title: `Message → ${args.to}` })
276
301
  return result
277
302
  },
@@ -284,6 +309,8 @@ const plugin: Plugin = async (input) => {
284
309
  },
285
310
  async execute(args, ctx) {
286
311
  const result = await executeTeamBroadcast(deps, args, ctx.sessionID)
312
+ // Track broadcast activity for stall detection
313
+ progressTracker.recordMessage(ctx.sessionID)
287
314
  ctx.metadata({ title: "Broadcast to team" })
288
315
  return result
289
316
  },
@@ -323,6 +350,8 @@ const plugin: Plugin = async (input) => {
323
350
  },
324
351
  async execute(args, ctx) {
325
352
  const result = await executeTeamTasksComplete(deps, args, ctx.sessionID)
353
+ // Track task completion for stall detection
354
+ progressTracker.recordTaskComplete(ctx.sessionID)
326
355
  ctx.metadata({ title: `Completed task` })
327
356
  return result
328
357
  },
@@ -361,6 +390,9 @@ const plugin: Plugin = async (input) => {
361
390
  },
362
391
  async execute(args, ctx) {
363
392
  const result = await executeTeamShutdown(deps, args, ctx.sessionID)
393
+ // Clean up progress tracking for the shut-down member
394
+ const member = deps.db.query("SELECT session_id FROM team_member WHERE name = ? AND status IN ('shutdown', 'shutdown_requested')").get(args.member) as { session_id: string } | null
395
+ if (member) progressTracker.remove(member.session_id)
364
396
  const hasWarning = result.includes("uncommitted")
365
397
  ctx.metadata({ title: hasWarning ? `${args.member} shut down — uncommitted changes` : `${args.member} shut down` })
366
398
  return result
@@ -374,7 +406,7 @@ const plugin: Plugin = async (input) => {
374
406
  acknowledge_uncommitted: tool.schema.boolean().default(false),
375
407
  },
376
408
  async execute(args, ctx) {
377
- const result = await executeTeamCleanup(deps, args, ctx.sessionID)
409
+ const result = await executeTeamCleanup(deps, args, ctx.sessionID, undefined, undefined, config.mergeOnCleanup)
378
410
  const blocked = result.includes("uncommitted")
379
411
  ctx.metadata({ title: blocked ? "Cleanup blocked — uncommitted changes" : "Team cleaned up" })
380
412
  return result
@@ -0,0 +1,85 @@
1
+ /** A single model step record. */
2
+ export interface StepRecord {
3
+ outputTokens: number
4
+ timestamp: number
5
+ }
6
+
7
+ /**
8
+ * In-memory tracker for teammate activity signals.
9
+ * Used by the watchdog for stall detection.
10
+ */
11
+ export class ProgressTracker {
12
+ private steps = new Map<string, StepRecord[]>()
13
+ private lastMessageAt = new Map<string, number>()
14
+ private lastTaskAt = new Map<string, number>()
15
+ private reported = new Set<string>()
16
+ private readonly maxSteps: number
17
+
18
+ constructor(maxSteps = 10) {
19
+ this.maxSteps = maxSteps
20
+ }
21
+
22
+ /** Record a model step completion with output token count. Keeps last `maxSteps` entries. */
23
+ recordStep(sessionId: string, outputTokens: number): void {
24
+ const records = this.steps.get(sessionId) ?? []
25
+ records.push({ outputTokens, timestamp: Date.now() })
26
+ if (records.length > this.maxSteps) records.shift()
27
+ this.steps.set(sessionId, records)
28
+ }
29
+
30
+ /** Record that this member sent a team_message. Clears stall report. */
31
+ recordMessage(sessionId: string): void {
32
+ this.lastMessageAt.set(sessionId, Date.now())
33
+ this.clearReport(sessionId)
34
+ }
35
+
36
+ /** Record that this member completed a task. Clears stall report. */
37
+ recordTaskComplete(sessionId: string): void {
38
+ this.lastTaskAt.set(sessionId, Date.now())
39
+ this.clearReport(sessionId)
40
+ }
41
+
42
+ /** Token-based stall: last `minSteps` steps all produced < `threshold` output tokens. */
43
+ isTokenStalled(sessionId: string, minSteps: number, threshold: number): boolean {
44
+ const records = this.steps.get(sessionId)
45
+ if (!records || records.length < minSteps) return false
46
+ const recent = records.slice(-minSteps)
47
+ return recent.every(r => r.outputTokens < threshold)
48
+ }
49
+
50
+ /** Time-based stall: no message or task completion within `thresholdMs` of now. */
51
+ isTimeStalled(sessionId: string, thresholdMs: number): boolean {
52
+ const records = this.steps.get(sessionId)
53
+ if (!records || records.length === 0) return false
54
+
55
+ const msgAt = this.lastMessageAt.get(sessionId) ?? 0
56
+ const taskAt = this.lastTaskAt.get(sessionId) ?? 0
57
+ const lastStep = records[records.length - 1]?.timestamp ?? 0
58
+ const baseline = Math.max(msgAt, taskAt, lastStep)
59
+
60
+ return Date.now() - baseline >= thresholdMs
61
+ }
62
+
63
+ /** Mark this session as reported-stalled (avoid spam). */
64
+ markReported(sessionId: string): void {
65
+ this.reported.add(sessionId)
66
+ }
67
+
68
+ /** Check if already reported. */
69
+ isReported(sessionId: string): boolean {
70
+ return this.reported.has(sessionId)
71
+ }
72
+
73
+ /** Clear stall report (called on new activity). */
74
+ clearReport(sessionId: string): void {
75
+ this.reported.delete(sessionId)
76
+ }
77
+
78
+ /** Remove all tracking for a session (on shutdown). */
79
+ remove(sessionId: string): void {
80
+ this.steps.delete(sessionId)
81
+ this.lastMessageAt.delete(sessionId)
82
+ this.lastTaskAt.delete(sessionId)
83
+ this.reported.delete(sessionId)
84
+ }
85
+ }
@@ -1,6 +1,11 @@
1
1
  import type { Database } from "bun:sqlite"
2
2
  import { markDelivered } from "./messaging"
3
3
 
4
+ /** Truncate a string to maxLen chars, appending "..." if truncated. */
5
+ function truncate(s: string, maxLen: number): string {
6
+ return s.length > maxLen ? `${s.slice(0, maxLen)}...` : s
7
+ }
8
+
4
9
  /** Status display mapping from DB status to human-readable label. */
5
10
  const STATUS_DISPLAY: Record<string, string> = {
6
11
  busy: "working",
@@ -51,6 +56,28 @@ export function buildLeadSystemPrompt(db: Database, teamId: string): string {
51
56
  `Tasks: ${completed} completed, ${inProgress} in progress, ${pending} pending`,
52
57
  ]
53
58
 
59
+ // Inline active and recently completed tasks
60
+ const activeTasks = db.query(
61
+ "SELECT content, assignee FROM team_task WHERE team_id = ? AND status = 'in_progress' ORDER BY time_updated DESC LIMIT 5"
62
+ ).all(teamId) as Array<{ content: string; assignee: string | null }>
63
+
64
+ const recentCompleted = db.query(
65
+ "SELECT content, assignee FROM team_task WHERE team_id = ? AND status = 'completed' ORDER BY time_updated DESC LIMIT 3"
66
+ ).all(teamId) as Array<{ content: string; assignee: string | null }>
67
+
68
+ if (activeTasks.length > 0) {
69
+ lines.push("Active tasks:")
70
+ for (const t of activeTasks) {
71
+ lines.push(` [in_progress] ${truncate(t.content, 120)}${t.assignee ? ` → ${t.assignee}` : ""}`)
72
+ }
73
+ }
74
+ if (recentCompleted.length > 0) {
75
+ lines.push("Recently completed:")
76
+ for (const t of recentCompleted) {
77
+ lines.push(` [completed] ${truncate(t.content, 120)}${t.assignee ? ` → ${t.assignee}` : ""}`)
78
+ }
79
+ }
80
+
54
81
  if (pendingMessages.length > 0) {
55
82
  lines.push("", "--- Team Messages ---")
56
83
  const MAX_MSG = 500
@@ -92,7 +119,7 @@ export function buildTeammateSystemPrompt(db: Database, teamId: string, memberNa
92
119
  const team = db.query("SELECT name FROM team WHERE id = ?").get(teamId) as { name: string } | null
93
120
  if (!team) return ""
94
121
 
95
- return `You are "${memberName}", a teammate in team "${team.name}". Use team_message to communicate.`
122
+ return `You are "${memberName}", a teammate in team "${team.name}". Use team_message to communicate. You MUST send your results to the lead via team_message before stopping.`
96
123
  }
97
124
 
98
125
  /**
@@ -135,9 +162,48 @@ export function buildTeamCompactionContext(
135
162
  ? `Members: ${memberList}`
136
163
  : "Members: none"
137
164
 
138
- return [
165
+ const lines = [
139
166
  roleLine,
140
167
  membersLine,
141
168
  `Tasks: ${completed} completed, ${inProgress} in progress, ${pending} pending`,
142
- ].join("\n")
169
+ ]
170
+
171
+ if (role === "member" && memberName) {
172
+ lines.push("IMPORTANT: You MUST send your results to the lead via team_message before stopping.")
173
+
174
+ // Include original task prompt
175
+ const member = db.query("SELECT prompt FROM team_member WHERE team_id = ? AND name = ?")
176
+ .get(teamId, memberName) as { prompt: string | null } | null
177
+ if (member?.prompt) {
178
+ lines.push(`Your original task: ${truncate(member.prompt, 300)}`)
179
+ }
180
+
181
+ // Include recent messages involving this member
182
+ const recentMsgs = db.query(
183
+ "SELECT from_name, content FROM team_message WHERE team_id = ? AND (from_name = ? OR to_name = ?) ORDER BY time_created DESC LIMIT 3"
184
+ ).all(teamId, memberName, memberName) as Array<{ from_name: string; content: string }>
185
+ if (recentMsgs.length > 0) {
186
+ lines.push("Recent context:")
187
+ for (const msg of recentMsgs) {
188
+ lines.push(` [${msg.from_name}]: ${truncate(msg.content, 200)}`)
189
+ }
190
+ }
191
+ } else if (role === "member") {
192
+ lines.push("IMPORTANT: You MUST send your results to the lead via team_message before stopping.")
193
+ }
194
+
195
+ if (role === "lead") {
196
+ // Include recently completed tasks
197
+ const completedTasks = db.query(
198
+ "SELECT content, assignee FROM team_task WHERE team_id = ? AND status = 'completed' ORDER BY time_updated DESC LIMIT 5"
199
+ ).all(teamId) as Array<{ content: string; assignee: string | null }>
200
+ if (completedTasks.length > 0) {
201
+ lines.push("Recently completed:")
202
+ for (const t of completedTasks) {
203
+ lines.push(` [completed] ${truncate(t.content, 120)}${t.assignee ? ` (by ${t.assignee})` : ""}`)
204
+ }
205
+ }
206
+ }
207
+
208
+ return lines.join("\n")
143
209
  }
@@ -1,8 +1,64 @@
1
1
  import type { ToolDeps } from "../types"
2
2
  import { requireLead, checkWorktreeDirty } from "./shared"
3
3
  import type { IsDirtyFn } from "./shared"
4
+ import { spawnFailures } from "./team-spawn"
4
5
  import { log } from "../log"
5
6
 
7
+ /** Result of merging all branches. */
8
+ export interface MergeAllResult {
9
+ merged: string[]
10
+ conflicted: string[]
11
+ }
12
+
13
+ /** Injectable function that merges all branches into the working directory. */
14
+ export type MergeAllFn = (branches: string[], cwd: string) => Promise<MergeAllResult>
15
+
16
+ /**
17
+ * Default merge implementation: stash user work, squash-merge each branch,
18
+ * unstage merged changes, restore user work. One stash/pop cycle for all branches.
19
+ */
20
+ export async function mergeAllBranches(branches: string[], cwd: string): Promise<MergeAllResult> {
21
+ const merged: string[] = []
22
+ const conflicted: string[] = []
23
+
24
+ // Stash existing work to preserve staged/unstaged state
25
+ const stash = Bun.spawn(["git", "stash", "--include-untracked"], { cwd, stdout: "pipe", stderr: "pipe" })
26
+ const stashOut = await new Response(stash.stdout).text()
27
+ const stashExit = await stash.exited
28
+ const didStash = stashExit === 0 && !stashOut.includes("No local changes")
29
+
30
+ for (const branch of branches) {
31
+ const merge = Bun.spawn(["git", "merge", "--squash", branch], { cwd, stdout: "pipe", stderr: "pipe" })
32
+ const stderrPromise = new Response(merge.stderr).text()
33
+ const exit = await merge.exited
34
+ if (exit !== 0) {
35
+ const stderr = await stderrPromise
36
+ log(`cleanup:merge:conflict branch=${branch} err=${stderr.trim()}`)
37
+ const abort = Bun.spawn(["git", "merge", "--abort"], { cwd, stdout: "pipe", stderr: "pipe" })
38
+ await abort.exited
39
+ conflicted.push(branch)
40
+ } else {
41
+ const del = Bun.spawn(["git", "branch", "-D", branch], { cwd, stdout: "pipe", stderr: "pipe" })
42
+ await del.exited
43
+ merged.push(branch)
44
+ }
45
+ }
46
+
47
+ // Unstage merged changes so user reviews with `git diff`
48
+ if (merged.length > 0) {
49
+ const reset = Bun.spawn(["git", "reset", "HEAD"], { cwd, stdout: "pipe", stderr: "pipe" })
50
+ await reset.exited
51
+ }
52
+
53
+ // Restore user's original staged/unstaged work
54
+ if (didStash) {
55
+ const pop = Bun.spawn(["git", "stash", "pop", "--index"], { cwd, stdout: "pipe", stderr: "pipe" })
56
+ await pop.exited
57
+ }
58
+
59
+ return { merged, conflicted }
60
+ }
61
+
6
62
  /**
7
63
  * Execute the team_cleanup tool. Archives the team and cleans up resources.
8
64
  * Checks for uncommitted changes BEFORE aborting sessions or removing worktrees.
@@ -14,6 +70,8 @@ export async function executeTeamCleanup(
14
70
  args: { force: boolean; acknowledge_uncommitted?: boolean },
15
71
  sessionId: string,
16
72
  isDirty: IsDirtyFn = checkWorktreeDirty,
73
+ mergeAll: MergeAllFn = mergeAllBranches,
74
+ mergeOnCleanup = true,
17
75
  ): Promise<string> {
18
76
  const teamInfo = requireLead(deps, sessionId)
19
77
 
@@ -57,8 +115,25 @@ export async function executeTeamCleanup(
57
115
  }
58
116
  }
59
117
 
60
- // Remove workspaces, worktrees, and collect branches for merging
61
- const branches: string[] = []
118
+ // Collect branches and merge BEFORE removing worktrees (worktree.remove may delete the branch ref)
119
+ const branches = members
120
+ .map(m => m.worktree_branch)
121
+ .filter((b): b is string => b !== null)
122
+
123
+ let mergeResult = ""
124
+ if (branches.length > 0 && mergeOnCleanup) {
125
+ const { merged, conflicted } = await mergeAll(branches, deps.directory)
126
+ const parts: string[] = []
127
+ if (merged.length > 0) {
128
+ parts.push(`Merged ${merged.length} branch(es) into working directory (unstaged). Review changes with: git diff`)
129
+ }
130
+ if (conflicted.length > 0) {
131
+ parts.push(`Could not auto-merge: ${conflicted.join(", ")}. Merge manually with: git merge <branch>`)
132
+ }
133
+ mergeResult = parts.join("\n")
134
+ }
135
+
136
+ // Remove workspaces and worktrees (after merging)
62
137
  for (const member of members) {
63
138
  if (member.workspace_id) {
64
139
  try {
@@ -72,16 +147,18 @@ export async function executeTeamCleanup(
72
147
  deps.db.run("UPDATE team_member SET worktree_dir = NULL WHERE team_id = ? AND name = ?", [teamInfo.teamId, member.name])
73
148
  } catch { /* best effort — worktree may already be gone */ }
74
149
  }
75
- if (member.worktree_branch) {
76
- branches.push(member.worktree_branch)
77
- }
78
150
  }
79
151
 
80
152
  // Archive team
81
153
  deps.db.run("UPDATE team SET status = 'archived', time_updated = ? WHERE id = ?", [Date.now(), teamInfo.teamId])
82
154
 
83
- // Clean up registry
155
+ // Clean up in-memory state
84
156
  deps.registry.unregisterTeam(teamInfo.teamId)
157
+ spawnFailures.delete(teamInfo.teamId)
158
+
159
+ if (mergeResult) {
160
+ return `Team "${teamInfo.teamName}" cleaned up.\n${mergeResult}`
161
+ }
85
162
 
86
163
  if (branches.length > 0) {
87
164
  const mergeCommands = branches.map(b => ` git merge ${b}`).join("\n")
@@ -4,6 +4,9 @@ import { requireLead } from "./shared"
4
4
  import { sendMessage } from "../messaging"
5
5
  import { log } from "../log"
6
6
 
7
+ /** Tracks consecutive spawn failures per team for circuit breaker. */
8
+ export const spawnFailures = new Map<string, { count: number; lastError: string }>()
9
+
7
10
  /** Timeout for worktree.create and session.create to prevent hanging on git lock contention. */
8
11
  function getSpawnTimeout(): number {
9
12
  return Number(process.env.SPAWN_TIMEOUT_MS) || 120_000
@@ -41,6 +44,12 @@ export async function executeTeamSpawn(
41
44
 
42
45
  const teamInfo = requireLead(deps, sessionId)
43
46
 
47
+ // Circuit breaker — stop retrying after 3 consecutive failures
48
+ const failures = spawnFailures.get(teamInfo.teamId)
49
+ if (failures && failures.count >= 3) {
50
+ throw new Error(`Spawn circuit breaker tripped for team "${teamInfo.teamName}": 3 consecutive failures. Last error: ${failures.lastError}. Investigate before retrying — the circuit breaker resets on the next successful spawn.`)
51
+ }
52
+
44
53
  // Check duplicate name
45
54
  const existing = deps.db.query("SELECT name FROM team_member WHERE team_id = ? AND name = ?")
46
55
  .get(teamInfo.teamId, args.name)
@@ -146,6 +155,10 @@ export async function executeTeamSpawn(
146
155
  log(`spawn:session:done name=${args.name} sessionId=${childSessionId}`)
147
156
  } catch (err) {
148
157
  log(`spawn:session:failed name=${args.name} err=${err instanceof Error ? err.message : String(err)}`)
158
+ // Track failure for circuit breaker
159
+ const errMsg = err instanceof Error ? err.message : String(err)
160
+ const prev = spawnFailures.get(teamInfo.teamId)
161
+ spawnFailures.set(teamInfo.teamId, { count: (prev?.count ?? 0) + 1, lastError: errMsg })
149
162
  // Rollback workspace and worktree if session creation failed
150
163
  if (workspaceId) {
151
164
  try { await deps.client.workspace.remove({ id: workspaceId }) } catch { /* best effort */ }
@@ -319,6 +332,8 @@ export async function executeTeamSpawn(
319
332
 
320
333
  const branchInfo = worktreeBranch ? ` (branch: ${worktreeBranch})` : ""
321
334
  const planInfo = usePlanApproval ? " [plan mode — will send plan for approval]" : ""
335
+ // Reset circuit breaker on success
336
+ spawnFailures.delete(teamInfo.teamId)
322
337
  log(`spawn:done name=${args.name} sessionId=${childSessionId}`)
323
338
  return `Teammate "${args.name}" spawned (agent: ${args.agent})${branchInfo}${planInfo}. They are working on: ${args.prompt.slice(0, 120)}${args.prompt.length > 120 ? "..." : ""}`
324
339
  }
@@ -6,6 +6,13 @@ export const lastCallTime = new Map<string, number>()
6
6
 
7
7
  const RATE_LIMIT_MS = 30_000
8
8
 
9
+ /** Format a duration in ms to a human-readable string. */
10
+ export function formatDuration(ms: number): string {
11
+ if (ms < 60_000) return `${Math.floor(ms / 1000)}s`
12
+ if (ms < 3_600_000) return `${Math.floor(ms / 60_000)}m`
13
+ return `${Math.floor(ms / 3_600_000)}h`
14
+ }
15
+
9
16
  /**
10
17
  * Execute the team_status tool. Shows team overview with member statuses and task summary.
11
18
  */
@@ -23,9 +30,9 @@ export async function executeTeamStatus(
23
30
  lastCallTime.set(teamInfo.teamId, now)
24
31
 
25
32
  const members = deps.db.query(
26
- "SELECT name, session_id, agent, status, execution_status, worktree_branch, worktree_dir, plan_approval FROM team_member WHERE team_id = ? ORDER BY time_created ASC"
33
+ "SELECT name, session_id, agent, status, execution_status, worktree_branch, worktree_dir, plan_approval, time_updated FROM team_member WHERE team_id = ? ORDER BY time_created ASC"
27
34
  ).all(teamInfo.teamId) as Array<{
28
- name: string; session_id: string; agent: string; status: string; execution_status: string; worktree_branch: string | null; worktree_dir: string | null; plan_approval: string
35
+ name: string; session_id: string; agent: string; status: string; execution_status: string; worktree_branch: string | null; worktree_dir: string | null; plan_approval: string; time_updated: number
29
36
  }>
30
37
 
31
38
  const tasks = deps.db.query(
@@ -42,9 +49,24 @@ export async function executeTeamStatus(
42
49
  lines.push("Members:")
43
50
  for (const m of members) {
44
51
  const statusIcon = m.status === "busy" ? "working" : m.status === "ready" ? "idle" : m.status
52
+ const duration = formatDuration(now - m.time_updated)
45
53
  const branch = m.worktree_branch ? ` branch: ${m.worktree_branch}` : ""
46
54
  const plan = m.plan_approval !== "none" ? `, plan: ${m.plan_approval}` : ""
47
- lines.push(` ${m.name} [${statusIcon}${plan}] agent: ${m.agent}${branch} session: ${m.session_id}`)
55
+
56
+ // Last message time
57
+ const lastMsg = deps.db.query("SELECT MAX(time_created) as last_msg FROM team_message WHERE team_id = ? AND from_name = ?")
58
+ .get(teamInfo.teamId, m.name) as { last_msg: number | null } | null
59
+ const msgInfo = lastMsg?.last_msg ? `last msg: ${formatDuration(now - lastMsg.last_msg)} ago` : "no messages yet"
60
+
61
+ lines.push(` ${m.name} [${statusIcon} ${duration}, ${msgInfo}${plan}] agent: ${m.agent}${branch}`)
62
+
63
+ // Current task
64
+ const task = deps.db.query("SELECT content FROM team_task WHERE team_id = ? AND assignee = ? AND status = 'in_progress' LIMIT 1")
65
+ .get(teamInfo.teamId, m.name) as { content: string } | null
66
+ if (task) {
67
+ const truncated = task.content.length > 80 ? `${task.content.slice(0, 80)}...` : task.content
68
+ lines.push(` task: ${truncated}`)
69
+ }
48
70
  if (m.worktree_dir) {
49
71
  lines.push(` worktree: ${m.worktree_dir}`)
50
72
  }
package/src/watchdog.ts CHANGED
@@ -1,6 +1,8 @@
1
1
  import type { Database } from "bun:sqlite"
2
2
  import type { PluginClient } from "./types"
3
3
  import type { MemberRegistry } from "./state"
4
+ import type { ProgressTracker } from "./progress"
5
+ import { sendMessage } from "./messaging"
4
6
 
5
7
  interface WatchdogOpts {
6
8
  db: Database
@@ -10,6 +12,14 @@ interface WatchdogOpts {
10
12
  ttlMs: number
11
13
  /** How often to run the check. Defaults to 60s. */
12
14
  checkIntervalMs?: number
15
+ /** Progress tracker for stall detection. */
16
+ progressTracker?: ProgressTracker
17
+ /** Stall detection threshold in ms. 0 disables. */
18
+ stallThresholdMs?: number
19
+ /** Min steps before token-based stall check. */
20
+ stallMinSteps?: number
21
+ /** Output token threshold for stall detection. */
22
+ stallTokenThreshold?: number
13
23
  }
14
24
 
15
25
  /**
@@ -22,6 +32,10 @@ export class Watchdog {
22
32
  private readonly registry: MemberRegistry
23
33
  private readonly ttlMs: number
24
34
  private readonly checkIntervalMs: number
35
+ private readonly progressTracker?: ProgressTracker
36
+ private readonly stallThresholdMs: number
37
+ private readonly stallMinSteps: number
38
+ private readonly stallTokenThreshold: number
25
39
  private timer: ReturnType<typeof setInterval> | undefined
26
40
 
27
41
  constructor(opts: WatchdogOpts) {
@@ -30,6 +44,10 @@ export class Watchdog {
30
44
  this.registry = opts.registry
31
45
  this.ttlMs = opts.ttlMs
32
46
  this.checkIntervalMs = opts.checkIntervalMs ?? 60_000
47
+ this.progressTracker = opts.progressTracker
48
+ this.stallThresholdMs = opts.stallThresholdMs ?? 0
49
+ this.stallMinSteps = opts.stallMinSteps ?? 3
50
+ this.stallTokenThreshold = opts.stallTokenThreshold ?? 500
33
51
  }
34
52
 
35
53
  private static STALE_THRESHOLD_MS = Number(process.env.STALE_WORKTREE_THRESHOLD_MS) || 300_000
@@ -61,9 +79,58 @@ export class Watchdog {
61
79
  }
62
80
  }
63
81
 
82
+ /** Check for stalled busy members and escalate to lead + nudge teammate. */
83
+ async checkStalled(): Promise<void> {
84
+ if (!this.progressTracker || this.stallThresholdMs === 0) return
85
+
86
+ const busy = this.db.query(
87
+ `SELECT tm.team_id, tm.name, tm.session_id
88
+ FROM team_member tm
89
+ JOIN team t ON tm.team_id = t.id
90
+ WHERE t.status = 'active' AND tm.status = 'busy'`
91
+ ).all() as Array<{ team_id: string; name: string; session_id: string }>
92
+
93
+ for (const member of busy) {
94
+ if (this.progressTracker.isReported(member.session_id)) continue
95
+
96
+ const tokenStalled = this.progressTracker.isTokenStalled(member.session_id, this.stallMinSteps, this.stallTokenThreshold)
97
+ const timeStalled = this.progressTracker.isTimeStalled(member.session_id, this.stallThresholdMs)
98
+
99
+ if (!tokenStalled && !timeStalled) continue
100
+
101
+ this.progressTracker.markReported(member.session_id)
102
+ const reason = tokenStalled ? "low output tokens" : "no communication"
103
+
104
+ // Nudge the teammate directly
105
+ this.client.session.promptAsync({
106
+ sessionID: member.session_id,
107
+ parts: [{ type: "text", text: "[System]: You appear stalled — no progress detected. Report your current status to the lead via team_message, or wrap up your work." }],
108
+ }).catch(() => { /* best effort */ })
109
+
110
+ // Notify the lead
111
+ sendMessage(this.db, {
112
+ teamId: member.team_id,
113
+ from: "system",
114
+ to: "lead",
115
+ content: `Teammate "${member.name}" appears stalled (${reason}). Consider checking on them via team_message or shutting them down.`,
116
+ })
117
+
118
+ // Toast for the user
119
+ try {
120
+ await this.client.tui.showToast({
121
+ title: "Team",
122
+ message: `${member.name} appears stalled`,
123
+ variant: "warning",
124
+ duration: 5000,
125
+ })
126
+ } catch { /* TUI may not be available */ }
127
+ }
128
+ }
129
+
64
130
  /** Run a single check for stale busy members. */
65
131
  async check(): Promise<void> {
66
132
  await this.cleanupStaleWorktrees()
133
+ await this.checkStalled()
67
134
  if (this.ttlMs === 0) return
68
135
 
69
136
  const cutoff = Date.now() - this.ttlMs