@hueyexe/opencode-ensemble 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -11
- package/dist/index.js +327 -22
- package/package.json +1 -1
- package/src/config.ts +75 -0
- package/src/hooks.ts +14 -0
- package/src/index.ts +46 -14
- package/src/progress.ts +85 -0
- package/src/system-prompt.ts +69 -3
- package/src/tools/team-cleanup.ts +83 -6
- package/src/tools/team-spawn.ts +15 -0
- package/src/tools/team-status.ts +25 -3
- package/src/watchdog.ts +67 -0
package/src/index.ts
CHANGED
|
@@ -8,11 +8,13 @@ import { wrapThrowingClient } from "./client"
|
|
|
8
8
|
import { recoverStaleMembers, recoverUndeliveredMessages, recoverOrphanedWorktrees } from "./recovery"
|
|
9
9
|
import { MemberRegistry, DescendantTracker } from "./state"
|
|
10
10
|
import { isWorktreeInstance } from "./util"
|
|
11
|
-
import { handleSessionStatusEvent, handleSessionCreatedEvent, checkToolIsolation } from "./hooks"
|
|
11
|
+
import { handleSessionStatusEvent, handleSessionCreatedEvent, checkToolIsolation, shouldNudgeIdleMember } from "./hooks"
|
|
12
12
|
import { notifyTeamEvent, notifyWorkingProgress } from "./notify"
|
|
13
13
|
import { buildLeadSystemPrompt, buildTeammateSystemPrompt, buildTeamCompactionContext } from "./system-prompt"
|
|
14
14
|
import { log, initLog } from "./log"
|
|
15
15
|
import { findTeamBySession } from "./types"
|
|
16
|
+
import { loadConfig } from "./config"
|
|
17
|
+
import { ProgressTracker } from "./progress"
|
|
16
18
|
import { executeTeamCreate } from "./tools/team-create"
|
|
17
19
|
import { executeTeamSpawn } from "./tools/team-spawn"
|
|
18
20
|
import { executeTeamMessage } from "./tools/team-message"
|
|
@@ -30,10 +32,8 @@ import type { ToolDeps, } from "./types"
|
|
|
30
32
|
import { TokenBucket } from "./rate-limit"
|
|
31
33
|
import { Watchdog } from "./watchdog"
|
|
32
34
|
|
|
33
|
-
const DEFAULT_RATE_LIMIT_CAPACITY = 10
|
|
34
35
|
const DEFAULT_RATE_LIMIT_REFILL = 2
|
|
35
36
|
const DEFAULT_RATE_LIMIT_INTERVAL_MS = 1000
|
|
36
|
-
const DEFAULT_WATCHDOG_TTL_MS = 30 * 60 * 1000 // 30 minutes
|
|
37
37
|
const DEFAULT_WATCHDOG_CHECK_MS = 60 * 1000 // 60 seconds
|
|
38
38
|
|
|
39
39
|
/**
|
|
@@ -47,9 +47,14 @@ const plugin: Plugin = async (input) => {
|
|
|
47
47
|
mkdirSync(path.dirname(dbPath), { recursive: true })
|
|
48
48
|
const db = createDb(dbPath)
|
|
49
49
|
|
|
50
|
+
// Load plugin configuration (global → project → env vars)
|
|
51
|
+
const config = loadConfig(input.directory)
|
|
52
|
+
|
|
50
53
|
// Initialize in-memory state
|
|
51
54
|
const registry = new MemberRegistry()
|
|
52
55
|
const tracker = new DescendantTracker()
|
|
56
|
+
const nudgedMembers = new Set<string>()
|
|
57
|
+
const progressTracker = new ProgressTracker()
|
|
53
58
|
|
|
54
59
|
// Extract the working HeyAPI transport from the plugin-provided v1 client and pass it
|
|
55
60
|
// to the v2 OpencodeClient. The plugin framework provides a v1 client which stores its
|
|
@@ -88,24 +93,22 @@ const plugin: Plugin = async (input) => {
|
|
|
88
93
|
log(`init:skip-recovery (worktree instance: ${input.directory})`)
|
|
89
94
|
}
|
|
90
95
|
|
|
91
|
-
// Initialize rate limiter
|
|
92
|
-
// OPENCODE_ENSEMBLE_RATE_LIMIT=0 disables it entirely
|
|
93
|
-
const rateLimitEnv = process.env.OPENCODE_ENSEMBLE_RATE_LIMIT
|
|
94
|
-
const rateLimitCapacity = rateLimitEnv === "0" ? 0 : (parseInt(rateLimitEnv ?? "", 10) || DEFAULT_RATE_LIMIT_CAPACITY)
|
|
96
|
+
// Initialize rate limiter — config value already accounts for env var override
|
|
95
97
|
const rateLimiter = new TokenBucket({
|
|
96
|
-
capacity: rateLimitCapacity,
|
|
98
|
+
capacity: config.rateLimitCapacity,
|
|
97
99
|
refillRate: DEFAULT_RATE_LIMIT_REFILL,
|
|
98
100
|
refillIntervalMs: DEFAULT_RATE_LIMIT_INTERVAL_MS,
|
|
99
101
|
})
|
|
100
102
|
|
|
101
|
-
// Initialize watchdog for
|
|
102
|
-
// OPENCODE_ENSEMBLE_TIMEOUT=0 disables it entirely
|
|
103
|
-
const timeoutEnv = process.env.OPENCODE_ENSEMBLE_TIMEOUT
|
|
104
|
-
const watchdogTtl = timeoutEnv === "0" ? 0 : (parseInt(timeoutEnv ?? "", 10) || DEFAULT_WATCHDOG_TTL_MS)
|
|
103
|
+
// Initialize watchdog — config value already accounts for env var override
|
|
105
104
|
const watchdog = new Watchdog({
|
|
106
105
|
db, client, registry,
|
|
107
|
-
ttlMs:
|
|
106
|
+
ttlMs: config.timeoutMs,
|
|
108
107
|
checkIntervalMs: DEFAULT_WATCHDOG_CHECK_MS,
|
|
108
|
+
progressTracker,
|
|
109
|
+
stallThresholdMs: config.stallThresholdMs,
|
|
110
|
+
stallMinSteps: config.stallMinSteps,
|
|
111
|
+
stallTokenThreshold: config.stallTokenThreshold,
|
|
109
112
|
})
|
|
110
113
|
watchdog.start()
|
|
111
114
|
|
|
@@ -123,6 +126,16 @@ const plugin: Plugin = async (input) => {
|
|
|
123
126
|
notifyTeamEvent(client, "shutdown", { memberName: transition.memberName })
|
|
124
127
|
} else if (transition.to === "ready" && transition.from === "busy") {
|
|
125
128
|
notifyTeamEvent(client, "completed", { memberName: transition.memberName })
|
|
129
|
+
// Nudge teammate if they went idle without reporting to the lead (once only)
|
|
130
|
+
const nudgeKey = `${transition.teamId}:${transition.memberName}`
|
|
131
|
+
if (!nudgedMembers.has(nudgeKey) && shouldNudgeIdleMember(db, transition.teamId, transition.memberName)) {
|
|
132
|
+
nudgedMembers.add(nudgeKey)
|
|
133
|
+
log(`nudge:idle-without-report name=${transition.memberName}`)
|
|
134
|
+
client.session.promptAsync({
|
|
135
|
+
sessionID,
|
|
136
|
+
parts: [{ type: "text", text: "[System]: You completed your work but did not report results. Send your findings to the lead via team_message now." }],
|
|
137
|
+
}).catch(() => { /* best effort */ })
|
|
138
|
+
}
|
|
126
139
|
} else if (transition.to === "error") {
|
|
127
140
|
notifyTeamEvent(client, "error", { memberName: transition.memberName })
|
|
128
141
|
} else if (transition.to === "retry") {
|
|
@@ -174,6 +187,16 @@ const plugin: Plugin = async (input) => {
|
|
|
174
187
|
handleSessionCreatedEvent(tracker, info.id, info.parentID)
|
|
175
188
|
}
|
|
176
189
|
}
|
|
190
|
+
|
|
191
|
+
// Track per-step output tokens for stall detection
|
|
192
|
+
if (event.type === "message.part.updated") {
|
|
193
|
+
const part = (event.properties as { part?: { type?: string; sessionID?: string; tokens?: { output?: number } } }).part
|
|
194
|
+
if (part?.type === "step-finish" && part.sessionID && part.tokens?.output !== undefined) {
|
|
195
|
+
if (registry.getBySession(part.sessionID)) {
|
|
196
|
+
progressTracker.recordStep(part.sessionID, part.tokens.output)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
177
200
|
},
|
|
178
201
|
|
|
179
202
|
// Sub-agent isolation + rate limiting hook
|
|
@@ -272,6 +295,8 @@ const plugin: Plugin = async (input) => {
|
|
|
272
295
|
},
|
|
273
296
|
async execute(args, ctx) {
|
|
274
297
|
const result = await executeTeamMessage(deps, args, ctx.sessionID)
|
|
298
|
+
// Track message activity for stall detection
|
|
299
|
+
progressTracker.recordMessage(ctx.sessionID)
|
|
275
300
|
ctx.metadata({ title: `Message → ${args.to}` })
|
|
276
301
|
return result
|
|
277
302
|
},
|
|
@@ -284,6 +309,8 @@ const plugin: Plugin = async (input) => {
|
|
|
284
309
|
},
|
|
285
310
|
async execute(args, ctx) {
|
|
286
311
|
const result = await executeTeamBroadcast(deps, args, ctx.sessionID)
|
|
312
|
+
// Track broadcast activity for stall detection
|
|
313
|
+
progressTracker.recordMessage(ctx.sessionID)
|
|
287
314
|
ctx.metadata({ title: "Broadcast to team" })
|
|
288
315
|
return result
|
|
289
316
|
},
|
|
@@ -323,6 +350,8 @@ const plugin: Plugin = async (input) => {
|
|
|
323
350
|
},
|
|
324
351
|
async execute(args, ctx) {
|
|
325
352
|
const result = await executeTeamTasksComplete(deps, args, ctx.sessionID)
|
|
353
|
+
// Track task completion for stall detection
|
|
354
|
+
progressTracker.recordTaskComplete(ctx.sessionID)
|
|
326
355
|
ctx.metadata({ title: `Completed task` })
|
|
327
356
|
return result
|
|
328
357
|
},
|
|
@@ -361,6 +390,9 @@ const plugin: Plugin = async (input) => {
|
|
|
361
390
|
},
|
|
362
391
|
async execute(args, ctx) {
|
|
363
392
|
const result = await executeTeamShutdown(deps, args, ctx.sessionID)
|
|
393
|
+
// Clean up progress tracking for the shut-down member
|
|
394
|
+
const member = deps.db.query("SELECT session_id FROM team_member WHERE name = ? AND status IN ('shutdown', 'shutdown_requested')").get(args.member) as { session_id: string } | null
|
|
395
|
+
if (member) progressTracker.remove(member.session_id)
|
|
364
396
|
const hasWarning = result.includes("uncommitted")
|
|
365
397
|
ctx.metadata({ title: hasWarning ? `${args.member} shut down — uncommitted changes` : `${args.member} shut down` })
|
|
366
398
|
return result
|
|
@@ -374,7 +406,7 @@ const plugin: Plugin = async (input) => {
|
|
|
374
406
|
acknowledge_uncommitted: tool.schema.boolean().default(false),
|
|
375
407
|
},
|
|
376
408
|
async execute(args, ctx) {
|
|
377
|
-
const result = await executeTeamCleanup(deps, args, ctx.sessionID)
|
|
409
|
+
const result = await executeTeamCleanup(deps, args, ctx.sessionID, undefined, undefined, config.mergeOnCleanup)
|
|
378
410
|
const blocked = result.includes("uncommitted")
|
|
379
411
|
ctx.metadata({ title: blocked ? "Cleanup blocked — uncommitted changes" : "Team cleaned up" })
|
|
380
412
|
return result
|
package/src/progress.ts
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/** A single model step record. */
|
|
2
|
+
export interface StepRecord {
|
|
3
|
+
outputTokens: number
|
|
4
|
+
timestamp: number
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* In-memory tracker for teammate activity signals.
|
|
9
|
+
* Used by the watchdog for stall detection.
|
|
10
|
+
*/
|
|
11
|
+
export class ProgressTracker {
|
|
12
|
+
private steps = new Map<string, StepRecord[]>()
|
|
13
|
+
private lastMessageAt = new Map<string, number>()
|
|
14
|
+
private lastTaskAt = new Map<string, number>()
|
|
15
|
+
private reported = new Set<string>()
|
|
16
|
+
private readonly maxSteps: number
|
|
17
|
+
|
|
18
|
+
constructor(maxSteps = 10) {
|
|
19
|
+
this.maxSteps = maxSteps
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Record a model step completion with output token count. Keeps last `maxSteps` entries. */
|
|
23
|
+
recordStep(sessionId: string, outputTokens: number): void {
|
|
24
|
+
const records = this.steps.get(sessionId) ?? []
|
|
25
|
+
records.push({ outputTokens, timestamp: Date.now() })
|
|
26
|
+
if (records.length > this.maxSteps) records.shift()
|
|
27
|
+
this.steps.set(sessionId, records)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Record that this member sent a team_message. Clears stall report. */
|
|
31
|
+
recordMessage(sessionId: string): void {
|
|
32
|
+
this.lastMessageAt.set(sessionId, Date.now())
|
|
33
|
+
this.clearReport(sessionId)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Record that this member completed a task. Clears stall report. */
|
|
37
|
+
recordTaskComplete(sessionId: string): void {
|
|
38
|
+
this.lastTaskAt.set(sessionId, Date.now())
|
|
39
|
+
this.clearReport(sessionId)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Token-based stall: last `minSteps` steps all produced < `threshold` output tokens. */
|
|
43
|
+
isTokenStalled(sessionId: string, minSteps: number, threshold: number): boolean {
|
|
44
|
+
const records = this.steps.get(sessionId)
|
|
45
|
+
if (!records || records.length < minSteps) return false
|
|
46
|
+
const recent = records.slice(-minSteps)
|
|
47
|
+
return recent.every(r => r.outputTokens < threshold)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Time-based stall: no message or task completion within `thresholdMs` of now. */
|
|
51
|
+
isTimeStalled(sessionId: string, thresholdMs: number): boolean {
|
|
52
|
+
const records = this.steps.get(sessionId)
|
|
53
|
+
if (!records || records.length === 0) return false
|
|
54
|
+
|
|
55
|
+
const msgAt = this.lastMessageAt.get(sessionId) ?? 0
|
|
56
|
+
const taskAt = this.lastTaskAt.get(sessionId) ?? 0
|
|
57
|
+
const lastStep = records[records.length - 1]?.timestamp ?? 0
|
|
58
|
+
const baseline = Math.max(msgAt, taskAt, lastStep)
|
|
59
|
+
|
|
60
|
+
return Date.now() - baseline >= thresholdMs
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Mark this session as reported-stalled (avoid spam). */
|
|
64
|
+
markReported(sessionId: string): void {
|
|
65
|
+
this.reported.add(sessionId)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Check if already reported. */
|
|
69
|
+
isReported(sessionId: string): boolean {
|
|
70
|
+
return this.reported.has(sessionId)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Clear stall report (called on new activity). */
|
|
74
|
+
clearReport(sessionId: string): void {
|
|
75
|
+
this.reported.delete(sessionId)
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Remove all tracking for a session (on shutdown). */
|
|
79
|
+
remove(sessionId: string): void {
|
|
80
|
+
this.steps.delete(sessionId)
|
|
81
|
+
this.lastMessageAt.delete(sessionId)
|
|
82
|
+
this.lastTaskAt.delete(sessionId)
|
|
83
|
+
this.reported.delete(sessionId)
|
|
84
|
+
}
|
|
85
|
+
}
|
package/src/system-prompt.ts
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite"
|
|
2
2
|
import { markDelivered } from "./messaging"
|
|
3
3
|
|
|
4
|
+
/** Truncate a string to maxLen chars, appending "..." if truncated. */
|
|
5
|
+
function truncate(s: string, maxLen: number): string {
|
|
6
|
+
return s.length > maxLen ? `${s.slice(0, maxLen)}...` : s
|
|
7
|
+
}
|
|
8
|
+
|
|
4
9
|
/** Status display mapping from DB status to human-readable label. */
|
|
5
10
|
const STATUS_DISPLAY: Record<string, string> = {
|
|
6
11
|
busy: "working",
|
|
@@ -51,6 +56,28 @@ export function buildLeadSystemPrompt(db: Database, teamId: string): string {
|
|
|
51
56
|
`Tasks: ${completed} completed, ${inProgress} in progress, ${pending} pending`,
|
|
52
57
|
]
|
|
53
58
|
|
|
59
|
+
// Inline active and recently completed tasks
|
|
60
|
+
const activeTasks = db.query(
|
|
61
|
+
"SELECT content, assignee FROM team_task WHERE team_id = ? AND status = 'in_progress' ORDER BY time_updated DESC LIMIT 5"
|
|
62
|
+
).all(teamId) as Array<{ content: string; assignee: string | null }>
|
|
63
|
+
|
|
64
|
+
const recentCompleted = db.query(
|
|
65
|
+
"SELECT content, assignee FROM team_task WHERE team_id = ? AND status = 'completed' ORDER BY time_updated DESC LIMIT 3"
|
|
66
|
+
).all(teamId) as Array<{ content: string; assignee: string | null }>
|
|
67
|
+
|
|
68
|
+
if (activeTasks.length > 0) {
|
|
69
|
+
lines.push("Active tasks:")
|
|
70
|
+
for (const t of activeTasks) {
|
|
71
|
+
lines.push(` [in_progress] ${truncate(t.content, 120)}${t.assignee ? ` → ${t.assignee}` : ""}`)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
if (recentCompleted.length > 0) {
|
|
75
|
+
lines.push("Recently completed:")
|
|
76
|
+
for (const t of recentCompleted) {
|
|
77
|
+
lines.push(` [completed] ${truncate(t.content, 120)}${t.assignee ? ` → ${t.assignee}` : ""}`)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
54
81
|
if (pendingMessages.length > 0) {
|
|
55
82
|
lines.push("", "--- Team Messages ---")
|
|
56
83
|
const MAX_MSG = 500
|
|
@@ -92,7 +119,7 @@ export function buildTeammateSystemPrompt(db: Database, teamId: string, memberNa
|
|
|
92
119
|
const team = db.query("SELECT name FROM team WHERE id = ?").get(teamId) as { name: string } | null
|
|
93
120
|
if (!team) return ""
|
|
94
121
|
|
|
95
|
-
return `You are "${memberName}", a teammate in team "${team.name}". Use team_message to communicate.`
|
|
122
|
+
return `You are "${memberName}", a teammate in team "${team.name}". Use team_message to communicate. You MUST send your results to the lead via team_message before stopping.`
|
|
96
123
|
}
|
|
97
124
|
|
|
98
125
|
/**
|
|
@@ -135,9 +162,48 @@ export function buildTeamCompactionContext(
|
|
|
135
162
|
? `Members: ${memberList}`
|
|
136
163
|
: "Members: none"
|
|
137
164
|
|
|
138
|
-
|
|
165
|
+
const lines = [
|
|
139
166
|
roleLine,
|
|
140
167
|
membersLine,
|
|
141
168
|
`Tasks: ${completed} completed, ${inProgress} in progress, ${pending} pending`,
|
|
142
|
-
]
|
|
169
|
+
]
|
|
170
|
+
|
|
171
|
+
if (role === "member" && memberName) {
|
|
172
|
+
lines.push("IMPORTANT: You MUST send your results to the lead via team_message before stopping.")
|
|
173
|
+
|
|
174
|
+
// Include original task prompt
|
|
175
|
+
const member = db.query("SELECT prompt FROM team_member WHERE team_id = ? AND name = ?")
|
|
176
|
+
.get(teamId, memberName) as { prompt: string | null } | null
|
|
177
|
+
if (member?.prompt) {
|
|
178
|
+
lines.push(`Your original task: ${truncate(member.prompt, 300)}`)
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Include recent messages involving this member
|
|
182
|
+
const recentMsgs = db.query(
|
|
183
|
+
"SELECT from_name, content FROM team_message WHERE team_id = ? AND (from_name = ? OR to_name = ?) ORDER BY time_created DESC LIMIT 3"
|
|
184
|
+
).all(teamId, memberName, memberName) as Array<{ from_name: string; content: string }>
|
|
185
|
+
if (recentMsgs.length > 0) {
|
|
186
|
+
lines.push("Recent context:")
|
|
187
|
+
for (const msg of recentMsgs) {
|
|
188
|
+
lines.push(` [${msg.from_name}]: ${truncate(msg.content, 200)}`)
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
} else if (role === "member") {
|
|
192
|
+
lines.push("IMPORTANT: You MUST send your results to the lead via team_message before stopping.")
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
if (role === "lead") {
|
|
196
|
+
// Include recently completed tasks
|
|
197
|
+
const completedTasks = db.query(
|
|
198
|
+
"SELECT content, assignee FROM team_task WHERE team_id = ? AND status = 'completed' ORDER BY time_updated DESC LIMIT 5"
|
|
199
|
+
).all(teamId) as Array<{ content: string; assignee: string | null }>
|
|
200
|
+
if (completedTasks.length > 0) {
|
|
201
|
+
lines.push("Recently completed:")
|
|
202
|
+
for (const t of completedTasks) {
|
|
203
|
+
lines.push(` [completed] ${truncate(t.content, 120)}${t.assignee ? ` (by ${t.assignee})` : ""}`)
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
return lines.join("\n")
|
|
143
209
|
}
|
|
@@ -1,8 +1,64 @@
|
|
|
1
1
|
import type { ToolDeps } from "../types"
|
|
2
2
|
import { requireLead, checkWorktreeDirty } from "./shared"
|
|
3
3
|
import type { IsDirtyFn } from "./shared"
|
|
4
|
+
import { spawnFailures } from "./team-spawn"
|
|
4
5
|
import { log } from "../log"
|
|
5
6
|
|
|
7
|
+
/** Result of merging all branches. */
|
|
8
|
+
export interface MergeAllResult {
|
|
9
|
+
merged: string[]
|
|
10
|
+
conflicted: string[]
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** Injectable function that merges all branches into the working directory. */
|
|
14
|
+
export type MergeAllFn = (branches: string[], cwd: string) => Promise<MergeAllResult>
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Default merge implementation: stash user work, squash-merge each branch,
|
|
18
|
+
* unstage merged changes, restore user work. One stash/pop cycle for all branches.
|
|
19
|
+
*/
|
|
20
|
+
export async function mergeAllBranches(branches: string[], cwd: string): Promise<MergeAllResult> {
|
|
21
|
+
const merged: string[] = []
|
|
22
|
+
const conflicted: string[] = []
|
|
23
|
+
|
|
24
|
+
// Stash existing work to preserve staged/unstaged state
|
|
25
|
+
const stash = Bun.spawn(["git", "stash", "--include-untracked"], { cwd, stdout: "pipe", stderr: "pipe" })
|
|
26
|
+
const stashOut = await new Response(stash.stdout).text()
|
|
27
|
+
const stashExit = await stash.exited
|
|
28
|
+
const didStash = stashExit === 0 && !stashOut.includes("No local changes")
|
|
29
|
+
|
|
30
|
+
for (const branch of branches) {
|
|
31
|
+
const merge = Bun.spawn(["git", "merge", "--squash", branch], { cwd, stdout: "pipe", stderr: "pipe" })
|
|
32
|
+
const stderrPromise = new Response(merge.stderr).text()
|
|
33
|
+
const exit = await merge.exited
|
|
34
|
+
if (exit !== 0) {
|
|
35
|
+
const stderr = await stderrPromise
|
|
36
|
+
log(`cleanup:merge:conflict branch=${branch} err=${stderr.trim()}`)
|
|
37
|
+
const abort = Bun.spawn(["git", "merge", "--abort"], { cwd, stdout: "pipe", stderr: "pipe" })
|
|
38
|
+
await abort.exited
|
|
39
|
+
conflicted.push(branch)
|
|
40
|
+
} else {
|
|
41
|
+
const del = Bun.spawn(["git", "branch", "-D", branch], { cwd, stdout: "pipe", stderr: "pipe" })
|
|
42
|
+
await del.exited
|
|
43
|
+
merged.push(branch)
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Unstage merged changes so user reviews with `git diff`
|
|
48
|
+
if (merged.length > 0) {
|
|
49
|
+
const reset = Bun.spawn(["git", "reset", "HEAD"], { cwd, stdout: "pipe", stderr: "pipe" })
|
|
50
|
+
await reset.exited
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Restore user's original staged/unstaged work
|
|
54
|
+
if (didStash) {
|
|
55
|
+
const pop = Bun.spawn(["git", "stash", "pop", "--index"], { cwd, stdout: "pipe", stderr: "pipe" })
|
|
56
|
+
await pop.exited
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return { merged, conflicted }
|
|
60
|
+
}
|
|
61
|
+
|
|
6
62
|
/**
|
|
7
63
|
* Execute the team_cleanup tool. Archives the team and cleans up resources.
|
|
8
64
|
* Checks for uncommitted changes BEFORE aborting sessions or removing worktrees.
|
|
@@ -14,6 +70,8 @@ export async function executeTeamCleanup(
|
|
|
14
70
|
args: { force: boolean; acknowledge_uncommitted?: boolean },
|
|
15
71
|
sessionId: string,
|
|
16
72
|
isDirty: IsDirtyFn = checkWorktreeDirty,
|
|
73
|
+
mergeAll: MergeAllFn = mergeAllBranches,
|
|
74
|
+
mergeOnCleanup = true,
|
|
17
75
|
): Promise<string> {
|
|
18
76
|
const teamInfo = requireLead(deps, sessionId)
|
|
19
77
|
|
|
@@ -57,8 +115,25 @@ export async function executeTeamCleanup(
|
|
|
57
115
|
}
|
|
58
116
|
}
|
|
59
117
|
|
|
60
|
-
//
|
|
61
|
-
const branches
|
|
118
|
+
// Collect branches and merge BEFORE removing worktrees (worktree.remove may delete the branch ref)
|
|
119
|
+
const branches = members
|
|
120
|
+
.map(m => m.worktree_branch)
|
|
121
|
+
.filter((b): b is string => b !== null)
|
|
122
|
+
|
|
123
|
+
let mergeResult = ""
|
|
124
|
+
if (branches.length > 0 && mergeOnCleanup) {
|
|
125
|
+
const { merged, conflicted } = await mergeAll(branches, deps.directory)
|
|
126
|
+
const parts: string[] = []
|
|
127
|
+
if (merged.length > 0) {
|
|
128
|
+
parts.push(`Merged ${merged.length} branch(es) into working directory (unstaged). Review changes with: git diff`)
|
|
129
|
+
}
|
|
130
|
+
if (conflicted.length > 0) {
|
|
131
|
+
parts.push(`Could not auto-merge: ${conflicted.join(", ")}. Merge manually with: git merge <branch>`)
|
|
132
|
+
}
|
|
133
|
+
mergeResult = parts.join("\n")
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Remove workspaces and worktrees (after merging)
|
|
62
137
|
for (const member of members) {
|
|
63
138
|
if (member.workspace_id) {
|
|
64
139
|
try {
|
|
@@ -72,16 +147,18 @@ export async function executeTeamCleanup(
|
|
|
72
147
|
deps.db.run("UPDATE team_member SET worktree_dir = NULL WHERE team_id = ? AND name = ?", [teamInfo.teamId, member.name])
|
|
73
148
|
} catch { /* best effort — worktree may already be gone */ }
|
|
74
149
|
}
|
|
75
|
-
if (member.worktree_branch) {
|
|
76
|
-
branches.push(member.worktree_branch)
|
|
77
|
-
}
|
|
78
150
|
}
|
|
79
151
|
|
|
80
152
|
// Archive team
|
|
81
153
|
deps.db.run("UPDATE team SET status = 'archived', time_updated = ? WHERE id = ?", [Date.now(), teamInfo.teamId])
|
|
82
154
|
|
|
83
|
-
// Clean up
|
|
155
|
+
// Clean up in-memory state
|
|
84
156
|
deps.registry.unregisterTeam(teamInfo.teamId)
|
|
157
|
+
spawnFailures.delete(teamInfo.teamId)
|
|
158
|
+
|
|
159
|
+
if (mergeResult) {
|
|
160
|
+
return `Team "${teamInfo.teamName}" cleaned up.\n${mergeResult}`
|
|
161
|
+
}
|
|
85
162
|
|
|
86
163
|
if (branches.length > 0) {
|
|
87
164
|
const mergeCommands = branches.map(b => ` git merge ${b}`).join("\n")
|
package/src/tools/team-spawn.ts
CHANGED
|
@@ -4,6 +4,9 @@ import { requireLead } from "./shared"
|
|
|
4
4
|
import { sendMessage } from "../messaging"
|
|
5
5
|
import { log } from "../log"
|
|
6
6
|
|
|
7
|
+
/** Tracks consecutive spawn failures per team for circuit breaker. */
|
|
8
|
+
export const spawnFailures = new Map<string, { count: number; lastError: string }>()
|
|
9
|
+
|
|
7
10
|
/** Timeout for worktree.create and session.create to prevent hanging on git lock contention. */
|
|
8
11
|
function getSpawnTimeout(): number {
|
|
9
12
|
return Number(process.env.SPAWN_TIMEOUT_MS) || 120_000
|
|
@@ -41,6 +44,12 @@ export async function executeTeamSpawn(
|
|
|
41
44
|
|
|
42
45
|
const teamInfo = requireLead(deps, sessionId)
|
|
43
46
|
|
|
47
|
+
// Circuit breaker — stop retrying after 3 consecutive failures
|
|
48
|
+
const failures = spawnFailures.get(teamInfo.teamId)
|
|
49
|
+
if (failures && failures.count >= 3) {
|
|
50
|
+
throw new Error(`Spawn circuit breaker tripped for team "${teamInfo.teamName}": 3 consecutive failures. Last error: ${failures.lastError}. Investigate before retrying — the circuit breaker resets on the next successful spawn.`)
|
|
51
|
+
}
|
|
52
|
+
|
|
44
53
|
// Check duplicate name
|
|
45
54
|
const existing = deps.db.query("SELECT name FROM team_member WHERE team_id = ? AND name = ?")
|
|
46
55
|
.get(teamInfo.teamId, args.name)
|
|
@@ -146,6 +155,10 @@ export async function executeTeamSpawn(
|
|
|
146
155
|
log(`spawn:session:done name=${args.name} sessionId=${childSessionId}`)
|
|
147
156
|
} catch (err) {
|
|
148
157
|
log(`spawn:session:failed name=${args.name} err=${err instanceof Error ? err.message : String(err)}`)
|
|
158
|
+
// Track failure for circuit breaker
|
|
159
|
+
const errMsg = err instanceof Error ? err.message : String(err)
|
|
160
|
+
const prev = spawnFailures.get(teamInfo.teamId)
|
|
161
|
+
spawnFailures.set(teamInfo.teamId, { count: (prev?.count ?? 0) + 1, lastError: errMsg })
|
|
149
162
|
// Rollback workspace and worktree if session creation failed
|
|
150
163
|
if (workspaceId) {
|
|
151
164
|
try { await deps.client.workspace.remove({ id: workspaceId }) } catch { /* best effort */ }
|
|
@@ -319,6 +332,8 @@ export async function executeTeamSpawn(
|
|
|
319
332
|
|
|
320
333
|
const branchInfo = worktreeBranch ? ` (branch: ${worktreeBranch})` : ""
|
|
321
334
|
const planInfo = usePlanApproval ? " [plan mode — will send plan for approval]" : ""
|
|
335
|
+
// Reset circuit breaker on success
|
|
336
|
+
spawnFailures.delete(teamInfo.teamId)
|
|
322
337
|
log(`spawn:done name=${args.name} sessionId=${childSessionId}`)
|
|
323
338
|
return `Teammate "${args.name}" spawned (agent: ${args.agent})${branchInfo}${planInfo}. They are working on: ${args.prompt.slice(0, 120)}${args.prompt.length > 120 ? "..." : ""}`
|
|
324
339
|
}
|
package/src/tools/team-status.ts
CHANGED
|
@@ -6,6 +6,13 @@ export const lastCallTime = new Map<string, number>()
|
|
|
6
6
|
|
|
7
7
|
const RATE_LIMIT_MS = 30_000
|
|
8
8
|
|
|
9
|
+
/** Format a duration in ms to a human-readable string. */
|
|
10
|
+
export function formatDuration(ms: number): string {
|
|
11
|
+
if (ms < 60_000) return `${Math.floor(ms / 1000)}s`
|
|
12
|
+
if (ms < 3_600_000) return `${Math.floor(ms / 60_000)}m`
|
|
13
|
+
return `${Math.floor(ms / 3_600_000)}h`
|
|
14
|
+
}
|
|
15
|
+
|
|
9
16
|
/**
|
|
10
17
|
* Execute the team_status tool. Shows team overview with member statuses and task summary.
|
|
11
18
|
*/
|
|
@@ -23,9 +30,9 @@ export async function executeTeamStatus(
|
|
|
23
30
|
lastCallTime.set(teamInfo.teamId, now)
|
|
24
31
|
|
|
25
32
|
const members = deps.db.query(
|
|
26
|
-
"SELECT name, session_id, agent, status, execution_status, worktree_branch, worktree_dir, plan_approval FROM team_member WHERE team_id = ? ORDER BY time_created ASC"
|
|
33
|
+
"SELECT name, session_id, agent, status, execution_status, worktree_branch, worktree_dir, plan_approval, time_updated FROM team_member WHERE team_id = ? ORDER BY time_created ASC"
|
|
27
34
|
).all(teamInfo.teamId) as Array<{
|
|
28
|
-
name: string; session_id: string; agent: string; status: string; execution_status: string; worktree_branch: string | null; worktree_dir: string | null; plan_approval: string
|
|
35
|
+
name: string; session_id: string; agent: string; status: string; execution_status: string; worktree_branch: string | null; worktree_dir: string | null; plan_approval: string; time_updated: number
|
|
29
36
|
}>
|
|
30
37
|
|
|
31
38
|
const tasks = deps.db.query(
|
|
@@ -42,9 +49,24 @@ export async function executeTeamStatus(
|
|
|
42
49
|
lines.push("Members:")
|
|
43
50
|
for (const m of members) {
|
|
44
51
|
const statusIcon = m.status === "busy" ? "working" : m.status === "ready" ? "idle" : m.status
|
|
52
|
+
const duration = formatDuration(now - m.time_updated)
|
|
45
53
|
const branch = m.worktree_branch ? ` branch: ${m.worktree_branch}` : ""
|
|
46
54
|
const plan = m.plan_approval !== "none" ? `, plan: ${m.plan_approval}` : ""
|
|
47
|
-
|
|
55
|
+
|
|
56
|
+
// Last message time
|
|
57
|
+
const lastMsg = deps.db.query("SELECT MAX(time_created) as last_msg FROM team_message WHERE team_id = ? AND from_name = ?")
|
|
58
|
+
.get(teamInfo.teamId, m.name) as { last_msg: number | null } | null
|
|
59
|
+
const msgInfo = lastMsg?.last_msg ? `last msg: ${formatDuration(now - lastMsg.last_msg)} ago` : "no messages yet"
|
|
60
|
+
|
|
61
|
+
lines.push(` ${m.name} [${statusIcon} ${duration}, ${msgInfo}${plan}] agent: ${m.agent}${branch}`)
|
|
62
|
+
|
|
63
|
+
// Current task
|
|
64
|
+
const task = deps.db.query("SELECT content FROM team_task WHERE team_id = ? AND assignee = ? AND status = 'in_progress' LIMIT 1")
|
|
65
|
+
.get(teamInfo.teamId, m.name) as { content: string } | null
|
|
66
|
+
if (task) {
|
|
67
|
+
const truncated = task.content.length > 80 ? `${task.content.slice(0, 80)}...` : task.content
|
|
68
|
+
lines.push(` task: ${truncated}`)
|
|
69
|
+
}
|
|
48
70
|
if (m.worktree_dir) {
|
|
49
71
|
lines.push(` worktree: ${m.worktree_dir}`)
|
|
50
72
|
}
|
package/src/watchdog.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite"
|
|
2
2
|
import type { PluginClient } from "./types"
|
|
3
3
|
import type { MemberRegistry } from "./state"
|
|
4
|
+
import type { ProgressTracker } from "./progress"
|
|
5
|
+
import { sendMessage } from "./messaging"
|
|
4
6
|
|
|
5
7
|
interface WatchdogOpts {
|
|
6
8
|
db: Database
|
|
@@ -10,6 +12,14 @@ interface WatchdogOpts {
|
|
|
10
12
|
ttlMs: number
|
|
11
13
|
/** How often to run the check. Defaults to 60s. */
|
|
12
14
|
checkIntervalMs?: number
|
|
15
|
+
/** Progress tracker for stall detection. */
|
|
16
|
+
progressTracker?: ProgressTracker
|
|
17
|
+
/** Stall detection threshold in ms. 0 disables. */
|
|
18
|
+
stallThresholdMs?: number
|
|
19
|
+
/** Min steps before token-based stall check. */
|
|
20
|
+
stallMinSteps?: number
|
|
21
|
+
/** Output token threshold for stall detection. */
|
|
22
|
+
stallTokenThreshold?: number
|
|
13
23
|
}
|
|
14
24
|
|
|
15
25
|
/**
|
|
@@ -22,6 +32,10 @@ export class Watchdog {
|
|
|
22
32
|
private readonly registry: MemberRegistry
|
|
23
33
|
private readonly ttlMs: number
|
|
24
34
|
private readonly checkIntervalMs: number
|
|
35
|
+
private readonly progressTracker?: ProgressTracker
|
|
36
|
+
private readonly stallThresholdMs: number
|
|
37
|
+
private readonly stallMinSteps: number
|
|
38
|
+
private readonly stallTokenThreshold: number
|
|
25
39
|
private timer: ReturnType<typeof setInterval> | undefined
|
|
26
40
|
|
|
27
41
|
constructor(opts: WatchdogOpts) {
|
|
@@ -30,6 +44,10 @@ export class Watchdog {
|
|
|
30
44
|
this.registry = opts.registry
|
|
31
45
|
this.ttlMs = opts.ttlMs
|
|
32
46
|
this.checkIntervalMs = opts.checkIntervalMs ?? 60_000
|
|
47
|
+
this.progressTracker = opts.progressTracker
|
|
48
|
+
this.stallThresholdMs = opts.stallThresholdMs ?? 0
|
|
49
|
+
this.stallMinSteps = opts.stallMinSteps ?? 3
|
|
50
|
+
this.stallTokenThreshold = opts.stallTokenThreshold ?? 500
|
|
33
51
|
}
|
|
34
52
|
|
|
35
53
|
private static STALE_THRESHOLD_MS = Number(process.env.STALE_WORKTREE_THRESHOLD_MS) || 300_000
|
|
@@ -61,9 +79,58 @@ export class Watchdog {
|
|
|
61
79
|
}
|
|
62
80
|
}
|
|
63
81
|
|
|
82
|
+
/** Check for stalled busy members and escalate to lead + nudge teammate. */
|
|
83
|
+
async checkStalled(): Promise<void> {
|
|
84
|
+
if (!this.progressTracker || this.stallThresholdMs === 0) return
|
|
85
|
+
|
|
86
|
+
const busy = this.db.query(
|
|
87
|
+
`SELECT tm.team_id, tm.name, tm.session_id
|
|
88
|
+
FROM team_member tm
|
|
89
|
+
JOIN team t ON tm.team_id = t.id
|
|
90
|
+
WHERE t.status = 'active' AND tm.status = 'busy'`
|
|
91
|
+
).all() as Array<{ team_id: string; name: string; session_id: string }>
|
|
92
|
+
|
|
93
|
+
for (const member of busy) {
|
|
94
|
+
if (this.progressTracker.isReported(member.session_id)) continue
|
|
95
|
+
|
|
96
|
+
const tokenStalled = this.progressTracker.isTokenStalled(member.session_id, this.stallMinSteps, this.stallTokenThreshold)
|
|
97
|
+
const timeStalled = this.progressTracker.isTimeStalled(member.session_id, this.stallThresholdMs)
|
|
98
|
+
|
|
99
|
+
if (!tokenStalled && !timeStalled) continue
|
|
100
|
+
|
|
101
|
+
this.progressTracker.markReported(member.session_id)
|
|
102
|
+
const reason = tokenStalled ? "low output tokens" : "no communication"
|
|
103
|
+
|
|
104
|
+
// Nudge the teammate directly
|
|
105
|
+
this.client.session.promptAsync({
|
|
106
|
+
sessionID: member.session_id,
|
|
107
|
+
parts: [{ type: "text", text: "[System]: You appear stalled — no progress detected. Report your current status to the lead via team_message, or wrap up your work." }],
|
|
108
|
+
}).catch(() => { /* best effort */ })
|
|
109
|
+
|
|
110
|
+
// Notify the lead
|
|
111
|
+
sendMessage(this.db, {
|
|
112
|
+
teamId: member.team_id,
|
|
113
|
+
from: "system",
|
|
114
|
+
to: "lead",
|
|
115
|
+
content: `Teammate "${member.name}" appears stalled (${reason}). Consider checking on them via team_message or shutting them down.`,
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
// Toast for the user
|
|
119
|
+
try {
|
|
120
|
+
await this.client.tui.showToast({
|
|
121
|
+
title: "Team",
|
|
122
|
+
message: `${member.name} appears stalled`,
|
|
123
|
+
variant: "warning",
|
|
124
|
+
duration: 5000,
|
|
125
|
+
})
|
|
126
|
+
} catch { /* TUI may not be available */ }
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
64
130
|
/** Run a single check for stale busy members. */
|
|
65
131
|
async check(): Promise<void> {
|
|
66
132
|
await this.cleanupStaleWorktrees()
|
|
133
|
+
await this.checkStalled()
|
|
67
134
|
if (this.ttlMs === 0) return
|
|
68
135
|
|
|
69
136
|
const cutoff = Date.now() - this.ttlMs
|