heyio 0.28.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/server.js +119 -1
- package/dist/copilot/agents.js +2 -2
- package/dist/copilot/orchestrator.js +24 -1
- package/dist/copilot/tools.js +183 -2
- package/dist/instance-watchdog.js +70 -0
- package/dist/instance-watchdog.test.js +112 -0
- package/dist/store/db.js +21 -0
- package/dist/store/instances.js +131 -0
- package/dist/store/instances.test.js +291 -0
- package/dist/store/tasks.js +2 -2
- package/dist/store/tasks.test.js +150 -0
- package/dist/store/worktrees.js +83 -0
- package/package.json +1 -1
- package/web-dist/assets/index-D3uXBVcQ.js +88 -0
- package/web-dist/assets/index-DmthMbtN.css +10 -0
- package/web-dist/index.html +2 -2
- package/web-dist/assets/index-C9tBcKPe.css +0 -10
- package/web-dist/assets/index-DZvNi6bo.js +0 -86
package/dist/api/server.js
CHANGED
|
@@ -4,7 +4,9 @@ import { existsSync, readFileSync } from "node:fs";
|
|
|
4
4
|
import express from "express";
|
|
5
5
|
import { config } from "../config.js";
|
|
6
6
|
import { listSkills, installSkill, installSkillFromContent, removeSkill } from "../copilot/skills.js";
|
|
7
|
-
import { listSquads, createSquad, listSquadAgents } from "../store/squads.js";
|
|
7
|
+
import { listSquads, createSquad, listSquadAgents, getSquad } from "../store/squads.js";
|
|
8
|
+
import { createInstance, getInstance, listInstances, updateInstanceStatus, getInstanceDecisions, mergeInstanceDecisions, buildContextSnapshot } from "../store/instances.js";
|
|
9
|
+
import { createWorktree, removeWorktree } from "../store/worktrees.js";
|
|
8
10
|
import { getAgentInfo, cancelAgentTask, getTaskEvents, subscribeToTaskEvents } from "../copilot/agents.js";
|
|
9
11
|
import { summarize, summarizeEvent } from "../copilot/event-summary.js";
|
|
10
12
|
import { abortOrchestrator } from "../copilot/orchestrator.js";
|
|
@@ -453,6 +455,122 @@ export async function startApiServer() {
|
|
|
453
455
|
res.status(500).json({ error: "Failed to list squad agents" });
|
|
454
456
|
}
|
|
455
457
|
});
|
|
458
|
+
// Squad Instances
|
|
459
|
+
api.get("/squads/:slug/instances", (req, res) => {
|
|
460
|
+
try {
|
|
461
|
+
const slug = Array.isArray(req.params.slug) ? req.params.slug[0] : req.params.slug;
|
|
462
|
+
const includeCompleted = req.query.include_completed === "true";
|
|
463
|
+
const instances = listInstances(slug, { includeCompleted });
|
|
464
|
+
res.json({ instances });
|
|
465
|
+
}
|
|
466
|
+
catch (e) {
|
|
467
|
+
console.error("Error listing instances:", e);
|
|
468
|
+
res.status(500).json({ error: e instanceof Error ? e.message : String(e) });
|
|
469
|
+
}
|
|
470
|
+
});
|
|
471
|
+
api.post("/squads/:slug/instances", (req, res) => {
|
|
472
|
+
try {
|
|
473
|
+
const slug = Array.isArray(req.params.slug) ? req.params.slug[0] : req.params.slug;
|
|
474
|
+
const { issue_ref, base_branch } = req.body;
|
|
475
|
+
const squad = getSquad(slug);
|
|
476
|
+
if (!squad) {
|
|
477
|
+
res.status(404).json({ error: "Squad not found" });
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
const sanitizedRef = (issue_ref ?? "task").replace(/[^a-zA-Z0-9_-]/g, "-").toLowerCase();
|
|
481
|
+
const instanceId = `${slug}--${sanitizedRef}`;
|
|
482
|
+
const branchName = `${slug}/instance/${sanitizedRef}`;
|
|
483
|
+
const contextSnapshot = buildContextSnapshot(slug);
|
|
484
|
+
const worktreePath = createWorktree(squad.project_path, instanceId, branchName, base_branch ?? "main");
|
|
485
|
+
let instance;
|
|
486
|
+
try {
|
|
487
|
+
instance = createInstance({
|
|
488
|
+
id: instanceId,
|
|
489
|
+
masterSquadSlug: slug,
|
|
490
|
+
issueRef: issue_ref,
|
|
491
|
+
worktreePath,
|
|
492
|
+
branchName,
|
|
493
|
+
contextSnapshot,
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
catch (createErr) {
|
|
497
|
+
// Roll back the worktree if DB insert fails (e.g. max instances exceeded)
|
|
498
|
+
removeWorktree(squad.project_path, worktreePath);
|
|
499
|
+
throw createErr;
|
|
500
|
+
}
|
|
501
|
+
updateInstanceStatus(instanceId, "active");
|
|
502
|
+
res.status(201).json({ instance: { ...instance, status: "active" } });
|
|
503
|
+
}
|
|
504
|
+
catch (e) {
|
|
505
|
+
console.error("Error creating instance:", e);
|
|
506
|
+
res.status(500).json({ error: e instanceof Error ? e.message : String(e) });
|
|
507
|
+
}
|
|
508
|
+
});
|
|
509
|
+
api.get("/squads/:slug/instances/:id", (req, res) => {
|
|
510
|
+
try {
|
|
511
|
+
const slug = Array.isArray(req.params.slug) ? req.params.slug[0] : req.params.slug;
|
|
512
|
+
const id = Array.isArray(req.params.id) ? req.params.id[0] : req.params.id;
|
|
513
|
+
const instance = getInstance(id);
|
|
514
|
+
if (!instance || instance.master_squad_slug !== slug) {
|
|
515
|
+
res.status(404).json({ error: "Instance not found" });
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
const decisions = getInstanceDecisions(id);
|
|
519
|
+
res.json({ instance, decisions });
|
|
520
|
+
}
|
|
521
|
+
catch (e) {
|
|
522
|
+
console.error("Error getting instance:", e);
|
|
523
|
+
res.status(500).json({ error: e instanceof Error ? e.message : String(e) });
|
|
524
|
+
}
|
|
525
|
+
});
|
|
526
|
+
api.post("/squads/:slug/instances/:id/complete", (req, res) => {
|
|
527
|
+
try {
|
|
528
|
+
const slug = Array.isArray(req.params.slug) ? req.params.slug[0] : req.params.slug;
|
|
529
|
+
const id = Array.isArray(req.params.id) ? req.params.id[0] : req.params.id;
|
|
530
|
+
const instance = getInstance(id);
|
|
531
|
+
if (!instance || instance.master_squad_slug !== slug) {
|
|
532
|
+
res.status(404).json({ error: "Instance not found" });
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
if (instance.status === "done") {
|
|
536
|
+
res.json({ message: "Already completed", merged: 0 });
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
updateInstanceStatus(id, "merging");
|
|
540
|
+
const merged = mergeInstanceDecisions(id, instance.master_squad_slug);
|
|
541
|
+
const squad = getSquad(instance.master_squad_slug);
|
|
542
|
+
if (squad) {
|
|
543
|
+
removeWorktree(squad.project_path, instance.worktree_path);
|
|
544
|
+
}
|
|
545
|
+
updateInstanceStatus(id, "done");
|
|
546
|
+
res.json({ message: "Instance completed", merged });
|
|
547
|
+
}
|
|
548
|
+
catch (e) {
|
|
549
|
+
console.error("Error completing instance:", e);
|
|
550
|
+
res.status(500).json({ error: e instanceof Error ? e.message : String(e) });
|
|
551
|
+
}
|
|
552
|
+
});
|
|
553
|
+
api.post("/squads/:slug/instances/:id/abort", (req, res) => {
|
|
554
|
+
try {
|
|
555
|
+
const slug = Array.isArray(req.params.slug) ? req.params.slug[0] : req.params.slug;
|
|
556
|
+
const id = Array.isArray(req.params.id) ? req.params.id[0] : req.params.id;
|
|
557
|
+
const instance = getInstance(id);
|
|
558
|
+
if (!instance || instance.master_squad_slug !== slug) {
|
|
559
|
+
res.status(404).json({ error: "Instance not found" });
|
|
560
|
+
return;
|
|
561
|
+
}
|
|
562
|
+
if (instance.status === "done" || instance.status === "failed") {
|
|
563
|
+
res.json({ message: `Already in terminal state: ${instance.status}` });
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
566
|
+
updateInstanceStatus(id, "failed");
|
|
567
|
+
res.json({ message: "Instance aborted", worktree_path: instance.worktree_path });
|
|
568
|
+
}
|
|
569
|
+
catch (e) {
|
|
570
|
+
console.error("Error aborting instance:", e);
|
|
571
|
+
res.status(500).json({ error: e instanceof Error ? e.message : String(e) });
|
|
572
|
+
}
|
|
573
|
+
});
|
|
456
574
|
// Agents endpoints
|
|
457
575
|
api.get("/agents", (_req, res) => {
|
|
458
576
|
try {
|
package/dist/copilot/agents.js
CHANGED
|
@@ -159,7 +159,7 @@ ${task}
|
|
|
159
159
|
|
|
160
160
|
${tail}`;
|
|
161
161
|
}
|
|
162
|
-
export async function delegateToAgent(squadSlug, task, onComplete, targetAgent) {
|
|
162
|
+
export async function delegateToAgent(squadSlug, task, onComplete, targetAgent, instanceId) {
|
|
163
163
|
const squad = getSquad(squadSlug);
|
|
164
164
|
if (!squad) {
|
|
165
165
|
throw new Error(`Squad not found: ${squadSlug}`);
|
|
@@ -206,7 +206,7 @@ export async function delegateToAgent(squadSlug, task, onComplete, targetAgent)
|
|
|
206
206
|
? await getOrCreateAgentSession(squadSlug, agent, task)
|
|
207
207
|
: await getOrCreateSession(squadSlug, task);
|
|
208
208
|
const taskId = randomUUID();
|
|
209
|
-
createTask(taskId, agentKey, task);
|
|
209
|
+
createTask(taskId, agentKey, task, undefined, instanceId);
|
|
210
210
|
updateSquadStatus(squadSlug, "working");
|
|
211
211
|
if (agent)
|
|
212
212
|
updateAgentStatus(squadSlug, agent.character_name, "working");
|
|
@@ -15,6 +15,9 @@ import { resetClient } from "./client.js";
|
|
|
15
15
|
import { delegateToAgent, getActiveAgentTasks, clearAgentInMemorySession } from "./agents.js";
|
|
16
16
|
import { saveConfig } from "../config.js";
|
|
17
17
|
import { checkForUpdate } from "../update.js";
|
|
18
|
+
import { startInstanceWatchdog } from "../instance-watchdog.js";
|
|
19
|
+
import { createInstance, getInstance, listInstances, updateInstanceStatus, logInstanceDecision, getInstanceDecisions, mergeInstanceDecisions, deleteInstance, buildContextSnapshot, reconcileInstances, ensureInstanceTables, } from "../store/instances.js";
|
|
20
|
+
import { createWorktree, removeWorktree } from "../store/worktrees.js";
|
|
18
21
|
// ---------------------------------------------------------------------------
|
|
19
22
|
// Constants
|
|
20
23
|
// ---------------------------------------------------------------------------
|
|
@@ -62,7 +65,7 @@ function getToolDeps() {
|
|
|
62
65
|
created_at: d.created_at,
|
|
63
66
|
})),
|
|
64
67
|
updateSquadStatus,
|
|
65
|
-
delegateToAgent,
|
|
68
|
+
delegateToAgent: (squadSlug, task, onComplete, targetAgent, instanceId) => delegateToAgent(squadSlug, task, onComplete, targetAgent, instanceId),
|
|
66
69
|
getTask,
|
|
67
70
|
getActiveAgentTasks: () => getActiveAgentTasks().map((t) => ({
|
|
68
71
|
taskId: t.taskId,
|
|
@@ -124,6 +127,20 @@ function getToolDeps() {
|
|
|
124
127
|
searchSkillsRegistry,
|
|
125
128
|
saveConfig,
|
|
126
129
|
checkForUpdate,
|
|
130
|
+
// Squad instance deps
|
|
131
|
+
createInstance,
|
|
132
|
+
getInstance,
|
|
133
|
+
listInstances,
|
|
134
|
+
updateInstanceStatus,
|
|
135
|
+
logInstanceDecision,
|
|
136
|
+
getInstanceDecisions,
|
|
137
|
+
mergeInstanceDecisions,
|
|
138
|
+
deleteInstance,
|
|
139
|
+
buildContextSnapshot,
|
|
140
|
+
reconcileInstances,
|
|
141
|
+
createWorktree,
|
|
142
|
+
removeWorktree,
|
|
143
|
+
activeInstanceId: undefined,
|
|
127
144
|
};
|
|
128
145
|
}
|
|
129
146
|
function getSessionConfig() {
|
|
@@ -407,6 +424,12 @@ async function processQueue() {
|
|
|
407
424
|
// ---------------------------------------------------------------------------
|
|
408
425
|
export async function initOrchestrator(copilotClient) {
|
|
409
426
|
client = copilotClient;
|
|
427
|
+
ensureInstanceTables();
|
|
428
|
+
const reconciledInstances = reconcileInstances();
|
|
429
|
+
if (reconciledInstances > 0) {
|
|
430
|
+
console.error(`[orchestrator] Reconciled ${reconciledInstances} stale instance(s) on startup`);
|
|
431
|
+
}
|
|
432
|
+
startInstanceWatchdog();
|
|
410
433
|
clearStaleTasks();
|
|
411
434
|
// Validate the configured model and resolve model tiers
|
|
412
435
|
try {
|
package/dist/copilot/tools.js
CHANGED
|
@@ -373,6 +373,11 @@ export function createTools(deps) {
|
|
|
373
373
|
}),
|
|
374
374
|
handler: async ({ slug, decision, context }) => {
|
|
375
375
|
try {
|
|
376
|
+
// If we're in an instance context, route to instance decisions
|
|
377
|
+
if (deps.activeInstanceId) {
|
|
378
|
+
deps.logInstanceDecision(deps.activeInstanceId, decision, context);
|
|
379
|
+
return `Decision logged for instance ${deps.activeInstanceId} (squad ${slug})`;
|
|
380
|
+
}
|
|
376
381
|
deps.logDecision(slug, decision, context);
|
|
377
382
|
return `Decision logged for squad ${slug}`;
|
|
378
383
|
}
|
|
@@ -405,7 +410,7 @@ export function createTools(deps) {
|
|
|
405
410
|
createFeedEntry({ type: "deliverable", title: `[${slug}] Task result`, body: result });
|
|
406
411
|
console.error(`[io] Task ${id} result routed to inbox`);
|
|
407
412
|
}
|
|
408
|
-
}, agent);
|
|
413
|
+
}, agent, deps.activeInstanceId);
|
|
409
414
|
const agentLabel = agent ? `agent "${agent}" in squad "${slug}"` : `squad "${slug}"`;
|
|
410
415
|
const warningPrefix = coverage.warning
|
|
411
416
|
? `${coverage.warning} A dedicated lead and a QA reviewer should both hold veto power on PR promotion — fix gaps before promoting work.\n\n`
|
|
@@ -1704,7 +1709,183 @@ export function createTools(deps) {
|
|
|
1704
1709
|
return `🚀 Fired IO schedule ${id} now.`;
|
|
1705
1710
|
},
|
|
1706
1711
|
});
|
|
1707
|
-
|
|
1712
|
+
// ---------------------------------------------------------------------------
|
|
1713
|
+
// Squad Instance tools (#231)
|
|
1714
|
+
// ---------------------------------------------------------------------------
|
|
1715
|
+
const squadInstanceCreate = defineTool("squad_instance_create", {
|
|
1716
|
+
description: "Spawn a parallel instance of a squad to work on a separate issue. Creates a git worktree for file isolation and snapshots the squad's current decisions as context.",
|
|
1717
|
+
skipPermission: true,
|
|
1718
|
+
parameters: z.object({
|
|
1719
|
+
squad_slug: z.string().describe("The squad to create an instance of"),
|
|
1720
|
+
issue_ref: z.string().describe("Issue reference or label (e.g. '#231', 'refactor-auth')"),
|
|
1721
|
+
base_branch: z.string().optional().describe("Branch to base the worktree on (default: 'main')"),
|
|
1722
|
+
}),
|
|
1723
|
+
handler: async ({ squad_slug, issue_ref, base_branch }) => {
|
|
1724
|
+
const squad = deps.getSquad(squad_slug);
|
|
1725
|
+
if (!squad)
|
|
1726
|
+
return `Squad not found: ${squad_slug}`;
|
|
1727
|
+
// Deterministic ID: allows idempotent re-creation if an instance for
|
|
1728
|
+
// the same issue_ref previously completed or failed.
|
|
1729
|
+
const sanitizedRef = issue_ref.replace(/[^a-zA-Z0-9_-]/g, "-").toLowerCase();
|
|
1730
|
+
const instanceId = `${squad_slug}--${sanitizedRef}`;
|
|
1731
|
+
const branchName = `${squad_slug}/instance/${sanitizedRef}`;
|
|
1732
|
+
const existing = deps.getInstance(instanceId);
|
|
1733
|
+
if (existing && existing.status !== "done" && existing.status !== "failed") {
|
|
1734
|
+
return `Instance "${instanceId}" already exists (status: ${existing.status})`;
|
|
1735
|
+
}
|
|
1736
|
+
try {
|
|
1737
|
+
const contextSnapshot = deps.buildContextSnapshot(squad_slug);
|
|
1738
|
+
const worktreePath = deps.createWorktree(squad.projectPath, instanceId, branchName, base_branch ?? "main");
|
|
1739
|
+
deps.createInstance({
|
|
1740
|
+
id: instanceId,
|
|
1741
|
+
masterSquadSlug: squad_slug,
|
|
1742
|
+
issueRef: issue_ref,
|
|
1743
|
+
worktreePath,
|
|
1744
|
+
branchName,
|
|
1745
|
+
contextSnapshot,
|
|
1746
|
+
});
|
|
1747
|
+
deps.updateInstanceStatus(instanceId, "active");
|
|
1748
|
+
const inherited = JSON.parse(contextSnapshot);
|
|
1749
|
+
return `Instance "${instanceId}" created.\nWorktree: ${worktreePath}\nBranch: ${branchName}\nStatus: active\nContext: ${inherited.length} decisions inherited`;
|
|
1750
|
+
}
|
|
1751
|
+
catch (err) {
|
|
1752
|
+
return `Error creating instance: ${err instanceof Error ? err.message : String(err)}`;
|
|
1753
|
+
}
|
|
1754
|
+
},
|
|
1755
|
+
});
|
|
1756
|
+
const squadInstanceList = defineTool("squad_instance_list", {
|
|
1757
|
+
description: "List active (and optionally completed) instances for a squad.",
|
|
1758
|
+
skipPermission: true,
|
|
1759
|
+
parameters: z.object({
|
|
1760
|
+
squad_slug: z.string().describe("Squad slug"),
|
|
1761
|
+
include_completed: z.boolean().optional().describe("Include done/failed instances (default: false)"),
|
|
1762
|
+
}),
|
|
1763
|
+
handler: async ({ squad_slug, include_completed }) => {
|
|
1764
|
+
const instances = deps.listInstances(squad_slug, { includeCompleted: include_completed ?? false });
|
|
1765
|
+
if (instances.length === 0)
|
|
1766
|
+
return `No instances for squad "${squad_slug}".`;
|
|
1767
|
+
return instances.map(i => `- **${i.id}** [${i.status}] — ${i.issue_ref ?? "no issue"} (branch: ${i.branch_name}, created: ${i.created_at})`).join("\n");
|
|
1768
|
+
},
|
|
1769
|
+
});
|
|
1770
|
+
const squadInstanceStatus = defineTool("squad_instance_status", {
|
|
1771
|
+
description: "Get detailed status of a specific squad instance.",
|
|
1772
|
+
skipPermission: true,
|
|
1773
|
+
parameters: z.object({
|
|
1774
|
+
instance_id: z.string().describe("Instance ID (e.g. 'my-squad--issue-42')"),
|
|
1775
|
+
}),
|
|
1776
|
+
handler: async ({ instance_id }) => {
|
|
1777
|
+
const instance = deps.getInstance(instance_id);
|
|
1778
|
+
if (!instance)
|
|
1779
|
+
return `Instance not found: ${instance_id}`;
|
|
1780
|
+
const decisions = deps.getInstanceDecisions(instance_id);
|
|
1781
|
+
return [
|
|
1782
|
+
`## Instance: ${instance.id}`,
|
|
1783
|
+
`- Squad: ${instance.master_squad_slug}`,
|
|
1784
|
+
`- Issue: ${instance.issue_ref ?? "none"}`,
|
|
1785
|
+
`- Status: ${instance.status}`,
|
|
1786
|
+
`- Branch: ${instance.branch_name}`,
|
|
1787
|
+
`- Worktree: ${instance.worktree_path}`,
|
|
1788
|
+
`- Created: ${instance.created_at}`,
|
|
1789
|
+
instance.completed_at ? `- Completed: ${instance.completed_at}` : null,
|
|
1790
|
+
`- Decisions: ${decisions.length} (${decisions.filter(d => d.merged_to_master).length} merged)`,
|
|
1791
|
+
].filter(Boolean).join("\n");
|
|
1792
|
+
},
|
|
1793
|
+
});
|
|
1794
|
+
const squadInstanceComplete = defineTool("squad_instance_complete", {
|
|
1795
|
+
description: "Complete a squad instance: merge its decisions back to the master squad and clean up the worktree.",
|
|
1796
|
+
skipPermission: true,
|
|
1797
|
+
parameters: z.object({
|
|
1798
|
+
instance_id: z.string().describe("Instance ID to complete"),
|
|
1799
|
+
}),
|
|
1800
|
+
handler: async ({ instance_id }) => {
|
|
1801
|
+
const instance = deps.getInstance(instance_id);
|
|
1802
|
+
if (!instance)
|
|
1803
|
+
return `Instance not found: ${instance_id}`;
|
|
1804
|
+
if (instance.status === "done")
|
|
1805
|
+
return `Instance already completed.`;
|
|
1806
|
+
try {
|
|
1807
|
+
deps.updateInstanceStatus(instance_id, "merging");
|
|
1808
|
+
const merged = deps.mergeInstanceDecisions(instance_id, instance.master_squad_slug);
|
|
1809
|
+
// Clean up worktree — use squad's project_path if available, fall back to stored path
|
|
1810
|
+
const squad = deps.getSquad(instance.master_squad_slug);
|
|
1811
|
+
const projectPath = squad?.projectPath ?? instance.worktree_path.replace(/\/\.io-worktrees\/.*$/, "");
|
|
1812
|
+
deps.removeWorktree(projectPath, instance.worktree_path);
|
|
1813
|
+
deps.updateInstanceStatus(instance_id, "done");
|
|
1814
|
+
return `Instance "${instance_id}" completed.\n- ${merged} decision(s) merged to master squad "${instance.master_squad_slug}"\n- Worktree cleaned up`;
|
|
1815
|
+
}
|
|
1816
|
+
catch (err) {
|
|
1817
|
+
return `Error completing instance: ${err instanceof Error ? err.message : String(err)}`;
|
|
1818
|
+
}
|
|
1819
|
+
},
|
|
1820
|
+
});
|
|
1821
|
+
const squadInstanceAbort = defineTool("squad_instance_abort", {
|
|
1822
|
+
description: "Abort a squad instance, marking it as failed. Worktree is preserved for debugging.",
|
|
1823
|
+
skipPermission: true,
|
|
1824
|
+
parameters: z.object({
|
|
1825
|
+
instance_id: z.string().describe("Instance ID to abort"),
|
|
1826
|
+
}),
|
|
1827
|
+
handler: async ({ instance_id }) => {
|
|
1828
|
+
const instance = deps.getInstance(instance_id);
|
|
1829
|
+
if (!instance)
|
|
1830
|
+
return `Instance not found: ${instance_id}`;
|
|
1831
|
+
if (instance.status === "done" || instance.status === "failed") {
|
|
1832
|
+
return `Instance already in terminal state: ${instance.status}`;
|
|
1833
|
+
}
|
|
1834
|
+
deps.updateInstanceStatus(instance_id, "failed");
|
|
1835
|
+
return `Instance "${instance_id}" aborted. Worktree preserved at: ${instance.worktree_path}\nUse squad_instance_cleanup to remove it.`;
|
|
1836
|
+
},
|
|
1837
|
+
});
|
|
1838
|
+
const squadInstanceCleanup = defineTool("squad_instance_cleanup", {
|
|
1839
|
+
description: "Force-remove a failed instance's worktree and delete the instance record.",
|
|
1840
|
+
skipPermission: true,
|
|
1841
|
+
parameters: z.object({
|
|
1842
|
+
instance_id: z.string().describe("Instance ID to clean up"),
|
|
1843
|
+
}),
|
|
1844
|
+
handler: async ({ instance_id }) => {
|
|
1845
|
+
const instance = deps.getInstance(instance_id);
|
|
1846
|
+
if (!instance)
|
|
1847
|
+
return `Instance not found: ${instance_id}`;
|
|
1848
|
+
if (instance.status !== "done" && instance.status !== "failed") {
|
|
1849
|
+
return `Cannot clean up instance in "${instance.status}" state. Abort it first.`;
|
|
1850
|
+
}
|
|
1851
|
+
const squad = deps.getSquad(instance.master_squad_slug);
|
|
1852
|
+
if (squad) {
|
|
1853
|
+
deps.removeWorktree(squad.projectPath, instance.worktree_path);
|
|
1854
|
+
}
|
|
1855
|
+
deps.deleteInstance(instance_id);
|
|
1856
|
+
return `Instance "${instance_id}" cleaned up and removed.`;
|
|
1857
|
+
},
|
|
1858
|
+
});
|
|
1859
|
+
// ---------------------------------------------------------------------------
|
|
1860
|
+
// Squad Instance context tools (#231 Phase 2)
|
|
1861
|
+
// ---------------------------------------------------------------------------
|
|
1862
|
+
const squadInstanceActivate = defineTool("squad_instance_activate", {
|
|
1863
|
+
description: "Activate an instance context. After activation, delegated tasks and decisions are scoped to this instance until deactivated.",
|
|
1864
|
+
skipPermission: true,
|
|
1865
|
+
parameters: z.object({
|
|
1866
|
+
instance_id: z.string().describe("Instance ID to activate"),
|
|
1867
|
+
}),
|
|
1868
|
+
handler: async ({ instance_id }) => {
|
|
1869
|
+
const instance = deps.getInstance(instance_id);
|
|
1870
|
+
if (!instance)
|
|
1871
|
+
return `Instance not found: ${instance_id}`;
|
|
1872
|
+
if (instance.status !== "active")
|
|
1873
|
+
return `Instance is not active (status: ${instance.status})`;
|
|
1874
|
+
deps.activeInstanceId = instance_id;
|
|
1875
|
+
return `Instance context activated: ${instance_id}. Tasks and decisions will be scoped to this instance.`;
|
|
1876
|
+
},
|
|
1877
|
+
});
|
|
1878
|
+
const squadInstanceDeactivate = defineTool("squad_instance_deactivate", {
|
|
1879
|
+
description: "Deactivate the current instance context, returning to master squad scope.",
|
|
1880
|
+
skipPermission: true,
|
|
1881
|
+
parameters: z.object({}),
|
|
1882
|
+
handler: async () => {
|
|
1883
|
+
const prev = deps.activeInstanceId;
|
|
1884
|
+
deps.activeInstanceId = undefined;
|
|
1885
|
+
return prev ? `Instance context deactivated (was: ${prev})` : `No instance context was active.`;
|
|
1886
|
+
},
|
|
1887
|
+
});
|
|
1888
|
+
return [wikiRead, wikiWrite, wikiSearch, wikiDelete, wikiList, squadCreate, squadRecall, squadStatus, squadLogDecision, squadDelegate, squadTaskStatus, squadDelete, squadAnalyze, squadAddAgent, squadAgents, squadRemoveAgent, squadResetAgent, squadSetLead, squadSetQA, squadTaskReviews, squadScheduleCreate, squadScheduleList, squadScheduleDelete, squadSchedulePause, squadScheduleResume, squadScheduleRunNow, scheduleCreate, scheduleList, scheduleDelete, schedulePause, scheduleResume, scheduleRunNow, skillList, skillInstall, skillRemove, skillSearch, configUpdate, checkUpdate, shell, fileOps, bash, readFile, viewTool, grepTool, strReplaceEditor, github, squadInstanceCreate, squadInstanceList, squadInstanceStatus, squadInstanceComplete, squadInstanceAbort, squadInstanceCleanup, squadInstanceActivate, squadInstanceDeactivate];
|
|
1708
1889
|
}
|
|
1709
1890
|
function walkDirectory(dir, maxDepth = 3, depth = 0) {
|
|
1710
1891
|
if (depth >= maxDepth)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Instance liveness watchdog.
|
|
3
|
+
*
|
|
4
|
+
* Periodically checks for active squad instances that haven't had any
|
|
5
|
+
* task activity beyond a configurable timeout and auto-aborts them.
|
|
6
|
+
*/
|
|
7
|
+
import { getDb } from "./store/db.js";
|
|
8
|
+
import { updateInstanceStatus } from "./store/instances.js";
|
|
9
|
+
import { createFeedEntry } from "./store/feed.js";
|
|
10
|
+
const DEFAULT_CHECK_INTERVAL_MS = 5 * 60_000; // Check every 5 minutes
|
|
11
|
+
const DEFAULT_STALE_THRESHOLD_MS = 30 * 60_000; // 30 minutes with no task activity
|
|
12
|
+
/**
|
|
13
|
+
* Find active instances whose last task activity exceeds the threshold.
|
|
14
|
+
* "Activity" is defined as the most recent started_at or completed_at
|
|
15
|
+
* in agent_tasks for that instance, or the instance's created_at if no tasks.
|
|
16
|
+
*/
|
|
17
|
+
export function findStaleInstances(thresholdMs) {
|
|
18
|
+
const db = getDb();
|
|
19
|
+
const now = Date.now();
|
|
20
|
+
const activeInstances = db.prepare("SELECT * FROM squad_instances WHERE status = 'active'").all();
|
|
21
|
+
const stale = [];
|
|
22
|
+
for (const instance of activeInstances) {
|
|
23
|
+
// Find the most recent task activity for this instance
|
|
24
|
+
const lastActivity = db.prepare(`
|
|
25
|
+
SELECT MAX(COALESCE(completed_at, started_at)) AS last_ts
|
|
26
|
+
FROM agent_tasks
|
|
27
|
+
WHERE instance_id = ?
|
|
28
|
+
`).get(instance.id);
|
|
29
|
+
const rawTs = lastActivity?.last_ts ?? instance.created_at;
|
|
30
|
+
const lastTs = new Date(rawTs.includes("T") ? rawTs : rawTs + "Z").getTime();
|
|
31
|
+
const idleMs = now - lastTs;
|
|
32
|
+
if (idleMs >= thresholdMs) {
|
|
33
|
+
stale.push({ instance, idleMs });
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return stale;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Start the instance watchdog. Returns a stop function.
|
|
40
|
+
*/
|
|
41
|
+
export function startInstanceWatchdog(opts = {}) {
|
|
42
|
+
const checkInterval = opts.checkIntervalMs ?? DEFAULT_CHECK_INTERVAL_MS;
|
|
43
|
+
const staleThreshold = opts.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
|
44
|
+
const timer = setInterval(() => {
|
|
45
|
+
try {
|
|
46
|
+
const staleInstances = findStaleInstances(staleThreshold);
|
|
47
|
+
for (const { instance, idleMs } of staleInstances) {
|
|
48
|
+
console.error(`[instance-watchdog] Auto-aborting stale instance "${instance.id}" — idle for ${Math.round(idleMs / 60_000)}m (threshold: ${Math.round(staleThreshold / 60_000)}m)`);
|
|
49
|
+
updateInstanceStatus(instance.id, "failed");
|
|
50
|
+
createFeedEntry({
|
|
51
|
+
type: "notification",
|
|
52
|
+
title: `[${instance.master_squad_slug}] Instance auto-aborted`,
|
|
53
|
+
body: `Instance "${instance.id}" was auto-aborted after ${Math.round(idleMs / 60_000)} minutes of inactivity. Worktree preserved at: ${instance.worktree_path}`,
|
|
54
|
+
source_type: "instance-watchdog",
|
|
55
|
+
});
|
|
56
|
+
if (opts.onAbort) {
|
|
57
|
+
opts.onAbort(instance, idleMs);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
catch (err) {
|
|
62
|
+
console.error("[instance-watchdog] Error during check:", err);
|
|
63
|
+
}
|
|
64
|
+
}, checkInterval);
|
|
65
|
+
timer.unref();
|
|
66
|
+
return () => {
|
|
67
|
+
clearInterval(timer);
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=instance-watchdog.js.map
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { before, after, beforeEach, describe, it } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { setDbPathForTests, closeDb, getDb } from "./store/db.js";
|
|
7
|
+
import { ensureInstanceTables } from "./store/instances.js";
|
|
8
|
+
import { findStaleInstances, startInstanceWatchdog } from "./instance-watchdog.js";
|
|
9
|
+
let tmpDir;
|
|
10
|
+
before(() => {
|
|
11
|
+
tmpDir = mkdtempSync(join(tmpdir(), "io-inst-watchdog-test-"));
|
|
12
|
+
setDbPathForTests(join(tmpDir, "io.db"));
|
|
13
|
+
ensureInstanceTables();
|
|
14
|
+
});
|
|
15
|
+
after(() => {
|
|
16
|
+
closeDb();
|
|
17
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
18
|
+
});
|
|
19
|
+
beforeEach(() => {
|
|
20
|
+
const db = getDb();
|
|
21
|
+
db.prepare("DELETE FROM agent_tasks").run();
|
|
22
|
+
db.prepare("DELETE FROM squad_instances").run();
|
|
23
|
+
db.prepare("DELETE FROM squads").run();
|
|
24
|
+
db.prepare("INSERT INTO squads (slug, name, project_path) VALUES (?, ?, ?)").run("test-squad", "Test", "/tmp/test");
|
|
25
|
+
});
|
|
26
|
+
describe("instance watchdog", () => {
|
|
27
|
+
describe("findStaleInstances", () => {
|
|
28
|
+
it("detects stale active instances with no task activity", () => {
|
|
29
|
+
const db = getDb();
|
|
30
|
+
db.prepare(`
|
|
31
|
+
INSERT INTO squad_instances (id, master_squad_slug, worktree_path, branch_name, status, created_at)
|
|
32
|
+
VALUES (?, ?, ?, ?, 'active', datetime('now', '-60 minutes'))
|
|
33
|
+
`).run("test-squad--old", "test-squad", "/tmp/wt", "test-squad/instance/old");
|
|
34
|
+
const stale = findStaleInstances(30 * 60_000);
|
|
35
|
+
assert.strictEqual(stale.length, 1);
|
|
36
|
+
assert.strictEqual(stale[0].instance.id, "test-squad--old");
|
|
37
|
+
});
|
|
38
|
+
it("does not flag instances with recent task activity", () => {
|
|
39
|
+
const db = getDb();
|
|
40
|
+
db.prepare(`
|
|
41
|
+
INSERT INTO squad_instances (id, master_squad_slug, worktree_path, branch_name, status, created_at)
|
|
42
|
+
VALUES (?, ?, ?, ?, 'active', datetime('now', '-60 minutes'))
|
|
43
|
+
`).run("test-squad--recent", "test-squad", "/tmp/wt2", "test-squad/instance/recent");
|
|
44
|
+
db.prepare(`
|
|
45
|
+
INSERT INTO agent_tasks (task_id, agent_slug, description, instance_id, started_at)
|
|
46
|
+
VALUES (?, ?, ?, ?, datetime('now', '-5 minutes'))
|
|
47
|
+
`).run("task-recent", "agent-1", "Work", "test-squad--recent");
|
|
48
|
+
const stale = findStaleInstances(30 * 60_000);
|
|
49
|
+
assert.strictEqual(stale.length, 0);
|
|
50
|
+
});
|
|
51
|
+
it("does not flag non-active instances", () => {
|
|
52
|
+
const db = getDb();
|
|
53
|
+
db.prepare(`
|
|
54
|
+
INSERT INTO squad_instances (id, master_squad_slug, worktree_path, branch_name, status, created_at)
|
|
55
|
+
VALUES (?, ?, ?, ?, 'done', datetime('now', '-120 minutes'))
|
|
56
|
+
`).run("test-squad--done", "test-squad", "/tmp/wt3", "test-squad/instance/done");
|
|
57
|
+
const stale = findStaleInstances(30 * 60_000);
|
|
58
|
+
assert.strictEqual(stale.length, 0);
|
|
59
|
+
});
|
|
60
|
+
it("calculates idleMs correctly relative to now", () => {
|
|
61
|
+
const db = getDb();
|
|
62
|
+
db.prepare(`
|
|
63
|
+
INSERT INTO squad_instances (id, master_squad_slug, worktree_path, branch_name, status, created_at)
|
|
64
|
+
VALUES (?, ?, ?, ?, 'active', datetime('now', '-45 minutes'))
|
|
65
|
+
`).run("test-squad--mid", "test-squad", "/tmp/wt4", "test-squad/instance/mid");
|
|
66
|
+
const stale = findStaleInstances(30 * 60_000);
|
|
67
|
+
assert.strictEqual(stale.length, 1);
|
|
68
|
+
assert.ok(stale[0].idleMs >= 44 * 60_000);
|
|
69
|
+
assert.ok(stale[0].idleMs <= 46 * 60_000);
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
describe("startInstanceWatchdog", () => {
|
|
73
|
+
it("calls onAbort for stale instances and stops cleanly", async () => {
|
|
74
|
+
const db = getDb();
|
|
75
|
+
db.prepare(`
|
|
76
|
+
INSERT INTO squad_instances (id, master_squad_slug, worktree_path, branch_name, status, created_at)
|
|
77
|
+
VALUES (?, ?, ?, ?, 'active', datetime('now', '-60 minutes'))
|
|
78
|
+
`).run("test-squad--stale", "test-squad", "/tmp/wt5", "test-squad/instance/stale");
|
|
79
|
+
const aborted = [];
|
|
80
|
+
const stop = startInstanceWatchdog({
|
|
81
|
+
checkIntervalMs: 50,
|
|
82
|
+
staleThresholdMs: 30 * 60_000,
|
|
83
|
+
onAbort: (inst) => aborted.push(inst.id),
|
|
84
|
+
});
|
|
85
|
+
// Wait for at least one interval to fire
|
|
86
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
87
|
+
stop();
|
|
88
|
+
assert.ok(aborted.includes("test-squad--stale"));
|
|
89
|
+
// Verify it was marked failed
|
|
90
|
+
const row = db.prepare("SELECT status FROM squad_instances WHERE id = ?").get("test-squad--stale");
|
|
91
|
+
assert.strictEqual(row.status, "failed");
|
|
92
|
+
});
|
|
93
|
+
it("stop function prevents further checks", async () => {
|
|
94
|
+
const db = getDb();
|
|
95
|
+
db.prepare(`
|
|
96
|
+
INSERT INTO squad_instances (id, master_squad_slug, worktree_path, branch_name, status, created_at)
|
|
97
|
+
VALUES (?, ?, ?, ?, 'active', datetime('now', '-60 minutes'))
|
|
98
|
+
`).run("test-squad--late", "test-squad", "/tmp/wt6", "test-squad/instance/late");
|
|
99
|
+
let abortCount = 0;
|
|
100
|
+
const stop = startInstanceWatchdog({
|
|
101
|
+
checkIntervalMs: 50,
|
|
102
|
+
staleThresholdMs: 30 * 60_000,
|
|
103
|
+
onAbort: () => abortCount++,
|
|
104
|
+
});
|
|
105
|
+
stop(); // Stop immediately before any tick fires
|
|
106
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
107
|
+
// Instance should NOT have been aborted since we stopped before first tick
|
|
108
|
+
assert.strictEqual(abortCount, 0);
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
//# sourceMappingURL=instance-watchdog.test.js.map
|
package/dist/store/db.js
CHANGED
|
@@ -172,6 +172,27 @@ GROUP BY agent_slug`,
|
|
|
172
172
|
)`,
|
|
173
173
|
`CREATE INDEX IF NOT EXISTS idx_unified_feed_type ON unified_feed(type, created_at)`,
|
|
174
174
|
`CREATE INDEX IF NOT EXISTS idx_unified_feed_unread ON unified_feed(read_at, created_at)`,
|
|
175
|
+
`CREATE TABLE IF NOT EXISTS squad_instances (
|
|
176
|
+
id TEXT PRIMARY KEY,
|
|
177
|
+
master_squad_slug TEXT NOT NULL,
|
|
178
|
+
issue_ref TEXT,
|
|
179
|
+
worktree_path TEXT NOT NULL,
|
|
180
|
+
branch_name TEXT NOT NULL,
|
|
181
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
182
|
+
context_snapshot TEXT,
|
|
183
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
184
|
+
completed_at DATETIME
|
|
185
|
+
)`,
|
|
186
|
+
`CREATE TABLE IF NOT EXISTS instance_decisions (
|
|
187
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
188
|
+
instance_id TEXT NOT NULL,
|
|
189
|
+
decision TEXT NOT NULL,
|
|
190
|
+
context TEXT,
|
|
191
|
+
merged_to_master INTEGER DEFAULT 0,
|
|
192
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
193
|
+
)`,
|
|
194
|
+
`ALTER TABLE agent_tasks ADD COLUMN instance_id TEXT`,
|
|
195
|
+
`CREATE INDEX IF NOT EXISTS idx_instance_decisions_instance ON instance_decisions(instance_id, merged_to_master)`,
|
|
175
196
|
];
|
|
176
197
|
for (const migration of migrations) {
|
|
177
198
|
try {
|