@agentmeshhq/agent 0.1.13 → 0.1.15

package/dist/core/daemon.js

@@ -2,15 +2,29 @@ import { execSync, spawn } from "node:child_process";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import { addAgentToState, getAgentState, loadConfig, updateAgentInState, } from "../config/loader.js";
+import { addAgentToState, getAgentState, loadConfig, resetAgentRestartCount, updateAgentInState, } from "../config/loader.js";
 import { loadContext, loadOrCreateContext, saveContext } from "../context/index.js";
 import { Heartbeat } from "./heartbeat.js";
 import { handleWebSocketEvent, injectRestoredContext, injectStartupMessage } from "./injector.js";
 import { checkInbox, fetchAssignments, registerAgent } from "./registry.js";
-import { buildRunnerConfig, getRunnerDisplayName, } from "./runner.js";
+import { buildRunnerConfig, getRunnerDisplayName } from "./runner.js";
 import { DockerSandbox } from "./sandbox.js";
-import { captureSessionContext, createSession, destroySession, getSessionName, sessionExists, updateSessionEnvironment, } from "./tmux.js";
+import { captureSessionContext, captureSessionOutput, createSession, destroySession, getSessionName, isSessionHealthy, sessionExists, updateSessionEnvironment, } from "./tmux.js";
+import { checkAgentProgress, cleanupOrphanContainers, isProcessRunning, sendNudge, } from "./watchdog.js";
 import { AgentWebSocket } from "./websocket.js";
+// Maximum number of auto-restart attempts
+const MAX_RESTART_ATTEMPTS = 3;
+// Time after which restart count resets (30 minutes of stable operation)
+const RESTART_COUNT_RESET_MS = 30 * 60 * 1000;
+// Time to wait after nudging before restarting (2 minutes)
+const NUDGE_WAIT_MS = 2 * 60 * 1000;
+// Path to the sandbox OpenCode config (permissive permissions)
+const SANDBOX_OPENCODE_CONFIG_PATH = path.join(os.homedir(), ".agentmesh", "opencode-sandbox.json");
+// Sandbox OpenCode config content - allow everything since container is sandboxed
+const SANDBOX_OPENCODE_CONFIG = {
+    $schema: "https://opencode.ai/config.json",
+    permission: "allow",
+};
 export class AgentDaemon {
     agentName;
     config;
@@ -27,17 +41,28 @@ export class AgentDaemon {
     serveMode;
     servePort;
     serveProcess = null;
-    serverContext;
     sandboxMode;
     sandboxImage;
     sandboxCpu;
     sandboxMemory;
     sandbox = null;
+    healthCheckInterval = null;
+    serverContext;
+    // Auto-restart tracking
+    restartCount = 0;
+    lastStableTime = null;
+    stuckSince = null;
+    nudgeSentAt = null;
     constructor(options) {
         const config = loadConfig();
         if (!config) {
             throw new Error("No config found. Run 'agentmesh init' first.");
         }
+        // Ensure config has required fields with defaults
+        if (!config.agents)
+            config.agents = [];
+        if (!config.defaults)
+            config.defaults = { command: "opencode", model: "claude-sonnet-4" };
         this.config = config;
         this.agentName = options.name;
         this.shouldRestoreContext = options.restoreContext !== false;
@@ -83,9 +108,28 @@ export class AgentDaemon {
             return;
         }
         console.log(`Starting agent: ${this.agentName}`);
+        // Check for duplicate process
+        const existingState = getAgentState(this.agentName);
+        if (existingState && existingState.pid > 0) {
+            if (isProcessRunning(existingState.pid)) {
+                throw new Error(`Agent "${this.agentName}" is already running (PID: ${existingState.pid}). ` +
+                    `Use 'agentmesh stop ${this.agentName}' first.`);
+            }
+            // Process not running, clean up stale state
+            console.log(`Cleaning up stale state for PID ${existingState.pid}`);
+        }
+        // Clean up orphan containers in sandbox mode
+        if (this.sandboxMode) {
+            const cleaned = cleanupOrphanContainers(this.agentName);
+            if (cleaned > 0) {
+                console.log(`Cleaned up ${cleaned} orphan container(s)`);
+            }
+        }
+        // Reset restart count on manual start
+        this.restartCount = 0;
+        this.lastStableTime = new Date();
         // Register with hub first (needed for assignment check)
         console.log("Registering with AgentMesh hub...");
-        const existingState = getAgentState(this.agentName);
         console.log(`Existing state: ${existingState ? `agentId=${existingState.agentId}` : "none"}`);
         const registration = await registerAgent({
             url: this.config.hubUrl,
@@ -155,6 +199,7 @@ export class AgentDaemon {
             assignedProject: this.assignedProject,
             runtimeModel: this.runnerConfig.model,
             runnerType: this.runnerConfig.type,
+            sandboxContainer: this.sandbox?.getContainerName(),
         });
         // Start heartbeat with auto-refresh
         console.log("Starting heartbeat...");
@@ -235,11 +280,11 @@ export class AgentDaemon {
             },
         });
         this.ws.connect();
-        // Check inbox and auto-nudge
+        // Check inbox and auto-nudge with full handoff details
         console.log("Checking inbox...");
         try {
             const inboxItems = await checkInbox(this.config.hubUrl, this.config.workspace, this.token);
-            injectStartupMessage(this.agentName, inboxItems.length);
+            injectStartupMessage(this.agentName, inboxItems.length, inboxItems);
         }
         catch (error) {
             console.error("Failed to check inbox:", error);
@@ -257,6 +302,8 @@ export class AgentDaemon {
             }
         }
         this.isRunning = true;
+        // Start session health monitoring (every 60 seconds)
+        this.startHealthMonitor();
         console.log(`
 Agent "${this.agentName}" is running.
 
@@ -273,9 +320,228 @@ Nudge agent:
         process.on("SIGINT", () => this.stop());
         process.on("SIGTERM", () => this.stop());
     }
+    /**
+     * Starts periodic health monitoring for the tmux session
+     * Includes auto-restart logic and progress watchdog
+     */
+    startHealthMonitor() {
+        // Skip health monitoring for serve mode (no tmux session)
+        if (this.serveMode)
+            return;
+        const logDir = path.join(os.homedir(), ".agentmesh", "logs");
+        if (!fs.existsSync(logDir)) {
+            fs.mkdirSync(logDir, { recursive: true });
+        }
+        this.healthCheckInterval = setInterval(async () => {
+            if (!this.isRunning)
+                return;
+            // Reset restart count after stable operation
+            if (this.lastStableTime && this.restartCount > 0) {
+                const stableTime = Date.now() - this.lastStableTime.getTime();
+                if (stableTime > RESTART_COUNT_RESET_MS) {
+                    console.log(`[HEALTH] Agent stable for 30+ minutes, resetting restart count`);
+                    this.restartCount = 0;
+                    resetAgentRestartCount(this.agentName);
+                }
+            }
+            // For sandbox mode, pass container name so health check looks inside container
+            const containerName = this.sandboxMode ? this.sandbox?.getContainerName() : undefined;
+            const health = isSessionHealthy(this.agentName, containerName);
+            if (!health.healthy) {
+                // Session died - attempt restart
+                await this.handleSessionDeath(health.reason || "unknown", logDir);
+                return;
+            }
+            // Session is alive - check progress watchdog
+            const progress = checkAgentProgress(this.agentName, containerName);
+            if (progress.status === "permission_blocked" || progress.status === "stuck") {
+                await this.handleStuckAgent(progress);
+            }
+            else if (progress.status === "active") {
+                // Agent is working - reset stuck tracking
+                if (this.stuckSince) {
+                    console.log(`[HEALTH] Agent resumed activity`);
+                    this.stuckSince = null;
+                    this.nudgeSentAt = null;
+                    updateAgentInState(this.agentName, { stuckSince: undefined, status: "running" });
+                }
+                this.lastStableTime = new Date();
+            }
+        }, 60000); // Check every 60 seconds
+    }
+    /**
+     * Handles session death - logs crash and attempts auto-restart
+     */
+    async handleSessionDeath(reason, logDir) {
+        const timestamp = new Date().toISOString();
+        const logFile = path.join(logDir, `crash-${this.agentName}.log`);
+        // Capture last session output before it's gone
+        let lastOutput = "";
+        try {
+            lastOutput = captureSessionOutput(this.agentName, 200) || "Unable to capture output";
+        }
+        catch {
+            lastOutput = "Failed to capture session output";
+        }
+        const crashLog = `
+================================================================================
+AGENT CRASH DETECTED
+================================================================================
+Timestamp: ${timestamp}
+Agent: ${this.agentName}
+Agent ID: ${this.agentId}
+Reason: ${reason}
+Restart Count: ${this.restartCount}/${MAX_RESTART_ATTEMPTS}
+Sandbox: ${this.sandboxMode ? this.sandbox?.getContainerName() : "none"}
+Workdir: ${this.agentConfig.workdir}
+Model: ${this.runnerConfig.model}
+
+--- Last Session Output ---
+${lastOutput}
+================================================================================
+
+`;
+        fs.appendFileSync(logFile, crashLog);
+        // Check if we can restart
+        if (this.restartCount < MAX_RESTART_ATTEMPTS) {
+            this.restartCount++;
+            console.error(`[CRASH] Session died: ${reason}. Attempting restart (${this.restartCount}/${MAX_RESTART_ATTEMPTS})...`);
+            updateAgentInState(this.agentName, {
+                restartCount: this.restartCount,
+                lastRestartAt: timestamp,
+                status: "running",
+            });
+            try {
+                await this.restartSession();
+                console.log(`[RESTART] Agent restarted successfully`);
+                this.lastStableTime = new Date();
+            }
+            catch (error) {
+                console.error(`[RESTART] Failed to restart: ${error.message}`);
+            }
+        }
+        else {
+            // Exceeded restart limit - mark as failed
+            console.error(`[FAILED] Agent exceeded restart limit (${MAX_RESTART_ATTEMPTS}). Manual intervention required.`);
+            // Terminal bell to alert user
+            process.stdout.write("\x07");
+            updateAgentInState(this.agentName, {
+                status: "failed",
+                restartCount: this.restartCount,
+            });
+            // Stop monitoring
+            this.isRunning = false;
+            if (this.healthCheckInterval) {
+                clearInterval(this.healthCheckInterval);
+                this.healthCheckInterval = null;
+            }
+        }
+    }
+    /**
+     * Handles stuck agent - sends nudge first, then restarts if still stuck
+     */
+    async handleStuckAgent(progress) {
+        const now = new Date();
+        if (!this.stuckSince) {
+            // First detection of stuck state
+            this.stuckSince = now;
+            console.log(`[HEALTH] Agent appears stuck: ${progress.details || progress.blockedOn || "no activity"}`);
+            updateAgentInState(this.agentName, {
+                stuckSince: now.toISOString(),
+                status: "stuck",
+            });
+        }
+        // If we haven't sent a nudge yet, send one
+        if (!this.nudgeSentAt) {
+            console.log(`[HEALTH] Sending nudge to unstick agent...`);
+            const nudgeMessage = progress.status === "permission_blocked"
+                ? "Please continue with your task. If you see a permission prompt, try an alternative approach that doesn't require that permission."
+                : "Please continue with your current task.";
+            const sent = sendNudge(this.agentName, nudgeMessage);
+            if (sent) {
+                this.nudgeSentAt = now;
+                console.log(`[HEALTH] Nudge sent successfully`);
+            }
+            else {
+                console.log(`[HEALTH] Failed to send nudge`);
+            }
+            return;
+        }
+        // Check if enough time has passed since nudge
+        const timeSinceNudge = now.getTime() - this.nudgeSentAt.getTime();
+        if (timeSinceNudge < NUDGE_WAIT_MS) {
+            // Still waiting for agent to respond to nudge
+            return;
+        }
+        // Agent still stuck after nudge - trigger restart
+        console.log(`[HEALTH] Agent still stuck after nudge, triggering restart...`);
+        this.stuckSince = null;
+        this.nudgeSentAt = null;
+        await this.handleSessionDeath("stuck_after_nudge", path.join(os.homedir(), ".agentmesh", "logs"));
+    }
+    /**
+     * Restarts the agent session (sandbox or non-sandbox)
+     */
+    async restartSession() {
+        // Destroy existing session
+        destroySession(this.agentName);
+        if (this.sandboxMode && this.sandbox) {
+            // Restart sandbox container
+            const newContainerId = await this.sandbox.restart();
+            console.log(`[RESTART] New container: ${newContainerId.substring(0, 12)}`);
+            // Recreate tmux session for sandbox
+            const containerName = this.sandbox.getContainerName();
+            const sessionName = getSessionName(this.agentName);
+            // Build environment args for docker exec
+            const envArgs = [];
+            const allEnv = {
+                ...this.runnerConfig.env,
+                AGENT_TOKEN: this.token,
+                AGENTMESH_AGENT_ID: this.agentId,
+            };
+            for (const [key, value] of Object.entries(allEnv)) {
+                if (value !== undefined && value !== "") {
+                    envArgs.push(`-e "${key}=${value}"`);
+                }
+            }
+            const envString = envArgs.join(" ");
+            const modelArg = this.runnerConfig.env?.OPENCODE_MODEL
+                ? ` --model ${this.runnerConfig.env.OPENCODE_MODEL}`
+                : "";
+            const dockerExecCommand = `docker exec -it ${envString} ${containerName} opencode${modelArg}`;
+            const created = createSession(this.agentName, dockerExecCommand, undefined, undefined);
+            if (!created) {
+                throw new Error("Failed to create tmux session for restarted sandbox");
+            }
+            // Update state with new container name
+            updateAgentInState(this.agentName, {
+                sandboxContainer: containerName,
+            });
+        }
+        else {
+            // Non-sandbox restart - just recreate tmux session
+            const created = createSession(this.agentName, this.agentConfig.command, this.agentConfig.workdir, this.runnerConfig.env);
+            if (!created) {
+                throw new Error("Failed to create tmux session");
+            }
+            // Re-inject environment
+            updateSessionEnvironment(this.agentName, {
+                AGENT_TOKEN: this.token,
+                AGENTMESH_AGENT_ID: this.agentId,
+                ...this.runnerConfig.env,
+            });
+        }
+        // Wait for session to be ready
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+    }
     async stop() {
         console.log(`\nStopping agent: ${this.agentName}`);
         this.isRunning = false;
+        // Stop health monitor
+        if (this.healthCheckInterval) {
+            clearInterval(this.healthCheckInterval);
+            this.healthCheckInterval = null;
+        }
         // Save context before stopping
         if (this.agentId) {
             console.log("Saving agent context...");
@@ -293,7 +559,11 @@ Nudge agent:
         }
         // Stop sandbox, serve process, or destroy tmux session
         if (this.sandboxMode && this.sandbox) {
-            console.log("Stopping sandbox container...");
+            console.log("Stopping sandbox...");
+            // In sandbox mode, we have both a tmux session (on host) and a Docker container
+            // Destroy tmux session first (this stops docker exec)
+            destroySession(this.agentName);
+            // Then destroy the container
             await this.sandbox.destroy();
             this.sandbox = null;
         }
@@ -322,12 +592,34 @@ Nudge agent:
     async startServeMode() {
         console.log(`Starting opencode serve mode on port ${this.servePort}...`);
         const workdir = this.agentConfig.workdir || process.cwd();
+        // Isolate OpenCode's SQLite database per agent to prevent WAL corruption.
+        // See docs/RCA-OPENCODE-SQLITE-CORRUPTION.md for details.
+        const agentDataDir = path.join(os.homedir(), ".agentmesh", "opencode-data", this.agentName);
+        const agentOpencodeDir = path.join(agentDataDir, "opencode");
+        if (!fs.existsSync(agentOpencodeDir)) {
+            fs.mkdirSync(agentOpencodeDir, { recursive: true });
+        }
+        // Copy auth.json from default OpenCode data dir so agents inherit API keys.
+        // Strips xAI provider to prevent OpenCode from defaulting to non-Anthropic models.
+        const agentAuthPath = path.join(agentOpencodeDir, "auth.json");
+        const sourceAuthPath = path.join(os.homedir(), ".local", "share", "opencode", "auth.json");
+        if (!fs.existsSync(agentAuthPath) && fs.existsSync(sourceAuthPath)) {
+            try {
+                const auth = JSON.parse(fs.readFileSync(sourceAuthPath, "utf-8"));
+                delete auth.xai;
+                fs.writeFileSync(agentAuthPath, JSON.stringify(auth, null, 2));
+            }
+            catch {
+                // Non-fatal — agent will just need manual auth
+            }
+        }
         // Build environment for opencode serve
         const env = {
             ...process.env,
             ...this.runnerConfig.env,
             AGENT_TOKEN: this.token,
             AGENTMESH_AGENT_ID: this.agentId,
+            XDG_DATA_HOME: agentDataDir,
         };
         // Spawn opencode serve as a child process
         this.serveProcess = spawn("opencode", ["serve", "--port", String(this.servePort), "--hostname", "0.0.0.0"], {
@@ -357,6 +649,10 @@ Nudge agent:
     /**
      * Starts agent in Docker sandbox mode
      * Provides filesystem isolation with only workspace mounted
+     *
+     * Strategy: Start Docker container with tail -f /dev/null, then create
+     * a tmux session on the HOST that runs `docker exec -it <container> opencode`.
+     * This way tmux provides the TTY that docker exec needs.
      */
     async startSandboxMode() {
         console.log("Starting in Docker sandbox mode...");
@@ -369,10 +665,54 @@ Nudge agent:
         const existingContainer = DockerSandbox.findExisting(this.agentName);
         if (existingContainer) {
             console.log(`Found existing sandbox container: ${existingContainer}`);
-            console.log("Stop it with: agentmesh stop " + this.agentName);
+            console.log(`Stop it with: agentmesh stop ${this.agentName}`);
             throw new Error("Sandbox container already exists");
         }
+        // Build additional mounts for credentials and config
+        // The entrypoint script copies these from /tmp/ to the correct locations
+        const additionalMounts = [];
+        // Mount git credentials
+        const gitCredentialsPath = path.join(os.homedir(), ".git-credentials");
+        if (fs.existsSync(gitCredentialsPath)) {
+            additionalMounts.push(`${gitCredentialsPath}:/tmp/.git-credentials-host:ro`);
+        }
+        // Mount OpenCode auth.json for API provider tokens (Anthropic, OpenAI, etc.)
+        const opencodeAuthPath = path.join(os.homedir(), ".local", "share", "opencode", "auth.json");
+        if (fs.existsSync(opencodeAuthPath)) {
+            additionalMounts.push(`${opencodeAuthPath}:/tmp/.opencode-auth-host:ro`);
+        }
+        // Mount AgentMesh config for hub URL, API key, workspace
+        const agentmeshConfigPath = path.join(os.homedir(), ".agentmesh", "config.json");
+        if (fs.existsSync(agentmeshConfigPath)) {
+            additionalMounts.push(`${agentmeshConfigPath}:/tmp/.agentmesh-config-host:ro`);
+        }
+        // Create and mount permissive OpenCode config for sandbox
+        // This allows all permissions since the container is already sandboxed
+        this.ensureSandboxOpencodeConfig();
+        additionalMounts.push(`${SANDBOX_OPENCODE_CONFIG_PATH}:/workspace/opencode.json:ro`);
+        // Pass GitHub token as environment variable for git operations
+        const gitCredentials = fs.existsSync(gitCredentialsPath)
+            ? fs.readFileSync(gitCredentialsPath, "utf-8").trim()
+            : "";
+        const gitHubToken = gitCredentials.match(/github_pat_[^\s@]+/)?.[0] || "";
+        // Build the command to run inside the container
+        // The agentmesh CLI inside the container will create tmux + opencode
+        const model = this.runnerConfig.env?.OPENCODE_MODEL || this.runnerConfig.model || "claude-sonnet-4";
+        const containerCommand = [
+            "agentmesh",
+            "start",
+            "--name",
+            this.agentName,
+            "--model",
+            model,
+            "--foreground",
+        ];
         // Create sandbox configuration
+        // Isolate OpenCode's SQLite database per agent to prevent WAL corruption.
+        const hostDataDir = path.join(os.homedir(), ".agentmesh", "opencode-data", this.agentName);
+        if (!fs.existsSync(hostDataDir)) {
+            fs.mkdirSync(hostDataDir, { recursive: true });
+        }
         this.sandbox = new DockerSandbox({
             agentName: this.agentName,
             image: this.sandboxImage,
@@ -383,38 +723,52 @@ Nudge agent:
                 ...this.runnerConfig.env,
                 AGENT_TOKEN: this.token,
                 AGENTMESH_AGENT_ID: this.agentId,
+                AGENT_NAME: this.agentName,
+                // XDG_DATA_HOME set by entrypoint based on AGENT_NAME
+                // Git credentials for pushing to GitHub
+                ...(gitHubToken && { GH_TOKEN: gitHubToken, GITHUB_TOKEN: gitHubToken }),
             },
             serveMode: this.serveMode,
             servePort: this.servePort,
             networkMode: "bridge",
+            additionalMounts: [
+                ...additionalMounts,
+                `${hostDataDir}:/home/node/.agentmesh/opencode-data/${this.agentName}:rw`,
+            ],
+            command: this.serveMode ? undefined : containerCommand,
         });
         // Validate mount policy (will throw if denied)
         this.sandbox.validateMountPolicy();
         // Pull image if needed
         await this.sandbox.pullImage();
-        // Start container
+        // Start container with agentmesh running inside
+        // The entrypoint script sets up credentials before agentmesh starts
         await this.sandbox.start();
+        const containerName = this.sandbox.getContainerName();
         console.log(`
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-🐳 SANDBOX MODE ACTIVE
+SANDBOX MODE ACTIVE
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
-Container: ${this.sandbox.getContainerName()}
+Container: ${containerName}
 Image:     ${this.sandboxImage}
 Workspace: ${workdir} -> /workspace
 CPU:       ${this.sandboxCpu} core(s)
 Memory:    ${this.sandboxMemory}
+Model:     ${model}
 
-The agent is running in an isolated Docker container.
-Only the workspace directory is accessible.
+The agent daemon is running INSIDE the Docker container.
+tmux session and OpenCode are managed inside the container.
+
+Attach:  agentmesh attach ${this.agentName}
+Nudge:   agentmesh nudge ${this.agentName} "message"
+Stop:    agentmesh stop ${this.agentName}
+Logs:    docker logs ${containerName}
 
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 `);
-        // Start opencode in the container
-        if (!this.serveMode) {
-            console.log("Starting opencode in sandbox container...");
-            await this.sandbox.spawnOpencode();
-        }
+        // No host tmux session needed - the container runs agentmesh which creates its own tmux
+        // Heartbeats are sent by the daemon running inside the container
     }
     /**
      * Saves the current agent context to disk
@@ -575,5 +929,27 @@ Option 3: Use --auto-setup to automatically clone the repository:
         console.log(`✓ Workspace ready: ${workspacePath}\n`);
         return workspacePath;
     }
+    /**
+     * Ensures the sandbox OpenCode config exists
+     * Creates ~/.agentmesh/opencode-sandbox.json with permissive permissions and model
+     */
+    ensureSandboxOpencodeConfig() {
+        const configDir = path.dirname(SANDBOX_OPENCODE_CONFIG_PATH);
+        if (!fs.existsSync(configDir)) {
+            fs.mkdirSync(configDir, { recursive: true });
+        }
+        // Build config with model if available
+        const config = {
+            ...SANDBOX_OPENCODE_CONFIG,
+        };
+        // Include model from runner config
+        const model = this.runnerConfig.env?.OPENCODE_MODEL;
+        if (model) {
+            config.model = model;
+        }
+        // Always write to ensure model is up to date
+        fs.writeFileSync(SANDBOX_OPENCODE_CONFIG_PATH, JSON.stringify(config, null, 2));
+        console.log(`Updated sandbox OpenCode config: ${SANDBOX_OPENCODE_CONFIG_PATH}`);
+    }
 }
 //# sourceMappingURL=daemon.js.map