@agentmeshhq/agent 0.4.16 → 0.4.20

package/dist/core/daemon.js

@@ -13,16 +13,21 @@ import { formatCrashLog } from "./daemon/crash-log.js";
 import { evaluateRestartState, filterActiveClaimsForAgent, filterCompletedHandoffsForAgent, formatRestartLifecycleLog, } from "./daemon/done-state-guard.js";
 import { cleanupGitAuth, setupGitAuth } from "./daemon/git-auth.js";
 import { getStuckDetail } from "./daemon/health-policy.js";
+import { verifyInjection } from "./daemon/injection-verify.js";
 import { runLeadTick } from "./daemon/lead-loop.js";
 import { isKnownRole, ROLE_DAEMON_BEHAVIOUR, VALID_ROLES } from "./daemon/roles.js";
 import { writeSandboxOpencodeConfig } from "./daemon/sandbox-config.js";
 import { isRecoverableSessionFailure } from "./daemon/session-recovery.js";
 import { captureAgentChildPids, persistRunningState } from "./daemon/state.js";
 import { startTmuxRuntimeSession } from "./daemon/tmux-session.js";
+import { handleWatcherWebSocketEvent, startWatcherLoop, } from "./daemon/watcher-loop.js";
+import { WatcherTickQueue } from "./daemon/watcher-queue.js";
 import { configureGitIdentity, setupWorkspace, updateWorkspaceFromRemote, validatePushAccess, } from "./daemon/workspace.js";
 import { findPendingHandoffBreaches } from "./handoff-sla.js";
 import { Heartbeat } from "./heartbeat.js";
 import { handleWebSocketEvent, injectInboxItems, injectOnboardMessage, injectRestoredContext, injectStartupMessage, } from "./injector.js";
+import { buildOpenCodeMetadata, fetchOpenCodeServerHealth, waitForOpenCodeServer, } from "./opencode-serve.js";
+import { shouldPersistOpenCodeSession, shouldRestoreOpenCodeSession, } from "./opencode-session-policy.js";
 import { checkInbox, createClaim, createSelfAssignment, fetchAssignments, fetchHandoffsForAgent, fetchOnboard, fetchProjectByCode, getAgentAutonomyState, getHandoff, listClaims, patchAgentMetadata, registerAgent, releaseClaim, updateHandoffStatusWithRetry, } from "./registry.js";
 import { getRunnerDisplayName } from "./runner.js";
 import { DockerSandbox } from "./sandbox.js";
@@ -61,7 +66,12 @@ export class AgentDaemon {
     autoSetup;
     serveMode;
     servePort;
+    serveHostname;
+    servePublicUrl;
+    serveUsername;
+    servePasswordEnv;
     serveProcess = null;
+    serveHealth = null;
     sandboxMode;
     sandboxImage;
     sandboxCpu;
@@ -74,8 +84,11 @@ export class AgentDaemon {
     autonomous;
     healthCheckInterval = null;
     inboxPollInterval = null;
+    commandPollInterval = null;
     leadInterval = null;
     leadContext = null;
+    watcherLoopHandle = null;
+    watcherQueue = null;
     stopCleanupScheduler = null;
     authHealthWatcher = null;
     // Session resume tracking
@@ -85,12 +98,14 @@ export class AgentDaemon {
     stuckSince = null;
     lastPendingHandoffAlertAt = null;
     remoteAutomationPaused = false;
+    rateLimitDetected = false;
     lastAutonomyPolicyFetchAt = null;
     teamId;
     pendingClaimCreations = new Set();
     sessionRecoveryAttempts = 0;
     lastSessionRecoveryAt = null;
     initialInboxCheckComplete = false;
+    commandPollInFlight = false;
     constructor(options) {
         const boot = bootstrapDaemon(options);
         this.config = boot.config;
@@ -101,6 +116,10 @@ export class AgentDaemon {
         this.agentConfig = boot.agentConfig;
         this.serveMode = boot.serveMode;
         this.servePort = boot.servePort;
+        this.serveHostname = boot.serveHostname;
+        this.servePublicUrl = boot.servePublicUrl;
+        this.serveUsername = boot.serveUsername;
+        this.servePasswordEnv = boot.servePasswordEnv;
         this.sandboxMode = boot.sandboxMode;
         this.sandboxImage = boot.sandboxImage;
         this.sandboxCpu = boot.sandboxCpu;
@@ -165,6 +184,14 @@ export class AgentDaemon {
         // - neither → "system" (hidden background agent)
         const effectiveAgentType = this.agentConfig.agentType ??
             (this.isWorkerAgent ? "worker" : this.autonomous ? "autonomous" : "system");
+        const spawnRequestId = process.env.AGENTMESH_SPAWN_REQUEST_ID?.trim() || null;
+        const targetId = process.env.AGENTMESH_TARGET_ID?.trim() || null;
+        const registrationMetadata = spawnRequestId || targetId
+            ? {
+                ...(spawnRequestId ? { spawn_request_id: spawnRequestId } : {}),
+                ...(targetId ? { target_id: targetId } : {}),
+            }
+            : undefined;
         const registration = await registerAgent({
             url: this.config.hubUrl,
             apiKey: this.config.apiKey,
@@ -175,15 +202,11 @@ export class AgentDaemon {
             restoreContext: this.shouldRestoreContext,
             agentType: effectiveAgentType,
             teamId: this.teamId,
+            metadata: registrationMetadata,
         });
         this.agentId = registration.agentId;
         this.token = registration.token;
-        // Best-effort: store role in hub agent metadata (once at startup, GH-829)
-        if (this.projectRole && isKnownRole(this.projectRole)) {
-            patchAgentMetadata(this.config.hubUrl, this.agentId, this.token, {
-                role: this.projectRole,
-            }).catch(() => { }); // fire-and-forget; non-fatal
-        }
+        void this.publishDurableAgentMetadata();
         // Persist the agentId back to config.json so it survives state.json wipes.
         // On next startup, agentConfig.agentId is the fallback when existingState is absent.
         upsertAgentConfig({ ...this.agentConfig, agentId: this.agentId });
@@ -285,7 +308,10 @@ export class AgentDaemon {
                 command: this.agentConfig.command,
                 workdir: this.agentConfig.workdir,
                 runnerEnv: this.runnerConfig.env,
+                runnerType: this.runnerConfig.type,
                 shouldRestoreContext: this.shouldRestoreContext,
+                serveMode: this.serveMode,
+                sandboxMode: this.sandboxMode,
                 autonomous: this.autonomous,
             });
             this._preStartSessionId = sessionStart.preStartSessionId;
@@ -310,6 +336,14 @@ export class AgentDaemon {
             sandboxContainer: this.sandbox?.getContainerName(),
             serveMode: this.serveMode,
             servePort: this.servePort,
+            serveHostname: this.serveHostname,
+            serveUrl: this.getServeLocalUrl(),
+            servePublicUrl: this.servePublicUrl,
+            serveAuthType: this.getServePassword() ? "basic" : "none",
+            serveUsername: this.serveUsername || (this.getServePassword() ? "opencode" : undefined),
+            servePasswordEnv: this.servePasswordEnv,
+            serveHealthy: this.serveHealth?.healthy,
+            serveVersion: this.serveHealth?.version,
         });
         // Track child PIDs for cleanup on restart/stop (tmux mode only — sandbox/serve manage their own)
         if (!this.sandboxMode && !this.serveMode) {
@@ -357,8 +391,14 @@ export class AgentDaemon {
                         token: newToken,
                         onMessage: (event) => {
                             console.log(`[WS] Received event: ${event.type}`);
+                            if (this.watcherQueue) {
+                                handleWatcherWebSocketEvent(event, this.watcherQueue);
+                            }
                             this.autoAcceptHandoffFromEvent(event);
-                            handleWebSocketEvent(this.agentName, event);
+                            handleWebSocketEvent(this.agentName, event, {
+                                hubUrl: this.config.hubUrl,
+                                token: this.token ?? undefined,
+                            });
                         },
                         onConnect: () => {
                             console.log("WebSocket reconnected with new token");
@@ -373,6 +413,7 @@ export class AgentDaemon {
                     });
                     this.ws.connect();
                 }
+                void this.publishDurableAgentMetadata();
             },
         });
         this.heartbeat.start();
@@ -384,6 +425,10 @@ export class AgentDaemon {
             token: this.token,
             onMessage: (event) => {
                 console.log(`[WS] Received event: ${event.type}`);
+                // Feed watcher queue before standard handlers (GH-883)
+                if (this.watcherQueue) {
+                    handleWatcherWebSocketEvent(event, this.watcherQueue);
+                }
                 this.autoAcceptHandoffFromEvent(event);
                 handleWebSocketEvent(this.agentName, event, {
                     hubUrl: this.config.hubUrl,
@@ -520,6 +565,7 @@ export class AgentDaemon {
         this.isRunning = true;
         // Start session health monitoring (every 60 seconds)
         this.startHealthMonitor();
+        this.startCommandPoller();
         // Start lead coordination loop for lead/coordinator roles (GH-824, GH-829)
         const roleBehaviour = isKnownRole(this.projectRole)
             ? ROLE_DAEMON_BEHAVIOUR[this.projectRole]
@@ -540,6 +586,19 @@ export class AgentDaemon {
             }, 120_000);
             console.log("[lead-loop] Lead coordination loop started (2-minute tick)");
         }
+        // Start LLM watcher loop for watcher role (GH-883)
+        if (roleBehaviour === "watcher-loop" && this.token) {
+            this.watcherQueue = new WatcherTickQueue();
+            this.watcherLoopHandle = startWatcherLoop({
+                hubUrl: this.config.hubUrl,
+                token: this.token,
+                workspace: this.config.workspace,
+                teamId: this.teamId ?? "",
+                agentName: this.agentName,
+                log: (msg) => console.log(msg),
+            }, this.watcherQueue);
+            console.log("[watcher-loop] LLM watcher loop started (2-minute tick)");
+        }
         // Start evicted-agent cleanup scheduler (GH-421)
         if (this.token) {
             this.stopCleanupScheduler = startCleanupScheduler({
@@ -579,6 +638,11 @@ Nudge agent:
             inboxItems.length === 0) {
             return inboxItems;
         }
+        // GH-887: Skip auto-accept if rate-limited
+        if (this.rateLimitDetected) {
+            console.log("[AUTO-ACCEPT] Skipping — agent is rate-limited");
+            return inboxItems;
+        }
         const accepted = new Set();
         for (const item of inboxItems) {
             if (item.type && item.type !== "handoff") {
@@ -590,6 +654,8 @@ Nudge agent:
                     accepted.add(item.id);
                     console.log(`[AUTO-ACCEPT] Accepted handoff ${item.id}`);
                     await this.ensureClaimForHandoff(item.id, item.scope);
+                    // GH-887: Verify the LLM actually started processing
+                    void this.verifyHandoffInjection(item.id, item.scope);
                 }
             }
             catch (error) {
@@ -598,6 +664,34 @@ Nudge agent:
         }
         return inboxItems.filter((item) => !accepted.has(item.id));
     }
+    /**
+     * GH-887: After accepting a handoff, verify the LLM acknowledged it.
+     * If rate-limited, pause auto-acceptance. If stuck after retries, log warning.
+     */
+    async verifyHandoffInjection(handoffId, scope) {
+        try {
+            const result = await verifyInjection(this.agentName, handoffId, scope);
+            if (result.rateLimited) {
+                this.rateLimitDetected = true;
+                console.warn(`[INJECT-VERIFY] Rate limit detected — pausing auto-accept for ${this.agentName}`);
+                // Clear rate limit after 5 minutes (check again then)
+                setTimeout(() => {
+                    this.rateLimitDetected = false;
+                    console.log("[INJECT-VERIFY] Rate limit cooldown expired — resuming auto-accept");
+                }, 300_000);
+                return;
+            }
+            if (result.verified) {
+                console.log(`[INJECT-VERIFY] Handoff ${handoffId} verified (${result.attempts} attempt${result.attempts > 1 ? "s" : ""})`);
+            }
+            else {
+                console.warn(`[INJECT-VERIFY] Handoff ${handoffId} NOT verified after ${result.attempts} attempts — LLM may be stuck`);
+            }
+        }
+        catch (error) {
+            console.warn(`[INJECT-VERIFY] Error verifying ${handoffId}: ${error.message}`);
+        }
+    }
     autoAcceptHandoffFromEvent(event) {
         if (!this.autoAcceptHandoffs || !this.token) {
             return;
@@ -836,9 +930,33 @@ Nudge agent:
         }
     }
     startHealthMonitor() {
-        // Skip health monitoring for serve mode (no tmux session)
-        if (this.serveMode)
+        if (this.serveMode) {
+            this.healthCheckInterval = setInterval(async () => {
+                if (!this.isRunning)
+                    return;
+                try {
+                    const health = await fetchOpenCodeServerHealth({
+                        hostname: this.serveHostname,
+                        port: this.servePort,
+                        username: this.serveUsername,
+                        password: this.getServePassword(),
+                    });
+                    await this.publishServeMetadata(health);
+                    updateAgentInState(this.agentName, {
+                        serveHealthy: health.healthy,
+                        serveVersion: health.version,
+                    });
+                }
+                catch (err) {
+                    console.error(`[SERVE] Health check failed: ${err.message}`);
+                    await this.publishServeMetadata({ healthy: false, version: this.serveHealth?.version });
+                    updateAgentInState(this.agentName, {
+                        serveHealthy: false,
+                    });
+                }
+            }, 30000);
             return;
+        }
         // Start periodic auth healthcheck for opencode runners (Epic #470)
         if (this.runnerConfig.type === "opencode") {
             this.authHealthWatcher = startAuthHealthWatcher(this.agentName, (event) => {
@@ -911,6 +1029,58 @@ Nudge agent:
             }, 5 * 60 * 1000); // Poll every 5 minutes
         }
     }
+    startCommandPoller() {
+        if (!this.token || !this.agentId || !this.isWorkerAgent) {
+            return;
+        }
+        // Poll every 5s so commands created during WS disconnects are still picked up.
+        this.commandPollInterval = setInterval(() => {
+            void this.pollNextCommand();
+        }, 5000);
+        void this.pollNextCommand();
+    }
+    async pollNextCommand() {
+        if (!this.isRunning || this.commandPollInFlight || !this.token || !this.agentId) {
+            return;
+        }
+        this.commandPollInFlight = true;
+        try {
+            const params = new URLSearchParams({ target_agent_id: this.agentId });
+            const res = await fetch(`${this.config.hubUrl}/api/v1/workspaces/${this.config.workspace}/commands/next?${params.toString()}`, {
+                headers: {
+                    Authorization: `Bearer ${this.token}`,
+                },
+            });
+            if (!res.ok) {
+                if (res.status !== 404) {
+                    const body = await res.text();
+                    console.warn(`[command-poller] hub returned ${res.status}: ${body}`);
+                }
+                return;
+            }
+            const payload = (await res.json());
+            if (!payload.command) {
+                return;
+            }
+            handleWebSocketEvent(this.agentName, {
+                type: "control.command",
+                command_id: payload.command.command_id,
+                workspace_id: payload.command.workspace_id,
+                target_agent_id: payload.command.target_agent_id,
+                command_type: payload.command.command_type,
+                payload: payload.command.payload ?? {},
+            }, {
+                hubUrl: this.config.hubUrl,
+                token: this.token ?? undefined,
+            });
+        }
+        catch (err) {
+            console.warn(`[command-poller] error: ${err.message}`);
+        }
+        finally {
+            this.commandPollInFlight = false;
+        }
+    }
     /**
      * Handles session death - logs crash and attempts auto-restart
      */
@@ -966,12 +1136,21 @@ Nudge agent:
             clearInterval(this.inboxPollInterval);
             this.inboxPollInterval = null;
         }
+        if (this.commandPollInterval) {
+            clearInterval(this.commandPollInterval);
+            this.commandPollInterval = null;
+        }
+        if (this.watcherLoopHandle) {
+            this.watcherLoopHandle.stop();
+            this.watcherLoopHandle = null;
+            this.watcherQueue = null;
+        }
     }
     async tryRecoverSession(reason) {
         if (!this.isWorkerAgent || this.serveMode || this.sandboxMode) {
             return false;
         }
-        if (!isRecoverableSessionFailure(reason)) {
+        if (!isRecoverableSessionFailure(reason, this.agentName)) {
             return false;
         }
         const now = Date.now();
@@ -990,7 +1169,10 @@ Nudge agent:
                 command: this.agentConfig.command,
                 workdir: this.agentConfig.workdir,
                 runnerEnv: this.runnerConfig.env,
+                runnerType: this.runnerConfig.type,
                 shouldRestoreContext: false,
+                serveMode: this.serveMode,
+                sandboxMode: this.sandboxMode,
                 autonomous: this.autonomous,
             });
             this._preStartSessionId = sessionStart.preStartSessionId;
@@ -1069,6 +1251,10 @@ Nudge agent:
             clearInterval(this.inboxPollInterval);
             this.inboxPollInterval = null;
         }
+        if (this.commandPollInterval) {
+            clearInterval(this.commandPollInterval);
+            this.commandPollInterval = null;
+        }
         if (this.leadInterval) {
             clearInterval(this.leadInterval);
             this.leadInterval = null;
@@ -1137,9 +1323,10 @@ Nudge agent:
      * Replaces tmux with a direct HTTP server
      */
     async startServeMode() {
-        console.log(`Starting opencode serve mode on port ${this.servePort}...`);
+        console.log(`Starting opencode serve mode on ${this.serveHostname}:${this.servePort}...`);
         const workdir = this.agentConfig.workdir || process.cwd();
         const agentDataDir = prepareOpenCodeRuntime(this.agentName);
+        const servePassword = this.getServePassword();
         // Build environment for opencode serve
         const env = {
             ...process.env,
@@ -1148,8 +1335,17 @@ Nudge agent:
             AGENTMESH_AGENT_ID: this.agentId,
             XDG_DATA_HOME: agentDataDir,
         };
+        if (this.serveUsername) {
+            env.OPENCODE_SERVER_USERNAME = this.serveUsername;
+        }
+        if (servePassword) {
+            env.OPENCODE_SERVER_PASSWORD = servePassword;
+            if (!env.OPENCODE_SERVER_USERNAME) {
+                env.OPENCODE_SERVER_USERNAME = this.serveUsername || "opencode";
+            }
+        }
         // Spawn opencode serve as a child process
-        this.serveProcess = spawn("opencode", ["serve", "--port", String(this.servePort), "--hostname", "0.0.0.0"], {
+        this.serveProcess = spawn("opencode", ["serve", "--port", String(this.servePort), "--hostname", this.serveHostname], {
             cwd: workdir,
             env,
             stdio: ["ignore", "inherit", "inherit"],
@@ -1157,6 +1353,7 @@ Nudge agent:
         // Handle process exit
         this.serveProcess.on("exit", (code, signal) => {
             console.error(`opencode serve exited with code ${code}, signal ${signal}`);
+            void this.publishServeMetadata({ healthy: false, version: this.serveHealth?.version });
             if (this.isRunning) {
                 console.log("Restarting opencode serve in 5 seconds...");
                 setTimeout(() => {
@@ -1169,19 +1366,81 @@ Nudge agent:
         this.serveProcess.on("error", (error) => {
             console.error("Failed to start opencode serve:", error);
         });
-        // Wait a moment for the server to start
-        await new Promise((resolve) => setTimeout(resolve, 2000));
-        console.log(`opencode serve started on http://0.0.0.0:${this.servePort}`);
+        let health;
+        try {
+            health = await waitForOpenCodeServer({
+                hostname: this.serveHostname,
+                port: this.servePort,
+                username: this.serveUsername,
+                password: servePassword,
+            }, { timeoutMs: 20000 });
+        }
+        catch (error) {
+            const failedProcess = this.serveProcess;
+            this.serveProcess = null;
+            if (failedProcess && !failedProcess.killed) {
+                failedProcess.kill("SIGTERM");
+            }
+            throw error;
+        }
+        this.serveHealth = health;
+        updateAgentInState(this.agentName, {
+            serveHealthy: health.healthy,
+            serveVersion: health.version,
+        });
+        await this.publishServeMetadata(health);
+        console.log(`opencode serve started on ${this.getServeLocalUrl()}`);
         // Store saved session ID for integration service reuse
         if (this.shouldRestoreContext && this.agentId) {
             const savedContext = loadContext(this.agentId);
             const savedSessionId = savedContext?.custom?.opencodeSessionId;
-            if (savedSessionId) {
+            if (savedSessionId &&
+                shouldRestoreOpenCodeSession({
+                    runnerType: this.runnerConfig.type,
+                    autonomous: this.autonomous,
+                    serveMode: this.serveMode,
+                    sandboxMode: this.sandboxMode,
+                })) {
                 console.log(`[SERVE] Saved OpenCode session available for reuse: ${savedSessionId}`);
                 updateAgentInState(this.agentName, { opencodeSessionId: savedSessionId });
             }
         }
     }
+    async publishServeMetadata(health) {
+        if (!this.agentId || !this.token)
+            return;
+        this.serveHealth = health;
+        await patchAgentMetadata(this.config.hubUrl, this.config.workspace, this.agentId, this.token, buildOpenCodeMetadata({
+            hostname: this.serveHostname,
+            port: this.servePort,
+            publicUrl: this.servePublicUrl,
+            configuredModel: this.runnerConfig.model,
+            username: this.serveUsername,
+            password: this.getServePassword(),
+        }, health));
+    }
+    async publishDurableAgentMetadata() {
+        if (!this.agentId || !this.token)
+            return;
+        if (this.projectRole && isKnownRole(this.projectRole)) {
+            await patchAgentMetadata(this.config.hubUrl, this.config.workspace, this.agentId, this.token, {
+                role: this.projectRole,
+            });
+        }
+        if (this.serveMode && this.serveHealth) {
+            await this.publishServeMetadata(this.serveHealth);
+        }
+    }
+    getServeLocalUrl() {
+        const effectiveHost = this.serveHostname === "0.0.0.0" ? "127.0.0.1" : this.serveHostname;
+        return `http://${effectiveHost}:${this.servePort}`;
+    }
+    getServePassword() {
+        if (this.servePasswordEnv) {
+            return process.env[this.servePasswordEnv];
+        }
+        return process.env.OPENCODE_SERVER_PASSWORD;
+    }
     /**
      * Starts agent in Docker sandbox mode
      * Provides filesystem isolation with only workspace mounted
@@ -1326,10 +1585,20 @@ Logs:    docker logs ${containerName}
                 };
             }
             // Capture OpenCode session ID for native resume on restart
-            const sessionId = getLatestSessionId(this.agentName);
-            if (sessionId) {
-                context.custom = { ...context.custom, opencodeSessionId: sessionId };
-                console.log(`[CONTEXT] Captured OpenCode session ID: ${sessionId}`);
+            if (shouldPersistOpenCodeSession({
+                runnerType: this.runnerConfig.type,
+                autonomous: this.autonomous,
+                serveMode: this.serveMode,
+                sandboxMode: this.sandboxMode,
+            })) {
+                const sessionId = getLatestSessionId(this.agentName);
+                if (sessionId) {
+                    context.custom = { ...context.custom, opencodeSessionId: sessionId };
+                    console.log(`[CONTEXT] Captured OpenCode session ID: ${sessionId}`);
+                }
+            }
+            else if (context.custom?.opencodeSessionId) {
+                context.custom = { ...context.custom, opencodeSessionId: undefined };
             }
             // Save updated context
             saveContext(context);