ework-daemon 0.1.3 → 0.2.0

package/src/opencode.ts

@@ -1,5 +1,5 @@
 import { spawn, type Subprocess } from "bun";
-import { mkdirSync, writeFileSync } from "fs";
+import { mkdirSync, writeFileSync, readdirSync } from "fs";
 import { join, resolve, isAbsolute } from "path";
 import { homedir } from "os";
 import { log } from "./logger";
@@ -14,6 +14,64 @@ interface TrackerRegistry {
   get(type: string): IssueTracker | undefined;
 }
 
+/**
+ * Pluggable strategy for taking over a session's workdir + opencode session.
+ * Phase 1 ships only RecloneStrategy (fresh clone + fresh opencode session).
+ * Phase 2 swaps in NAS-backed / OpenCode-server strategies without touching
+ * the coordination layer.
+ */
+export interface TakeoverStrategy {
+  /** Resolve (and ensure exists) the workdir for this session+issue. */
+  acquireWorkdir(session: OpSession, issue: Issue): Promise<string>;
+  /** Return an opencode session id to resume, or null for a fresh session. */
+  resumeOpenCodeSession(session: OpSession): Promise<string | null>;
+}
+
+/**
+ * Default TakeoverStrategy: deterministic per-issue workdir under
+ * `<baseWorkdir>/<owner>--<repo>/<issueId>/<sessionName>`, with a best-effort
+ * `git clone` when the directory is empty. Resume always returns null
+ * (fresh opencode session — accepts memory loss on takeover).
+ */
+export class RecloneStrategy implements TakeoverStrategy {
+  constructor(private cfg: Config) {}
+
+  async acquireWorkdir(session: OpSession, issue: Issue): Promise<string> {
+    if (session.workdir) {
+      let dir = session.workdir;
+      if (dir.startsWith("~")) dir = join(homedir(), dir.slice(1));
+      dir = isAbsolute(dir) ? dir : resolve(this.cfg.opencode.baseWorkdir, dir);
+      mkdirSync(dir, { recursive: true });
+      return dir;
+    }
+    const parts = issue.trackerScopeKey.split("/");
+    const owner = issue.trackerScope["owner"] ?? parts[0] ?? "default";
+    const repo = issue.trackerScope["repo"] ?? parts[parts.length - 1] ?? "default";
+    const dir = join(
+      this.cfg.opencode.baseWorkdir,
+      `${owner}--${repo}`,
+      String(issue.trackerIssueId),
+      session.name,
+    );
+    mkdirSync(dir, { recursive: true });
+    try {
+      const entries = readdirSync(dir);
+      if (entries.length === 0) {
+        const base = this.cfg.gitea.url.replace(/\/$/, "");
+        const url = `${base}/${owner}/${repo}.git`;
+        Bun.spawnSync({ cmd: ["git", "clone", url, dir], stdout: "ignore", stderr: "ignore" });
+      }
+    } catch {
+      // directory access failed — leave it; the agent's own tools can clone
+    }
+    return dir;
+  }
+
+  async resumeOpenCodeSession(_session: OpSession): Promise<string | null> {
+    return null;
+  }
+}
+
 /**
  * Extract the target name of an @mention from comment text.
  *
@@ -61,10 +119,20 @@ export function pickLastActive(sessions: OpSession[]): OpSession | undefined {
 
 // ─── Engine ───
 
+export interface EngineOptions {
+  /** DB-allocated logical daemon id (from Store.registerDaemon). */
+  daemonId: number;
+  /** Workdir + session-resume strategy; defaults to RecloneStrategy. */
+  takeover?: TakeoverStrategy;
+}
+
 export class Engine {
   private cfg: Config;
   private store: Store;
   private trackers: TrackerRegistry;
+  private readonly daemonId: number;
+  private readonly takeover: TakeoverStrategy;
+  private heartbeatTimer?: ReturnType<typeof setInterval>;
 
   // Runtime state keyed by session key (trackerType:scopeKey#issueId@sessionName)
   private processes = new Map<string, Subprocess<"ignore", "pipe", "pipe">>();
@@ -97,14 +165,52 @@ export class Engine {
   private static STUCK_THRESHOLD_MS = 30 * 60 * 1000;
   private static MAX_PROCESS_EXIT_NUDGE_ROUNDS = 1;
 
-  constructor(cfg: Config, store: Store, trackers: TrackerRegistry) {
+  constructor(cfg: Config, store: Store, trackers: TrackerRegistry, opts: EngineOptions) {
     this.cfg = cfg;
     this.store = store;
     this.trackers = trackers;
+    this.daemonId = opts.daemonId;
+    this.takeover = opts.takeover ?? new RecloneStrategy(cfg);
     this.startGlobalObserver();
     void this.recover();
   }
 
+  /** Start the lease heartbeat. Must be called once after registerDaemon. */
+  startHeartbeat(intervalMs: number): void {
+    if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
+    this.heartbeatTimer = setInterval(() => {
+      this.store.heartbeat(this.daemonId).catch((e) => {
+        log.error(`engine: heartbeat failed for daemon ${this.daemonId}:`, (e as Error).message);
+      });
+    }, intervalMs);
+  }
+
+  stopHeartbeat(): void {
+    if (this.heartbeatTimer) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = undefined;
+    }
+  }
+
+  getDaemonId(): number {
+    return this.daemonId;
+  }
+
+  /**
+   * Ensure this engine owns the issue before doing work on it. Returns true
+   * if we own it (either already, or just claimed). Returns false if another
+   * daemon won the claim — caller must skip.
+   */
+  private async ensureOwned(issue: Issue): Promise<boolean> {
+    if (issue.ownerDaemonId === this.daemonId) return true;
+    const won = await this.store.claimIssue(issue.id, this.daemonId);
+    if (!won) {
+      log.info(`engine: lost claim on issue ${issue.id} to another daemon (owner=${issue.ownerDaemonId})`);
+      return false;
+    }
+    return true;
+  }
+
   private get stuckThresholdMs(): number {
     return this.cfg.stuck?.thresholdMs ?? Engine.STUCK_THRESHOLD_MS;
   }
@@ -127,27 +233,24 @@ export class Engine {
     return { trackerType: issue.trackerType, scope: issue.trackerScope, issueId: issue.trackerIssueId };
   }
 
-  private resolveWorkdir(session: OpSession, issue: Issue): string {
-    if (session.workdir) {
-      let dir = session.workdir;
-      if (dir.startsWith("~")) dir = join(homedir(), dir.slice(1));
-      return isAbsolute(dir) ? dir : resolve(this.cfg.opencode.baseWorkdir, dir);
-    }
-    // Fallback: use repo name from scope
-    const repoName = issue.trackerScope["repo"] ?? issue.trackerScopeKey.split("/").pop() ?? "default";
-    return join(this.cfg.opencode.baseWorkdir, repoName);
+  private async resolveWorkdir(session: OpSession, issue: Issue): Promise<string> {
+    return this.takeover.acquireWorkdir(session, issue);
   }
 
-  private persistRuntimeState(sessionId: string) {
-    const session = this.store.getSession(sessionId);
+  private async persistRuntimeState(sessionId: string) {
+    const session = await this.store.getSession(sessionId);
     if (!session) return;
-    const issue = this.store.getIssue(session.issueId);
+    const issue = await this.store.getIssue(session.issueId);
     if (!issue) return;
     const k = this.sessionKey(session, issue);
-    this.store.updateSession(sessionId, {
+    await this.store.updateSession(sessionId, {
       startedAt: this.startedAt.get(k),
       progressCommentId: this.progressCommentId.get(k),
       currentPrompt: this.currentPrompt.get(k),
+      lastOutputAt: this.lastOutputAt.get(k),
+      nudgeRounds: this.nudgeRounds.get(k) ?? 0,
+      stuckNudgeRounds: this.stuckNudgeRounds.get(k) ?? 0,
+      generation: this.generation.get(k) ?? 0,
     });
   }
 
@@ -234,33 +337,38 @@ export class Engine {
     model?: string,
   ) {
     // Create or find issue
-    const issue = this.store.findOrCreateIssue(ref, scopeKey, issueData.title);
+    const issue = await this.store.findOrCreateIssue(ref, scopeKey, issueData.title);
     if (issue.state === "closed") {
       // Issue was closed before, now reopened
-      this.store.updateIssueState(issue.id, "active");
+      await this.store.updateIssueState(issue.id, "active");
       issue.state = "active";
     } else if (issue.state === "created") {
-      this.store.updateIssueState(issue.id, "active");
+      await this.store.updateIssueState(issue.id, "active");
       issue.state = "active";
     }
 
+    // Multi-machine: claim before doing work. If another daemon already owns
+    // this issue, skip — they will handle it.
+    if (!(await this.ensureOwned(issue))) return;
+    issue.ownerDaemonId = this.daemonId;
+
     // Start observer for this issue
     this.startObserver(issue);
 
     // Create default session for bot user
     const defaultSessionName = this.cfg.bot.username;
-    let session = this.store.getSessionByName(issue.id, defaultSessionName);
+    let session = await this.store.getSessionByName(issue.id, defaultSessionName);
     if (session && session.state === "running") {
       log.info(`engine: duplicate issue_opened — session already running for ${scopeKey}#${ref.issueId}`);
       this.startObserver(issue);
       return;
     }
     if (!session) {
-      session = this.store.createSession(issue.id, defaultSessionName);
+      session = await this.store.createSession(issue.id, defaultSessionName);
     }
 
     const k = this.sessionKey(session, issue);
-    const workdir = this.resolveWorkdir(session, issue);
+    const workdir = await this.resolveWorkdir(session, issue);
     log.info(`engine: session "${session.name}" created for ${k}, workdir=${workdir}`);
 
     await tracker.createComment(ref, `[system] 🔄 **${session.name}** picked up this issue.\n> session: \`${session.id}\` | workdir: \`${workdir}\``);
@@ -271,7 +379,7 @@ export class Engine {
       this.handleLargeContent(workdir, issueData.body, "issue-body.txt"),
       issueData.author, workdir, instructions
     );
-    this.enqueueOrRun(session, issue, prompt, undefined, model);
+    await this.enqueueOrRun(session, issue, prompt, undefined, model);
   }
 
   private async handleCommented(
@@ -292,7 +400,7 @@ export class Engine {
     if (issueData.state !== "open") return;
 
     if (comment.id) {
-      if (this.processingComments.has(comment.id) || this.store.findMessageByCommentId(comment.id)) {
+      if (this.processingComments.has(comment.id) || await this.store.findMessageByCommentId(comment.id)) {
         log.info(`engine: duplicate comment ${comment.id}, skipping`);
         return;
       }
@@ -301,11 +409,11 @@ export class Engine {
 
     try {
     // Find issue
-    let issue = this.store.findIssue(ref.trackerType, scopeKey, ref.issueId);
+    let issue = await this.store.findIssue(ref.trackerType, scopeKey, ref.issueId);
     if (!issue) {
       // Issue not tracked yet — auto-track it
-      issue = this.store.findOrCreateIssue(ref, scopeKey, issueData.title);
-      this.store.updateIssueState(issue.id, "active");
+      issue = await this.store.findOrCreateIssue(ref, scopeKey, issueData.title);
+      await this.store.updateIssueState(issue.id, "active");
       issue.state = "active";
       this.startObserver(issue);
     } else if (issue.state === "closed") {
@@ -313,6 +421,10 @@ export class Engine {
       return;
     }
 
+    // Multi-machine: claim before doing work.
+    if (!(await this.ensureOwned(issue))) return;
+    issue.ownerDaemonId = this.daemonId;
+
     const dirPath = this.parseDirCommand(comment.body);
     const rawMention = this.extractMentionName(comment.body);
     // Gate extracted @mentions through the agent whitelist. An unknown name
@@ -325,15 +437,15 @@ export class Engine {
 
     if (mentionName) {
       // @mention → targeted delivery
-      let session = this.store.getSessionByName(issue.id, mentionName);
+      let session = await this.store.getSessionByName(issue.id, mentionName);
 
       if (session) {
         // Forward to existing session
         if (dirPath) {
-          this.store.updateSession(session.id, { workdir: dirPath });
+          await this.store.updateSession(session.id, { workdir: dirPath });
           session.workdir = dirPath;
         }
-        const workdir = this.resolveWorkdir(session, issue);
+        const workdir = await this.resolveWorkdir(session, issue);
         const instructions = tracker.getTrackerInstructions(ref);
         const prompt = this.buildForwardPrompt(
           session.name, this.handleLargeContent(workdir, comment.body, `comment-${comment.id}.txt`),
@@ -343,15 +455,15 @@ export class Engine {
         // Immediate ack
         await tracker.createComment(ref, `[system] ✓ Message forwarded to **${session.name}**${this.running.has(this.sessionKey(session, issue)) ? " (running)" : ""}.\n> session: \`${session.id}\` | workdir: \`${workdir}\``);
 
-        this.enqueueOrRun(session, issue, prompt, comment.id, model);
+        await this.enqueueOrRun(session, issue, prompt, comment.id, model);
       } else {
         // Create new session
-        session = this.store.createSession(issue.id, mentionName);
+        session = await this.store.createSession(issue.id, mentionName);
         if (dirPath) {
-          this.store.updateSession(session.id, { workdir: dirPath });
+          await this.store.updateSession(session.id, { workdir: dirPath });
           session.workdir = dirPath;
         }
-        const workdir = this.resolveWorkdir(session, issue);
+        const workdir = await this.resolveWorkdir(session, issue);
 
         await tracker.createComment(ref, `[system] 🔄 **${session.name}** joined the conversation.`);
 
@@ -361,30 +473,30 @@ export class Engine {
           this.handleLargeContent(workdir, issueData.body, "issue-body.txt"),
           issueData.author, workdir, instructions
         );
-        this.enqueueOrRun(session, issue, prompt, comment.id, model);
+        await this.enqueueOrRun(session, issue, prompt, comment.id, model);
       }
     } else {
       // No valid @mention → forward to the most recently active session only.
       // Broadcasting to all sessions makes multiple AIs race on the same request;
       // routing to the last-active lets the user continue without re-@mentioning,
       // while @mention switches to a different AI.
-      const sessions = this.store.getSessionsForIssue(issue.id);
+      const sessions = await this.store.getSessionsForIssue(issue.id);
       if (sessions.length === 0) return;
 
       const session = pickLastActive(sessions);
       if (!session) return;
       if (dirPath) {
-        this.store.updateSession(session.id, { workdir: dirPath });
+        await this.store.updateSession(session.id, { workdir: dirPath });
         session.workdir = dirPath;
       }
-      const workdir = this.resolveWorkdir(session, issue);
+      const workdir = await this.resolveWorkdir(session, issue);
       const instructions = tracker.getTrackerInstructions(ref);
       const prompt = this.buildForwardPrompt(
         session.name, this.handleLargeContent(workdir, comment.body, `comment-${comment.id}.txt`),
         comment.author, issueData.title, workdir, instructions
       );
       await tracker.createComment(ref, `[system] ✓ Message forwarded to **${session.name}**${this.running.has(this.sessionKey(session, issue)) ? " (running)" : ""}.\n> session: \`${session.id}\` | workdir: \`${workdir}\``);
-      this.enqueueOrRun(session, issue, prompt, comment.id, model);
+      await this.enqueueOrRun(session, issue, prompt, comment.id, model);
     }
     } finally {
       if (comment.id) this.processingComments.delete(comment.id);
@@ -396,14 +508,14 @@ export class Engine {
     scopeKey: string,
     tracker: IssueTracker
   ) {
-    const issue = this.store.findIssue(ref.trackerType, scopeKey, ref.issueId);
+    const issue = await this.store.findIssue(ref.trackerType, scopeKey, ref.issueId);
     if (!issue) return;
 
-    this.store.updateIssueState(issue.id, "closed");
+    await this.store.updateIssueState(issue.id, "closed");
     this.stopObserver(issue.id);
 
     // Kill all running processes for this issue's sessions
-    const sessions = this.store.getSessionsForIssue(issue.id);
+    const sessions = await this.store.getSessionsForIssue(issue.id);
     for (const session of sessions) {
       const k = this.sessionKey(session, issue);
       const proc = this.processes.get(k);
@@ -414,14 +526,14 @@ export class Engine {
       // Clear runtime state
       this.clearRuntimeState(k);
       // Mark pending/running messages as interrupted
-      const msgs = this.store.getMessagesForSession(session.id);
+      const msgs = await this.store.getMessagesForSession(session.id);
       for (const msg of msgs) {
         if (msg.status === "pending" || msg.status === "running") {
-          this.store.updateMessageStatus(msg.id, "interrupted", "issue closed");
+          await this.store.updateMessageStatus(msg.id, "interrupted", "issue closed");
         }
       }
       // Update session state
-      this.store.updateSession(session.id, { state: "idle", opencodePid: undefined });
+      await this.store.updateSession(session.id, { state: "idle", opencodePid: undefined });
     }
 
     log.info(`engine: issue closed, ${sessions.length} sessions paused for ${scopeKey}#${ref.issueId}`);
@@ -444,32 +556,32 @@ export class Engine {
 
   // ─── Preemptive Scheduler ───
 
-  private enqueueOrRun(session: OpSession, issue: Issue, prompt: string, sourceCommentId?: string, model?: string) {
+  private async enqueueOrRun(session: OpSession, issue: Issue, prompt: string, sourceCommentId?: string, model?: string) {
     const k = this.sessionKey(session, issue);
 
     this.stuckNudgeRounds.delete(k);
     this.processExitNudgeRounds.delete(k);
 
-    const msg = this.store.createMessage(session.id, prompt, sourceCommentId, undefined, model);
+    const msg = await this.store.createMessage(session.id, prompt, sourceCommentId, undefined, model);
 
     if (this.running.has(k)) {
       // PREEMPTIVE: Kill running process, new message takes priority
       log.info(`engine: preempting ${k} with new message ${msg.id.slice(0, 8)}`);
-      this.preemptSession(k, session, issue, msg);
+      await this.preemptSession(k, session, issue, msg);
       return;
     }
 
     // Not running — execute directly
-    this.executeMessage(k, session, issue, msg);
+    await this.executeMessage(k, session, issue, msg);
   }
 
-  private preemptSession(k: string, session: OpSession, issue: Issue, newMsg: Message) {
+  private async preemptSession(k: string, session: OpSession, issue: Issue, newMsg: Message) {
     const proc = this.processes.get(k);
     const oldMsgId = this.currentMessage.get(k);
 
     // Mark old message as interrupted
     if (oldMsgId) {
-      this.store.updateMessageStatus(oldMsgId, "interrupted", "preempted by new message");
+      await this.store.updateMessageStatus(oldMsgId, "interrupted", "preempted by new message");
     }
 
     // Kill running process
@@ -486,19 +598,28 @@ export class Engine {
     this.startedAt.delete(k);
 
     // Execute new message
-    this.executeMessage(k, session, issue, newMsg);
+    await this.executeMessage(k, session, issue, newMsg);
   }
 
-  private executeMessage(k: string, session: OpSession, issue: Issue, msg: Message) {
+  private async executeMessage(k: string, session: OpSession, issue: Issue, msg: Message) {
     log.info(`engine: executing msg ${msg.id.slice(0, 8)} for ${k}`);
+
+    // Multi-machine: atomic message claim. Pending → running, only if we win.
+    // Locally-created messages always succeed (no contention); this gates the
+    // cross-daemon race when peer daemons share the session.
+    const won = await this.store.claimMessage(msg.id);
+    if (!won) {
+      log.info(`engine: lost message claim for ${msg.id.slice(0, 8)}, another daemon took it`);
+      return;
+    }
+
     this.running.add(k);
-    this.store.updateMessageStatus(msg.id, "running");
     this.currentMessage.set(k, msg.id);
 
     // Update session state
-    this.store.updateSession(session.id, { state: "running" });
+    await this.store.updateSession(session.id, { state: "running" });
 
-    this.execProcess(k, session, issue, msg);
+    void this.execProcess(k, session, issue, msg);
   }
 
   // ─── Process Manager ───
@@ -507,15 +628,22 @@ export class Engine {
     const gen = (this.generation.get(k) ?? 0) + 1;
     this.generation.set(k, gen);
 
-    const workdir = this.resolveWorkdir(session, issue);
-    mkdirSync(workdir, { recursive: true });
+    const workdir = await this.resolveWorkdir(session, issue);
 
     const ref = this.sessionToRef(session, issue);
     const tracker = this.getTracker(issue.trackerType);
 
     const args = [this.cfg.opencode.binary, "run", "--format", "json", "--dir", workdir];
-    if (session.opencodeSessionId) {
-      args.push("--session", session.opencodeSessionId);
+    // Prefer the captured session id (resume our own previous run). Otherwise
+    // ask the takeover strategy whether a resumable session exists elsewhere
+    // (e.g. NAS-backed). Default strategy returns null → fresh session.
+    let resumeSessionId = session.opencodeSessionId;
+    if (!resumeSessionId) {
+      const fromStrategy = await this.takeover.resumeOpenCodeSession(session);
+      if (fromStrategy) resumeSessionId = fromStrategy;
+    }
+    if (resumeSessionId) {
+      args.push("--session", resumeSessionId);
     }
     // Push --model BEFORE the message content. Defends against env-var-
     // registered providers stealing the slot (the original bug). Empty/
@@ -562,8 +690,8 @@ export class Engine {
       this.currentPrompt.set(k, msg.content);
 
       // Persist PID for crash recovery
-      this.store.updateSession(session.id, { opencodePid: proc.pid });
-      this.persistRuntimeState(session.id);
+      await this.store.updateSession(session.id, { opencodePid: proc.pid });
+      await this.persistRuntimeState(session.id);
 
       log.info(`engine: spawned pid=${proc.pid} for ${k}`);
 
@@ -594,7 +722,7 @@ export class Engine {
                 // Persist now, not at exit: a preempt/crash before exit must
                 // not lose the ID, otherwise the re-run opens a fresh session.
                 if (!session.opencodeSessionId) {
-                  this.store.updateSession(session.id, { opencodeSessionId: sid });
+                  await this.store.updateSession(session.id, { opencodeSessionId: sid });
                   session.opencodeSessionId = sid;
                   log.info(`engine: captured sessionID=${sid.slice(0, 8)} for ${k} (early persist)`);
                 }
@@ -617,26 +745,26 @@ export class Engine {
 
       this.processes.delete(k);
       this.lastOutputAt.delete(k);
-      this.store.updateSession(session.id, { opencodePid: undefined });
+      await this.store.updateSession(session.id, { opencodePid: undefined });
 
       if (exitCode !== 0) {
         log.error(`engine: pid=${proc.pid} exited ${exitCode} for ${k}`);
         log.error(`  stderr: ${stderr.slice(0, 2000)}`);
-        this.store.updateMessageStatus(msg.id, "failed", `exit ${exitCode}: ${stderr.slice(0, 500)}`);
+        await this.store.updateMessageStatus(msg.id, "failed", `exit ${exitCode}: ${stderr.slice(0, 500)}`);
       } else {
         log.info(`engine: pid=${proc.pid} completed for ${k}`);
         if (stderr) log.warn(`engine: pid=${proc.pid} stderr on exit 0: ${stderr.slice(0, 500)}`);
         if (!opencodeSessionId) log.warn(`engine: pid=${proc.pid} produced NO sessionID (no stdout output)`);
-        this.store.updateMessageStatus(msg.id, "done");
+        await this.store.updateMessageStatus(msg.id, "done");
       }
 
       // Save opencode session ID for continuity
       if (opencodeSessionId && !session.opencodeSessionId) {
-        this.store.updateSession(session.id, { opencodeSessionId });
+        await this.store.updateSession(session.id, { opencodeSessionId });
       }
     } catch (err) {
       log.error(`engine: exec failed for ${k}:`, err);
-      this.store.updateMessageStatus(msg.id, "failed", (err as Error).message);
+      await this.store.updateMessageStatus(msg.id, "failed", (err as Error).message);
     }
 
     await this.finishRun(k, session, issue, exitCode, gen);
@@ -686,14 +814,14 @@ export class Engine {
     }
     this.progressCommentId.delete(k);
     this.currentPrompt.delete(k);
-    this.persistRuntimeState(session.id);
+    await this.persistRuntimeState(session.id);
 
 
     // spawn failed (exitCode === null) → skip completion check
     if (exitCode === null) {
       log.info(`engine: spawn failed for ${k}, skipping completion check`);
-      this.store.updateSession(session.id, { opencodePid: undefined });
-      this.deactivateIfIdle(k, session, issue);
+      await this.store.updateSession(session.id, { opencodePid: undefined });
+      await this.deactivateIfIdle(k, session, issue);
       return;
     }
     if (superseded()) {
@@ -712,19 +840,19 @@ export class Engine {
     if (hasRecent) {
       log.info(`engine: recent [bot] reply found for ${k}, marking done`);
       this.nudgeRounds.delete(k);
-      this.persistRuntimeState(session.id);
+      await this.persistRuntimeState(session.id);
     } else {
       const nudgeRound = this.nudgeRounds.get(k) ?? 0;
       if (exitCode === 0 && nudgeRound < Engine.MAX_NUDGE_ROUNDS) {
         log.info(`engine: no recent [bot] reply for ${k}, nudging (round ${nudgeRound + 1}/${Engine.MAX_NUDGE_ROUNDS})`);
         this.nudgeRounds.set(k, nudgeRound + 1);
         this.currentPrompt.delete(k);
-        this.persistRuntimeState(session.id);
+        await this.persistRuntimeState(session.id);
 
         const instructions = tracker.getTrackerInstructions(ref);
         const nudgePrompt = this.buildNudgePrompt(session, issue, instructions);
-        const nudgeMsg = this.store.createMessage(session.id, nudgePrompt);
-        this.dequeueOrIdle(k, session, issue, nudgeMsg);
+        const nudgeMsg = await this.store.createMessage(session.id, nudgePrompt);
+        await this.dequeueOrIdle(k, session, issue, nudgeMsg);
         return;
       }
       log.info(`engine: no recent [bot] reply for ${k}, marking done (nudge exhausted or process failed)`);
@@ -734,7 +862,7 @@ export class Engine {
     }
 
     // Remove eyes on source comment; react +1/-1 on the bot's last reply (fallback: source)
-    const recentMsgs = this.store.getRecentMessages(session.id, 1);
+    const recentMsgs = await this.store.getRecentMessages(session.id, 1);
     const lastMsg = recentMsgs[0];
     if (lastMsg?.sourceCommentId) {
       // Check if any other session is still running on this issue
@@ -755,30 +883,30 @@ export class Engine {
       log.info(`engine: finishRun aborted (superseded) for ${k}`);
       return;
     }
-    this.deactivateIfIdle(k, session, issue);
+    await this.deactivateIfIdle(k, session, issue);
   }
 
-  private deactivateIfIdle(k: string, session: OpSession, issue: Issue) {
-    const nextMsg = this.store.getNextPendingMessage(session.id);
+  private async deactivateIfIdle(k: string, session: OpSession, issue: Issue) {
+    const nextMsg = await this.store.getNextPendingMessage(session.id);
     if (nextMsg) {
-      const current = this.store.getSession(session.id);
+      const current = await this.store.getSession(session.id);
       if (current && current.state !== "idle") {
-        this.dequeueOrIdle(k, current, issue, nextMsg);
+        await this.dequeueOrIdle(k, current, issue, nextMsg);
         return;
       }
     }
 
     this.clearRuntimeState(k);
-    this.store.updateSession(session.id, { state: "idle" });
+    await this.store.updateSession(session.id, { state: "idle" });
   }
 
-  private dequeueOrIdle(k: string, session: OpSession, issue: Issue, msg: Message) {
+  private async dequeueOrIdle(k: string, session: OpSession, issue: Issue, msg: Message) {
     log.info(`engine: running dequeued msg ${msg.id.slice(0, 8)} for ${k}`);
-    this.store.updateMessageStatus(msg.id, "running");
+    await this.store.updateMessageStatus(msg.id, "running");
     this.running.add(k);
     this.currentMessage.set(k, msg.id);
-    this.store.updateSession(session.id, { state: "running" });
-    this.execProcess(k, session, issue, msg);
+    await this.store.updateSession(session.id, { state: "running" });
+    void this.execProcess(k, session, issue, msg);
   }
 
   // ─── Prompts ───
@@ -918,10 +1046,18 @@ export class Engine {
   }
 
   private async runObserverCycle() {
-    const activeIssues = this.store.listActiveIssues();
+    // Multi-machine: periodically release stale owners so we can adopt their
+    // work, and only iterate issues/sessions this daemon owns.
+    try {
+      await this.store.releaseDeadOwners(this.cfg.work.leaseTtlMs);
+    } catch (err) {
+      log.error("engine: releaseDeadOwners failed:", (err as Error).message);
+    }
+
+    const ownedIssues = (await this.store.listOwnedIssues(this.daemonId))
+      .filter((i) => this.observedIssues.has(i.id));
 
-    for (const issue of activeIssues) {
-      if (!this.observedIssues.has(issue.id)) continue;
+    for (const issue of ownedIssues) {
       try {
         await this.observeIssue(issue);
       } catch (err) {
@@ -932,7 +1068,7 @@ export class Engine {
 
   private async observeIssue(issue: Issue) {
     const tracker = this.getTracker(issue.trackerType);
-    const sessions = this.store.getSessionsForIssue(issue.id);
+    const sessions = await this.store.getSessionsForIssue(issue.id);
 
     for (const session of sessions) {
       if (session.state !== "running") continue;
@@ -952,12 +1088,12 @@ export class Engine {
           this.processes.delete(k);
           this.lastOutputAt.delete(k);
           this.running.delete(k);
-          this.store.updateSession(session.id, { state: "idle", opencodePid: undefined });
+          await this.store.updateSession(session.id, { state: "idle", opencodePid: undefined });
 
           // Mark running message as failed
           const msgId = this.currentMessage.get(k);
           if (msgId) {
-            this.store.updateMessageStatus(msgId, "failed", "process died unexpectedly");
+            await this.store.updateMessageStatus(msgId, "failed", "process died unexpectedly");
             this.currentMessage.delete(k);
           }
 
@@ -977,8 +1113,8 @@ export class Engine {
 
               const instructions = tracker.getTrackerInstructions(ref);
               const nudgePrompt = this.buildProcessExitNudgePrompt(session, issue, instructions);
-              const nudgeMsg = this.store.createMessage(session.id, nudgePrompt);
-              this.dequeueOrIdle(k, session, issue, nudgeMsg);
+              const nudgeMsg = await this.store.createMessage(session.id, nudgePrompt);
+              await this.dequeueOrIdle(k, session, issue, nudgeMsg);
               continue;
             } else {
               // AI already replied before dying — no need to nudge, but user must be
@@ -997,9 +1133,9 @@ export class Engine {
           }
 
           // Try to dequeue next message
-          const nextMsg = this.store.getNextPendingMessage(session.id);
+          const nextMsg = await this.store.getNextPendingMessage(session.id);
           if (nextMsg) {
-            this.dequeueOrIdle(k, session, issue, nextMsg);
+            await this.dequeueOrIdle(k, session, issue, nextMsg);
           }
           continue;
         }
@@ -1021,8 +1157,8 @@ export class Engine {
 
             const instructions = tracker.getTrackerInstructions(ref);
             const nudgePrompt = this.buildStuckNudgePrompt(session, issue, instructions, minutes);
-            const nudgeMsg = this.store.createMessage(session.id, nudgePrompt);
-            this.dequeueOrIdle(k, session, issue, nudgeMsg);
+            const nudgeMsg = await this.store.createMessage(session.id, nudgePrompt);
+            await this.dequeueOrIdle(k, session, issue, nudgeMsg);
           } else {
             log.warn(`engine: stuck for ${minutes}min on ${k}, stuck nudge exhausted (${stuckNudgeRound}/${this.maxStuckNudges}), giving up`);
             await tracker.createComment(ref, `[system] ⛔ **${session.name}** stuck for ${minutes} min, gave up after ${this.maxStuckNudges} restart(s).`).catch(
@@ -1035,7 +1171,7 @@ export class Engine {
       } else if (session.state === "running" && !this.running.has(k)) {
         log.warn(`engine: observer fixing orphaned running state for ${k}`);
         this.running.delete(k);
-        this.store.updateSession(session.id, { state: "idle" });
+        await this.store.updateSession(session.id, { state: "idle" });
       }
     }
 
@@ -1057,7 +1193,7 @@ export class Engine {
         } else {
           const result = await tracker.createComment(ref, body);
           this.progressCommentId.set(k, result.id);
-          this.persistRuntimeState(session.id);
+          await this.persistRuntimeState(session.id);
         }
       } catch (err) {
         log.error(`engine: progress report failed for ${k}:`, (err as Error).message);
@@ -1071,8 +1207,20 @@ export class Engine {
   // ─── Recovery ───
 
   private async recover() {
-    const allSessions = this.store.listAllSessions();
-    for (const session of allSessions) {
+    // Release stale owners first so we can adopt orphaned issues that just
+    // became available (this daemon is fresh; any dead daemon's slots are now
+    // reclaimable).
+    try {
+      await this.store.releaseDeadOwners(this.cfg.work.leaseTtlMs);
+    } catch (err) {
+      log.error("engine: releaseDeadOwners at boot failed:", (err as Error).message);
+    }
+
+    // Multi-machine: recover ONLY this daemon's sessions. Other daemons own
+    // the rest; touching their state would race them.
+    const ownedSessions = await this.store.listOwnedSessions(this.daemonId);
+
+    for (const session of ownedSessions) {
       if (session.opencodePid) {
         try {
           process.kill(session.opencodePid, 0);
@@ -1088,19 +1236,23 @@ export class Engine {
             try { process.kill(session.opencodePid, "SIGKILL"); } catch { /* dead */ }
           }
         } catch { /* already dead */ }
-        this.store.updateSession(session.id, { opencodePid: undefined });
+        await this.store.updateSession(session.id, { opencodePid: undefined });
       }
     }
 
-    // Restore runtime state from DB
-    for (const session of allSessions) {
-      const issue = this.store.getIssue(session.issueId);
+    // Restore runtime state (now persisted in op_sessions) from DB.
+    for (const session of ownedSessions) {
+      const issue = await this.store.getIssue(session.issueId);
       if (!issue || issue.state === "closed") continue;
       const k = this.sessionKey(session, issue);
       if (session.startedAt != null) this.startedAt.set(k, session.startedAt);
       if (session.progressCommentId) this.progressCommentId.set(k, session.progressCommentId);
       if (session.currentPrompt) this.currentPrompt.set(k, session.currentPrompt);
-      // Start observer for active issues
+      if (session.lastOutputAt != null) this.lastOutputAt.set(k, session.lastOutputAt);
+      if (session.nudgeRounds != null) this.nudgeRounds.set(k, session.nudgeRounds);
+      if (session.stuckNudgeRounds != null) this.stuckNudgeRounds.set(k, session.stuckNudgeRounds);
+      if (session.generation != null) this.generation.set(k, session.generation);
+      // Start observer for active issues we own
       this.startObserver(issue);
     }
 
@@ -1109,8 +1261,8 @@ export class Engine {
       log.info(`engine: restored runtime state for ${restored} sessions from DB`);
     }
 
-    // Recover stuck messages
-    const stuck = this.store.getPendingOrRunningMessages();
+    // Recover stuck messages scoped to this daemon's issues.
+    const stuck = await this.store.getOwnedPendingOrRunningMessages(this.daemonId);
     if (stuck.length === 0) return;
 
     log.info(`engine: recovering ${stuck.length} stuck messages`);
@@ -1118,7 +1270,7 @@ export class Engine {
     // Reset running messages to pending
     for (const msg of stuck) {
       if (msg.status === "running") {
-        this.store.updateMessageStatus(msg.id, "pending");
+        await this.store.updateMessageStatus(msg.id, "pending");
       }
     }
 
@@ -1131,9 +1283,9 @@ export class Engine {
     }
 
     for (const [sessionId, msgs] of bySession) {
-      const session = this.store.getSession(sessionId);
+      const session = await this.store.getSession(sessionId);
       if (!session) continue;
-      const issue = this.store.getIssue(session.issueId);
+      const issue = await this.store.getIssue(session.issueId);
       if (!issue || issue.state === "closed") continue;
 
       const k = this.sessionKey(session, issue);
@@ -1147,53 +1299,54 @@ export class Engine {
       const comments = await tracker.listComments(ref).catch((): TrackerComment[] => []);
       if (this.hasRecentBotReply(comments, tracker)) {
         log.info(`engine: recovered msg ${first.id.slice(0, 8)} for ${k} — bot reply detected, marking done`);
-        this.store.updateMessageStatus(first.id, "done");
-        const next = this.store.getNextPendingMessage(session.id);
-        if (next) { this.dequeueOrIdle(k, session, issue, next); }
+        await this.store.updateMessageStatus(first.id, "done");
+        const next = await this.store.getNextPendingMessage(session.id);
+        if (next) { await this.dequeueOrIdle(k, session, issue, next); }
         continue;
       }
 
       log.info(`engine: recovering msg ${first.id.slice(0, 8)} for ${k}`);
-      this.dequeueOrIdle(k, session, issue, first);
+      await this.dequeueOrIdle(k, session, issue, first);
     }
   }
 
   // ─── API Methods ───
 
-  retryMessage(messageId: string): boolean {
-    const msg = this.store.getMessage(messageId);
+  async retryMessage(messageId: string): Promise<boolean> {
+    const msg = await this.store.getMessage(messageId);
     if (!msg || msg.status !== "failed") return false;
-    const session = this.store.getSession(msg.sessionId);
+    const session = await this.store.getSession(msg.sessionId);
     if (!session) return false;
-    const issue = this.store.getIssue(session.issueId);
+    const issue = await this.store.getIssue(session.issueId);
     if (!issue || issue.state === "closed") return false;
 
-    this.store.updateMessageStatus(messageId, "pending");
+    await this.store.updateMessageStatus(messageId, "pending");
     const k = this.sessionKey(session, issue);
     if (!this.running.has(k)) {
-      this.dequeueOrIdle(k, session, issue, msg);
+      await this.dequeueOrIdle(k, session, issue, msg);
     }
     return true;
   }
 
-  getStatus() {
-    const pendingCount = this.store.getPendingOrRunningMessages().filter(m => m.status === "pending").length;
+  async getStatus() {
+    const pendingCount = (await this.store.getOwnedPendingOrRunningMessages(this.daemonId)).filter(m => m.status === "pending").length;
     return {
       runningCount: this.running.size,
       runningKeys: [...this.running],
       pendingCount,
       processCount: this.processes.size,
       observedIssues: this.observedIssues.size,
+      daemonId: this.daemonId,
     };
   }
 
-  getQueue(): Record<string, number> {
+  async getQueue(): Promise<Record<string, number>> {
     const result: Record<string, number> = {};
-    const allPending = this.store.getPendingOrRunningMessages().filter(m => m.status === "pending");
+    const allPending = (await this.store.getOwnedPendingOrRunningMessages(this.daemonId)).filter(m => m.status === "pending");
     for (const msg of allPending) {
-      const session = this.store.getSession(msg.sessionId);
+      const session = await this.store.getSession(msg.sessionId);
       if (!session) continue;
-      const issue = this.store.getIssue(session.issueId);
+      const issue = await this.store.getIssue(session.issueId);
       if (!issue) continue;
       const k = this.sessionKey(session, issue);
       result[k] = (result[k] ?? 0) + 1;
@@ -1224,7 +1377,7 @@ export class Engine {
     try { process.kill(pid, signal); } catch { /* already dead */ }
   }
 
-  forceStop(key: string): boolean {
+  async forceStop(key: string): Promise<boolean> {
     const proc = this.processes.get(key);
     this.stopping.add(key);
     log.warn(`engine: forceStop ${key}, pid=${proc?.pid ?? "none"}`);
@@ -1248,22 +1401,22 @@ export class Engine {
     // Update session and messages
     const parsed = parseKey(key);
     if (parsed) {
-      const issue = this.store.findIssue(parsed.trackerType, parsed.scopeKey, parsed.issueId);
+      const issue = await this.store.findIssue(parsed.trackerType, parsed.scopeKey, parsed.issueId);
       if (issue) {
-        const session = this.store.getSessionByName(issue.id, parsed.sessionName);
+        const session = await this.store.getSessionByName(issue.id, parsed.sessionName);
         if (session) {
           if (progressId) {
             const ref = this.sessionToRef(session, issue);
             const tracker = this.getTracker(issue.trackerType);
             void tracker.editComment(ref, progressId, `[system] ⛔ **${session.name}** force-stopped.`).catch(() => {});
           }
-          const msgs = this.store.getMessagesForSession(session.id);
+          const msgs = await this.store.getMessagesForSession(session.id);
           for (const msg of msgs) {
             if (msg.status === "pending" || msg.status === "running") {
-              this.store.updateMessageStatus(msg.id, "failed", "force stopped");
+              await this.store.updateMessageStatus(msg.id, "failed", "force stopped");
             }
           }
-          this.store.updateSession(session.id, {
+          await this.store.updateSession(session.id, {
             state: "idle",
             opencodePid: undefined,
             startedAt: undefined,
@@ -1278,6 +1431,7 @@ export class Engine {
   }
 
   destroy() {
+    this.stopHeartbeat();
     if (this.observerTimer) clearInterval(this.observerTimer);
     this.observedIssues.clear();
     for (const [, proc] of this.processes) {