ework-daemon 0.4.28 → 0.4.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ework-daemon",
-  "version": "0.4.28",
+  "version": "0.4.30",
   "description": "Issue-driven AI development daemon. Spawns opencode subprocesses to resolve Gitea issues.",
   "module": "src/index.ts",
   "type": "module",

package/src/db.ts

@@ -18,6 +18,30 @@ import { createPool, type Pool, type PoolConnection, type ResultSetHeader } from
 import { mkdirSync, readFileSync, existsSync } from "fs";
 import { dirname, join } from "path";
 import { homedir } from "os";
+import { log } from "./logger";
+
+const RETRYABLE_CODES = ["ETIMEDOUT", "ECONNRESET", "PROTOCOL_CONNECTION_LOST", "PROTOCOL_SEQUENCE_TIMEOUT", "EPIPE"];
+const MAX_DB_RETRIES = 2;
+const DB_RETRY_BASE_MS = 300;
+
+async function withDbRetry<T>(fn: () => Promise<T>, label: string): Promise<T> {
+  let lastErr: unknown;
+  for (let attempt = 0; attempt <= MAX_DB_RETRIES; attempt++) {
+    try {
+      return await fn();
+    } catch (e) {
+      lastErr = e;
+      const code = (e as { code?: string })?.code;
+      if (!code || !RETRYABLE_CODES.includes(code)) throw e;
+      if (attempt < MAX_DB_RETRIES) {
+        const delay = DB_RETRY_BASE_MS * Math.pow(2, attempt);
+        log.warn(`db: ${label} failed (${code}), retry ${attempt + 1}/${MAX_DB_RETRIES} in ${delay}ms`);
+        await new Promise((r) => setTimeout(r, delay));
+      }
+    }
+  }
+  throw lastErr;
+}
 
 // ---- public async interface (driver-agnostic) ----
 export interface DbRunResult {
@@ -224,21 +248,29 @@ class MysqlDriver implements AsyncDatabase {
   }
 
   async all<T = unknown>(sql: string, params: unknown[] = []): Promise<T[]> {
-    const [rows] = await this.conn.query(this.prepare(sql), params);
-    return rows as T[];
+    return withDbRetry(async () => {
+      const [rows] = await this.conn.query(this.prepare(sql), params);
+      return rows as T[];
+    }, "all");
   }
   async get<T = unknown>(sql: string, params: unknown[] = []): Promise<T | null> {
-    const [rows] = await this.conn.query(this.prepare(sql), params);
-    const arr = rows as T[];
-    return arr[0] ?? null;
+    return withDbRetry(async () => {
+      const [rows] = await this.conn.query(this.prepare(sql), params);
+      const arr = rows as T[];
+      return arr[0] ?? null;
+    }, "get");
   }
   async run(sql: string, params: unknown[] = []): Promise<DbRunResult> {
-    const [result] = await this.conn.query(this.prepare(sql), params);
-    const r = result as ResultSetHeader;
-    return { insertId: Number(r.insertId), changes: r.affectedRows };
+    return withDbRetry(async () => {
+      const [result] = await this.conn.query(this.prepare(sql), params);
+      const r = result as ResultSetHeader;
+      return { insertId: Number(r.insertId), changes: r.affectedRows };
+    }, "run");
   }
   async exec(sql: string): Promise<void> {
-    await this.conn.query(this.prepare(sql));
+    return withDbRetry(async () => {
+      await this.conn.query(this.prepare(sql));
+    }, "exec");
   }
   async transaction<T>(fn: () => Promise<T>): Promise<T> {
     if (this.txConn) return fn();

package/src/index.ts

@@ -88,6 +88,13 @@ async function boot() {
   process.on("SIGTERM", () => shutdown("SIGTERM"));
   process.on("SIGINT", () => shutdown("SIGINT"));
 
+  process.on("unhandledRejection", (reason) => {
+    log.error("unhandledRejection (continuing):", reason);
+  });
+  process.on("uncaughtException", (err) => {
+    log.error("uncaughtException (continuing):", err);
+  });
+
   const activeCount = (await store.listActiveIssues()).length;
   log.info(`\n${isTest ? "🧪" : "✅"} ework-daemon ready at http://${server.hostname}:${server.port}/webhook`);
   log.info(`   Configure Gitea webhook to POST to /webhook/gitea`);

package/src/op.ts

@@ -420,6 +420,21 @@ export class Store {
     );
   }
 
+  async getDaemonCapacity(daemonId: number): Promise<number | null> {
+    const row = await getDB().get<{ capacity: number }>(
+      "SELECT capacity FROM {{daemons}} WHERE id = ?",
+      [daemonId]
+    );
+    return row ? row.capacity : null;
+  }
+
+  async updateDaemonCapacity(daemonId: number, capacity: number): Promise<void> {
+    await getDB().run(
+      "UPDATE {{daemons}} SET capacity = ? WHERE id = ?",
+      [capacity, daemonId]
+    );
+  }
+
   async markDaemonStatus(daemonId: number, status: "active" | "drained" | "dead"): Promise<void> {
     await getDB().run(
       "UPDATE {{daemons}} SET status = ? WHERE id = ?",

package/src/opencode.ts

@@ -300,6 +300,7 @@ export class Engine {
   private readonly takeover: TakeoverStrategy;
   private readonly backend: RuntimeBackend;
   private heartbeatTimer?: ReturnType<typeof setInterval>;
+  private maxConcurrent: number;
 
   // Runtime state keyed by session key (trackerType:scopeKey#issueId@sessionName)
   private processes = new Map<string, RuntimeHandle>();
@@ -348,6 +349,7 @@ export class Engine {
     this.daemonId = opts.daemonId;
     this.takeover = opts.takeover ?? new RecloneStrategy(cfg);
     this.backend = opts.backend ?? createDefaultBackend(cfg);
+    this.maxConcurrent = cfg.work.maxConcurrent;
     this.startGlobalObserver();
     void this.recover();
   }
@@ -369,9 +371,30 @@ export class Engine {
       this.store.heartbeat(this.daemonId).catch((e) => {
         log.error(`engine: heartbeat failed for daemon ${this.daemonId}:`, (e as Error).message);
       });
+      void this.syncMaxConcurrent();
     }, intervalMs);
   }
 
+  private async syncMaxConcurrent(): Promise<void> {
+    try {
+      const cap = await this.store.getDaemonCapacity(this.daemonId);
+      if (cap != null && cap > 0 && cap !== this.maxConcurrent) {
+        log.info(`engine: maxConcurrent updated ${this.maxConcurrent} → ${cap} (DB sync)`);
+        this.maxConcurrent = cap;
+      }
+    } catch { /* non-critical */ }
+  }
+
+  setMaxConcurrent(n: number): void {
+    if (!Number.isFinite(n) || n < 1) return;
+    this.maxConcurrent = Math.floor(n);
+    log.info(`engine: maxConcurrent set to ${this.maxConcurrent}`);
+  }
+
+  getMaxConcurrent(): number {
+    return this.maxConcurrent;
+  }
+
   stopHeartbeat(): void {
     if (this.heartbeatTimer) {
       clearInterval(this.heartbeatTimer);
@@ -562,8 +585,8 @@ export class Engine {
     const tracker = this.getTracker(ref.trackerType);
     const scopeKey = tracker.formatScopeKey(ref.scope);
 
-    if (this.paused && event.type === "issue_opened") {
-      log.info(`engine: paused — skipping issue_opened for ${ref.trackerType}:${scopeKey}#${ref.issueId}`);
+    if (this.paused && (event.type === "issue_opened" || event.type === "comment_created")) {
+      log.info(`engine: paused — skipping ${event.type} for ${ref.trackerType}:${scopeKey}#${ref.issueId}`);
       return;
     }
 
@@ -796,13 +819,11 @@ export class Engine {
 
     // Kill all running processes for this issue's sessions
     const sessions = await this.store.getSessionsForIssue(issue.id);
+    let killedCount = 0;
     for (const session of sessions) {
       const k = this.sessionKey(session, issue);
-      const proc = this.processes.get(k);
-      if (proc) {
-        this.stopping.add(k);
-        try { this.killProcessTree(proc.pid, "SIGTERM"); } catch { /* already dead */ }
-      }
+      const killed = await this.killSessionProcess(session, k);
+      if (killed) killedCount++;
       this.clearRuntimeState(k);
       const msgs = await this.store.getMessagesForSession(session.id);
       for (const msg of msgs) {
@@ -829,7 +850,7 @@ export class Engine {
     this.cloneUrls.delete(gcKey);
     this.senders.delete(gcKey);
 
-    log.info(`engine: issue closed, ${sessions.length} sessions paused for ${scopeKey}#${ref.issueId}`);
+    log.info(`engine: issue closed, ${killedCount}/${sessions.length} sessions killed for ${scopeKey}#${ref.issueId}`);
     void tracker.updateStatus(ref, "completed");
   }
 
@@ -842,13 +863,11 @@ export class Engine {
     if (!issue) return;
     this.stopObserver(issue.id);
     const sessions = await this.store.getSessionsForIssue(issue.id);
+    let killedCount = 0;
     for (const session of sessions) {
       const k = this.sessionKey(session, issue);
-      const proc = this.processes.get(k);
-      if (proc) {
-        this.stopping.add(k);
-        try { this.killProcessTree(proc.pid, "SIGTERM"); } catch { /* already dead */ }
-      }
+      const killed = await this.killSessionProcess(session, k);
+      if (killed) killedCount++;
       this.clearRuntimeState(k);
       const msgs = await this.store.getMessagesForSession(session.id);
       for (const msg of msgs) {
@@ -858,7 +877,7 @@ export class Engine {
       }
       await this.store.updateSession(session.id, { state: "idle", opencodePid: undefined });
     }
-    log.info(`engine: issue halted, ${sessions.length} sessions killed for ${scopeKey}#${ref.issueId}`);
+    log.info(`engine: issue halted, ${killedCount}/${sessions.length} sessions killed for ${scopeKey}#${ref.issueId}`);
     try { await tracker.createComment(ref, "[system] ⏸️ AI processing halted by user."); } catch { /* tracker unavailable */ }
   }
 
@@ -878,6 +897,36 @@ export class Engine {
     this.generation.delete(k);
   }
 
+  private async killSessionProcess(session: OpSession, k: string): Promise<boolean> {
+    const handle = this.processes.get(k);
+    if (handle) {
+      this.stopping.add(k);
+      try { this.killProcessTree(handle.pid, "SIGTERM"); } catch { /* already dead */ }
+      this.processes.delete(k);
+      return true;
+    }
+
+    const pid = session.opencodePid;
+    if (!pid) return false;
+    try {
+      process.kill(pid, 0);
+    } catch {
+      return false;
+    }
+
+    log.info(`engine: killing orphaned pid=${pid} for ${k} (cross-restart)`);
+    this.stopping.add(k);
+    try {
+      this.killProcessTree(pid, "SIGTERM");
+      for (let i = 0; i < 30; i++) {
+        await new Promise(r => setTimeout(r, 100));
+        try { process.kill(pid, 0); } catch { break; }
+      }
+      try { process.kill(pid, "SIGKILL"); } catch { /* dead */ }
+    } catch { /* already dead */ }
+    return true;
+  }
+
   // ─── Preemptive Scheduler ───
 
   private async enqueueOrRun(session: OpSession, issue: Issue, prompt: string, sourceCommentId?: string, model?: string) {
@@ -895,8 +944,8 @@ export class Engine {
       return;
     }
 
-    if (this.running.size >= this.cfg.work.maxConcurrent) {
-      log.info(`engine: concurrency limit reached (${this.running.size}/${this.cfg.work.maxConcurrent}), message ${msg.id.slice(0, 8)} queued for ${k}`);
+    if (this.running.size >= this.maxConcurrent) {
+      log.info(`engine: concurrency limit reached (${this.running.size}/${this.maxConcurrent}), message ${msg.id.slice(0, 8)} queued for ${k}`);
       return;
     }
 
@@ -1213,8 +1262,8 @@ export class Engine {
     if (nextMsg) {
       const current = await this.store.getSession(session.id);
       if (current && current.state !== "idle") {
-        if (this.running.size >= this.cfg.work.maxConcurrent) {
-          log.info(`engine: concurrency limit (${this.running.size}/${this.cfg.work.maxConcurrent}), keeping msg ${nextMsg.id.slice(0, 8)} pending for ${k}`);
+        if (this.running.size >= this.maxConcurrent) {
+          log.info(`engine: concurrency limit (${this.running.size}/${this.maxConcurrent}), keeping msg ${nextMsg.id.slice(0, 8)} pending for ${k}`);
         } else {
           await this.dequeueOrIdle(k, current, issue, nextMsg);
           return;
@@ -1229,11 +1278,11 @@ export class Engine {
   }
 
   private async drainGlobalPending(): Promise<void> {
-    const slotsAvailable = this.cfg.work.maxConcurrent - this.running.size;
+    const slotsAvailable = this.maxConcurrent - this.running.size;
     if (slotsAvailable <= 0) return;
     const pending = await this.store.getGlobalPendingMessages(slotsAvailable);
     for (const msg of pending) {
-      if (this.running.size >= this.cfg.work.maxConcurrent) break;
+      if (this.running.size >= this.maxConcurrent) break;
       const session = await this.store.getSession(msg.sessionId);
       if (!session || session.state === "running") continue;
       const issue = await this.store.getIssue(session.issueId);
@@ -1393,16 +1442,20 @@ export class Engine {
   }
 
   private async runObserverCycle() {
-    // Multi-machine: periodically release stale owners so we can adopt their
-    // work, and only iterate issues/sessions this daemon owns.
     try {
       await this.store.releaseDeadOwners(this.cfg.work.leaseTtlMs);
     } catch (err) {
       log.error("engine: releaseDeadOwners failed:", (err as Error).message);
     }
 
-    const ownedIssues = (await this.store.listOwnedIssues(this.daemonId))
-      .filter((i) => this.observedIssues.has(i.id));
+    let ownedIssues;
+    try {
+      ownedIssues = (await this.store.listOwnedIssues(this.daemonId))
+        .filter((i) => this.observedIssues.has(i.id));
+    } catch (err) {
+      log.error("engine: listOwnedIssues failed:", (err as Error).message);
+      return;
+    }
 
     for (const issue of ownedIssues) {
       try {

package/src/server.ts

@@ -213,6 +213,15 @@ export function createServer(
       await engine.resume();
       return json({ ok: true, paused: false });
     }
+    if (pathname === "/api/admin/max-concurrent" && req.method === "POST") {
+      const body = await req.json().catch(() => ({} as unknown));
+      const value = (body as { value?: unknown })?.value;
+      if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
+        return json({ error: "value must be a positive number" }, 400);
+      }
+      engine.setMaxConcurrent(Math.floor(value));
+      return json({ ok: true, maxConcurrent: engine.getMaxConcurrent() });
+    }
 
     return json({ error: "not found" }, 404);
   }