ework-daemon 0.4.29 → 0.4.31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ework-daemon",
-  "version": "0.4.29",
+  "version": "0.4.31",
   "description": "Issue-driven AI development daemon. Spawns opencode subprocesses to resolve Gitea issues.",
   "module": "src/index.ts",
   "type": "module",

package/src/db.ts

@@ -18,6 +18,30 @@ import { createPool, type Pool, type PoolConnection, type ResultSetHeader } from
 import { mkdirSync, readFileSync, existsSync } from "fs";
 import { dirname, join } from "path";
 import { homedir } from "os";
+import { log } from "./logger";
+
+const RETRYABLE_CODES = ["ETIMEDOUT", "ECONNRESET", "PROTOCOL_CONNECTION_LOST", "PROTOCOL_SEQUENCE_TIMEOUT", "EPIPE"];
+const MAX_DB_RETRIES = 2;
+const DB_RETRY_BASE_MS = 300;
+
+async function withDbRetry<T>(fn: () => Promise<T>, label: string): Promise<T> {
+  let lastErr: unknown;
+  for (let attempt = 0; attempt <= MAX_DB_RETRIES; attempt++) {
+    try {
+      return await fn();
+    } catch (e) {
+      lastErr = e;
+      const code = (e as { code?: string })?.code;
+      if (!code || !RETRYABLE_CODES.includes(code)) throw e;
+      if (attempt < MAX_DB_RETRIES) {
+        const delay = DB_RETRY_BASE_MS * Math.pow(2, attempt);
+        log.warn(`db: ${label} failed (${code}), retry ${attempt + 1}/${MAX_DB_RETRIES} in ${delay}ms`);
+        await new Promise((r) => setTimeout(r, delay));
+      }
+    }
+  }
+  throw lastErr;
+}
 
 // ---- public async interface (driver-agnostic) ----
 export interface DbRunResult {
@@ -224,21 +248,29 @@ class MysqlDriver implements AsyncDatabase {
   }
 
   async all<T = unknown>(sql: string, params: unknown[] = []): Promise<T[]> {
-    const [rows] = await this.conn.query(this.prepare(sql), params);
-    return rows as T[];
+    return withDbRetry(async () => {
+      const [rows] = await this.conn.query(this.prepare(sql), params);
+      return rows as T[];
+    }, "all");
   }
   async get<T = unknown>(sql: string, params: unknown[] = []): Promise<T | null> {
-    const [rows] = await this.conn.query(this.prepare(sql), params);
-    const arr = rows as T[];
-    return arr[0] ?? null;
+    return withDbRetry(async () => {
+      const [rows] = await this.conn.query(this.prepare(sql), params);
+      const arr = rows as T[];
+      return arr[0] ?? null;
+    }, "get");
   }
   async run(sql: string, params: unknown[] = []): Promise<DbRunResult> {
-    const [result] = await this.conn.query(this.prepare(sql), params);
-    const r = result as ResultSetHeader;
-    return { insertId: Number(r.insertId), changes: r.affectedRows };
+    return withDbRetry(async () => {
+      const [result] = await this.conn.query(this.prepare(sql), params);
+      const r = result as ResultSetHeader;
+      return { insertId: Number(r.insertId), changes: r.affectedRows };
+    }, "run");
   }
   async exec(sql: string): Promise<void> {
-    await this.conn.query(this.prepare(sql));
+    return withDbRetry(async () => {
+      await this.conn.query(this.prepare(sql));
+    }, "exec");
   }
   async transaction<T>(fn: () => Promise<T>): Promise<T> {
     if (this.txConn) return fn();

package/src/op.ts

@@ -260,6 +260,13 @@ export class Store {
     return rows.map(rowToSession);
   }
 
+  async listSessionsWithPid(): Promise<Array<{ id: string; opencodePid: number }>> {
+    const rows = await getDB().all<{ uid: string; opencode_pid: number }>(
+      "SELECT uid, opencode_pid FROM {{op_sessions}} WHERE opencode_pid IS NOT NULL"
+    );
+    return rows.map((r) => ({ id: r.uid, opencodePid: r.opencode_pid }));
+  }
+
   async listNonIdleSessions(): Promise<OpSession[]> {
     const rows = await getDB().all<SessionRow>("SELECT * FROM {{op_sessions}} WHERE state != 'idle'");
     return rows.map(rowToSession);
@@ -420,6 +427,21 @@ export class Store {
     );
   }
 
+  async getDaemonCapacity(daemonId: number): Promise<number | null> {
+    const row = await getDB().get<{ capacity: number }>(
+      "SELECT capacity FROM {{daemons}} WHERE id = ?",
+      [daemonId]
+    );
+    return row ? row.capacity : null;
+  }
+
+  async updateDaemonCapacity(daemonId: number, capacity: number): Promise<void> {
+    await getDB().run(
+      "UPDATE {{daemons}} SET capacity = ? WHERE id = ?",
+      [capacity, daemonId]
+    );
+  }
+
   async markDaemonStatus(daemonId: number, status: "active" | "drained" | "dead"): Promise<void> {
     await getDB().run(
       "UPDATE {{daemons}} SET status = ? WHERE id = ?",

package/src/opencode.ts

@@ -1,5 +1,5 @@
 import { Database } from "bun:sqlite";
-import { mkdirSync, writeFileSync, readdirSync, existsSync } from "fs";
+import { mkdirSync, writeFileSync, readdirSync, existsSync, readFileSync } from "fs";
 import { join, resolve, isAbsolute } from "path";
 import { homedir } from "os";
 import { log } from "./logger";
@@ -300,6 +300,7 @@ export class Engine {
   private readonly takeover: TakeoverStrategy;
   private readonly backend: RuntimeBackend;
   private heartbeatTimer?: ReturnType<typeof setInterval>;
+  private maxConcurrent: number;
 
   // Runtime state keyed by session key (trackerType:scopeKey#issueId@sessionName)
   private processes = new Map<string, RuntimeHandle>();
@@ -348,6 +349,7 @@ export class Engine {
     this.daemonId = opts.daemonId;
     this.takeover = opts.takeover ?? new RecloneStrategy(cfg);
     this.backend = opts.backend ?? createDefaultBackend(cfg);
+    this.maxConcurrent = cfg.work.maxConcurrent;
     this.startGlobalObserver();
     void this.recover();
   }
@@ -369,9 +371,30 @@ export class Engine {
       this.store.heartbeat(this.daemonId).catch((e) => {
         log.error(`engine: heartbeat failed for daemon ${this.daemonId}:`, (e as Error).message);
       });
+      void this.syncMaxConcurrent();
     }, intervalMs);
   }
 
+  private async syncMaxConcurrent(): Promise<void> {
+    try {
+      const cap = await this.store.getDaemonCapacity(this.daemonId);
+      if (cap != null && cap > 0 && cap !== this.maxConcurrent) {
+        log.info(`engine: maxConcurrent updated ${this.maxConcurrent} → ${cap} (DB sync)`);
+        this.maxConcurrent = cap;
+      }
+    } catch { /* non-critical */ }
+  }
+
+  setMaxConcurrent(n: number): void {
+    if (!Number.isFinite(n) || n < 1) return;
+    this.maxConcurrent = Math.floor(n);
+    log.info(`engine: maxConcurrent set to ${this.maxConcurrent}`);
+  }
+
+  getMaxConcurrent(): number {
+    return this.maxConcurrent;
+  }
+
   stopHeartbeat(): void {
     if (this.heartbeatTimer) {
       clearInterval(this.heartbeatTimer);
@@ -921,8 +944,8 @@ export class Engine {
       return;
     }
 
-    if (this.running.size >= this.cfg.work.maxConcurrent) {
-      log.info(`engine: concurrency limit reached (${this.running.size}/${this.cfg.work.maxConcurrent}), message ${msg.id.slice(0, 8)} queued for ${k}`);
+    if (this.running.size >= this.maxConcurrent) {
+      log.info(`engine: concurrency limit reached (${this.running.size}/${this.maxConcurrent}), message ${msg.id.slice(0, 8)} queued for ${k}`);
       return;
     }
 
@@ -1239,8 +1262,8 @@ export class Engine {
     if (nextMsg) {
       const current = await this.store.getSession(session.id);
       if (current && current.state !== "idle") {
-        if (this.running.size >= this.cfg.work.maxConcurrent) {
-          log.info(`engine: concurrency limit (${this.running.size}/${this.cfg.work.maxConcurrent}), keeping msg ${nextMsg.id.slice(0, 8)} pending for ${k}`);
+        if (this.running.size >= this.maxConcurrent) {
+          log.info(`engine: concurrency limit (${this.running.size}/${this.maxConcurrent}), keeping msg ${nextMsg.id.slice(0, 8)} pending for ${k}`);
         } else {
           await this.dequeueOrIdle(k, current, issue, nextMsg);
           return;
@@ -1255,11 +1278,11 @@ export class Engine {
   }
 
   private async drainGlobalPending(): Promise<void> {
-    const slotsAvailable = this.cfg.work.maxConcurrent - this.running.size;
+    const slotsAvailable = this.maxConcurrent - this.running.size;
     if (slotsAvailable <= 0) return;
     const pending = await this.store.getGlobalPendingMessages(slotsAvailable);
     for (const msg of pending) {
-      if (this.running.size >= this.cfg.work.maxConcurrent) break;
+      if (this.running.size >= this.maxConcurrent) break;
       const session = await this.store.getSession(msg.sessionId);
       if (!session || session.state === "running") continue;
       const issue = await this.store.getIssue(session.issueId);
@@ -1594,6 +1617,8 @@ export class Engine {
       log.error("engine: releaseDeadOwners at boot failed:", (err as Error).message);
     }
 
+    await this.cleanupGlobalOrphans();
+
     // Multi-machine: recover ONLY this daemon's sessions. Other daemons own
     // the rest; touching their state would race them.
     const ownedSessions = await this.store.listOwnedSessions(this.daemonId);
@@ -1688,6 +1713,33 @@ export class Engine {
     }
   }
 
+  private async cleanupGlobalOrphans(): Promise<void> {
+    let sessions: Array<{ id: string; opencodePid: number }>;
+    try {
+      sessions = await this.store.listSessionsWithPid();
+    } catch { return; }
+    let killed = 0;
+    for (const s of sessions) {
+      let alive = false;
+      try { process.kill(s.opencodePid, 0); alive = true; } catch { /* dead */ }
+      if (!alive) continue;
+
+      let ppid = -1;
+      try {
+        const stat = readFileSync(`/proc/${s.opencodePid}/stat`, "utf8");
+        const m = stat.match(/\)\s+\S+\s+(\d+)/);
+        ppid = m ? Number(m[1]) : -1;
+      } catch { continue; }
+
+      if (ppid === 1) {
+        log.info(`engine: killing global orphan pid=${s.opencodePid} (PPID=1, session ${s.id})`);
+        try { this.killProcessTree(s.opencodePid); } catch { /* dead */ }
+        killed++;
+      }
+    }
+    if (killed > 0) log.info(`engine: cleaned up ${killed} orphaned processes (PPID=1)`);
+  }
+
   // ─── API Methods ───
 
   async retryMessage(messageId: string): Promise<boolean> {

package/src/server.ts

@@ -213,6 +213,15 @@ export function createServer(
       await engine.resume();
       return json({ ok: true, paused: false });
     }
+    if (pathname === "/api/admin/max-concurrent" && req.method === "POST") {
+      const body = await req.json().catch(() => ({} as unknown));
+      const value = (body as { value?: unknown })?.value;
+      if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
+        return json({ error: "value must be a positive number" }, 400);
+      }
+      engine.setMaxConcurrent(Math.floor(value));
+      return json({ ok: true, maxConcurrent: engine.getMaxConcurrent() });
+    }
 
     return json({ error: "not found" }, 404);
   }