@kody-ade/kody-engine 0.4.115 → 0.4.116

package/dist/bin/kody.js

@@ -880,7 +880,7 @@ var init_loadPriorArt = __esm({
 // package.json
 var package_default = {
   name: "@kody-ade/kody-engine",
-  version: "0.4.115",
+  version: "0.4.116",
   description: "kody \u2014 autonomous development engine. Single-session Claude Code agent behind a generic executor + declarative executable profiles.",
   license: "MIT",
   type: "module",
@@ -4884,6 +4884,7 @@ function sleep2(ms) {
 }
 
 // src/pool/manager.ts
+var MAX_CLAIM_ATTEMPTS = 3;
 var PoolManager = class {
   constructor(deps) {
     this.deps = deps;
@@ -4923,43 +4924,84 @@ var PoolManager = class {
     await this.refill();
   }
   /**
-   * Claim a warm machine for a job. Returns ok:false (caller falls back to
-   * create-fresh) when the pool is empty or the woken machine fails to take
-   * the job. The pick is synchronous — the atomic step.
+   * Claim a warm machine for a job. Tries free machines in turn: if a woken
+   * machine is stale/unhealthy/rejecting (e.g. it vanished out-of-band), it's
+   * destroyed and the next free one is tried, up to MAX_CLAIM_ATTEMPTS. Only
+   * when none work (or the pool is empty) does it return ok:false so the
+   * caller falls back to create-fresh. The pick (shift) is synchronous — the
+   * atomic step that prevents two concurrent claims grabbing the same machine.
    */
   async claim(job) {
-    const machine = this.free.shift();
-    if (!machine) {
-      this.log("claim: pool empty");
-      void this.refill();
-      return { ok: false, reason: "pool empty" };
-    }
-    this.claimsInFlight++;
-    try {
-      await this.deps.fly.start(machine.id);
-      const base = this.baseUrl(machine);
-      const healthy = await this.deps.fly.waitHealthy(base, { timeoutMs: this.deps.config.healthTimeoutMs });
-      if (!healthy) {
-        this.log(`claim: machine ${machine.id} unhealthy after wake \u2014 destroying`);
-        await this.safeDestroy(machine.id);
-        return { ok: false, reason: "woken machine unhealthy" };
-      }
-      const accepted = await this.postRun(machine, job, this.deps.config);
-      if (!accepted) {
-        this.log(`claim: machine ${machine.id} rejected job \u2014 destroying`);
+    let lastReason = "pool empty";
+    for (let attempt = 0; attempt < MAX_CLAIM_ATTEMPTS; attempt++) {
+      const machine = this.free.shift();
+      if (!machine) break;
+      this.claimsInFlight++;
+      try {
+        await this.deps.fly.start(machine.id);
+        const healthy = await this.deps.fly.waitHealthy(this.baseUrl(machine), {
+          timeoutMs: this.deps.config.healthTimeoutMs
+        });
+        if (!healthy) {
+          this.log(`claim: machine ${machine.id} unhealthy after wake \u2014 destroying, trying next`);
+          await this.safeDestroy(machine.id);
+          lastReason = "woken machine unhealthy";
+          continue;
+        }
+        const accepted = await this.postRun(machine, job, this.deps.config);
+        if (!accepted) {
+          this.log(`claim: machine ${machine.id} rejected job \u2014 destroying, trying next`);
+          await this.safeDestroy(machine.id);
+          lastReason = "machine rejected job";
+          continue;
+        }
+        this.log(`claim: machine ${machine.id} took job ${job.jobId}`);
+        void this.refill();
+        return { ok: true, machineId: machine.id };
+      } catch (err) {
+        this.log(`claim: error on ${machine.id}: ${errMsg2(err)} \u2014 destroying, trying next`);
         await this.safeDestroy(machine.id);
-        return { ok: false, reason: "machine rejected job" };
+        lastReason = errMsg2(err);
+      } finally {
+        this.claimsInFlight--;
       }
-      this.log(`claim: machine ${machine.id} took job ${job.jobId}`);
-      return { ok: true, machineId: machine.id };
+    }
+    void this.refill();
+    return { ok: false, reason: lastReason };
+  }
+  /**
+   * Periodic self-heal: reconcile the in-memory free list against actual Fly
+   * state. Prunes free entries whose machine vanished out-of-band (auto-destroy
+   * after a job, manual ops) so a later claim never tries a dead machine, and
+   * adopts any suspended machines we lost track of. Then tops up. Unlike
+   * reconcile() this MERGES rather than rebuilds, so it won't drop a machine
+   * that's momentarily not yet reflected as suspended by Fly's eventual
+   * consistency.
+   */
+  async resync() {
+    let machines;
+    try {
+      machines = await this.deps.fly.listPooled();
     } catch (err) {
-      this.log(`claim: error on ${machine.id}: ${errMsg2(err)} \u2014 destroying`);
-      await this.safeDestroy(machine.id);
-      return { ok: false, reason: errMsg2(err) };
-    } finally {
-      this.claimsInFlight--;
-      void this.refill();
+      this.log(`resync: listPooled failed: ${errMsg2(err)}`);
+      return;
     }
+    const liveIds = new Set(machines.map((m) => m.id));
+    const before = this.free.length;
+    this.free = this.free.filter((f) => liveIds.has(f.id));
+    const pruned = before - this.free.length;
+    const tracked = new Set(this.free.map((f) => f.id));
+    let adopted = 0;
+    for (const m of machines) {
+      if ((m.state === "suspended" || m.state === "suspending") && m.private_ip && !tracked.has(m.id)) {
+        this.free.push({ id: m.id, privateIp: m.private_ip });
+        adopted++;
+      }
+    }
+    if (pruned > 0 || adopted > 0) {
+      this.log(`resync: pruned ${pruned} stale, adopted ${adopted} (free=${this.free.length})`);
+    }
+    await this.refill();
   }
   /** Top up free machines to `min`. Serialized so it never overshoots. */
   async refill() {
@@ -5180,7 +5222,7 @@ var poolServe = async (ctx) => {
   manager.reconcile().catch((err) => log(`reconcile failed: ${err instanceof Error ? err.message : String(err)}`));
   const refillMs = envInt("POOL_REFILL_INTERVAL_MS", 6e4);
   const tick = setInterval(() => {
-    manager.refill().catch((err) => log(`refill tick failed: ${err instanceof Error ? err.message : String(err)}`));
+    manager.resync().catch((err) => log(`resync tick failed: ${err instanceof Error ? err.message : String(err)}`));
   }, refillMs);
   const server = createServer3(async (req, res) => {
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kody-ade/kody-engine",
-  "version": "0.4.115",
+  "version": "0.4.116",
   "description": "kody — autonomous development engine. Single-session Claude Code agent behind a generic executor + declarative executable profiles.",
   "license": "MIT",
   "type": "module",