@cabane/companion 0.6.126 → 0.6.128

package/dist/cli.js

@@ -1037,6 +1037,11 @@ function leaseRefusal(err) {
 function isWriteFenceRefusal(err) {
   return err instanceof ApiError && err.status === 409 && apiErrorCode(err) === "not_running";
 }
+function isTurnEndedRefusal(err) {
+  if (!(err instanceof ApiError) || err.status !== 401) return false;
+  const body = err.body;
+  return typeof body === "object" && body !== null && body.reason === "turn_ended";
+}
 function describeErrorBody(body) {
   if (body === void 0 || body === null) return void 0;
   let text;
@@ -1286,6 +1291,15 @@ var init_api = __esm({
               progressed = true;
             } catch (err) {
               if (err instanceof ApiError && err.status >= 400 && err.status < 500) {
+                if (entry.kind === "active-run" && isTurnEndedRefusal(err)) {
+                  this.opts.log?.debug(
+                    { kind: entry.kind, turnId: entry.turnId, ...apiErrorLogFields(err) },
+                    "companion: queued settle refused; the turn had already ended"
+                  );
+                  outbox.remove(entry.turnId, entry.seq);
+                  progressed = true;
+                  continue;
+                }
                 this.opts.log?.warn(
                   { kind: entry.kind, turnId: entry.turnId, seq: entry.seq, ...apiErrorLogFields(err) },
                   "Cabane rejected a saved update; it could not be delivered. This companion may need an update: npm i -g @cabane/companion@latest"
@@ -8998,10 +9012,17 @@ var init_turn_execution = __esm({
             body
           );
         } catch (err) {
-          turnLog.warn(
-            { ...apiErrorLogFields(err), turnId },
-            "Couldn't save the turn's result to Cabane. This companion may need an update: npm i -g @cabane/companion@latest"
-          );
+          if (isTurnEndedRefusal(err)) {
+            turnLog.debug(
+              { ...apiErrorLogFields(err), turnId },
+              "companion: settle refused; the turn had already ended"
+            );
+          } else {
+            turnLog.warn(
+              { ...apiErrorLogFields(err), turnId },
+              "Couldn't save the turn's result to Cabane. This companion may need an update: npm i -g @cabane/companion@latest"
+            );
+          }
         }
         const durationMs = Date.now() - startedAt;
         this.supervisor.notifyEnd({ id: payload.messageId, ok: false, durationMs, reason });
@@ -9809,10 +9830,17 @@ var init_turn_execution = __esm({
               body
             );
           } catch (err) {
-            turnLog.warn(
-              { ...apiErrorLogFields(err) },
-              "Couldn't save the turn's result to Cabane. This companion may need an update: npm i -g @cabane/companion@latest"
-            );
+            if (isTurnEndedRefusal(err)) {
+              turnLog.debug(
+                { ...apiErrorLogFields(err) },
+                "companion: settle refused; the turn had already ended"
+              );
+            } else {
+              turnLog.warn(
+                { ...apiErrorLogFields(err) },
+                "Couldn't save the turn's result to Cabane. This companion may need an update: npm i -g @cabane/companion@latest"
+              );
+            }
           }
           this.supervisor.releaseAbort(turnId, abortController);
         }
@@ -13810,6 +13838,15 @@ var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 function containerName(attempt) {
   return `cabane-runner-${attempt.conversationId}-${attempt.agentId}`;
 }
+var networkName = containerName;
+function networkCreateError(stderr) {
+  const said = stderr.trim();
+  if (/address pool|fully subnetted/i.test(said))
+    return new Error(
+      `docker: couldn't create this conversation's runner network: the engine has no address pool left for another network. Each conversation and agent with a runner container holds one; remove kept runner containers, or widen \`default-address-pools\` in the daemon.json. Docker said: ${said}`
+    );
+  return new Error(`docker: creating the runner network failed: ${said || "no reason given"}`);
+}
 function sessionArchivePath(deviceDir, attemptId) {
   if (!UUID_RE.test(attemptId)) throw new Error(`docker: not an attempt id: ${attemptId}`);
   return join17(deviceDir, "attempts", attemptId, "session.tar");
@@ -13886,7 +13923,7 @@ function createDockerProvider(opts) {
   const killGraceMs = opts.killGraceMs ?? DEFAULT_KILL_GRACE_MS2;
   const sessionCap = opts.sessionCapBytes ?? DEFAULT_SESSION_CAP_BYTES;
   const collectCap = opts.collectCapBytes ?? COLLECT_CAP_BYTES;
-  const network = `cabane-runners-${opts.deviceKey}`;
+  const sharedNetwork = `cabane-runners-${opts.deviceKey}`;
   const companionLabel = createHash4("sha256").update(opts.companion.key).digest("hex").slice(0, 16);
   const proxy = `cabane-egress-${opts.deviceKey}`;
   const held = /* @__PURE__ */ new Map();
@@ -13919,32 +13956,14 @@ function createDockerProvider(opts) {
     }
     throw new Error("docker: the egress proxy never started listening");
   };
-  let networkReady = null;
+  let proxyReady = null;
   const proxyKey = async () => {
     const id = (await docker.run(["image", "inspect", "-f", "{{.Id}}", opts.image])).stdout.trim();
     return createHash4("sha256").update(JSON.stringify([id || opts.image, opts.companion.key, [...opts.egressAllow].sort()])).digest("hex").slice(0, 16);
   };
-  const ensureNetwork = () => {
-    networkReady ??= (async () => {
+  const ensureProxy = () => {
+    proxyReady ??= (async () => {
       const allowKey = await proxyKey();
-      if ((await docker.run(["network", "inspect", network])).code !== 0) {
-        await must(
-          [
-            "network",
-            "create",
-            // No route off the host, and no address of the host's on the
-            // bridge: the host's own ports are out of reach too. The proxy is
-            // the only other thing on the network.
-            "--internal",
-            "-o",
-            "com.docker.network.bridge.inhibit_ipv4=true",
-            "--label",
-            `cabane.device=${opts.deviceKey}`,
-            network
-          ],
-          "creating the runners network"
-        );
-      }
       const seen = await docker.run([
         "inspect",
         "-f",
@@ -13953,6 +13972,7 @@ function createDockerProvider(opts) {
       ]);
       const [key, running] = seen.stdout.trim().split(" ");
       if (seen.code === 0 && key === allowKey) {
+        await retireSharedNetwork();
         if (running !== "true") {
           await must(["start", proxy], "starting the egress proxy");
           await proxyListening();
@@ -13987,17 +14007,61 @@ function createDockerProvider(opts) {
         ],
         "starting the egress proxy"
       );
-      await must(
-        ["network", "connect", "--alias", PROXY_ALIAS, network, proxy],
-        "connecting the proxy"
-      );
+      await retireSharedNetwork();
       await proxyListening();
-      log.info({ network, proxy, allow: opts.egressAllow }, "runners: egress proxy ready");
+      log.info({ proxy, allow: opts.egressAllow }, "runners: egress proxy ready");
     })().catch((err) => {
-      networkReady = null;
+      proxyReady = null;
       throw err;
     });
-    return networkReady;
+    return proxyReady;
+  };
+  const retireSharedNetwork = async () => {
+    if ((await docker.run(["network", "inspect", sharedNetwork])).code !== 0) return;
+    await docker.run(["network", "disconnect", "-f", sharedNetwork, proxy]);
+    const on = await docker.run(["ps", "-aq", "--filter", `network=${sharedNetwork}`]);
+    if (on.code === 0 && !on.stdout.trim()) await docker.run(["network", "rm", sharedNetwork]);
+  };
+  const networksOf = async (container) => (await docker.run([
+    "inspect",
+    "-f",
+    "{{range $k, $_ := .NetworkSettings.Networks}}{{$k}} {{end}}",
+    container
+  ])).stdout.split(/\s+/).filter(Boolean);
+  const ensurePairNetwork = async (net, attempt) => {
+    if ((await docker.run(["network", "inspect", net])).code !== 0) {
+      const made = await docker.run([
+        "network",
+        "create",
+        // No route off the host, and no address of the host's on the bridge:
+        // the host's own ports are out of reach too. The proxy is the only
+        // other thing on the network.
+        "--internal",
+        "-o",
+        "com.docker.network.bridge.inhibit_ipv4=true",
+        "--label",
+        `cabane.device=${opts.deviceKey}`,
+        "--label",
+        `cabane.conversation=${attempt.conversationId}`,
+        "--label",
+        `cabane.agent=${attempt.agentId}`,
+        net
+      ]);
+      if (made.code !== 0) throw networkCreateError(made.stderr);
+    }
+    if (!(await networksOf(proxy)).includes(net))
+      await must(
+        ["network", "connect", "--alias", PROXY_ALIAS, net, proxy],
+        "connecting the proxy"
+      );
+  };
+  const settleNetworks = async (name, net) => {
+    const on = await networksOf(name);
+    if (!on.includes(net)) await must(["network", "connect", net, name], "moving the container");
+    const others = on.filter((n) => n !== net);
+    for (const other of others)
+      await must(["network", "disconnect", "-f", other, name], "moving the container");
+    if (others.includes(sharedNetwork)) await retireSharedNetwork();
   };
   const socketRoot = opts.socketRoot ?? join17(
     process.env.XDG_RUNTIME_DIR && existsSync13(process.env.XDG_RUNTIME_DIR) ? process.env.XDG_RUNTIME_DIR : tmpdir2(),
@@ -14029,7 +14093,7 @@ function createDockerProvider(opts) {
         "--label",
         `cabane.companion=${companionLabel}`,
         "--network",
-        network,
+        networkName(attempt),
         "--mount",
         `type=volume,src=${name}-home,dst=${CONTAINER_HOME}`,
         "--mount",
@@ -14055,6 +14119,17 @@ function createDockerProvider(opts) {
     await docker.run(["rm", "-f", "-v", name]);
     await docker.run(["volume", "rm", "-f", `${name}-home`, `${name}-work`]);
     rmSync9(socketDir(name), { recursive: true, force: true });
+    await dropNetwork(name);
+  };
+  const dropNetwork = async (name) => {
+    if ((await docker.run(["network", "inspect", name])).code !== 0) return;
+    await docker.run(["network", "disconnect", "-f", name, proxy]);
+    const gone = await docker.run(["network", "rm", name]);
+    if (gone.code !== 0)
+      log.warn(
+        { network: name, err: gone.stderr.trim() },
+        "runners: couldn't remove a pair's network"
+      );
   };
   const containerEnv = () => {
     const proxyUrl = `http://${PROXY_ALIAS}:${PROXY_PORT}`;
@@ -14221,7 +14296,7 @@ function createDockerProvider(opts) {
       });
       await before;
       try {
-        await ensureNetwork();
+        await ensureProxy();
         mkdirSync10(socketRoot, { recursive: true, mode: 448 });
         mkdirSync10(socketDir(name), { recursive: true, mode: 457 });
         let seen = await docker.run([
@@ -14251,9 +14326,12 @@ function createDockerProvider(opts) {
           );
           seen = { code: 1, stdout: "", stderr: "" };
         }
+        await ensurePairNetwork(networkName(attempt), attempt);
         if (seen.code !== 0) {
           await create(name, attempt);
           log.debug({ container: name }, "runners: container created");
+        } else {
+          await settleNetworks(name, networkName(attempt));
         }
         if (seen.stdout.trim().split(" ")[0] !== "true")
           await must(["start", name], "starting the container");
@@ -14267,7 +14345,8 @@ function createDockerProvider(opts) {
         return { provider: "docker", id: name };
       } catch (err) {
         const keep = opts.keepContainers || readPending(name).length > 0;
-        await (keep ? docker.run(["stop", "-t", "1", name]) : destroy(name)).catch(() => {
+        const kept = keep && (await docker.run(["inspect", "-f", "{{.Id}}", name])).code === 0;
+        await (kept ? docker.run(["stop", "-t", "1", name]) : keep ? dropNetwork(networkName(attempt)) : destroy(name)).catch(() => {
         });
         release();
         throw err;

package/dist/runtime.js

@@ -2279,6 +2279,11 @@ function leaseRefusal(err) {
 function isWriteFenceRefusal(err) {
   return err instanceof ApiError && err.status === 409 && apiErrorCode(err) === "not_running";
 }
+function isTurnEndedRefusal(err) {
+  if (!(err instanceof ApiError) || err.status !== 401) return false;
+  const body = err.body;
+  return typeof body === "object" && body !== null && body.reason === "turn_ended";
+}
 var ERROR_BODY_LOG_CAP = 2e3;
 function describeErrorBody(body) {
   if (body === void 0 || body === null) return void 0;
@@ -2469,6 +2474,15 @@ var CabaneApi = class {
           progressed = true;
         } catch (err) {
           if (err instanceof ApiError && err.status >= 400 && err.status < 500) {
+            if (entry.kind === "active-run" && isTurnEndedRefusal(err)) {
+              this.opts.log?.debug(
+                { kind: entry.kind, turnId: entry.turnId, ...apiErrorLogFields(err) },
+                "companion: queued settle refused; the turn had already ended"
+              );
+              outbox.remove(entry.turnId, entry.seq);
+              progressed = true;
+              continue;
+            }
             this.opts.log?.warn(
               { kind: entry.kind, turnId: entry.turnId, seq: entry.seq, ...apiErrorLogFields(err) },
               "Cabane rejected a saved update; it could not be delivered. This companion may need an update: npm i -g @cabane/companion@latest"
@@ -9984,10 +9998,17 @@ var TurnExecution = class {
         body
       );
     } catch (err) {
-      turnLog.warn(
-        { ...apiErrorLogFields(err), turnId },
-        "Couldn't save the turn's result to Cabane. This companion may need an update: npm i -g @cabane/companion@latest"
-      );
+      if (isTurnEndedRefusal(err)) {
+        turnLog.debug(
+          { ...apiErrorLogFields(err), turnId },
+          "companion: settle refused; the turn had already ended"
+        );
+      } else {
+        turnLog.warn(
+          { ...apiErrorLogFields(err), turnId },
+          "Couldn't save the turn's result to Cabane. This companion may need an update: npm i -g @cabane/companion@latest"
+        );
+      }
     }
     const durationMs = Date.now() - startedAt;
     this.supervisor.notifyEnd({ id: payload.messageId, ok: false, durationMs, reason });
@@ -10795,10 +10816,17 @@ var TurnExecution = class {
           body
         );
       } catch (err) {
-        turnLog.warn(
-          { ...apiErrorLogFields(err) },
-          "Couldn't save the turn's result to Cabane. This companion may need an update: npm i -g @cabane/companion@latest"
-        );
+        if (isTurnEndedRefusal(err)) {
+          turnLog.debug(
+            { ...apiErrorLogFields(err) },
+            "companion: settle refused; the turn had already ended"
+          );
+        } else {
+          turnLog.warn(
+            { ...apiErrorLogFields(err) },
+            "Couldn't save the turn's result to Cabane. This companion may need an update: npm i -g @cabane/companion@latest"
+          );
+        }
       }
       this.supervisor.releaseAbort(turnId, abortController);
     }
@@ -11527,6 +11555,15 @@ var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 function containerName(attempt) {
   return `cabane-runner-${attempt.conversationId}-${attempt.agentId}`;
 }
+var networkName = containerName;
+function networkCreateError(stderr) {
+  const said = stderr.trim();
+  if (/address pool|fully subnetted/i.test(said))
+    return new Error(
+      `docker: couldn't create this conversation's runner network: the engine has no address pool left for another network. Each conversation and agent with a runner container holds one; remove kept runner containers, or widen \`default-address-pools\` in the daemon.json. Docker said: ${said}`
+    );
+  return new Error(`docker: creating the runner network failed: ${said || "no reason given"}`);
+}
 function sessionArchivePath(deviceDir, attemptId) {
   if (!UUID_RE.test(attemptId)) throw new Error(`docker: not an attempt id: ${attemptId}`);
   return join17(deviceDir, "attempts", attemptId, "session.tar");
@@ -11603,7 +11640,7 @@ function createDockerProvider(opts) {
   const killGraceMs = opts.killGraceMs ?? DEFAULT_KILL_GRACE_MS2;
   const sessionCap = opts.sessionCapBytes ?? DEFAULT_SESSION_CAP_BYTES;
   const collectCap = opts.collectCapBytes ?? COLLECT_CAP_BYTES;
-  const network = `cabane-runners-${opts.deviceKey}`;
+  const sharedNetwork = `cabane-runners-${opts.deviceKey}`;
   const companionLabel = createHash4("sha256").update(opts.companion.key).digest("hex").slice(0, 16);
   const proxy = `cabane-egress-${opts.deviceKey}`;
   const held = /* @__PURE__ */ new Map();
@@ -11636,32 +11673,14 @@ function createDockerProvider(opts) {
     }
     throw new Error("docker: the egress proxy never started listening");
   };
-  let networkReady = null;
+  let proxyReady = null;
   const proxyKey = async () => {
     const id = (await docker.run(["image", "inspect", "-f", "{{.Id}}", opts.image])).stdout.trim();
     return createHash4("sha256").update(JSON.stringify([id || opts.image, opts.companion.key, [...opts.egressAllow].sort()])).digest("hex").slice(0, 16);
   };
-  const ensureNetwork = () => {
-    networkReady ??= (async () => {
+  const ensureProxy = () => {
+    proxyReady ??= (async () => {
       const allowKey = await proxyKey();
-      if ((await docker.run(["network", "inspect", network])).code !== 0) {
-        await must(
-          [
-            "network",
-            "create",
-            // No route off the host, and no address of the host's on the
-            // bridge: the host's own ports are out of reach too. The proxy is
-            // the only other thing on the network.
-            "--internal",
-            "-o",
-            "com.docker.network.bridge.inhibit_ipv4=true",
-            "--label",
-            `cabane.device=${opts.deviceKey}`,
-            network
-          ],
-          "creating the runners network"
-        );
-      }
       const seen = await docker.run([
         "inspect",
         "-f",
@@ -11670,6 +11689,7 @@ function createDockerProvider(opts) {
       ]);
       const [key, running] = seen.stdout.trim().split(" ");
       if (seen.code === 0 && key === allowKey) {
+        await retireSharedNetwork();
         if (running !== "true") {
           await must(["start", proxy], "starting the egress proxy");
           await proxyListening();
@@ -11704,17 +11724,61 @@ function createDockerProvider(opts) {
         ],
         "starting the egress proxy"
       );
-      await must(
-        ["network", "connect", "--alias", PROXY_ALIAS, network, proxy],
-        "connecting the proxy"
-      );
+      await retireSharedNetwork();
       await proxyListening();
-      log.info({ network, proxy, allow: opts.egressAllow }, "runners: egress proxy ready");
+      log.info({ proxy, allow: opts.egressAllow }, "runners: egress proxy ready");
     })().catch((err) => {
-      networkReady = null;
+      proxyReady = null;
       throw err;
     });
-    return networkReady;
+    return proxyReady;
+  };
+  const retireSharedNetwork = async () => {
+    if ((await docker.run(["network", "inspect", sharedNetwork])).code !== 0) return;
+    await docker.run(["network", "disconnect", "-f", sharedNetwork, proxy]);
+    const on = await docker.run(["ps", "-aq", "--filter", `network=${sharedNetwork}`]);
+    if (on.code === 0 && !on.stdout.trim()) await docker.run(["network", "rm", sharedNetwork]);
+  };
+  const networksOf = async (container) => (await docker.run([
+    "inspect",
+    "-f",
+    "{{range $k, $_ := .NetworkSettings.Networks}}{{$k}} {{end}}",
+    container
+  ])).stdout.split(/\s+/).filter(Boolean);
+  const ensurePairNetwork = async (net, attempt) => {
+    if ((await docker.run(["network", "inspect", net])).code !== 0) {
+      const made = await docker.run([
+        "network",
+        "create",
+        // No route off the host, and no address of the host's on the bridge:
+        // the host's own ports are out of reach too. The proxy is the only
+        // other thing on the network.
+        "--internal",
+        "-o",
+        "com.docker.network.bridge.inhibit_ipv4=true",
+        "--label",
+        `cabane.device=${opts.deviceKey}`,
+        "--label",
+        `cabane.conversation=${attempt.conversationId}`,
+        "--label",
+        `cabane.agent=${attempt.agentId}`,
+        net
+      ]);
+      if (made.code !== 0) throw networkCreateError(made.stderr);
+    }
+    if (!(await networksOf(proxy)).includes(net))
+      await must(
+        ["network", "connect", "--alias", PROXY_ALIAS, net, proxy],
+        "connecting the proxy"
+      );
+  };
+  const settleNetworks = async (name, net) => {
+    const on = await networksOf(name);
+    if (!on.includes(net)) await must(["network", "connect", net, name], "moving the container");
+    const others = on.filter((n) => n !== net);
+    for (const other of others)
+      await must(["network", "disconnect", "-f", other, name], "moving the container");
+    if (others.includes(sharedNetwork)) await retireSharedNetwork();
   };
   const socketRoot = opts.socketRoot ?? join17(
     process.env.XDG_RUNTIME_DIR && existsSync13(process.env.XDG_RUNTIME_DIR) ? process.env.XDG_RUNTIME_DIR : tmpdir2(),
@@ -11746,7 +11810,7 @@ function createDockerProvider(opts) {
         "--label",
         `cabane.companion=${companionLabel}`,
         "--network",
-        network,
+        networkName(attempt),
         "--mount",
         `type=volume,src=${name}-home,dst=${CONTAINER_HOME}`,
         "--mount",
@@ -11772,6 +11836,17 @@ function createDockerProvider(opts) {
     await docker.run(["rm", "-f", "-v", name]);
     await docker.run(["volume", "rm", "-f", `${name}-home`, `${name}-work`]);
     rmSync9(socketDir(name), { recursive: true, force: true });
+    await dropNetwork(name);
+  };
+  const dropNetwork = async (name) => {
+    if ((await docker.run(["network", "inspect", name])).code !== 0) return;
+    await docker.run(["network", "disconnect", "-f", name, proxy]);
+    const gone = await docker.run(["network", "rm", name]);
+    if (gone.code !== 0)
+      log.warn(
+        { network: name, err: gone.stderr.trim() },
+        "runners: couldn't remove a pair's network"
+      );
   };
   const containerEnv = () => {
     const proxyUrl = `http://${PROXY_ALIAS}:${PROXY_PORT}`;
@@ -11938,7 +12013,7 @@ function createDockerProvider(opts) {
       });
       await before;
       try {
-        await ensureNetwork();
+        await ensureProxy();
         mkdirSync10(socketRoot, { recursive: true, mode: 448 });
         mkdirSync10(socketDir(name), { recursive: true, mode: 457 });
         let seen = await docker.run([
@@ -11968,9 +12043,12 @@ function createDockerProvider(opts) {
           );
           seen = { code: 1, stdout: "", stderr: "" };
         }
+        await ensurePairNetwork(networkName(attempt), attempt);
         if (seen.code !== 0) {
           await create(name, attempt);
           log.debug({ container: name }, "runners: container created");
+        } else {
+          await settleNetworks(name, networkName(attempt));
         }
         if (seen.stdout.trim().split(" ")[0] !== "true")
           await must(["start", name], "starting the container");
@@ -11984,7 +12062,8 @@ function createDockerProvider(opts) {
         return { provider: "docker", id: name };
       } catch (err) {
         const keep = opts.keepContainers || readPending(name).length > 0;
-        await (keep ? docker.run(["stop", "-t", "1", name]) : destroy(name)).catch(() => {
+        const kept = keep && (await docker.run(["inspect", "-f", "{{.Id}}", name])).code === 0;
+        await (kept ? docker.run(["stop", "-t", "1", name]) : keep ? dropNetwork(networkName(attempt)) : destroy(name)).catch(() => {
         });
         release();
         throw err;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cabane/companion",
-  "version": "0.6.126",
+  "version": "0.6.128",
   "type": "module",
   "description": "The Cabane Companion (headless): connect a coding agent on your machine to your Cabane workspace as a responder — drive work against your own codebase, files, and MCP servers without putting any of it in Cabane.",
   "license": "UNLICENSED",