@pushpalsdev/cli 1.0.38 → 1.0.39

package/dist/pushpals-cli.js

@@ -122,6 +122,7 @@ class ServiceManager {
   onServiceDegraded;
   onEvent;
   timer;
+  shutdownBegun = false;
   stopped = false;
   constructor(options = {}) {
     this.pollMs = Math.max(50, Math.floor(options.pollMs ?? DEFAULT_SERVICE_MANAGER_POLL_MS));
@@ -161,10 +162,10 @@ class ServiceManager {
       action: this.degradedAction
     };
   }
-  stop() {
-    if (this.stopped)
+  beginShutdown() {
+    if (this.shutdownBegun || this.stopped)
       return;
-    this.stopped = true;
+    this.shutdownBegun = true;
     clearInterval(this.timer);
     for (const state of this.stateByService.values()) {
       if (!state.pendingRestartTimer)
@@ -172,6 +173,12 @@ class ServiceManager {
       clearTimeout(state.pendingRestartTimer);
       state.pendingRestartTimer = null;
     }
+  }
+  stop() {
+    if (this.stopped)
+      return;
+    this.beginShutdown();
+    this.stopped = true;
     for (const service of this.services.values()) {
       try {
         const pid = service.proc.pid;
@@ -207,7 +214,7 @@ class ServiceManager {
     this.onEvent?.(level, line);
   }
   tick() {
-    if (this.stopped)
+    if (this.shutdownBegun || this.stopped)
       return;
     const now = Date.now();
     for (const [name, service] of this.services.entries()) {
@@ -250,7 +257,7 @@ class ServiceManager {
       state.pendingRestartTimer = setTimeout(() => {
         state.pendingRestartTimer = null;
         state.nextRestartAtMs = 0;
-        if (this.stopped)
+        if (this.shutdownBegun || this.stopped)
           return;
         const current = this.services.get(name);
         if (!current || !current.exited)
@@ -4792,6 +4799,7 @@ async function main() {
       return;
     const serviceManager = autoStartedServiceManager;
     autoStartedServiceManager = null;
+    serviceManager.beginShutdown();
     const shutdown = await requestLocalRuntimeShutdown(serverUrl, repoRoot, reason);
     if (shutdown.attempted && shutdown.accepted) {
       console.log("[pushpals] Local runtime shutdown accepted; waiting for services to exit...");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushpalsdev/cli",
-  "version": "1.0.38",
+  "version": "1.0.39",
   "description": "PushPals terminal CLI for LocalBuddy -> RemoteBuddy orchestration",
   "license": "MIT",
   "repository": {

package/runtime/sandbox/apps/workerpals/src/common/server_transport.ts

@@ -0,0 +1,323 @@
+import type { CommandRequest } from "protocol";
+
+export type WorkerHeartbeatStatus = "idle" | "busy" | "error" | "offline";
+
+type QueuePriority = "high" | "normal";
+
+type TransportTask = {
+  label: string;
+  priority: QueuePriority;
+  droppable: boolean;
+  run: () => Promise<void>;
+  resolve: () => void;
+};
+
+export type WorkerServerTransportOptions = {
+  server: string;
+  headers: Record<string, string>;
+  workerId: string;
+  pollMs: number;
+  heartbeatMs: number;
+  staleClaimTtlMs: number;
+  fetchFn?: typeof fetch;
+  logInfo?: (message: string) => void;
+  logWarn?: (message: string) => void;
+  nowFn?: () => number;
+};
+
+export type WorkerTransportHealthSnapshot = {
+  heartbeatInFlight: boolean;
+  consecutiveHeartbeatFailures: number;
+  lastHeartbeatAttemptAt: number;
+  lastHeartbeatSuccessAt: number;
+  queuedRequests: number;
+  droppedLogRequests: number;
+};
+
+export type WorkerHeartbeatPayload = {
+  status: WorkerHeartbeatStatus;
+  currentJobId: string | null;
+  capabilities?: Record<string, unknown>;
+  details?: Record<string, unknown>;
+};
+
+function computeHeartbeatTimeoutMs(heartbeatMs: number): number {
+  return Math.max(1_500, Math.min(4_000, Math.floor(heartbeatMs * 0.8)));
+}
+
+function computeRequestTimeoutMs(heartbeatMs: number): number {
+  return Math.max(4_000, Math.min(10_000, Math.floor(heartbeatMs * 2)));
+}
+
+async function readResponseDetail(response: Response): Promise<string> {
+  const text = await response.text().catch(() => "");
+  return text.trim();
+}
+
+export class WorkerServerTransport {
+  private readonly server: string;
+  private readonly headers: Record<string, string>;
+  private readonly workerId: string;
+  private readonly pollMs: number;
+  private readonly staleClaimTtlMs: number;
+  private readonly fetchFn: typeof fetch;
+  private readonly logInfo: (message: string) => void;
+  private readonly logWarn: (message: string) => void;
+  private readonly nowFn: () => number;
+  private readonly heartbeatTimeoutMs: number;
+  private readonly requestTimeoutMs: number;
+  private readonly maxQueuedRequests = 256;
+
+  private queuedRequests: TransportTask[] = [];
+  private queueDrainInFlight = false;
+  private queueFlushWaiters: Array<() => void> = [];
+  private droppedLogRequests = 0;
+  private heartbeatInFlight = false;
+  private lastHeartbeatAttemptAt = 0;
+  private lastHeartbeatSuccessAt = 0;
+  private consecutiveHeartbeatFailures = 0;
+  private firstHeartbeatFailureAt = -1;
+  private lastHeartbeatFailureDetail = "";
+
+  constructor(options: WorkerServerTransportOptions) {
+    this.server = options.server;
+    this.headers = options.headers;
+    this.workerId = options.workerId;
+    this.pollMs = options.pollMs;
+    this.staleClaimTtlMs = options.staleClaimTtlMs;
+    this.fetchFn = options.fetchFn ?? fetch;
+    this.logInfo = options.logInfo ?? ((message) => console.log(message));
+    this.logWarn = options.logWarn ?? ((message) => console.warn(message));
+    this.nowFn = options.nowFn ?? (() => Date.now());
+    this.heartbeatTimeoutMs = computeHeartbeatTimeoutMs(options.heartbeatMs);
+    this.requestTimeoutMs = computeRequestTimeoutMs(options.heartbeatMs);
+  }
+
+  getHealthSnapshot(): WorkerTransportHealthSnapshot {
+    return {
+      heartbeatInFlight: this.heartbeatInFlight,
+      consecutiveHeartbeatFailures: this.consecutiveHeartbeatFailures,
+      lastHeartbeatAttemptAt: this.lastHeartbeatAttemptAt,
+      lastHeartbeatSuccessAt: this.lastHeartbeatSuccessAt,
+      queuedRequests: this.queuedRequests.length,
+      droppedLogRequests: this.droppedLogRequests,
+    };
+  }
+
+  getHeartbeatStaleAgeMs(nowMs = this.nowFn()): number {
+    if (this.lastHeartbeatSuccessAt <= 0) return Number.POSITIVE_INFINITY;
+    return Math.max(0, nowMs - this.lastHeartbeatSuccessAt);
+  }
+
+  shouldRecycleBusyWorker(nowMs = this.nowFn()): boolean {
+    const failureAgeMs =
+      this.firstHeartbeatFailureAt >= 0
+        ? Math.max(0, nowMs - this.firstHeartbeatFailureAt)
+        : null;
+    if (failureAgeMs == null) return false;
+    const threshold = Math.min(
+      this.staleClaimTtlMs,
+      Math.max(
+        30_000,
+        Math.min(
+          this.staleClaimTtlMs - this.heartbeatTimeoutMs,
+          Math.floor(this.staleClaimTtlMs * 0.75),
+        ),
+      ),
+    );
+    return failureAgeMs >= threshold;
+  }
+
+  async sendHeartbeat(payload: WorkerHeartbeatPayload): Promise<boolean> {
+    if (this.heartbeatInFlight) {
+      return false;
+    }
+    this.heartbeatInFlight = true;
+    this.lastHeartbeatAttemptAt = this.nowFn();
+    try {
+      const response = await this.postJson("/workers/heartbeat", {
+        workerId: this.workerId,
+        status: payload.status,
+        currentJobId: payload.currentJobId,
+        pollMs: this.pollMs,
+        capabilities: payload.capabilities ?? {},
+        details: payload.details ?? {},
+      }, this.heartbeatTimeoutMs);
+      if (!response.ok) {
+        const detail = await readResponseDetail(response);
+        throw new Error(
+          `heartbeat rejected (${response.status})${detail ? `: ${detail}` : ""}`,
+        );
+      }
+      const previousFailures = this.consecutiveHeartbeatFailures;
+      this.lastHeartbeatSuccessAt = this.nowFn();
+      this.consecutiveHeartbeatFailures = 0;
+      this.firstHeartbeatFailureAt = -1;
+      this.lastHeartbeatFailureDetail = "";
+      if (previousFailures > 0) {
+        this.logInfo(
+          `[WorkerPals] Heartbeat recovered for ${this.workerId} after ${previousFailures} failed attempt(s).`,
+        );
+      }
+      return true;
+    } catch (error) {
+      if (this.consecutiveHeartbeatFailures === 0) {
+        this.firstHeartbeatFailureAt = this.nowFn();
+      }
+      this.consecutiveHeartbeatFailures += 1;
+      this.lastHeartbeatFailureDetail = error instanceof Error ? error.message : String(error);
+      const staleAgeMs = this.getHeartbeatStaleAgeMs();
+      this.logWarn(
+        `[WorkerPals] Heartbeat failure ${this.consecutiveHeartbeatFailures} for ${this.workerId}: ${this.lastHeartbeatFailureDetail} (lastSuccessAgeMs=${Number.isFinite(staleAgeMs) ? staleAgeMs : -1}).`,
+      );
+      return false;
+    } finally {
+      this.heartbeatInFlight = false;
+    }
+  }
+
+  queueSessionCommand(
+    sessionId: string,
+    cmd: CommandRequest,
+    options: { priority?: QueuePriority; droppable?: boolean } = {},
+  ): Promise<void> {
+    return this.enqueueTask({
+      label: `command:${cmd.type}`,
+      priority: options.priority ?? "normal",
+      droppable: options.droppable ?? false,
+      run: async () => {
+        const response = await this.postJson(
+          `/sessions/${sessionId}/command`,
+          cmd,
+          this.requestTimeoutMs,
+        );
+        if (!response.ok) {
+          const detail = await readResponseDetail(response);
+          this.logWarn(
+            `[WorkerPals] Command ${cmd.type} failed: ${response.status}${detail ? ` ${detail}` : ""}`,
+          );
+        }
+      },
+    });
+  }
+
+  queueJobLog(
+    jobId: string,
+    payload: { stream: "stdout" | "stderr"; seq: number; message: string; ts: string },
+  ): Promise<void> {
+    return this.enqueueTask({
+      label: "job_log",
+      priority: "normal",
+      droppable: true,
+      run: async () => {
+        const response = await this.postJson(`/jobs/${jobId}/log`, payload, this.requestTimeoutMs);
+        if (!response.ok) {
+          const detail = await readResponseDetail(response);
+          this.logWarn(
+            `[WorkerPals] Job log delivery failed for ${jobId}: ${response.status}${detail ? ` ${detail}` : ""}`,
+          );
+        }
+      },
+    });
+  }
+
+  async flush(timeoutMs = 15_000): Promise<void> {
+    if (this.queuedRequests.length === 0 && !this.queueDrainInFlight) return;
+    await new Promise<void>((resolve) => {
+      let settled = false;
+      const timer = setTimeout(() => {
+        if (settled) return;
+        settled = true;
+        this.logWarn(
+          `[WorkerPals] Timed out flushing queued server transport requests after ${timeoutMs}ms (queued=${this.queuedRequests.length}).`,
+        );
+        resolve();
+      }, timeoutMs);
+      this.queueFlushWaiters.push(() => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        resolve();
+      });
+      this.maybeResolveFlushWaiters();
+    });
+  }
+
+  private enqueueTask(task: Omit<TransportTask, "resolve">): Promise<void> {
+    if (task.droppable && this.queuedRequests.length >= this.maxQueuedRequests) {
+      this.droppedLogRequests += 1;
+      if (this.droppedLogRequests === 1 || this.droppedLogRequests % 25 === 0) {
+        this.logWarn(
+          `[WorkerPals] Dropped ${this.droppedLogRequests} queued low-priority transport request(s) because the queue is saturated (limit=${this.maxQueuedRequests}).`,
+        );
+      }
+      return Promise.resolve();
+    }
+
+    return new Promise((resolve) => {
+      const queued: TransportTask = { ...task, resolve };
+      if (queued.priority === "high") {
+        const firstNormalIndex = this.queuedRequests.findIndex((entry) => entry.priority !== "high");
+        if (firstNormalIndex === -1) {
+          this.queuedRequests.push(queued);
+        } else {
+          this.queuedRequests.splice(firstNormalIndex, 0, queued);
+        }
+      } else {
+        this.queuedRequests.push(queued);
+      }
+      void this.drainQueue();
+    });
+  }
+
+  private async drainQueue(): Promise<void> {
+    if (this.queueDrainInFlight) return;
+    this.queueDrainInFlight = true;
+    try {
+      while (this.queuedRequests.length > 0) {
+        const task = this.queuedRequests.shift();
+        if (!task) break;
+        try {
+          await task.run();
+        } catch (error) {
+          this.logWarn(
+            `[WorkerPals] Transport request ${task.label} failed: ${
+              error instanceof Error ? error.message : String(error)
+            }`,
+          );
+        } finally {
+          task.resolve();
+        }
+      }
+    } finally {
+      this.queueDrainInFlight = false;
+      this.maybeResolveFlushWaiters();
+    }
+  }
+
+  private maybeResolveFlushWaiters(): void {
+    if (this.queuedRequests.length > 0 || this.queueDrainInFlight) return;
+    const waiters = this.queueFlushWaiters.splice(0);
+    for (const waiter of waiters) waiter();
+  }
+
+  private async postJson(path: string, payload: unknown, timeoutMs: number): Promise<Response> {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort("timeout"), timeoutMs);
+    try {
+      return await this.fetchFn(`${this.server}${path}`, {
+        method: "POST",
+        headers: this.headers,
+        body: JSON.stringify(payload),
+        signal: controller.signal,
+      });
+    } catch (error) {
+      if (controller.signal.aborted) {
+        throw new Error(`request timed out after ${timeoutMs}ms (${path})`);
+      }
+      throw error;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}

package/runtime/sandbox/apps/workerpals/src/workerpals_main.ts

@@ -20,7 +20,6 @@
  * JobRunner executes a single job, streams logs, and outputs a final result with a sentinel line.
  */
 
-import type { CommandRequest } from "protocol";
 import { randomUUID } from "crypto";
 import { mkdirSync } from "fs";
 import { resolve } from "path";
@@ -44,6 +43,7 @@ import {
 } from "./execute_job.js";
 import { DockerExecutionExhaustedError, DockerExecutor } from "./docker_executor.js";
 import { forceDeleteWorktreePath } from "./common/worktree_cleanup.js";
+import { WorkerServerTransport, type WorkerHeartbeatPayload } from "./common/server_transport.js";
 import { DEFAULT_DOCKER_TIMEOUT_MS, parseDockerTimeoutMs } from "./timeout_policy.js";
 
 type CommitRef = {
@@ -95,6 +95,31 @@ function estimateTokensFromText(text: string): number {
   return Math.max(0, Math.ceil(String(text ?? "").length / 3));
 }
 
+async function postJsonWithTimeout(
+  url: string,
+  headers: Record<string, string>,
+  body: unknown,
+  timeoutMs = 10_000,
+): Promise<Response> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort("timeout"), timeoutMs);
+  try {
+    return await fetch(url, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    });
+  } catch (error) {
+    if (controller.signal.aborted) {
+      throw new Error(`request timed out after ${timeoutMs}ms: ${url}`);
+    }
+    throw error;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
 function buildWorkerLlmUsageEvent(
   job: {
     kind: string;
@@ -172,11 +197,7 @@ async function reportWorkerLlmUsage(
 ): Promise<void> {
   const payload = buildWorkerLlmUsageEvent(job, result);
   if (!payload) return;
-  const response = await fetch(`${server}/telemetry/llm-usage`, {
-    method: "POST",
-    headers,
-    body: JSON.stringify(payload),
-  });
+  const response = await postJsonWithTimeout(`${server}/telemetry/llm-usage`, headers, payload);
   if (!response.ok) {
     const detail = await response.text().catch(() => "");
     throw new Error(
@@ -223,6 +244,16 @@ export function shouldEmitDirectSessionJobEvent(options: {
   return !options.statusPersistedToServer;
 }
 
+export function shouldRecycleWorkerForHeartbeatDegradation(options: {
+  heartbeatDelivered: boolean;
+  allowHeartbeatRecycle: boolean;
+  transportStale: boolean;
+}): boolean {
+  if (options.heartbeatDelivered) return false;
+  if (!options.allowHeartbeatRecycle) return false;
+  return options.transportStale;
+}
+
 function shouldRecycleWorkerForCodexUnavailableFailure(
   summary: string,
   stderr?: string | null,
@@ -853,19 +884,15 @@ async function enqueueCompletion(
       resultSummary,
     });
 
-    const response = await fetch(`${server}/completions/enqueue`, {
-      method: "POST",
-      headers,
-      body: JSON.stringify({
-        jobId: job.id,
-        sessionId: job.sessionId,
-        commitSha: commit.sha,
-        branch: commit.branch,
-        message: `${job.kind}: ${job.taskId} (worker PR metadata attached)`,
-        prUrl,
-        prTitle: pr.title,
-        prBody: pr.body,
-      }),
+    const response = await postJsonWithTimeout(`${server}/completions/enqueue`, headers, {
+      jobId: job.id,
+      sessionId: job.sessionId,
+      commitSha: commit.sha,
+      branch: commit.branch,
+      message: `${job.kind}: ${job.taskId} (worker PR metadata attached)`,
+      prUrl,
+      prTitle: pr.title,
+      prBody: pr.body,
     });
 
     if (response.ok) {
@@ -883,24 +910,6 @@ async function enqueueCompletion(
   }
 }
 
-function sendCommand(
-  server: string,
-  sessionId: string,
-  headers: Record<string, string>,
-  cmd: CommandRequest,
-): Promise<void> {
-  return fetch(`${server}/sessions/${sessionId}/command`, {
-    method: "POST",
-    headers,
-    body: JSON.stringify(cmd),
-  })
-    .then((res) => {
-      if (!res.ok) console.error(`[WorkerPals] Command ${cmd.type} failed: ${res.status}`);
-    })
-    .catch((err) => console.error(`[WorkerPals] Command ${cmd.type} error:`, err));
-}
-
-type WorkerHeartbeatStatus = "idle" | "busy" | "error" | "offline";
 type WorkerRuntimeState = {
   currentJobId: string | null;
   currentSessionId: string | null;
@@ -913,44 +922,11 @@ function buildWorkerHeaders(authToken: string | null): Record<string, string> {
   return headers;
 }
 
-async function sendWorkerHeartbeat(
-  opts: ReturnType<typeof parseArgs>,
-  headers: Record<string, string>,
-  status: WorkerHeartbeatStatus,
-  currentJobId: string | null = null,
-): Promise<void> {
-  try {
-    await fetch(`${opts.server}/workers/heartbeat`, {
-      method: "POST",
-      headers,
-      body: JSON.stringify({
-        workerId: opts.workerId,
-        status,
-        currentJobId,
-        pollMs: opts.pollMs,
-        capabilities: {
-          docker: opts.docker,
-          labels: opts.labels,
-          executor: resolveExecutor(CONFIG),
-          requireDocker: opts.requireDocker,
-        },
-        details: {
-          repo: opts.repo,
-          baseRef: opts.worktreeBaseRef,
-          dockerImage: opts.docker ? opts.dockerImage : null,
-          dockerNetworkMode: opts.docker ? opts.dockerNetworkMode : null,
-        },
-      }),
-    });
-  } catch (err) {
-    console.error(`[WorkerPals] Heartbeat error:`, err);
-  }
-}
-
 async function failActiveJobOnShutdown(
   opts: ReturnType<typeof parseArgs>,
   headers: Record<string, string>,
   runtimeState: WorkerRuntimeState,
+  transport: WorkerServerTransport,
   signalName: string,
 ): Promise<void> {
   const activeJobId = runtimeState.currentJobId;
@@ -961,10 +937,9 @@ async function failActiveJobOnShutdown(
   let statusPersistedToServer = false;
 
   try {
-    const response = await fetch(`${opts.server}/jobs/${activeJobId}/fail`, {
-      method: "POST",
-      headers,
-      body: JSON.stringify({ message, detail }),
+    const response = await postJsonWithTimeout(`${opts.server}/jobs/${activeJobId}/fail`, headers, {
+      message,
+      detail,
     });
     statusPersistedToServer = response.ok;
   } catch (err) {
@@ -978,7 +953,7 @@ async function failActiveJobOnShutdown(
     runtimeState.currentSessionId &&
     shouldEmitDirectSessionJobEvent({ ok: false, statusPersistedToServer })
   ) {
-    await sendCommand(opts.server, runtimeState.currentSessionId, headers, {
+    await transport.queueSessionCommand(runtimeState.currentSessionId, {
       type: "job_failed",
       payload: {
         jobId: activeJobId,
@@ -986,7 +961,7 @@ async function failActiveJobOnShutdown(
         detail,
       },
       from: `worker:${opts.workerId}`,
-    });
+    }, { priority: "high" });
   }
 }
 
@@ -994,6 +969,8 @@ async function workerLoop(
   opts: ReturnType<typeof parseArgs>,
   dockerExecutor: DockerExecutor | null,
   runtimeState: WorkerRuntimeState,
+  transport: WorkerServerTransport,
+  requestWorkerRestart: (reason: string) => void,
 ): Promise<void> {
   const headers = buildWorkerHeaders(opts.authToken);
 
@@ -1007,17 +984,37 @@ async function workerLoop(
   }
   console.log(`[WorkerPals ${opts.workerId}] Executor backend: ${resolveExecutor(CONFIG)}`);
   const heartbeatEveryMs = Math.max(1000, opts.heartbeatMs);
+  const claimTimeoutMs = Math.max(4_000, Math.min(15_000, opts.pollMs * 3));
   let lastHeartbeatAt = 0;
+  const buildHeartbeatPayload = (
+    status: WorkerHeartbeatPayload["status"],
+    currentJobId: string | null,
+  ): WorkerHeartbeatPayload => ({
+    status,
+    currentJobId,
+    capabilities: {
+      docker: opts.docker,
+      labels: opts.labels,
+      executor: resolveExecutor(CONFIG),
+      requireDocker: opts.requireDocker,
+    },
+    details: {
+      repo: opts.repo,
+      baseRef: opts.worktreeBaseRef,
+      dockerImage: opts.docker ? opts.dockerImage : null,
+      dockerNetworkMode: opts.docker ? opts.dockerNetworkMode : null,
+    },
+  });
 
   const maybeHeartbeat = async (
-    status: WorkerHeartbeatStatus,
+    status: WorkerHeartbeatPayload["status"],
     currentJobId: string | null = null,
     force = false,
   ) => {
     const now = Date.now();
     if (!force && now - lastHeartbeatAt < heartbeatEveryMs) return;
-    await sendWorkerHeartbeat(opts, headers, status, currentJobId);
-    lastHeartbeatAt = now;
+    const ok = await transport.sendHeartbeat(buildHeartbeatPayload(status, currentJobId));
+    if (ok) lastHeartbeatAt = now;
   };
 
   await maybeHeartbeat("idle", null, true);
@@ -1025,11 +1022,12 @@ async function workerLoop(
   while (!runtimeState.shutdownRequested) {
     try {
       await maybeHeartbeat("idle");
-      const claimRes = await fetch(`${opts.server}/jobs/claim`, {
-        method: "POST",
+      const claimRes = await postJsonWithTimeout(
+        `${opts.server}/jobs/claim`,
         headers,
-        body: JSON.stringify({ workerId: opts.workerId }),
-      });
+        { workerId: opts.workerId },
+        claimTimeoutMs,
+      );
 
       if (claimRes.ok) {
         const data = (await claimRes.json()) as any;
@@ -1040,22 +1038,35 @@ async function workerLoop(
           runtimeState.currentSessionId = job.sessionId ?? null;
           console.log(`[WorkerPals] Claimed job ${job.id} (${job.kind})`);
           await maybeHeartbeat("busy", job.id, true);
+          let allowHeartbeatRecycle = true;
 
           const busyHeartbeat = setInterval(() => {
-            void sendWorkerHeartbeat(opts, headers, "busy", job.id);
+            void transport.sendHeartbeat(buildHeartbeatPayload("busy", job.id)).then((ok) => {
+              if (
+                !shouldRecycleWorkerForHeartbeatDegradation({
+                  heartbeatDelivered: ok,
+                  allowHeartbeatRecycle,
+                  transportStale: transport.shouldRecycleBusyWorker(),
+                })
+              ) {
+                return;
+              }
+              requestWorkerRestart(
+                `heartbeat transport stale while claimed job ${job.id} is still running`,
+              );
+            });
           }, heartbeatEveryMs);
 
           if (job.sessionId) {
-            await sendCommand(opts.server, job.sessionId, headers, {
+            await transport.queueSessionCommand(job.sessionId, {
               type: "job_claimed",
               payload: { jobId: job.id, workerId: opts.workerId },
               from: `worker:${opts.workerId}`,
-            });
+            }, { priority: "high" });
           }
 
           let stdoutSeq = 0;
           let stderrSeq = 0;
-          let logChain: Promise<void> = Promise.resolve();
           let lastCleanLog = "";
           let lastCleanLogAt = 0;
 
@@ -1077,20 +1088,17 @@ async function workerLoop(
                 const logTs = new Date(now).toISOString();
 
                 const seq = stream === "stdout" ? ++stdoutSeq : ++stderrSeq;
-                logChain = logChain.then(() =>
-                  Promise.allSettled([
-                    sendCommand(opts.server, job.sessionId, headers, {
-                      type: "job_log",
-                      payload: { jobId: job.id, stream, seq, line: cleaned, ts: logTs },
-                      from: `worker:${opts.workerId}`,
-                    }),
-                    fetch(`${opts.server}/jobs/${job.id}/log`, {
-                      method: "POST",
-                      headers,
-                      body: JSON.stringify({ stream, seq, message: cleaned, ts: logTs }),
-                    }),
-                  ]).then(() => undefined),
-                );
+                void transport.queueSessionCommand(job.sessionId, {
+                  type: "job_log",
+                  payload: { jobId: job.id, stream, seq, line: cleaned, ts: logTs },
+                  from: `worker:${opts.workerId}`,
+                }, { droppable: true });
+                void transport.queueJobLog(job.id, {
+                  stream,
+                  seq,
+                  message: cleaned,
+                  ts: logTs,
+                });
               }
             : undefined;
 
@@ -1153,7 +1161,8 @@ async function workerLoop(
             }
             const jobDurationMs = Math.max(0, Date.now() - jobStartedAtMs);
 
-            await logChain;
+            allowHeartbeatRecycle = false;
+            await transport.flush();
             try {
               await reportWorkerLlmUsage(opts.server, headers, jobData, result);
             } catch (err) {
@@ -1267,10 +1276,10 @@ async function workerLoop(
                 reviewAgent.prUrl.trim().length > 0
                   ? reviewAgent.prUrl.trim()
                   : null;
-              const response = await fetch(`${opts.server}/jobs/${job.id}/complete`, {
-                method: "POST",
+              const response = await postJsonWithTimeout(
+                `${opts.server}/jobs/${job.id}/complete`,
                 headers,
-                body: JSON.stringify({
+                {
                   summary: result.summary,
                   durationMs: jobDurationMs,
                   prUrl: jobPrUrl,
@@ -1278,21 +1287,17 @@ async function workerLoop(
                     ...(result.stdout ? [{ kind: "stdout", text: result.stdout }] : []),
                     ...(result.stderr ? [{ kind: "stderr", text: result.stderr }] : []),
                   ],
-                }),
-              });
+                },
+              );
               statusPersistedToServer = response.ok;
               console.log(
                 `[WorkerPals] Job ${job.id} completed in ${formatDurationMs(jobDurationMs)}: ${result.summary}`,
               );
             } else {
-              const response = await fetch(`${opts.server}/jobs/${job.id}/fail`, {
-                method: "POST",
-                headers,
-                body: JSON.stringify({
-                  message: result.summary,
-                  detail: redactSensitiveText(result.stderr ?? ""),
-                  durationMs: jobDurationMs,
-                }),
+              const response = await postJsonWithTimeout(`${opts.server}/jobs/${job.id}/fail`, headers, {
+                message: result.summary,
+                detail: redactSensitiveText(result.stderr ?? ""),
+                durationMs: jobDurationMs,
               });
               statusPersistedToServer = response.ok;
               console.log(
@@ -1327,11 +1332,11 @@ async function workerLoop(
                     ? `${rawText.slice(0, maxResponseChars - 3)}...`
                     : rawText;
                 if (assistantText) {
-                  await sendCommand(opts.server, job.sessionId, headers, {
+                  await transport.queueSessionCommand(job.sessionId, {
                     type: "assistant_message",
                     payload: { text: assistantText },
                     from: `worker:${opts.workerId}`,
-                  });
+                  }, { priority: "high" });
                 }
               }
 
@@ -1358,7 +1363,9 @@ async function workerLoop(
                       from: `worker:${opts.workerId}`,
                     };
 
-                await sendCommand(opts.server, job.sessionId, headers, eventCmd);
+                await transport.queueSessionCommand(job.sessionId, eventCmd, {
+                  priority: "high",
+                });
               }
             }
           } finally {
@@ -1369,13 +1376,13 @@ async function workerLoop(
               result?.cooldownMs &&
               result.cooldownMs > 0
             ) {
-              await sendCommand(opts.server, job.sessionId, headers, {
+              await transport.queueSessionCommand(job.sessionId, {
                 type: "assistant_message",
                 payload: {
                   text: `WorkerPal is cooling down for ${formatDurationMs(result.cooldownMs)} after transient infrastructure failures.`,
                 },
                 from: `worker:${opts.workerId}`,
-              });
+              }, { priority: "high" });
             }
             if (!recycleWorkerAfterJob && result?.cooldownMs && result.cooldownMs > 0) {
               const cooldownMs = Math.max(0, Math.floor(result.cooldownMs));
@@ -1501,6 +1508,14 @@ async function main(): Promise<void> {
     shutdownRequested: false,
   };
   const headers = buildWorkerHeaders(opts.authToken);
+  const transport = new WorkerServerTransport({
+    server: opts.server,
+    headers,
+    workerId: opts.workerId,
+    pollMs: opts.pollMs,
+    heartbeatMs: opts.heartbeatMs,
+    staleClaimTtlMs: CONFIG.server.staleClaimTtlMs,
+  });
   let shutdownTriggered = false;
   const shutdownAndExit = (signalName: string, code: number) => {
     if (shutdownTriggered) return;
@@ -1517,9 +1532,25 @@ async function main(): Promise<void> {
 
     void (async () => {
       await withTimeout(
-        sendWorkerHeartbeat(opts, headers, "offline", runtimeState.currentJobId ?? null),
+        transport.sendHeartbeat({
+          status: "offline",
+          currentJobId: runtimeState.currentJobId ?? null,
+          capabilities: {
+            docker: opts.docker,
+            labels: opts.labels,
+            executor: resolveExecutor(CONFIG),
+            requireDocker: opts.requireDocker,
+          },
+          details: {
+            repo: opts.repo,
+            baseRef: opts.worktreeBaseRef,
+            dockerImage: opts.docker ? opts.dockerImage : null,
+            dockerNetworkMode: opts.docker ? opts.dockerNetworkMode : null,
+          },
+        }),
       );
-      await withTimeout(failActiveJobOnShutdown(opts, headers, runtimeState, signalName));
+      await withTimeout(failActiveJobOnShutdown(opts, headers, runtimeState, transport, signalName));
+      await withTimeout(transport.flush());
       if (dockerExecutor) {
         await withTimeout(
           dockerExecutor.shutdown().catch((err) => {
@@ -1548,7 +1579,13 @@ async function main(): Promise<void> {
     }
   });
 
-  workerLoop(opts, dockerExecutor, runtimeState).catch((err) => {
+  const requestWorkerRestart = (reason: string) => {
+    if (shutdownTriggered) return;
+    console.error(`[WorkerPals] Control plane unhealthy: ${reason}. Recycling worker.`);
+    shutdownAndExit("CONTROL_PLANE_UNHEALTHY", 91);
+  };
+
+  workerLoop(opts, dockerExecutor, runtimeState, transport, requestWorkerRestart).catch((err) => {
     console.error("[WorkerPals] Fatal:", err);
     process.exit(1);
   });