bunqueue 2.8.5 → 2.8.6

package/dist/client/queue/queue.js

@@ -63,6 +63,7 @@ export class Queue {
                     poolSize,
                     pingInterval: connOpts.pingInterval,
                     commandTimeout: connOpts.commandTimeout,
+                    maxCommandTimeouts: connOpts.maxCommandTimeouts,
                     pipelining: connOpts.pipelining,
                     maxInFlight: connOpts.maxInFlight,
                 });
@@ -76,6 +77,7 @@ export class Queue {
                     poolSize,
                     pingInterval: connOpts.pingInterval,
                     commandTimeout: connOpts.commandTimeout,
+                    maxCommandTimeouts: connOpts.maxCommandTimeouts,
                     pipelining: connOpts.pipelining,
                     maxInFlight: connOpts.maxInFlight,
                 });

package/dist/client/tcp/client.d.ts

@@ -73,6 +73,15 @@ export declare class TcpClient extends EventEmitter {
     /** Send ping to check connection health */
     ping(): Promise<boolean>;
     private handlePingFailure;
+    /**
+     * A command timed out. On a half-open socket (peer gone, no FIN/RST) writes
+     * keep succeeding but no response ever returns — every command times out
+     * while the socket still looks "connected". The health-check ping is one way
+     * to notice, but it can be disabled or slower than real traffic, leaving a
+     * worker's PULL loop to time out forever without ever reconnecting (#94).
+     * Treat a sustained run of timeouts as a dead link and force a reconnect.
+     */
+    private handleCommandTimeout;
     private forceReconnect;
     /** Get connection health metrics */
     getHealth(): ConnectionHealth;

package/dist/client/tcp/client.js

@@ -79,6 +79,7 @@ export class TcpClient extends EventEmitter {
         this.health = new HealthTracker({
             pingInterval: this.options.pingInterval,
             maxPingFailures: this.options.maxPingFailures,
+            maxCommandTimeouts: this.options.maxCommandTimeouts,
         });
         this.reconnect = new ReconnectManager({
             maxReconnectAttempts: this.options.maxReconnectAttempts,
@@ -258,15 +259,44 @@ export class TcpClient extends EventEmitter {
             this.emit('health', { type: 'ping_failed' });
         }
     }
+    /**
+     * A command timed out. On a half-open socket (peer gone, no FIN/RST) writes
+     * keep succeeding but no response ever returns — every command times out
+     * while the socket still looks "connected". The health-check ping is one way
+     * to notice, but it can be disabled or slower than real traffic, leaving a
+     * worker's PULL loop to time out forever without ever reconnecting (#94).
+     * Treat a sustained run of timeouts as a dead link and force a reconnect.
+     */
+    handleCommandTimeout() {
+        if (this.health.recordCommandTimeout()) {
+            this.emit('health', { type: 'unhealthy', reason: 'max_command_timeouts' });
+            this.forceReconnect();
+        }
+    }
     forceReconnect() {
         if (this.reconnect.isClosed())
             return;
         if (this.socket) {
-            this.socket.end();
+            // end() can throw on an already-errored/half-dead socket. Swallow it:
+            // failing to close the corpse must NOT abort the reconnect path below,
+            // or the connection would stay wedged forever (the #94 failure mode).
+            try {
+                this.socket.end();
+            }
+            catch {
+                /* socket already torn down */
+            }
             this.socket = null;
         }
         this.connected = false;
         this.health.stopPing();
+        // Settle every in-flight/queued command NOW. Otherwise their per-command
+        // timeouts keep ticking and fire AFTER the fresh socket is up — each stale
+        // timeout bumps the new connection's dead-link counter and can re-trigger
+        // forceReconnect in a loop (a reconnect storm that never stabilises). It
+        // also unblocks awaiting callers (e.g. a Worker's PULL) immediately instead
+        // of making them wait out the full commandTimeout on a corpse.
+        this.commands.rejectAll(new Error('Connection lost'));
         if (this.reconnect.canReconnect())
             this.reconnect.scheduleReconnect(() => this.connect());
     }
@@ -343,6 +373,8 @@ export class TcpClient extends EventEmitter {
                 if (removed) {
                     this.health.recordError();
                     next.reject(new Error('Command timeout'));
+                    // In-flight command got no response: count it toward dead-link detection.
+                    this.handleCommandTimeout();
                 }
             }, this.options.commandTimeout);
             next.timeout = newTimeout;
@@ -365,17 +397,20 @@ export class TcpClient extends EventEmitter {
         let pendingRef;
         const promise = new Promise((resolve, reject) => {
             const timeout = setTimeout(() => {
-                // Try to remove from queue first
+                // Try to remove from queue first. A still-queued command never reached
+                // the socket (e.g. waiting on connect), so it is NOT evidence of a dead
+                // link — reject it but don't count it toward dead-link detection.
                 if (this.commands.remove(id)) {
                     this.health.recordError();
                     reject(new Error('Command timeout'));
                     return;
                 }
-                // Try to remove from in-flight
+                // Try to remove from in-flight: this one WAS sent and got no response.
                 const removed = this.commands.removeByReqId(reqId);
                 if (removed) {
                     this.health.recordError();
                     reject(new Error('Command timeout'));
+                    this.handleCommandTimeout();
                 }
             }, this.options.commandTimeout);
             pendingRef = {

package/dist/client/tcp/connection.js

@@ -42,6 +42,17 @@ export async function createConnection(target, connectTimeout, events) {
             },
             open(sock) {
                 cleanup();
+                // Enable TCP keepalive so the OS probes idle connections and surfaces a
+                // dead peer (suspended host, NAT/LB drop) via an error/close event,
+                // instead of a half-open socket lingering until tcp_retries2 (~15 min).
+                // Best-effort: not all platforms honor the delay, and older Bun builds
+                // may lack the method — never let it abort connection setup. See #94.
+                try {
+                    sock.setKeepAlive?.(true, 15000);
+                }
+                catch {
+                    /* keepalive unsupported on this platform/runtime */
+                }
                 socketData.write = (d) => sock.write(d);
                 socketData.end = () => sock.end();
                 connectionResolved = true;

package/dist/client/tcp/health.d.ts

@@ -7,6 +7,11 @@ import type { ConnectionHealth } from './types';
 export interface HealthConfig {
     pingInterval: number;
     maxPingFailures: number;
+    /**
+     * Consecutive command timeouts before the link is concluded dead (0 = off).
+     * Optional for backward compatibility; defaults to 3 when omitted.
+     */
+    maxCommandTimeouts?: number;
 }
 /**
  * Tracks connection health metrics
@@ -14,6 +19,7 @@ export interface HealthConfig {
 export declare class HealthTracker {
     private readonly config;
     private consecutivePingFailures;
+    private consecutiveCommandTimeouts;
     private lastSuccessAt;
     private lastErrorAt;
     private connectedAt;
@@ -35,6 +41,14 @@ export declare class HealthTracker {
     recordPingSuccess(latencyMs: number): void;
     /** Record ping failure, returns true if max failures reached */
     recordPingFailure(): boolean;
+    /**
+     * Record a command timeout. Returns true when the configured consecutive
+     * threshold is reached (and the feature is enabled), signalling the caller to
+     * force a reconnect. Any intervening success resets the counter, so this only
+     * fires on a sustained run of timeouts — the signature of a dead/half-open
+     * socket where writes succeed but no response ever comes back.
+     */
+    recordCommandTimeout(): boolean;
     /** Get current health metrics */
     getHealth(state: 'connected' | 'connecting' | 'disconnected' | 'closed'): ConnectionHealth;
     /** Start ping timer */

package/dist/client/tcp/health.js

@@ -2,12 +2,15 @@
  * TCP Health Tracker
  * Monitors connection health with ping and latency tracking
  */
+/** Default consecutive command-timeout threshold when not configured. */
+const DEFAULT_MAX_COMMAND_TIMEOUTS = 3;
 /**
  * Tracks connection health metrics
  */
 export class HealthTracker {
     config;
     consecutivePingFailures = 0;
+    consecutiveCommandTimeouts = 0;
     lastSuccessAt = null;
     lastErrorAt = null;
     connectedAt = null;
@@ -23,6 +26,9 @@ export class HealthTracker {
     recordSuccess(latencyMs) {
         this.lastSuccessAt = Date.now();
         this.totalCommands++;
+        // A real response proves the link is alive: the prior timeouts were not a
+        // sustained run, so reset the dead-link counter ("consecutive" must mean it).
+        this.consecutiveCommandTimeouts = 0;
         this.recordLatency(latencyMs);
     }
     /** Record command error */
@@ -38,10 +44,13 @@ export class HealthTracker {
     recordConnected() {
         this.connectedAt = Date.now();
         this.consecutivePingFailures = 0;
+        this.consecutiveCommandTimeouts = 0;
     }
     /** Record ping success */
     recordPingSuccess(latencyMs) {
+        // A successful ping is also proof the link is alive — clear both suspicions.
         this.consecutivePingFailures = 0;
+        this.consecutiveCommandTimeouts = 0;
         this.recordLatency(latencyMs);
     }
     /** Record ping failure, returns true if max failures reached */
@@ -51,6 +60,20 @@ export class HealthTracker {
         this.totalErrors++;
         return this.consecutivePingFailures >= this.config.maxPingFailures;
     }
+    /**
+     * Record a command timeout. Returns true when the configured consecutive
+     * threshold is reached (and the feature is enabled), signalling the caller to
+     * force a reconnect. Any intervening success resets the counter, so this only
+     * fires on a sustained run of timeouts — the signature of a dead/half-open
+     * socket where writes succeed but no response ever comes back.
+     */
+    recordCommandTimeout() {
+        const max = this.config.maxCommandTimeouts ?? DEFAULT_MAX_COMMAND_TIMEOUTS;
+        if (max <= 0)
+            return false;
+        this.consecutiveCommandTimeouts++;
+        return this.consecutiveCommandTimeouts >= max;
+    }
     /** Get current health metrics */
     getHealth(state) {
         const avgLatency = this.latencyHistory.length > 0
@@ -63,6 +86,7 @@ export class HealthTracker {
             lastErrorAt: this.lastErrorAt,
             avgLatencyMs: Math.round(avgLatency * 100) / 100,
             consecutivePingFailures: this.consecutivePingFailures,
+            consecutiveCommandTimeouts: this.consecutiveCommandTimeouts,
             totalCommands: this.totalCommands,
             totalErrors: this.totalErrors,
             uptimeMs: this.connectedAt ? Date.now() - this.connectedAt : 0,

package/dist/client/tcp/types.d.ts

@@ -26,6 +26,14 @@ export interface ConnectionOptions {
     pingInterval?: number;
     /** Max consecutive ping failures before forcing reconnect (default: 3) */
     maxPingFailures?: number;
+    /**
+     * Max consecutive command timeouts (with no intervening success) before the
+     * connection is concluded dead and reconnect is forced (default: 3, 0 to
+     * disable). This is the recovery path for a half-open socket when the
+     * health-check ping is disabled or slower than real traffic — a worker whose
+     * PULLs keep timing out no longer stalls forever waiting on the ping. See #94.
+     */
+    maxCommandTimeouts?: number;
     /** Enable pipelining - multiple commands in flight (default: true) */
     pipelining?: boolean;
     /** Max commands in flight when pipelining (default: 100) */
@@ -45,6 +53,8 @@ export interface ConnectionHealth {
     avgLatencyMs: number;
     /** Consecutive ping failures */
     consecutivePingFailures: number;
+    /** Consecutive command timeouts with no intervening success */
+    consecutiveCommandTimeouts: number;
     /** Total commands sent */
     totalCommands: number;
     /** Total errors */

package/dist/client/tcp/types.js

@@ -15,6 +15,7 @@ export const DEFAULT_CONNECTION = {
     autoReconnect: true,
     pingInterval: 30000,
     maxPingFailures: 3,
+    maxCommandTimeouts: 3,
     pipelining: true,
     maxInFlight: 100,
 };

package/dist/client/tcpPool.js

@@ -29,6 +29,7 @@ export class TcpConnectionPool {
             autoReconnect: options.autoReconnect ?? true,
             pingInterval: options.pingInterval ?? 30000,
             maxPingFailures: options.maxPingFailures ?? 3,
+            maxCommandTimeouts: options.maxCommandTimeouts ?? 3,
             pipelining: options.pipelining ?? true,
             maxInFlight: options.maxInFlight ?? 100,
         };
@@ -46,6 +47,7 @@ export class TcpConnectionPool {
                 autoReconnect: this.options.autoReconnect,
                 pingInterval: this.options.pingInterval,
                 maxPingFailures: this.options.maxPingFailures,
+                maxCommandTimeouts: this.options.maxCommandTimeouts,
             });
             this.clients.push(client);
         }

package/dist/client/types.d.ts

@@ -376,6 +376,12 @@ export interface ConnectionOptions {
     pingInterval?: number;
     /** Command timeout in ms (default: 30000) */
     commandTimeout?: number;
+    /**
+     * Consecutive command timeouts (no intervening success) before the connection
+     * is concluded dead and a reconnect is forced (default: 3, 0 to disable).
+     * Recovery path for a half-open socket independent of the health-check ping. See #94.
+     */
+    maxCommandTimeouts?: number;
     /** Enable TCP pipelining (default: true) */
     pipelining?: boolean;
     /** Max commands in flight per connection (default: 100) */

package/dist/client/worker/worker.js

@@ -47,6 +47,7 @@ function createTcpPool(opts, concurrency) {
         poolSize,
         pingInterval: connOpts.pingInterval,
         commandTimeout: connOpts.commandTimeout,
+        maxCommandTimeouts: connOpts.maxCommandTimeouts,
         pipelining: connOpts.pipelining,
         maxInFlight: connOpts.maxInFlight,
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bunqueue",
-  "version": "2.8.5",
+  "version": "2.8.6",
   "description": "High-performance job queue for Bun & AI agents. SQLite persistence, cron scheduling, priorities, retries, DLQ, webhooks, native MCP server. Zero external dependencies.",
   "type": "module",
   "main": "dist/main.js",