@spooled/sdk 1.2.0 → 1.3.0

package/dist/index.js

@@ -1,5 +1,3 @@
-import { createHmac } from 'crypto';
-import { Buffer } from 'buffer';
 import * as grpc2 from '@grpc/grpc-js';
 import * as protoLoader from '@grpc/proto-loader';
 import { existsSync } from 'fs';
@@ -26,10 +24,10 @@ var DEFAULT_CONFIG = {
     successThreshold: 3,
     timeout: 3e4
   },
-  userAgent: "@spooled/sdk-nodejs/1.2.0",
+  userAgent: "@spooled/sdk-nodejs/1.3.0",
   autoRefreshToken: true
 };
-var SDK_VERSION = "1.2.0";
+var SDK_VERSION = "1.3.0";
 var API_VERSION = "v1";
 var API_BASE_PATH = `/api/${API_VERSION}`;
 function normalizeCredential(value) {
@@ -86,9 +84,9 @@ function resolveConfig(options) {
   };
 }
 function validateConfig(config) {
-  if (!config.apiKey && !config.accessToken) {
+  if (!config.apiKey && !config.accessToken && !config.adminKey) {
     throw new Error(
-      "SpooledClient requires either apiKey or accessToken for authentication"
+      "SpooledClient requires either apiKey or accessToken for authentication (or adminKey for admin-only use)"
     );
   }
   if (config.apiKey && !config.apiKey.startsWith("sp_") && !config.apiKey.startsWith("sk_")) {
@@ -151,9 +149,15 @@ var SpooledError = class extends Error {
       Error.captureStackTrace(this, this.constructor);
     }
   }
-  /** Whether this error is retryable */
+  /**
+   * Whether this error is retryable.
+   *
+   * Only 408 (Request Timeout) is retryable among the plain 4xx statuses: the
+   * server gave up waiting for the request, so it never ran. Subclasses
+   * override this (5xx, network, timeout, rate limit).
+   */
   isRetryable() {
-    return false;
+    return this.statusCode === 408;
   }
   /** Convert to plain object for logging */
   toJSON() {
@@ -192,8 +196,8 @@ var ConflictError = class extends SpooledError {
   }
 };
 var ValidationError = class extends SpooledError {
-  constructor(message = "Validation failed", code = "VALIDATION_ERROR", details, requestId) {
-    super(message, 400, code, details, requestId);
+  constructor(message = "Validation failed", code = "VALIDATION_ERROR", details, requestId, statusCode = 400) {
+    super(message, statusCode, code, details, requestId);
     this.name = "ValidationError";
   }
 };
@@ -203,6 +207,7 @@ var PayloadTooLargeError = class extends SpooledError {
     this.name = "PayloadTooLargeError";
   }
 };
+var QUOTA_MESSAGE = /\blimit reached\b/i;
 var RateLimitError = class extends SpooledError {
   /** Rate limit information */
   rateLimitInfo;
@@ -211,8 +216,27 @@ var RateLimitError = class extends SpooledError {
     this.name = "RateLimitError";
     this.rateLimitInfo = rateLimitInfo;
   }
+  /**
+   * True when this 429 is a plan quota or limit (daily job cap, active jobs,
+   * queue/worker/webhook caps) rather than per-second rate limiting.
+   *
+   * A quota does not clear for minutes or hours, so retrying it only blocks the
+   * caller for the whole backoff and, via the circuit breaker, takes every
+   * other call on the client down with it. It is never retried and never
+   * counted as a circuit breaker failure.
+   *
+   * Detection: the `QUOTA_EXCEEDED` code, or, for older plan-limit paths that
+   * answer with plain text wrapped as `RATE_LIMIT_EXCEEDED`, a "... limit
+   * reached" message that carries no `Retry-After`.
+   */
+  get isQuota() {
+    if (this.code === "QUOTA_EXCEEDED") {
+      return true;
+    }
+    return this.rateLimitInfo.retryAfter === void 0 && QUOTA_MESSAGE.test(this.message);
+  }
   isRetryable() {
-    return true;
+    return !this.isQuota;
   }
   /** Get the number of seconds to wait before retrying */
   getRetryAfter() {
@@ -272,6 +296,20 @@ var CircuitBreakerOpenError = class extends SpooledError {
     return false;
   }
 };
+var UnsupportedOperationError = class extends SpooledError {
+  /** The SDK method that is not supported, e.g. `ingest.github` */
+  operation;
+  constructor(operation, reason) {
+    super(
+      `${operation} is not supported by the Spooled API: ${reason}`,
+      0,
+      "UNSUPPORTED_OPERATION",
+      { operation }
+    );
+    this.name = "UnsupportedOperationError";
+    this.operation = operation;
+  }
+};
 function parseRateLimitHeaders(headers) {
   const info = {};
   const retryAfter = headers.get("Retry-After");
@@ -343,6 +381,10 @@ async function createErrorFromResponse(response) {
   switch (response.status) {
     case 400:
       return new ValidationError(message, code, details, requestId);
+    // A missing or mistyped JSON field is a 422 from the API's extractor; it is
+    // the same class of caller error as a 400 field-rule failure.
+    case 422:
+      return new ValidationError(message, code, details, requestId, 422);
     case 401:
       return new AuthenticationError(message, code, details, requestId);
     case 403:
@@ -571,8 +613,12 @@ var HttpClient = class {
   authToken;
   /** Optional function to mint a fresh access token (set when auto-refresh is enabled) */
   refreshTokenFn = null;
-  /** Guard to prevent a refresh request from recursively triggering another refresh */
-  isRefreshing = false;
+  /**
+   * The refresh currently in flight, shared by every request that gets a 401
+   * meanwhile. A boolean guard here made the second concurrent 401 skip the
+   * refresh and fail even though the token was about to be replaced.
+   */
+  refreshInFlight = null;
   constructor(config, circuitBreaker) {
     this.config = config;
     this.circuitBreaker = circuitBreaker;
@@ -594,6 +640,30 @@ var HttpClient = class {
   setRefreshTokenFn(fn) {
     this.refreshTokenFn = fn;
   }
+  /**
+   * Refresh the access token once for all callers.
+   *
+   * Resolves true when a new token is installed, false when the refresh failed
+   * (the caller then surfaces its original 401).
+   */
+  refreshAuthToken() {
+    if (!this.refreshInFlight) {
+      const refresh = this.refreshTokenFn;
+      this.refreshInFlight = (async () => {
+        try {
+          this.setAuthToken(await refresh());
+          return true;
+        } catch (refreshError) {
+          const message = refreshError instanceof Error ? refreshError.message : "unknown";
+          this.debug?.("Token refresh failed", { error: message });
+          return false;
+        } finally {
+          this.refreshInFlight = null;
+        }
+      })();
+    }
+    return this.refreshInFlight;
+  }
   /**
    * Build full URL with query parameters
    */
@@ -669,6 +739,7 @@ var HttpClient = class {
    */
   async executeRequest(url, options, hasRefreshed = false) {
     const method = options.method ?? "GET";
+    const tokenUsed = this.authToken;
     const headers = this.buildHeaders(options.headers);
     const timeout = options.timeout ?? this.config.timeout;
     const init = {
@@ -683,6 +754,9 @@ var HttpClient = class {
         init.body = JSON.stringify(body);
       }
     }
+    if (init.body === void 0) {
+      headers.delete("Content-Type");
+    }
     this.debug?.(`${method} ${url}`, { timeout, hasBody: !!options.body });
     let response;
     try {
@@ -704,20 +778,9 @@ var HttpClient = class {
     }
     this.debug?.(`Response: ${response.status}`, { url });
     if (!response.ok) {
-      if (response.status === 401 && this.refreshTokenFn && !hasRefreshed && !this.isRefreshing) {
-        let refreshed = false;
-        this.isRefreshing = true;
-        try {
-          const newToken = await this.refreshTokenFn();
-          this.setAuthToken(newToken);
-          refreshed = true;
-        } catch (refreshError) {
-          const message = refreshError instanceof Error ? refreshError.message : "unknown";
-          this.debug?.("Token refresh failed", { error: message });
-        } finally {
-          this.isRefreshing = false;
-        }
-        if (refreshed) {
+      if (response.status === 401 && this.refreshTokenFn && !hasRefreshed && !options.skipAuthRefresh) {
+        const alreadyRefreshed = this.authToken !== tokenUsed;
+        if (alreadyRefreshed || await this.refreshAuthToken()) {
           return this.executeRequest(url, options, true);
         }
       }
@@ -779,22 +842,26 @@ var HttpClient = class {
   /**
    * Resolve the retry configuration for a request.
    *
-   * Idempotent methods (GET/PUT/DELETE) and any request carrying an
-   * idempotency key retry normally. Non-idempotent methods (POST/PATCH)
-   * without an idempotency key must NOT be retried on ambiguous failures
+   * Idempotent methods (GET/PUT/DELETE), any request carrying an idempotency
+   * key and any request marked `idempotent` retry normally. Non-idempotent
+   * methods (POST/PATCH) otherwise must NOT be retried on ambiguous failures
    * (network/timeout/5xx) — doing so risks duplicate jobs, double claims,
-   * etc. They are still retried on 429, which is rejected before the
-   * server processes the request and is therefore side-effect-free.
+   * etc. They are still retried on a rate-limit 429, which is rejected before
+   * the server processes the request and is therefore side-effect-free. A
+   * quota 429 is never retried: it does not clear for hours.
    */
   resolveRetryConfig(method, options) {
     const base = this.config.retry;
-    if (IDEMPOTENT_METHODS.has(method) || this.hasIdempotencyKey(options)) {
+    if (options.idempotent || IDEMPOTENT_METHODS.has(method) || this.hasIdempotencyKey(options)) {
       return base;
     }
     return {
       ...base,
       retryOn: (error, attempt) => {
         if (error instanceof RateLimitError) {
+          if (error.isQuota) {
+            return false;
+          }
           return base.retryOn ? base.retryOn(error, attempt) : true;
         }
         return false;
@@ -1025,13 +1092,17 @@ var AuthResource = class {
    * Exchange API key for JWT tokens
    */
   async login(params) {
-    return this.http.post("/auth/login", params);
+    return this.http.post("/auth/login", params, {
+      skipAuthRefresh: true
+    });
   }
   /**
    * Refresh access token using refresh token
    */
   async refresh(params) {
-    return this.http.post("/auth/refresh", params);
+    return this.http.post("/auth/refresh", params, {
+      skipAuthRefresh: true
+    });
   }
   /**
    * Logout and revoke the refresh token server-side.
@@ -1102,6 +1173,21 @@ function mapValidate(raw) {
   };
 }
 
+// src/utils/failure.ts
+var MAX_FAILURE_MESSAGE_CHARS = 2048;
+var DEFAULT_FAILURE_MESSAGE = "Job failed";
+function coerceFailureMessage(message) {
+  const text = typeof message === "string" ? message : message == null ? "" : `${message}`;
+  if (text === "") {
+    return DEFAULT_FAILURE_MESSAGE;
+  }
+  if (text.length <= MAX_FAILURE_MESSAGE_CHARS) {
+    return text;
+  }
+  const codePoints = Array.from(text);
+  return codePoints.length <= MAX_FAILURE_MESSAGE_CHARS ? text : codePoints.slice(0, MAX_FAILURE_MESSAGE_CHARS).join("");
+}
+
 // src/resources/jobs.ts
 var JobsResource = class {
   constructor(http) {
@@ -1118,7 +1204,9 @@ var JobsResource = class {
    * Create a new job
    */
   async create(params) {
-    return this.http.post("/jobs", params);
+    return this.http.post("/jobs", params, {
+      idempotent: params.idempotencyKey ? true : void 0
+    });
   }
   /**
    * Create a job and return the full job object
@@ -1202,26 +1290,43 @@ var JobsResource = class {
   }
   /**
    * Complete a job (worker ack)
+   *
+   * Lease-fenced and idempotent on the server, so a network error, timeout or
+   * 5xx is retried: a lost ack would otherwise re-run a finished job.
    */
   async complete(id, params) {
-    return this.http.post(`/jobs/${id}/complete`, params);
+    return this.http.post(
+      `/jobs/${id}/complete`,
+      params,
+      { idempotent: true }
+    );
   }
   /**
    * Fail a job (worker nack)
+   *
+   * `params.error` is made acceptable first: empty text becomes "Job failed"
+   * and text over 2048 characters is truncated, because the API rejects both
+   * with a 400 that would leave the job processing until its lease expires.
+   * Retried like {@link JobsResource.complete}.
    */
   async fail(id, params) {
     return this.http.post(
       `/jobs/${id}/fail`,
-      params
+      { ...params, error: coerceFailureMessage(params.error) },
+      { idempotent: true }
     );
   }
   /**
    * Extend job lease (heartbeat)
+   *
+   * Retried like {@link JobsResource.complete}: with the default 30 s lease and
+   * 15 s heartbeat, one lost heartbeat leaves no slack.
    */
   async heartbeat(id, params) {
     return this.http.post(
       `/jobs/${id}/heartbeat`,
-      params
+      params,
+      { idempotent: true }
     );
   }
   // DLQ operations (private implementations)
@@ -1317,9 +1422,17 @@ var QueuesResource = class {
   }
   /**
    * Delete a queue configuration
-   */
-  async delete(name) {
-    await this.http.delete(`/queues/${encodeURIComponent(name)}`);
+   *
+   * Without `deleteJobs` the API only deletes an explicit queue config of an
+   * idle queue, and answers 409 otherwise: a queue that exists only because
+   * jobs were posted to it has no config to delete, and a queue with pending
+   * or processing jobs is not idle. Pass `{ deleteJobs: true }` to delete the
+   * queue's jobs (and DLQ rows) along with it.
+   */
+  async delete(name, options) {
+    await this.http.delete(`/queues/${encodeURIComponent(name)}`, {
+      params: options?.deleteJobs ? { deleteJobs: true } : void 0
+    });
   }
 };
 
@@ -1348,9 +1461,13 @@ var WorkersResource = class {
   }
   /**
    * Send worker heartbeat
+   *
+   * Idempotent on the server, so network errors, timeouts and 5xx are retried.
    */
   async heartbeat(id, params) {
-    await this.http.post(`/workers/${id}/heartbeat`, params);
+    await this.http.post(`/workers/${id}/heartbeat`, params, {
+      idempotent: true
+    });
   }
   /**
    * Deregister a worker
@@ -1499,7 +1616,7 @@ var WorkflowsResource = class {
     const jobs = await this.listWorkflowJobs(workflowId);
     return jobs.map((job) => ({
       jobId: job.id,
-      key: job.key,
+      ...job.key !== void 0 && { key: job.key },
       status: job.status
     }));
   }
@@ -1557,7 +1674,10 @@ function mapWorkflowDetailJobs(detail) {
     return {
       id,
       workflowId: String(job.workflowId ?? workflowId),
-      key: String(job.key ?? ""),
+      // The API does not return the creation-time key (jobs store only a
+      // workflow step), so there is nothing to put here unless a future
+      // version sends one. Never fabricate "".
+      ...typeof job.key === "string" && job.key !== "" && { key: job.key },
       queueName: String(job.queueName ?? job.queue ?? ""),
       status: job.status,
       payload: job.payload ?? {},
@@ -1769,19 +1889,18 @@ var OrganizationsResource = class {
   /**
    * Clear/delete the webhook token for the current organization.
    *
-   * POST /organizations/webhook-token/clear requires `{ confirm: true }`.
-   * An empty body 422s. The API currently rejects the operation and tells
-   * callers to use regenerateWebhookToken() instead.
-   *
-   * @example
-   * ```typescript
-   * await client.organizations.clearWebhookToken();
-   * ```
+   * @deprecated The API refuses this operation on purpose (an org without a
+   * webhook token would have an open public ingestion endpoint) and always
+   * answers with an error. This method now throws
+   * {@link UnsupportedOperationError} without a network call; rotate the token
+   * with {@link OrganizationsResource.regenerateWebhookToken} instead. It will
+   * be removed in the next major release.
    */
   async clearWebhookToken() {
-    await this.http.post("/organizations/webhook-token/clear", {
-      confirm: true
-    });
+    throw new UnsupportedOperationError(
+      "organizations.clearWebhookToken",
+      "clearing the webhook token is not allowed; use organizations.regenerateWebhookToken() to rotate it"
+    );
   }
 };
 
@@ -1989,24 +2108,8 @@ var AdminResource = class {
     });
   }
 };
-function toBytes(body) {
-  if (typeof body === "string") {
-    return Buffer.from(body, "utf8");
-  }
-  if (body instanceof Uint8Array) {
-    return body;
-  }
-  return new Uint8Array(body);
-}
-function githubSignature(secret, body) {
-  const hex = createHmac("sha256", secret).update(body).digest("hex");
-  return `sha256=${hex}`;
-}
-function stripeSignature(secret, body, timestamp) {
-  const payload = `${timestamp}.${Buffer.from(body).toString("utf8")}`;
-  const hex = createHmac("sha256", secret).update(payload, "utf8").digest("hex");
-  return `t=${timestamp},v1=${hex}`;
-}
+
+// src/resources/webhook-ingestion.ts
 var WebhookIngestionResource = class {
   constructor(http) {
     this.http = http;
@@ -2014,66 +2117,32 @@ var WebhookIngestionResource = class {
   /**
    * Ingest a GitHub webhook.
    *
-   * POST /api/v1/webhooks/{org_id}/github
-   */
-  async github(orgId, body, options) {
-    const bytes = toBytes(body);
-    const signature = options.signature ?? (options.secret ? githubSignature(options.secret, bytes) : void 0);
-    if (!signature) {
-      throw new ValidationError(
-        "GitHub webhook signature is required (provide `signature` or `secret`)"
-      );
-    }
-    const headers = {
-      "Content-Type": "application/json",
-      "X-GitHub-Event": options.githubEvent,
-      "X-Hub-Signature-256": signature
-    };
-    if (options.webhookToken) {
-      headers["X-Webhook-Token"] = options.webhookToken;
-    }
-    if (options.forwardedProto) {
-      headers["X-Forwarded-Proto"] = options.forwardedProto;
-    }
-    await this.http.request(`/webhooks/${orgId}/github`, {
-      method: "POST",
-      rawBody: bytes,
-      headers,
-      skipRequestConversion: true,
-      skipResponseConversion: true
-    });
+   * @deprecated The API has no `/webhooks/{org_id}/github` route (it always
+   * answered 404). This method now throws {@link UnsupportedOperationError}
+   * without a network call; point the GitHub webhook at `ingest.custom` /
+   * `POST /api/v1/webhooks/{org_id}/custom` instead. It will be removed in the
+   * next major release.
+   */
+  async github(_orgId, _body, _options) {
+    throw new UnsupportedOperationError(
+      "ingest.github",
+      "the API has no GitHub webhook route; use ingest.custom (POST /webhooks/{org_id}/custom)"
+    );
   }
   /**
    * Ingest a Stripe webhook.
    *
-   * POST /api/v1/webhooks/{org_id}/stripe
-   */
-  async stripe(orgId, body, options = {}) {
-    const bytes = toBytes(body);
-    const timestamp = options.timestamp ?? Math.floor(Date.now() / 1e3);
-    const signature = options.signature ?? (options.secret ? stripeSignature(options.secret, bytes, timestamp) : void 0);
-    if (!signature) {
-      throw new ValidationError(
-        "Stripe webhook signature is required (provide `signature` or `secret`)"
-      );
-    }
-    const headers = {
-      "Content-Type": "application/json",
-      "Stripe-Signature": signature
-    };
-    if (options.webhookToken) {
-      headers["X-Webhook-Token"] = options.webhookToken;
-    }
-    if (options.forwardedProto) {
-      headers["X-Forwarded-Proto"] = options.forwardedProto;
-    }
-    await this.http.request(`/webhooks/${orgId}/stripe`, {
-      method: "POST",
-      rawBody: bytes,
-      headers,
-      skipRequestConversion: true,
-      skipResponseConversion: true
-    });
+   * @deprecated The API has no `/webhooks/{org_id}/stripe` route (it always
+   * answered 404). This method now throws {@link UnsupportedOperationError}
+   * without a network call; point the Stripe webhook at `ingest.custom` /
+   * `POST /api/v1/webhooks/{org_id}/custom` instead. It will be removed in the
+   * next major release.
+   */
+  async stripe(_orgId, _body, _options = {}) {
+    throw new UnsupportedOperationError(
+      "ingest.stripe",
+      "the API has no Stripe webhook route; use ingest.custom (POST /webhooks/{org_id}/custom)"
+    );
   }
   /**
    * Ingest a custom webhook.
@@ -2437,6 +2506,26 @@ function isAuthFailureEvent(event) {
 }
 
 // src/realtime/sse.ts
+var SSE_EVENT_NAMES = [
+  "job.created",
+  "job.status",
+  "job.completed",
+  "job.failed",
+  "queue.stats",
+  "worker.heartbeat",
+  "worker.registered",
+  "worker.deregistered",
+  "system.health",
+  "ping",
+  "error"
+];
+var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set([
+  "completed",
+  "failed",
+  "deadletter",
+  "cancelled"
+]);
+var STABLE_CONNECTION_MS = 3e4;
 var SseRealtimeClient = class {
   options;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -2445,6 +2534,12 @@ var SseRealtimeClient = class {
   reconnectAttempts = 0;
   reconnectTimer = null;
   filter = null;
+  /** When the current connection opened (for the stable-connection check). */
+  connectedAt = null;
+  /** The server finished this stream on purpose (terminal job); do not reconnect. */
+  streamEnded = false;
+  /** The server rejected our token; mint a fresh one for the next connect. */
+  forceTokenRefresh = false;
   // Event handlers
   eventHandlers = /* @__PURE__ */ new Map();
   allEventsHandlers = /* @__PURE__ */ new Set();
@@ -2483,6 +2578,7 @@ var SseRealtimeClient = class {
       return;
     }
     this.filter = filter || null;
+    this.streamEnded = false;
     this.setState("connecting");
     const sseUrl = this.buildSseUrl(filter);
     this.options.debug(`Connecting to SSE: ${sseUrl}`);
@@ -2502,7 +2598,9 @@ var SseRealtimeClient = class {
     }
     let token;
     try {
-      token = await this.options.tokenProvider(this.reconnectAttempts > 0);
+      const force = this.forceTokenRefresh;
+      this.forceTokenRefresh = false;
+      token = await this.options.tokenProvider(force);
     } catch (error) {
       this.setState("disconnected");
       throw new Error(`Failed to acquire realtime token: ${error}`);
@@ -2522,14 +2620,22 @@ var SseRealtimeClient = class {
         this.eventSource.onopen = () => {
           this.options.debug("SSE connected");
           this.setState("connected");
-          this.reconnectAttempts = 0;
+          this.connectedAt = Date.now();
           resolve();
         };
         this.eventSource.onmessage = (event) => {
           this.handleMessage(event.data);
         };
-        this.eventSource.onerror = () => {
+        for (const name of SSE_EVENT_NAMES) {
+          this.eventSource.addEventListener(name, (event) => {
+            if (typeof event?.data === "string") this.handleMessage(event.data);
+          });
+        }
+        this.eventSource.onerror = (event) => {
           this.options.debug("SSE error");
+          if (event?.code === 401 || event?.code === 403) {
+            this.forceTokenRefresh = true;
+          }
           if (this.state === "connecting") {
             this.setState("disconnected");
             if (this.eventSource) {
@@ -2561,6 +2667,8 @@ var SseRealtimeClient = class {
     }
     this.setState("disconnected");
     this.filter = null;
+    this.connectedAt = null;
+    this.streamEnded = false;
   }
   /**
    * Reconnect with a new filter (SSE requires reconnection to change subscriptions)
@@ -2608,7 +2716,7 @@ var SseRealtimeClient = class {
       return `${this.options.baseUrl}/api/v1/events/jobs/${encodeURIComponent(filter.jobId)}`;
     }
     if (filter?.queueName) {
-      return `${this.options.baseUrl}/api/v1/events/queues/${encodeURIComponent(filter.queueName)}`;
+      return `${this.options.baseUrl}/api/v1/events?queue=${encodeURIComponent(filter.queueName)}`;
     }
     return `${this.options.baseUrl}/api/v1/events`;
   }
@@ -2625,6 +2733,9 @@ var SseRealtimeClient = class {
       parsed.type = eventType;
       const event = parsed;
       this.options.debug(`Received SSE event: ${eventType}`, event);
+      if (this.filter?.jobId && isStreamEndingEvent(eventType, parsed.data)) {
+        this.streamEnded = true;
+      }
       const handlers = this.eventHandlers.get(eventType);
       if (handlers) {
         const eventData = event.data;
@@ -2652,6 +2763,15 @@ var SseRealtimeClient = class {
       this.eventSource.close();
       this.eventSource = null;
     }
+    if (this.streamEnded) {
+      this.connectedAt = null;
+      this.setState("disconnected");
+      return;
+    }
+    if (this.connectedAt !== null && Date.now() - this.connectedAt >= STABLE_CONNECTION_MS) {
+      this.reconnectAttempts = 0;
+    }
+    this.connectedAt = null;
     if (this.options.autoReconnect && this.reconnectAttempts < this.options.maxReconnectAttempts) {
       this.scheduleReconnect();
     } else {
@@ -2683,6 +2803,19 @@ var SseRealtimeClient = class {
     }, delay);
   }
 };
+function isStreamEndingEvent(eventType, data) {
+  if (eventType === "error" || eventType === "job.completed") {
+    return true;
+  }
+  const record = data ?? {};
+  if (eventType === "job.failed") {
+    return record.willRetry === false;
+  }
+  if (eventType === "job.status_changed" || eventType === "job.status") {
+    return typeof record.newStatus === "string" && TERMINAL_JOB_STATUSES.has(record.newStatus);
+  }
+  return false;
+}
 
 // src/realtime/index.ts
 var SpooledRealtime = class {
@@ -2869,6 +3002,11 @@ function clearProtoCache() {
   cachedDefinition = null;
 }
 
+// src/grpc/encode.ts
+function encodeFailRequest(params) {
+  return { ...params, retry: params.retry ?? true };
+}
+
 // src/grpc/streaming.ts
 async function* asyncIterableFromStream(stream, options) {
   const { signal, onConnected, onError, onEnd } = options ?? {};
@@ -2941,7 +3079,9 @@ function createProcessJobsStream(call, options) {
   }
   return {
     send(request) {
-      call.write(request);
+      call.write(
+        request.fail ? { ...request, fail: encodeFailRequest(request.fail) } : request
+      );
     },
     receive() {
       return new AsyncIterableStream(call, { onError, onEnd });
@@ -3129,12 +3269,19 @@ function mapGrpcError(error, timeoutMs) {
       return new ServerError(message, 500, "GRPC_ERROR");
   }
 }
+function grpcHost(address) {
+  if (address.startsWith("[")) {
+    const end = address.indexOf("]");
+    return end === -1 ? address : address.slice(1, end);
+  }
+  return address.split(":").length > 2 ? address : address.split(":")[0];
+}
 function shouldUseTls(address, explicitTls) {
   if (explicitTls !== void 0) {
     return explicitTls;
   }
-  const host = address.split(":")[0].toLowerCase();
-  return host !== "localhost" && host !== "127.0.0.1" && !host.startsWith("[::1]");
+  const host = grpcHost(address).toLowerCase();
+  return host !== "localhost" && host !== "127.0.0.1" && host !== "::1";
 }
 var GrpcQueueOperations = class {
   constructor(client, metadata, timeoutMs) {
@@ -3186,7 +3333,7 @@ var GrpcQueueOperations = class {
     return promisify(
       this.client,
       this.client.Fail,
-      params,
+      encodeFailRequest(params),
       this.metadata,
       this.timeoutMs
     );
@@ -3576,6 +3723,13 @@ var SpooledClient = class _SpooledClient {
   /**
    * Exchange the API key for a JWT via /auth/login and cache the result.
    *
+   * The JWT belongs to the realtime transport only. It must NOT replace the
+   * REST bearer: an API-key client has no refresh token wired into its HTTP
+   * layer, so after the JWT's 24 h lifetime every REST call (claim, complete,
+   * heartbeat, create) would have failed with 401 for good. REST keeps using
+   * the API key; realtime re-mints its own JWT near expiry (see
+   * `getJwtToken`).
+   *
    * Concurrent callers share a single in-flight login so a burst of reconnects
    * (e.g. WebSocket and SSE at once) issues just one request.
    */
@@ -3585,9 +3739,6 @@ var SpooledClient = class _SpooledClient {
     }
     this.jwtLoginPromise = (async () => {
       const response = await this.auth.login({ apiKey: this.config.apiKey });
-      this.http.setAuthToken(response.accessToken);
-      this.tokenExpiresAt = Date.now() + response.expiresIn * 1e3;
-      this.config.refreshToken = response.refreshToken;
       this.cachedJwt = response.accessToken;
       return response.accessToken;
     })();
@@ -3783,6 +3934,9 @@ var DEFAULT_OPTIONS = {
   autoStart: false,
   shutdownTimeout: 3e4
 };
+var JOB_TIMEOUT_MESSAGE = "Job timeout exceeded";
+var SHUTDOWN_TIMEOUT_MESSAGE = "Worker shutdown timeout";
+var MAX_TIMER_MS = 2 ** 31 - 1;
 var SpooledWorker = class {
   client;
   /**
@@ -3799,6 +3953,10 @@ var SpooledWorker = class {
   pollTimer = null;
   workerHeartbeatTimer = null;
   shutdownPromise = null;
+  /** The claim request currently on the wire, so a stop can wait for its jobs. */
+  pollInFlight = null;
+  /** Wakes the shutdown drain when the last active job has been acked. */
+  drainWaiter = null;
   // Event handlers
   eventHandlers = /* @__PURE__ */ new Map();
   constructor(client, options) {
@@ -3815,7 +3973,11 @@ var SpooledWorker = class {
     this.debug = config.debug ?? (() => {
     });
     if (this.options.autoStart) {
-      setImmediate(() => this.start());
+      setImmediate(() => {
+        this.start().catch((error) => {
+          this.debug("Auto-start failed", error);
+        });
+      });
     }
   }
   /**
@@ -3886,7 +4048,15 @@ var SpooledWorker = class {
     }
   }
   /**
-   * Stop the worker gracefully
+   * Stop the worker gracefully.
+   *
+   * Drains: the worker stops claiming, then lets running handlers finish for
+   * up to `shutdownTimeout`. A handler that returns completes its job and one
+   * that throws fails it, even though a stop was requested meanwhile; its
+   * `signal` is NOT aborted during the grace period. Only handlers still
+   * running when the timeout ends are aborted and failed with "Worker shutdown
+   * timeout". The worker deregisters last, because deregistering returns every
+   * job the worker still holds to the queue.
    */
   async stop() {
     if (this.state !== "running") {
@@ -3899,39 +4069,37 @@ var SpooledWorker = class {
     return this.shutdownPromise;
   }
   async doStop() {
+    const deadline = Date.now() + this.options.shutdownTimeout;
     this.state = "stopping";
     this.debug("Stopping worker...");
     if (this.pollTimer) {
       clearTimeout(this.pollTimer);
       this.pollTimer = null;
     }
-    if (this.workerHeartbeatTimer) {
-      clearInterval(this.workerHeartbeatTimer);
-      this.workerHeartbeatTimer = null;
-    }
-    for (const active of this.activeJobs.values()) {
-      active.abortController.abort();
+    if (this.pollInFlight) {
+      await this.pollInFlight;
     }
     if (this.activeJobs.size > 0) {
       this.debug(
         `Waiting for ${this.activeJobs.size} active jobs to complete...`
       );
-      await Promise.race([
-        this.waitForActiveJobs(),
-        this.sleep(this.options.shutdownTimeout)
-      ]);
-      const remaining = Array.from(this.activeJobs.values());
-      for (const active of remaining) {
-        this.debug(
-          `Force-failing job ${active.job.id} due to shutdown timeout`
-        );
-        await this.failJob(active.job, "Worker shutdown timeout");
-        this.cleanupJob(active);
-      }
+      await this.waitForDrain(Math.max(0, deadline - Date.now()));
+      const stragglers = Array.from(this.activeJobs.values()).filter(
+        (active) => !active.settled
+      );
+      await Promise.all(stragglers.map((active) => this.abandonJob(active)));
+      await Promise.all(
+        Array.from(this.activeJobs.values()).map((active) => active.ack)
+      );
+    }
+    if (this.workerHeartbeatTimer) {
+      clearInterval(this.workerHeartbeatTimer);
+      this.workerHeartbeatTimer = null;
     }
-    if (this.workerId) {
+    const workerId = this.workerId;
+    if (workerId) {
       try {
-        await this.client.workers.deregister(this.workerId);
+        await this.client.workers.deregister(workerId);
         this.debug("Worker deregistered");
       } catch (error) {
         this.debug("Failed to deregister worker", error);
@@ -3939,7 +4107,7 @@ var SpooledWorker = class {
     }
     this.state = "stopped";
     this.workerId = null;
-    this.emit("stopped", { workerId: this.workerId ?? "", reason: "graceful" });
+    this.emit("stopped", { workerId: workerId ?? "", reason: "graceful" });
     this.shutdownPromise = null;
   }
   /**
@@ -3978,7 +4146,17 @@ var SpooledWorker = class {
     if (this.state !== "running") {
       return;
     }
-    this.pollTimer = setTimeout(() => this.poll(), this.options.pollInterval);
+    this.pollTimer = setTimeout(() => {
+      this.pollTimer = null;
+      const run = this.poll();
+      this.pollInFlight = run;
+      const done = () => {
+        if (this.pollInFlight === run) {
+          this.pollInFlight = null;
+        }
+      };
+      run.then(done, done);
+    }, this.options.pollInterval);
   }
   async poll() {
     if (this.state !== "running" || !this.workerId) {
@@ -4012,7 +4190,8 @@ var SpooledWorker = class {
       executionId: Symbol(job.id),
       job,
       startedAt: /* @__PURE__ */ new Date(),
-      abortController
+      abortController,
+      settled: false
     };
     this.activeJobs.set(activeJob.executionId, activeJob);
     const heartbeatInterval = this.options.leaseDuration * this.options.heartbeatFraction * 1e3;
@@ -4041,23 +4220,72 @@ var SpooledWorker = class {
       }
     };
     try {
-      const result = await this.handler(context);
-      if (signal.aborted) {
-        this.debug(`Job ${job.id} was aborted`);
+      const outcome = await this.runHandler(activeJob, context);
+      if (activeJob.settled) {
+        this.debug(`Job ${job.id} was already settled; dropping late result`);
         return;
       }
-      await this.completeJob(job, result ?? void 0);
-    } catch (error) {
-      if (signal.aborted) {
-        this.debug(`Job ${job.id} was aborted`);
-        return;
+      activeJob.settled = true;
+      switch (outcome.kind) {
+        case "returned":
+          activeJob.ack = this.completeJob(job, outcome.value ?? void 0);
+          break;
+        case "threw":
+          activeJob.ack = this.failJob(
+            job,
+            outcome.error instanceof Error ? outcome.error.message : String(outcome.error)
+          );
+          break;
+        case "timeout":
+          abortController.abort(new Error(JOB_TIMEOUT_MESSAGE));
+          this.emit("job:timeout", {
+            jobId: job.id,
+            queueName: job.queueName
+          });
+          activeJob.ack = this.failJob(job, JOB_TIMEOUT_MESSAGE);
+          break;
       }
-      const errorMessage = error instanceof Error ? error.message : String(error);
-      await this.failJob(job, errorMessage);
+      await activeJob.ack;
     } finally {
       this.cleanupJob(activeJob);
     }
   }
+  /**
+   * Run the handler, racing it against the job's `timeoutSeconds`.
+   * Never rejects: a throw (sync or async) is reported as an outcome.
+   */
+  runHandler(activeJob, context) {
+    const run = (async () => {
+      try {
+        return { kind: "returned", value: await this.handler(context) };
+      } catch (error) {
+        return { kind: "threw", error };
+      }
+    })();
+    const { timeoutSeconds } = activeJob.job;
+    if (typeof timeoutSeconds !== "number" || !(timeoutSeconds > 0)) {
+      return run;
+    }
+    return new Promise((resolve) => {
+      activeJob.timeoutTimer = setTimeout(
+        () => resolve({ kind: "timeout" }),
+        Math.min(timeoutSeconds * 1e3, MAX_TIMER_MS)
+      );
+      run.then((outcome) => {
+        this.clearTimeoutTimer(activeJob);
+        resolve(outcome);
+      });
+    });
+  }
+  /** Abort a handler that outlived the shutdown grace period and fail its job. */
+  async abandonJob(activeJob) {
+    this.debug(`Force-failing job ${activeJob.job.id} due to shutdown timeout`);
+    activeJob.settled = true;
+    activeJob.abortController.abort(new Error(SHUTDOWN_TIMEOUT_MESSAGE));
+    activeJob.ack = this.failJob(activeJob.job, SHUTDOWN_TIMEOUT_MESSAGE);
+    await activeJob.ack;
+    this.cleanupJob(activeJob);
+  }
   async completeJob(job, result) {
     if (!this.workerId) return;
     try {
@@ -4073,25 +4301,34 @@ var SpooledWorker = class {
       });
     } catch (error) {
       this.debug(`Failed to complete job ${job.id}`, error);
+      this.emit("error", { error });
     }
   }
   async failJob(job, errorMessage) {
     if (!this.workerId) return;
     const willRetry = job.retryCount < job.maxRetries;
+    const message = coerceFailureMessage(errorMessage);
     try {
       await this.client.jobs.fail(job.id, {
         workerId: this.workerId,
-        error: errorMessage,
+        error: message,
         ...job.leaseId != null && { leaseId: job.leaseId }
       });
       this.emit("job:failed", {
         jobId: job.id,
         queueName: job.queueName,
-        error: errorMessage,
+        error: message,
         willRetry
       });
     } catch (error) {
       this.debug(`Failed to fail job ${job.id}`, error);
+      this.emit("error", { error });
+    }
+  }
+  clearTimeoutTimer(activeJob) {
+    if (activeJob.timeoutTimer) {
+      clearTimeout(activeJob.timeoutTimer);
+      activeJob.timeoutTimer = void 0;
     }
   }
   cleanupJob(activeJob) {
@@ -4099,8 +4336,12 @@ var SpooledWorker = class {
       clearInterval(activeJob.heartbeatTimer);
       activeJob.heartbeatTimer = void 0;
     }
+    this.clearTimeoutTimer(activeJob);
     if (this.activeJobs.get(activeJob.executionId) === activeJob) {
       this.activeJobs.delete(activeJob.executionId);
+      if (this.activeJobs.size === 0) {
+        this.drainWaiter?.();
+      }
     }
   }
   async sendJobHeartbeat(activeJob) {
@@ -4117,7 +4358,9 @@ var SpooledWorker = class {
     }
   }
   async sendWorkerHeartbeat() {
-    if (!this.workerId || this.state !== "running") return;
+    if (!this.workerId || this.state !== "running" && this.state !== "stopping") {
+      return;
+    }
     try {
       await this.client.workers.heartbeat(this.workerId, {
         currentJobs: this.activeJobs.size,
@@ -4127,16 +4370,28 @@ var SpooledWorker = class {
       this.debug("Worker heartbeat failed", error);
     }
   }
-  async waitForActiveJobs() {
-    while (this.activeJobs.size > 0) {
-      await this.sleep(100);
+  /**
+   * Resolve once no job is active, or after `timeoutMs`. The timer is always
+   * cleared, so a clean stop leaves nothing that keeps the process alive.
+   */
+  waitForDrain(timeoutMs) {
+    if (this.activeJobs.size === 0) {
+      return Promise.resolve();
     }
-  }
-  sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
+    return new Promise((resolve) => {
+      const timer = setTimeout(() => {
+        this.drainWaiter = null;
+        resolve();
+      }, timeoutMs);
+      this.drainWaiter = () => {
+        clearTimeout(timer);
+        this.drainWaiter = null;
+        resolve();
+      };
+    });
   }
 };
 
-export { API_VERSION, AdminResource, ApiKeysResource, AuthResource, AuthenticationError, AuthorizationError, BillingResource, CircuitBreaker, CircuitBreakerOpenError, CircuitState, ConflictError, DEFAULT_CONFIG, DashboardResource, GrpcJobStatus, GrpcJobStatusNumber, GrpcQueueOperations, GrpcWorkerOperations, HealthResource, HttpClient, JobsResource, MetricsResource, NetworkError, NotFoundError, OrganizationsResource, PROTO_LOADER_OPTIONS, PayloadTooLargeError, QueuesResource, RateLimitError, SDK_VERSION, SchedulesResource, ServerError, SpooledClient, SpooledError, SpooledGrpcClient, SpooledRealtime, SpooledWorker, SseRealtimeClient, TimeoutError, ValidationError, WebSocketRealtimeClient, WebhookIngestionResource, WebhooksResource, WorkersResource, WorkflowsResource, asyncIterableFromStream, calculateDelay, camelToSnake, clearProtoCache, convertQueryParams, convertRequest, convertResponse, createCircuitBreaker, createClient, createErrorFromResponse, createHttpClient, createJobStream, createPaginatedResponse, createProcessJobsStream, createRetryWrapper, dateToTimestamp, getProtoPath, isRetryableError, isSpooledError, loadProtoDefinition, loadProtoDefinitionAsync, parseRateLimitHeaders, resolveConfig, shouldRetry, sleep, snakeToCamel, timestampToDate, toCamelCase, toSnakeCase, validateConfig, withRetry };
+export { API_VERSION, AdminResource, ApiKeysResource, AuthResource, AuthenticationError, AuthorizationError, BillingResource, CircuitBreaker, CircuitBreakerOpenError, CircuitState, ConflictError, DEFAULT_CONFIG, DashboardResource, GrpcJobStatus, GrpcJobStatusNumber, GrpcQueueOperations, GrpcWorkerOperations, HealthResource, HttpClient, JobsResource, MetricsResource, NetworkError, NotFoundError, OrganizationsResource, PROTO_LOADER_OPTIONS, PayloadTooLargeError, QueuesResource, RateLimitError, SDK_VERSION, SchedulesResource, ServerError, SpooledClient, SpooledError, SpooledGrpcClient, SpooledRealtime, SpooledWorker, SseRealtimeClient, TimeoutError, UnsupportedOperationError, ValidationError, WebSocketRealtimeClient, WebhookIngestionResource, WebhooksResource, WorkersResource, WorkflowsResource, asyncIterableFromStream, calculateDelay, camelToSnake, clearProtoCache, convertQueryParams, convertRequest, convertResponse, createCircuitBreaker, createClient, createErrorFromResponse, createHttpClient, createJobStream, createPaginatedResponse, createProcessJobsStream, createRetryWrapper, dateToTimestamp, getProtoPath, isRetryableError, isSpooledError, loadProtoDefinition, loadProtoDefinitionAsync, parseRateLimitHeaders, resolveConfig, shouldRetry, sleep, snakeToCamel, timestampToDate, toCamelCase, toSnakeCase, validateConfig, withRetry };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map