@lionhummer/constellation 0.10.0 → 0.11.0

package/README.md

@@ -152,7 +152,7 @@ const receiver = new EventReceiver(client, hooks, {
   // One process: leave this out (an in-memory lock is used).
   // Several processes or pods: share the lock so only one works on an event at a time.
   lock: redisEventLock((args) => redis.call(...args)), // ioredis; node-redis: (args) => redis.sendCommand(args)
-  onFailure: (eventId, error) => logger.warn({ eventId, error }, "event will be re-sent"),
+  onFailure: (eventId, error) => logger.warn({ eventId, error }, error.willRetry ? "retrying event" : "event will be re-sent"),
 });
 
 export async function webhook(request: Request): Promise<Response> {
@@ -177,6 +177,8 @@ Don't run tools inside the webhook call. Constellation waits 5 seconds for the a
 
 ### Recovery
 
+A failure that may pass, such as a database lock timeout or a network error, is retried in the same process after 1, 3 and 9 seconds while it still holds the event (`retryDelaysMs`; `[]` turns this off). A re-send arriving meanwhile is answered `202`. Your own code can throw `PermanentEventError` to skip these retries.
+
 If your process dies while handling an event, you don't need a poller or a store to recover it. Constellation sends an unanswered `model.completed` event again after the response window (10 minutes by default; operators can lower it, for example to 2 minutes), up to 3 times. A re-sent event has the same `id`:
 
 - while another process still holds the event's lock, `claim` returns `null` and you answer `202`. Constellation then checks again after another window without counting a re-send;
@@ -201,7 +203,8 @@ If you answer an event twice, for example because the reply to your first answer
 | `200`, `outcome: "already_accepted"` | This event was already answered by you. `bodyMatched: false` means the earlier answer differed and stands. | `answered` |
 | `200`, `outcome: "event_closed"` | The request was cancelled or expired before your continuation arrived. Your telemetry is still recorded (`observationsRecorded`). | `answered`, `ended: "closed"` |
 | A permanent rejection (`PERMANENT_RESPONSE_CODES`, for example `400 bad_request`) of a continuation | The continuation can never be accepted. | The SDK sends an `observe` response with the error instead, which ends the request as failed: `ended: "failed"` |
-| Anything else: network error, timeout, `5xx`, `429`, `409 conflict`, `401`, `403` | Temporary. | `onFailure` runs and the call throws. The event stays unanswered and is re-sent. |
+| `401`, `403` | Key or scope problem. | `onFailure` runs and the call throws. The event stays unanswered and is re-sent. |
+| Anything else: network error, timeout, `5xx`, `429`, `409 conflict`, or an error thrown while preparing | Temporary. | `onFailure` runs with `willRetry: true` and the attempt is repeated after the next delay. After the last one, `onFailure` runs with `willRetry: false`, the call throws, and the event is re-sent. |
 
 While processing, the SDK renews the lock every third of its lease. If renewal fails, it aborts its own work through `signal`, so a re-send taken over elsewhere doesn't run alongside it.
 
@@ -223,7 +226,7 @@ Delivery is at least once. A crash after a tool ran but before the answer was ac
 
 ## Live output
 
-`requests.streamEvents(id, { after, signal })` streams `model.delta` events (text and reasoning as it is generated), followed by each step's final event. It reconnects automatically. To resume after a restart, pass the last `event.sequence` you processed as `after`. Streaming is for display only: run tools from the webhook, never from the stream. Stopping the stream does not cancel the request; `requests.cancel(id)` does.
+`requests.streamEvents(id, { after, signal })` streams `model.delta` events (text and reasoning as it is generated), followed by each step's final event. It reconnects automatically and ends when the request has ended: the server closes the stream with an end frame after the request's last event, so the loop finishes without any state polling. To resume after a restart, pass the last `event.sequence` you processed as `after`. Streaming is for display only: run tools from the webhook, never from the stream. Stopping the stream does not cancel the request; `requests.cancel(id)` does.
 
 ## Telemetry
 

package/dist/index.js

@@ -253,8 +253,11 @@ class RequestClient {
 `)) >= 0) {
             const frame = buffer.slice(0, boundary);
             buffer = buffer.slice(boundary + 2);
-            const data = frame.split(`
-`).filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trimStart()).join(`
+            const lines = frame.split(`
+`);
+            if (lines.some((line) => line === "event: end" || line === "event:end"))
+              return;
+            const data = lines.filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trimStart()).join(`
 `);
             if (!data)
               continue;
@@ -270,41 +273,9 @@ class RequestClient {
       } finally {
         await reader.cancel().catch(() => {});
       }
-      const inspection = await this.#replayRead(`/v1/requests/${encodeURIComponent(id)}/state`, options.signal);
-      if (["completed", "failed", "cancelled"].includes(inspection.request.status)) {
-        while (true) {
-          const before = after;
-          const tail = await this.#replayRead(`/v1/requests/${encodeURIComponent(id)}/events?after=${after}`, options.signal);
-          for (const event of tail.items) {
-            if (event.sequence <= after)
-              continue;
-            after = event.sequence;
-            yield event;
-          }
-          if (tail.items.length === 0)
-            return;
-          if (after === before)
-            throw new Error("Replay page did not advance the event cursor");
-        }
-      }
       await pause(500, options.signal);
     }
   }
-  async#replayRead(path, signal) {
-    for (;; ) {
-      signal?.throwIfAborted();
-      try {
-        return await this.#call(path, "GET", undefined, signal);
-      } catch (error) {
-        signal?.throwIfAborted();
-        if (error instanceof ConstellationError && error.status !== 429 && error.status < 500)
-          throw error;
-        if (error instanceof SyntaxError)
-          throw error;
-        await pause(500, signal);
-      }
-    }
-  }
   get models() {
     return {
       list: (scope, signal) => this.#call(`/v1/models${queryString(scope)}`, "GET", undefined, signal),
@@ -493,19 +464,13 @@ class EventReceiver {
     const leaseMs = this.options.leaseMs ?? DEFAULT_LEASE_MS;
     if (!await this.#lock.acquire(event.id, leaseMs))
       return null;
-    return async (signal) => {
-      let hooks;
-      try {
-        hooks = typeof this.hooks === "function" ? await this.hooks(event) : this.hooks;
-      } catch (error) {
-        await this.#lock.release(event.id).catch(() => {});
-        throw error;
-      }
-      return runClaimed(this.client, this.#lock, event, hooks, { ...this.options, signal: signal ?? this.options.signal });
-    };
+    const hooks = this.hooks;
+    const load = typeof hooks === "function" ? () => hooks(event) : async () => hooks;
+    return (signal) => runClaimed(this.client, this.#lock, event, load, { ...this.options, signal: signal ?? this.options.signal });
   }
 }
 var DEFAULT_LEASE_MS = 60000;
+var DEFAULT_RETRY_DELAYS_MS = [1000, 3000, 9000];
 
 class ClientTelemetry {
   parentSpanId;
@@ -699,10 +664,33 @@ async function prepareResponse(event, hooks, signal) {
 async function processEvent(client, lock, event, hooks, options = {}) {
   if (!await lock.acquire(event.id, options.leaseMs ?? DEFAULT_LEASE_MS))
     return { status: "busy" };
-  return runClaimed(client, lock, event, hooks, options);
+  return runClaimed(client, lock, event, async () => hooks, options);
+}
+function retryable(error) {
+  if (error instanceof PermanentEventError)
+    return false;
+  if (error instanceof ConstellationError) {
+    return !PERMANENT_RESPONSE_CODES.has(error.code) && (error.status === 0 || [408, 409, 425, 429].includes(error.status) || error.status >= 500);
+  }
+  return true;
 }
-async function runClaimed(client, lock, event, hooks, options) {
+function wait(ms, signal) {
+  signal.throwIfAborted();
+  return new Promise((resolve, reject) => {
+    const abort = () => {
+      clearTimeout(timer);
+      reject(signal.reason);
+    };
+    const timer = setTimeout(() => {
+      signal.removeEventListener("abort", abort);
+      resolve();
+    }, ms);
+    signal.addEventListener("abort", abort, { once: true });
+  });
+}
+async function runClaimed(client, lock, event, loadHooks, options) {
   const leaseMs = options.leaseMs ?? DEFAULT_LEASE_MS;
+  const delays = options.retryDelaysMs ?? DEFAULT_RETRY_DELAYS_MS;
   const fence = new AbortController;
   const signal = options.signal ? AbortSignal.any([options.signal, fence.signal]) : fence.signal;
   const renewal = setInterval(() => {
@@ -711,48 +699,58 @@ async function runClaimed(client, lock, event, hooks, options) {
         fence.abort(new Error("Event lock lost"));
     }, () => fence.abort(new Error("Event lock unavailable")));
   }, Math.max(1000, Math.floor(leaseMs / 3)));
-  let stage = "prepare";
   try {
-    const response = await prepareResponse(event, hooks, signal);
-    if (!response)
-      return { status: "answered", action: "stop", receipt: null, ended: null };
-    stage = "submit";
-    signal.throwIfAborted();
-    const observe = "action" in response && response.action === "observe";
-    let receipt;
-    try {
-      receipt = await client.requests.submit(response, signal);
-    } catch (error) {
-      if (!(error instanceof ConstellationError) || !PERMANENT_RESPONSE_CODES.has(error.code) || observe)
-        throw error;
-      receipt = await client.requests.submit({
-        action: "observe",
-        idempotencyKey: event.id,
-        inReplyTo: { requestId: event.requestId, eventId: event.id },
-        telemetry: response.telemetry ?? { spans: [], logs: [] },
-        error: { code: error.code, message: error.message.slice(0, 8192) || error.code }
-      }, signal);
-      return { status: "answered", action: "stop", receipt, ended: receipt.outcome === "event_closed" ? "closed" : "failed" };
-    }
-    const failed = observe && "error" in response && !!response.error;
-    const ended = receipt.outcome === "event_closed" ? "closed" : observe ? failed ? "failed" : "completed" : null;
-    return { status: "answered", action: observe ? "stop" : "continue", receipt, ended };
-  } catch (error) {
-    if (options.onFailure) {
-      const detail = error instanceof ConstellationError ? { status: error.status, code: error.code, message: error.message } : { status: 0, code: error instanceof Error && error.name === "TimeoutError" ? "timeout" : stage === "submit" ? "transport_error" : "event_prepare_failed", message: error instanceof Error ? error.message : "Event processing failed" };
+    for (let attempt = 1;; attempt++) {
+      const progress = { stage: "prepare" };
       try {
-        await options.onFailure(event.id, detail);
-      } catch (monitorError) {
-        if (error && typeof error === "object")
-          Object.assign(error, { failureHookError: monitorError });
+        return await answer(client, event, await loadHooks(), signal, progress);
+      } catch (error) {
+        const willRetry = attempt <= delays.length && !signal.aborted && retryable(error);
+        if (options.onFailure) {
+          const detail = error instanceof ConstellationError ? { status: error.status, code: error.code, message: error.message } : { status: 0, code: error instanceof Error && error.name === "TimeoutError" ? "timeout" : progress.stage === "submit" ? "transport_error" : "event_prepare_failed", message: error instanceof Error ? error.message : "Event processing failed" };
+          try {
+            await options.onFailure(event.id, { ...detail, attempt, willRetry });
+          } catch (monitorError) {
+            if (error && typeof error === "object")
+              Object.assign(error, { failureHookError: monitorError });
+          }
+        }
+        if (!willRetry)
+          throw error;
+        await wait(delays[attempt - 1] ?? 0, signal);
       }
     }
-    throw error;
   } finally {
     clearInterval(renewal);
     await lock.release(event.id).catch(() => {});
   }
 }
+async function answer(client, event, hooks, signal, progress) {
+  const response = await prepareResponse(event, hooks, signal);
+  if (!response)
+    return { status: "answered", action: "stop", receipt: null, ended: null };
+  progress.stage = "submit";
+  signal.throwIfAborted();
+  const observe = "action" in response && response.action === "observe";
+  let receipt;
+  try {
+    receipt = await client.requests.submit(response, signal);
+  } catch (error) {
+    if (!(error instanceof ConstellationError) || !PERMANENT_RESPONSE_CODES.has(error.code) || observe)
+      throw error;
+    receipt = await client.requests.submit({
+      action: "observe",
+      idempotencyKey: event.id,
+      inReplyTo: { requestId: event.requestId, eventId: event.id },
+      telemetry: response.telemetry ?? { spans: [], logs: [] },
+      error: { code: error.code, message: error.message.slice(0, 8192) || error.code }
+    }, signal);
+    return { status: "answered", action: "stop", receipt, ended: receipt.outcome === "event_closed" ? "closed" : "failed" };
+  }
+  const failed = observe && "error" in response && !!response.error;
+  const ended = receipt.outcome === "event_closed" ? "closed" : observe ? failed ? "failed" : "completed" : null;
+  return { status: "answered", action: observe ? "stop" : "continue", receipt, ended };
+}
 
 // src/index.ts
 function queryString2(values) {

package/dist/types/client-core/src/agent.d.ts

@@ -46,11 +46,17 @@ export type EventOptions = {
     signal?: AbortSignal;
     /** Lock lease, renewed every third of it while processing. Keep it well below the response window. */
     leaseMs?: number;
-    /** Reports a failure that leaves the event unanswered; Constellation will re-send it. */
+    /** Pauses before retrying a transient failure while still holding the event, before leaving
+     * it to Constellation's re-send. Default 1 s, 3 s and 9 s; `[]` leaves every failure to the re-send. */
+    retryDelaysMs?: readonly number[];
+    /** Reports each failed attempt. `willRetry` is false for the last one: the event stays
+     * unanswered and Constellation re-sends it after its response window. */
     onFailure?: (eventId: string, error: {
         status: number;
         code: string;
         message: string;
+        attempt: number;
+        willRetry: boolean;
     }) => void | Promise<void>;
 };
 /** The error result returned to the model for a call whose arguments were not valid JSON. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lionhummer/constellation",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "TypeScript client for durable model requests and model events",
   "type": "module",
   "license": "UNLICENSED",