@lionhummer/constellation 0.9.4 → 0.11.0

package/README.md

@@ -70,12 +70,11 @@ console.log(request.status, request.output?.text);
 
 ## Build an agent with tools
 
-A tool-using agent needs four pieces:
+A tool-using agent needs three pieces:
 
-1. **A webhook** that receives events and stores them.
+1. **A webhook** that receives events.
 2. **Hooks** that run your tools and build the next input.
-3. **An `EventStore`**: a table in your database that remembers each event and the response you prepared for it.
-4. **`processEvent`**, which ties the other three together.
+3. **An `EventReceiver`**, which takes each event, runs your hooks in the background and sends the response. You don't store events or responses: Constellation re-sends an event that stays unanswered, and deduplicates your answers.
 
 ### 1. Turn on events
 
@@ -142,112 +141,19 @@ const hooks: AgentHooks = {
 
 `conversation.settings` must repeat the first request's `workspaceId`, `userId`, `externalId`, `model`, `planId`, `events` and `timeoutMs`. A continuation that changes any of them is rejected. You may change `messages`, `instructions`, `tools`, `parameters` and `cache`.
 
-### 3. Implement the `EventStore`
-
-The store is a small table in your own database. It guarantees two things:
-
-- an event you acknowledged is never lost;
-- a response you prepared is sent unchanged, even after a crash.
-
-Do not use an in-memory map in production. Here is a complete PostgreSQL version using [`pg`](https://node-postgres.com/):
-
-```sql
-CREATE TABLE constellation_events (
-  event_id          text PRIMARY KEY,
-  payload           jsonb NOT NULL,
-  prepared          boolean NOT NULL DEFAULT false,
-  response          jsonb,                         -- the exact body we send (or resend)
-  state             text NOT NULL DEFAULT 'pending', -- pending | done | rejected
-  rejection         jsonb,
-  failures          integer NOT NULL DEFAULT 0,
-  last_error        jsonb,
-  first_failure_at  timestamptz,
-  last_failure_at   timestamptz,
-  received_at       timestamptz NOT NULL DEFAULT now()
-);
-```
-
-```ts
-import pg from "pg";
-import { PermanentEventError, type EventStore, type WebhookEvent } from "@lionhummer/constellation";
-
-const db = new pg.Pool();
-
-export const store: EventStore & { pending(): Promise<WebhookEvent[]> } = {
-  // Save the event before acknowledging the webhook. Duplicates must match.
-  async accept(event) {
-    await db.query(
-      `INSERT INTO constellation_events (event_id, payload) VALUES ($1, $2)
-       ON CONFLICT (event_id) DO NOTHING`, [event.id, event]);
-    const { rows } = await db.query(
-      `SELECT payload = $2::jsonb AS same FROM constellation_events WHERE event_id = $1`, [event.id, event]);
-    if (!rows[0]?.same) throw new Error(`event ${event.id} redelivered with a different payload`);
-  },
-
-  // Run `prepare` at most once per event at a time, and save its result before returning it.
-  async prepare(event, prepare) {
-    const tx = await db.connect();
-    try {
-      await tx.query("BEGIN");
-      const { rows } = await tx.query(
-        `SELECT prepared, response, state, rejection FROM constellation_events
-         WHERE event_id = $1 FOR UPDATE`, [event.id]); // serializes concurrent processing
-      const row = rows[0];
-      if (row.state === "rejected") {
-        const r = row.rejection;
-        throw new PermanentEventError(r.status, r.code, r.message);
-      }
-      if (row.prepared) { await tx.query("COMMIT"); return row.response; }
-      const response = await prepare(); // runs your tools and hooks
-      await tx.query(
-        `UPDATE constellation_events SET prepared = true, response = $2 WHERE event_id = $1`,
-        [event.id, response]);
-      await tx.query("COMMIT");
-      return response;
-    } catch (error) {
-      await tx.query("ROLLBACK");
-      throw error;
-    } finally {
-      tx.release();
-    }
-  },
-
-  async complete(eventId) {
-    await db.query(`UPDATE constellation_events SET state = 'done' WHERE event_id = $1`, [eventId]);
-  },
-
-  // A permanent rejection: never resend this response.
-  async reject(eventId, error) {
-    await db.query(
-      `UPDATE constellation_events SET state = 'rejected', rejection = $2 WHERE event_id = $1`,
-      [eventId, { status: error.status, code: error.code, message: error.message }]);
-  },
-
-  // Optional: a retryable failure. Alert when failures pile up or get old.
-  async retry(eventId, error) {
-    await db.query(
-      `UPDATE constellation_events SET failures = failures + 1, last_error = $2,
-         first_failure_at = coalesce(first_failure_at, now()), last_failure_at = now()
-       WHERE event_id = $1`, [eventId, error]);
-  },
-
-  // Not part of EventStore: used by the recovery sweep below.
-  async pending() {
-    const { rows } = await db.query(
-      `SELECT payload FROM constellation_events WHERE state = 'pending' ORDER BY received_at LIMIT 100`);
-    return rows.map((row) => row.payload);
-  },
-};
-```
-
-`prepare` keeps a row lock while your tools run. That is acceptable for tools that finish in seconds or minutes. For very long work, replace the lock with a lease column.
+### 3. Receive webhooks
 
-### 4. Receive webhooks and process events
-
-Acknowledge quickly, then process in the background. The webhook handler below works with any framework that uses the web `Request`/`Response` types (Hono, Next.js route handlers, Bun, Deno):
+Acknowledge quickly, then process in the background. The handler below works with any framework that uses the web `Request`/`Response` types (Hono, Next.js route handlers, Bun, Deno):
 
 ```ts
-import { processEvent, PermanentEventError } from "@lionhummer/constellation";
+import { EventReceiver, redisEventLock } from "@lionhummer/constellation";
+
+const receiver = new EventReceiver(client, hooks, {
+  // One process: leave this out (an in-memory lock is used).
+  // Several processes or pods: share the lock so only one works on an event at a time.
+  lock: redisEventLock((args) => redis.call(...args)), // ioredis; node-redis: (args) => redis.sendCommand(args)
+  onFailure: (eventId, error) => logger.warn({ eventId, error }, error.willRetry ? "retrying event" : "event will be re-sent"),
+});
 
 export async function webhook(request: Request): Promise<Response> {
   let event;
@@ -256,68 +162,63 @@ export async function webhook(request: Request): Promise<Response> {
   } catch {
     return new Response(null, { status: 401 });
   }
-  await store.accept(event); // if this throws, the 5xx makes Constellation redeliver
-  void handle(event);        // don't make the webhook wait for your tools
-  return new Response(null, { status: 202 });
+  const run = await receiver.claim(event);
+  if (!run) return new Response(null, { status: 202 }); // another process is working on it
+  void run().then((outcome) => {
+    if (outcome.status === "answered" && outcome.ended) markConversationEnded(event.requestId, outcome.ended);
+  }, () => {}); // a failure leaves the event unanswered; Constellation re-sends it
+  return new Response(null, { status: 204 });
 }
-
-async function handle(event: WebhookEvent) {
-  try {
-    await processEvent(client, store, event, hooks);
-  } catch (error) {
-    if (error instanceof PermanentEventError) {
-      await markConversationFailed(event.requestId, error); // will not be retried
-    } else {
-      logger.warn({ eventId: event.id, error }, "event processing will be retried");
-    }
-  }
-}
-
-// Optional: resume unfinished events right after a restart.
-for (const event of await store.pending()) void handle(event);
 ```
 
 Don't run tools inside the webhook call. Constellation waits 5 seconds for the acknowledgement by default, and nothing in Constellation waits for your tools.
 
-If your process dies while handling an event, you don't need a poller to recover it. Constellation sends an unanswered `model.completed` event again after the response window (10 minutes by default), up to 3 times, and stops as soon as your response arrives. A re-sent event has the same `id`, so `store.accept` and `store.prepare` resume it from your saved checkpoint. If the event is still being processed when the re-send arrives, acknowledge it anyway. Keep your processing lease shorter than the response window and renew it while work continues, so a re-send after a crash can take the event over.
+`outcome.ended` tells you when the request is over: `completed` or `failed` after your final answer, `closed` when it had already been cancelled or had expired. Requests that fail or are cancelled on the Constellation side arrive as `model.failed` and `model.cancelled` events. You never need to read the request back.
 
-## What `processEvent` does
+### Recovery
 
-For each event:
+A failure that may pass, such as a database lock timeout or a network error, is retried in the same process after 1, 3 and 9 seconds while it still holds the event (`retryDelaysMs`; `[]` turns this off). A re-send arriving meanwhile is answered `202`. Your own code can throw `PermanentEventError` to skip these retries.
 
-1. **Prepare the response, at most once.** `store.prepare` returns the saved response if one exists. Otherwise the SDK builds one:
-   - For `model.completed`, it runs every tool call in parallel and calls `onStepComplete`. To continue, it calls `prepareRequest`, which produces a continuation. To stop, it produces an `observe` response.
-   - For `model.failed` and `model.cancelled`, it calls `onError(event.data)` and produces an `observe` response.
+If your process dies while handling an event, you don't need a poller or a store to recover it. Constellation sends an unanswered `model.completed` event again after the response window (10 minutes by default; operators can lower it, for example to 2 minutes), up to 3 times. A re-sent event has the same `id`:
 
-   The response carries the idempotency key `event.id`, `inReplyTo: { requestId, eventId }` and the telemetry recorded while preparing. The store saves it before anything is sent.
-2. **Submit it** through `requests.submit`. Retries send the saved body unchanged.
-3. **Record the outcome**:
+- while another process still holds the event's lock, `claim` returns `null` and you answer `202`. Constellation then checks again after another window without counting a re-send;
+- once the holder is gone, its lock expires (60 s by default, renewed while it works) and the re-send is processed from the start.
 
-| Server answer | Meaning | `processEvent` |
-| --- | --- | --- |
-| `200`, `outcome: "accepted"` | Your response was accepted. A continuation starts the next step (`scheduled: true`). | `store.complete`, resolves |
-| `200`, `outcome: "already_accepted"` | A retry of a response that was already accepted, for example after a lost reply. Nothing runs twice. | `store.complete`, resolves |
-| `200`, `outcome: "event_closed"` | The request was cancelled or expired before your continuation arrived. Nothing continues; your telemetry is still recorded (`observationsRecorded`). | `store.complete`, resolves |
-| `400 bad_request` | The response itself is invalid, for example a changed model or a bad message. | `store.reject`, throws `PermanentEventError` |
-| `409 response_conflict` | A *different* response to this event was already accepted. | `store.reject`, throws `PermanentEventError` |
-| `410 observations_expired` | The request ended more than 24 hours ago. | `store.reject`, throws `PermanentEventError` |
-| Anything else: network error, timeout, `5xx`, `429`, `409 conflict`, `401`, `403` | Temporary. | `store.retry`, throws. The next attempt resends the same body. |
+If you answer an event twice, for example because the reply to your first answer was lost, Constellation keeps the first answer and reports the second as `already_accepted` with `bodyMatched: false`. Nothing continues twice.
+
+## What happens to each event
 
-The permanent codes are exported as `PERMANENT_RESPONSE_CODES`. The receipt passed to `store.complete` includes `outcome`, so your store can record that a request ended without continuing.
+1. **Take the lock** for the event. If another process holds it: `busy` (answer `202`).
+2. **Prepare the response:**
+   - For `model.completed`, run every tool call in parallel and call `onStepComplete`. To continue, call `prepareRequest`, which produces a continuation. To stop, produce an `observe` response.
+   - A tool call marked `invalid` (its arguments were not valid JSON) is never run. Its result is an error from `invalidToolInputResult` with `isError: true`, so the model can correct the call on the next step, as the AI SDK does.
+   - For `model.failed` and `model.cancelled`, call `onError(event.data)` and produce an `observe` response.
+
+   The response carries the idempotency key `event.id`, `inReplyTo: { requestId, eventId }` and the telemetry recorded while preparing.
+3. **Submit it** and release the lock:
+
+| Server answer | Meaning | Result |
+| --- | --- | --- |
+| `200`, `outcome: "accepted"` | Your response was accepted. A continuation starts the next step (`scheduled: true`). | `answered` |
+| `200`, `outcome: "already_accepted"` | This event was already answered by you. `bodyMatched: false` means the earlier answer differed and stands. | `answered` |
+| `200`, `outcome: "event_closed"` | The request was cancelled or expired before your continuation arrived. Your telemetry is still recorded (`observationsRecorded`). | `answered`, `ended: "closed"` |
+| A permanent rejection (`PERMANENT_RESPONSE_CODES`, for example `400 bad_request`) of a continuation | The continuation can never be accepted. | The SDK sends an `observe` response with the error instead, which ends the request as failed: `ended: "failed"` |
+| `401`, `403` | Key or scope problem. | `onFailure` runs and the call throws. The event stays unanswered and is re-sent. |
+| Anything else: network error, timeout, `5xx`, `429`, `409 conflict`, or an error thrown while preparing | Temporary. | `onFailure` runs with `willRetry: true` and the attempt is repeated after the next delay. After the last one, `onFailure` runs with `willRetry: false`, the call throws, and the event is re-sent. |
 
-Fix `401` and `403` by correcting the key: the saved response is still sent unchanged afterwards.
+While processing, the SDK renews the lock every third of its lease. If renewal fails, it aborts its own work through `signal`, so a re-send taken over elsewhere doesn't run alongside it.
 
 ## When your code fails
 
 | What happened | Without `onError` | With `onError` |
 | --- | --- | --- |
-| A tool or hook throws | `processEvent` throws and the event is retried later. Tools may run again, so use `effectId`. | `onError` runs. If it returns, the request stops and the error is recorded in telemetry. If it throws, the event is retried. |
-| No handler for a tool the model called, or `prepareRequest` is missing when continuing | The request is marked **failed** and `HookConfigurationError` (a `PermanentEventError`) is thrown. | `onError` runs, as above. |
+| A tool or hook throws | The event stays unanswered and is re-sent. Tools may run again, so use `effectId`. | `onError` runs. If it returns, the request stops and the error is recorded in telemetry. If it throws, the event is re-sent. |
+| No handler for a tool the model called, or `prepareRequest` is missing when continuing | The request ends as **failed** (`invalid_hook_configuration`). | `onError` runs, as above. |
 | The model step failed or was cancelled (`model.failed`, `model.cancelled`) | An `observe` response records your telemetry. | `onError(event.data)` runs first. |
 
 ## Side effects and duplicates
 
-Hooks can run more than once. A crash after a tool ran, but before the prepared response was saved, means the tool runs again. Constellation cannot make your side effects exactly-once; your code must:
+Delivery is at least once. A crash after a tool ran but before the answer was accepted means the tool runs again when the event is re-sent. Constellation cannot make your side effects exactly-once; your code must:
 
 - pass `effectId` to anything with side effects (payments, emails, writes). It is stable for one tool call of one event;
 - settle per-step billing or usage once per `event.id`;
@@ -325,7 +226,7 @@ Hooks can run more than once. A crash after a tool ran, but before the prepared
 
 ## Live output
 
-`requests.streamEvents(id, { after, signal })` streams `model.delta` events (text and reasoning as it is generated), followed by each step's final event. It reconnects automatically. To resume after a restart, pass the last `event.sequence` you processed as `after`. Streaming is for display only: run tools from the webhook, never from the stream. Stopping the stream does not cancel the request; `requests.cancel(id)` does.
+`requests.streamEvents(id, { after, signal })` streams `model.delta` events (text and reasoning as it is generated), followed by each step's final event. It reconnects automatically and ends when the request has ended: the server closes the stream with an end frame after the request's last event, so the loop finishes without any state polling. To resume after a restart, pass the last `event.sequence` you processed as `after`. Streaming is for display only: run tools from the webhook, never from the stream. Stopping the stream does not cancel the request; `requests.cancel(id)` does.
 
 ## Telemetry
 
@@ -427,7 +328,7 @@ A failed provider call falls back to the next provider only before any output ha
 | `requests.preview(input)` | Validate a complete input and show routing, without calling a provider. |
 | `requests.cancel(id)` | Cancel a request. |
 | `openers.create(input)`, `openers.sentence(id)` | Start an opening line for a just-sent message (null when openers are turned off), then wait for its sentence (null when skipped or failed). |
-| `requests.getState(id)` | Committed state: status, steps, attempt outcomes, usage and cost. |
+| `requests.getState(id)` | Committed state: status, steps, attempt outcomes, usage and cost. For inspection and support tools; event processing never needs it, because boundary events carry the step number, step input and request identity. |
 | `requests.getSummary(id)`, `requests.get(id)` | Request detail, without or with spans and logs. |
 | `requests.list(query)` | Page through requests, e.g. by `externalId` or `status`. |
 | `requests.listEvents(id, after)`, `requests.streamEvents(id)` | Replay events after a sequence number, or stream them live. |
@@ -440,7 +341,7 @@ A failed provider call falls back to the next provider only before any output ha
 | `management.registry.*` | Edit the model registry. Needs management permission. |
 | `discover()` | The OpenAPI document for exactly what this key may call. |
 
-Exports for event processing: `processEvent`, `prepareResponse` (build a response without submitting it), `EventReceiver`, `ClientTelemetry`, `PermanentEventError`, `HookConfigurationError`, `PERMANENT_RESPONSE_CODES`, `ConstellationError` and `uuid7`.
+Exports for event processing: `EventReceiver`, `processEvent` (inline: lock, prepare, submit), `prepareResponse` (build a response without submitting it), `MemoryEventLock`, `redisEventLock`, `invalidToolInputResult`, `ClientTelemetry`, `PermanentEventError`, `HookConfigurationError`, `PERMANENT_RESPONSE_CODES`, `ConstellationError` and `uuid7`.
 
 ## Troubleshooting
 
@@ -448,7 +349,7 @@ Exports for event processing: `processEvent`, `prepareResponse` (build a respons
 | --- | --- |
 | `models.list` is empty | The key's workspace/user scope, an access rule for that workspace, and an enabled model and offering. |
 | `terminal: "notify"` is rejected | The execution key has no webhook URL configured. |
-| The same webhook keeps arriving | Your endpoint returned non-2xx or took longer than the delivery timeout (5 s by default). Return 202 after `store.accept` and process in the background. |
+| The same webhook keeps arriving | Your endpoint returned non-2xx or took longer than the delivery timeout (5 s by default). Answer after `receiver.claim` and process in the background. |
 | `400` on a continuation | Model, workspace, user, `externalId`, plan, events or `timeoutMs` differ from the first request, or the message history is malformed. |
 | `409 response_conflict` | Your code produced a different response for an event that was already answered. Answers must come from the saved response. |
 | Tools ran twice | Expected after a crash. Make side effects idempotent with `effectId`. |

package/dist/index.js

@@ -253,8 +253,11 @@ class RequestClient {
 `)) >= 0) {
             const frame = buffer.slice(0, boundary);
             buffer = buffer.slice(boundary + 2);
-            const data = frame.split(`
-`).filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trimStart()).join(`
+            const lines = frame.split(`
+`);
+            if (lines.some((line) => line === "event: end" || line === "event:end"))
+              return;
+            const data = lines.filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trimStart()).join(`
 `);
             if (!data)
               continue;
@@ -270,41 +273,9 @@ class RequestClient {
       } finally {
         await reader.cancel().catch(() => {});
       }
-      const inspection = await this.#replayRead(`/v1/requests/${encodeURIComponent(id)}/state`, options.signal);
-      if (["completed", "failed", "cancelled"].includes(inspection.request.status)) {
-        while (true) {
-          const before = after;
-          const tail = await this.#replayRead(`/v1/requests/${encodeURIComponent(id)}/events?after=${after}`, options.signal);
-          for (const event of tail.items) {
-            if (event.sequence <= after)
-              continue;
-            after = event.sequence;
-            yield event;
-          }
-          if (tail.items.length === 0)
-            return;
-          if (after === before)
-            throw new Error("Replay page did not advance the event cursor");
-        }
-      }
       await pause(500, options.signal);
     }
   }
-  async#replayRead(path, signal) {
-    for (;; ) {
-      signal?.throwIfAborted();
-      try {
-        return await this.#call(path, "GET", undefined, signal);
-      } catch (error) {
-        signal?.throwIfAborted();
-        if (error instanceof ConstellationError && error.status !== 429 && error.status < 500)
-          throw error;
-        if (error instanceof SyntaxError)
-          throw error;
-        await pause(500, signal);
-      }
-    }
-  }
   get models() {
     return {
       list: (scope, signal) => this.#call(`/v1/models${queryString(scope)}`, "GET", undefined, signal),
@@ -424,15 +395,82 @@ class HookConfigurationError extends PermanentEventError {
   }
 }
 
-class EventReceiver {
-  store;
-  constructor(store) {
-    this.store = store;
+class MemoryEventLock {
+  #held = new Map;
+  async acquire(eventId, ttlMs) {
+    const until = this.#held.get(eventId);
+    if (until !== undefined && until > Date.now())
+      return false;
+    this.#held.set(eventId, Date.now() + ttlMs);
+    return true;
+  }
+  async renew(eventId, ttlMs) {
+    if (!this.#held.has(eventId))
+      return false;
+    this.#held.set(eventId, Date.now() + ttlMs);
+    return true;
+  }
+  async release(eventId) {
+    this.#held.delete(eventId);
   }
-  async receive(event) {
-    await this.store.accept(event);
+}
+var RENEW_SCRIPT = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end";
+var RELEASE_SCRIPT = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end";
+function redisEventLock(command, options = {}) {
+  const prefix = options.prefix ?? "constellation:event-lock:";
+  const tokens = new Map;
+  return {
+    async acquire(eventId, ttlMs) {
+      const token = uuid7();
+      const reply = await command(["SET", prefix + eventId, token, "NX", "PX", String(ttlMs)]);
+      if (reply !== "OK")
+        return false;
+      tokens.set(eventId, token);
+      return true;
+    },
+    async renew(eventId, ttlMs) {
+      const token = tokens.get(eventId);
+      if (!token)
+        return false;
+      return Number(await command(["EVAL", RENEW_SCRIPT, "1", prefix + eventId, token, String(ttlMs)])) === 1;
+    },
+    async release(eventId) {
+      const token = tokens.get(eventId);
+      tokens.delete(eventId);
+      if (token)
+        await command(["EVAL", RELEASE_SCRIPT, "1", prefix + eventId, token]);
+    }
+  };
+}
+function invalidToolInputResult(call) {
+  return {
+    error: "invalid_tool_input",
+    message: `The arguments for ${call.name} were not valid JSON (${call.invalid?.error ?? "unparseable"}). Call ${call.name} again with complete, valid JSON arguments.`
+  };
+}
+
+class EventReceiver {
+  client;
+  hooks;
+  options;
+  #lock;
+  constructor(client, hooks, options = {}) {
+    this.client = client;
+    this.hooks = hooks;
+    this.options = options;
+    this.#lock = options.lock ?? new MemoryEventLock;
+  }
+  async claim(event) {
+    const leaseMs = this.options.leaseMs ?? DEFAULT_LEASE_MS;
+    if (!await this.#lock.acquire(event.id, leaseMs))
+      return null;
+    const hooks = this.hooks;
+    const load = typeof hooks === "function" ? () => hooks(event) : async () => hooks;
+    return (signal) => runClaimed(this.client, this.#lock, event, load, { ...this.options, signal: signal ?? this.options.signal });
   }
 }
+var DEFAULT_LEASE_MS = 60000;
+var DEFAULT_RETRY_DELAYS_MS = [1000, 3000, 9000];
 
 class ClientTelemetry {
   parentSpanId;
@@ -563,6 +601,10 @@ async function prepareResponseBody(event, hooks, telemetry, signal) {
           return;
         signal?.throwIfAborted();
         const call = calls[index];
+        if (call.invalid) {
+          results[index] = { id: call.id, result: invalidToolInputResult(call), isError: true };
+          continue;
+        }
         const handler = hooks.tools?.[call.name];
         if (!handler)
           throw new HookConfigurationError(`No handler for tool ${call.name}`);
@@ -619,44 +661,96 @@ async function prepareResponse(event, hooks, signal) {
     response.telemetry = telemetry.snapshot();
   return response;
 }
-async function processEvent(client, store, event, hooks, signal) {
-  let stage = "prepare";
+async function processEvent(client, lock, event, hooks, options = {}) {
+  if (!await lock.acquire(event.id, options.leaseMs ?? DEFAULT_LEASE_MS))
+    return { status: "busy" };
+  return runClaimed(client, lock, event, async () => hooks, options);
+}
+function retryable(error) {
+  if (error instanceof PermanentEventError)
+    return false;
+  if (error instanceof ConstellationError) {
+    return !PERMANENT_RESPONSE_CODES.has(error.code) && (error.status === 0 || [408, 409, 425, 429].includes(error.status) || error.status >= 500);
+  }
+  return true;
+}
+function wait(ms, signal) {
+  signal.throwIfAborted();
+  return new Promise((resolve, reject) => {
+    const abort = () => {
+      clearTimeout(timer);
+      reject(signal.reason);
+    };
+    const timer = setTimeout(() => {
+      signal.removeEventListener("abort", abort);
+      resolve();
+    }, ms);
+    signal.addEventListener("abort", abort, { once: true });
+  });
+}
+async function runClaimed(client, lock, event, loadHooks, options) {
+  const leaseMs = options.leaseMs ?? DEFAULT_LEASE_MS;
+  const delays = options.retryDelaysMs ?? DEFAULT_RETRY_DELAYS_MS;
+  const fence = new AbortController;
+  const signal = options.signal ? AbortSignal.any([options.signal, fence.signal]) : fence.signal;
+  const renewal = setInterval(() => {
+    lock.renew(event.id, leaseMs).then((held) => {
+      if (!held)
+        fence.abort(new Error("Event lock lost"));
+    }, () => fence.abort(new Error("Event lock unavailable")));
+  }, Math.max(1000, Math.floor(leaseMs / 3)));
   try {
-    const response = await store.prepare(event, () => prepareResponse(event, hooks, signal));
-    let receipt = null;
-    if (response) {
-      stage = "submit";
+    for (let attempt = 1;; attempt++) {
+      const progress = { stage: "prepare" };
       try {
-        receipt = await client.requests.submit(response, signal);
+        return await answer(client, event, await loadHooks(), signal, progress);
       } catch (error) {
-        if (error instanceof ConstellationError && PERMANENT_RESPONSE_CODES.has(error.code)) {
-          const rejection = new PermanentEventError(error.status, error.code, error.message);
-          await store.reject(event.id, rejection);
-          throw rejection;
+        const willRetry = attempt <= delays.length && !signal.aborted && retryable(error);
+        if (options.onFailure) {
+          const detail = error instanceof ConstellationError ? { status: error.status, code: error.code, message: error.message } : { status: 0, code: error instanceof Error && error.name === "TimeoutError" ? "timeout" : progress.stage === "submit" ? "transport_error" : "event_prepare_failed", message: error instanceof Error ? error.message : "Event processing failed" };
+          try {
+            await options.onFailure(event.id, { ...detail, attempt, willRetry });
+          } catch (monitorError) {
+            if (error && typeof error === "object")
+              Object.assign(error, { failureHookError: monitorError });
+          }
         }
-        throw error;
-      }
-      if ("action" in response && response.action === "observe" && response.error) {
-        const failure = new HookConfigurationError(response.error.message);
-        await store.reject(event.id, failure);
-        throw failure;
-      }
-    }
-    stage = "checkpoint";
-    await store.complete(event.id, receipt);
-  } catch (error) {
-    if (!(error instanceof PermanentEventError) && store.retry) {
-      const detail = error instanceof ConstellationError ? { status: error.status, code: error.code, message: error.message } : { status: 0, code: error instanceof Error && error.name === "TimeoutError" ? "timeout" : stage === "submit" ? "transport_error" : `event_${stage}_failed`, message: error instanceof Error ? error.message : "Event processing failed" };
-      try {
-        await store.retry(event.id, detail);
-      } catch (monitorError) {
-        if (error && typeof error === "object")
-          Object.assign(error, { retryHookError: monitorError });
+        if (!willRetry)
+          throw error;
+        await wait(delays[attempt - 1] ?? 0, signal);
       }
     }
-    throw error;
+  } finally {
+    clearInterval(renewal);
+    await lock.release(event.id).catch(() => {});
   }
 }
+async function answer(client, event, hooks, signal, progress) {
+  const response = await prepareResponse(event, hooks, signal);
+  if (!response)
+    return { status: "answered", action: "stop", receipt: null, ended: null };
+  progress.stage = "submit";
+  signal.throwIfAborted();
+  const observe = "action" in response && response.action === "observe";
+  let receipt;
+  try {
+    receipt = await client.requests.submit(response, signal);
+  } catch (error) {
+    if (!(error instanceof ConstellationError) || !PERMANENT_RESPONSE_CODES.has(error.code) || observe)
+      throw error;
+    receipt = await client.requests.submit({
+      action: "observe",
+      idempotencyKey: event.id,
+      inReplyTo: { requestId: event.requestId, eventId: event.id },
+      telemetry: response.telemetry ?? { spans: [], logs: [] },
+      error: { code: error.code, message: error.message.slice(0, 8192) || error.code }
+    }, signal);
+    return { status: "answered", action: "stop", receipt, ended: receipt.outcome === "event_closed" ? "closed" : "failed" };
+  }
+  const failed = observe && "error" in response && !!response.error;
+  const ended = receipt.outcome === "event_closed" ? "closed" : observe ? failed ? "failed" : "completed" : null;
+  return { status: "answered", action: observe ? "stop" : "continue", receipt, ended };
+}
 
 // src/index.ts
 function queryString2(values) {
@@ -807,12 +901,15 @@ class Constellation extends RequestClient {
 }
 export {
   uuid7,
+  redisEventLock,
   processEvent,
   prepareResponse,
   parseOpenerSentence,
   normalizeTimestamp,
+  invalidToolInputResult,
   PermanentEventError,
   PERMANENT_RESPONSE_CODES,
+  MemoryEventLock,
   HookConfigurationError,
   EventReceiver,
   ConstellationError,

package/dist/types/client-core/src/agent.d.ts

@@ -10,34 +10,75 @@ export declare class PermanentEventError extends Error {
 export declare class HookConfigurationError extends PermanentEventError {
     constructor(message: string);
 }
-/** Application-owned durable event checkpoints. All methods must survive process restart. */
-export interface EventStore {
-    /** Atomically retain the event before processing it.
-     * Duplicate IDs must compare equal payloads. Resolve only after durable receipt. */
-    accept(event: WebhookEvent): Promise<void>;
-    /** Serialize processing per event. Return the saved response if present; otherwise
-     * run prepare and durably store its exact result before returning. A crash can
-     * rerun prepare: tool/billing effects must use the supplied stable identities. */
-    prepare(event: WebhookEvent, prepare: () => Promise<Admission | null>): Promise<Admission | null>;
-    /** Atomically mark the event processed after its saved response settled. `receipt.outcome` is
-     * `accepted`, `already_accepted` or `event_closed`; for `event_closed`, `receipt.observationsRecorded`
-     * says whether the continuation's telemetry was kept. `null` when no response was submitted. */
-    complete(eventId: string, receipt: AdmissionReceipt | null): Promise<void>;
-    /** Durably stop retries of a rejected saved response. Replays must rethrow this rejection. */
-    reject(eventId: string, error: PermanentEventError): Promise<void>;
-    /** Persist retry count, first/last failure timestamps and latest error in the existing inbox. Non-HTTP failures use status 0; alert on age/repeated failures.
-     * If this hook throws, processEvent still rethrows the original failure, with the hook error as its `retryHookError` property. */
-    retry?(eventId: string, error: {
+/** Exclusive, expiring ownership of one event across the application's processes.
+ * Constellation re-sends an event that stays unanswered; whoever holds the lock
+ * answers that re-send with 202 instead of starting a second run. */
+export interface EventLock {
+    /** Take the event for `ttlMs`. False while another holder's lease is live. */
+    acquire(eventId: string, ttlMs: number): Promise<boolean>;
+    /** Extend this holder's lease. False if it expired or another holder took it. */
+    renew(eventId: string, ttlMs: number): Promise<boolean>;
+    release(eventId: string): Promise<void>;
+}
+/** Correct for a single process. Use `redisEventLock` when several processes receive events. */
+export declare class MemoryEventLock implements EventLock {
+    #private;
+    acquire(eventId: string, ttlMs: number): Promise<boolean>;
+    renew(eventId: string, ttlMs: number): Promise<boolean>;
+    release(eventId: string): Promise<void>;
+}
+/** A lock shared through Redis. `command` sends one raw command and resolves its reply,
+ * for example `(args) => redis.call(...args)` (ioredis) or `(args) => client.sendCommand(args)` (node-redis). */
+export declare function redisEventLock(command: (args: string[]) => Promise<unknown>, options?: {
+    prefix?: string;
+}): EventLock;
+/** What processing an event did. `ended` says the request is over after this response:
+ * `completed` or `failed` from this answer, `closed` when the request had already ended. */
+export type EventOutcome = {
+    status: "busy";
+} | {
+    status: "answered";
+    action: "continue" | "stop";
+    receipt: AdmissionReceipt | null;
+    ended: "completed" | "failed" | "closed" | null;
+};
+export type EventOptions = {
+    signal?: AbortSignal;
+    /** Lock lease, renewed every third of it while processing. Keep it well below the response window. */
+    leaseMs?: number;
+    /** Pauses before retrying a transient failure while still holding the event, before leaving
+     * it to Constellation's re-send. Default 1 s, 3 s and 9 s; `[]` leaves every failure to the re-send. */
+    retryDelaysMs?: readonly number[];
+    /** Reports each failed attempt. `willRetry` is false for the last one: the event stays
+     * unanswered and Constellation re-sends it after its response window. */
+    onFailure?: (eventId: string, error: {
         status: number;
         code: string;
         message: string;
-    }): Promise<void>;
-}
+        attempt: number;
+        willRetry: boolean;
+    }) => void | Promise<void>;
+};
+/** The error result returned to the model for a call whose arguments were not valid JSON. */
+export declare function invalidToolInputResult(call: {
+    name: string;
+    invalid?: {
+        error: string;
+    };
+}): ToolResult["result"];
+/** Webhook-side processing. After authenticating an event, call `claim`: `null` means another
+ * process holds it (answer 202); otherwise answer 204 and run the returned function in the
+ * background. Tools never run inside the webhook call. */
 export declare class EventReceiver {
-    private readonly store;
-    constructor(store: EventStore);
-    /** Call after webhook authentication. Return a successful HTTP response only after this resolves. */
-    receive(event: WebhookEvent): Promise<void>;
+    #private;
+    private readonly client;
+    private readonly hooks;
+    private readonly options;
+    /** `hooks` may be loaded per event, for example to bind them to the request's own records. */
+    constructor(client: RequestClient, hooks: AgentHooks | ((event: WebhookEvent) => Promise<AgentHooks>), options?: EventOptions & {
+        lock?: EventLock;
+    });
+    claim(event: WebhookEvent): Promise<((signal?: AbortSignal) => Promise<EventOutcome>) | null>;
 }
 export declare class ClientTelemetry {
     readonly parentSpanId?: string | undefined;
@@ -75,7 +116,6 @@ export interface AgentHooks {
 }
 /** Record the client processing interval and its actual child-span hierarchy. */
 export declare function prepareResponse(event: WebhookEvent, hooks: AgentHooks, signal?: AbortSignal): Promise<Admission | null>;
-/** Process an authenticated event using an application-owned durable checkpoint.
- * Any receipt (`accepted`, `already_accepted`, `event_closed`) completes the checkpoint; a permanent
- * error code rejects it with PermanentEventError; anything else reports through `store.retry` and throws. */
-export declare function processEvent(client: RequestClient, store: EventStore, event: WebhookEvent, hooks: AgentHooks, signal?: AbortSignal): Promise<void>;
+/** Process an authenticated event inline: take its lock, prepare and submit the response, release.
+ * Returns `busy` without doing anything while another process holds the event. */
+export declare function processEvent(client: RequestClient, lock: EventLock, event: WebhookEvent, hooks: AgentHooks, options?: EventOptions): Promise<EventOutcome>;

package/dist/types/client-core/src/index.d.ts

@@ -7,10 +7,17 @@ export type Tool = {
     description?: string;
     inputSchema: Json;
 };
+/** The provider's tool arguments were not valid JSON. `raw` is truncated to 16 KiB. */
+export type InvalidToolInput = {
+    raw: string;
+    error: string;
+};
+/** With `invalid` set, `arguments` is `{}`: answer with an error result instead of running the tool. */
 export type ToolCall = {
     id: string;
     name: string;
     arguments: Json;
+    invalid?: InvalidToolInput;
 };
 export type Message = {
     role: "system" | "user" | "assistant" | "tool";
@@ -122,6 +129,9 @@ export type AdmissionReceipt = {
     outcome?: ResponseOutcome;
     /** With `event_closed`: whether the continuation's telemetry was kept as late observations. */
     observationsRecorded?: boolean;
+    /** With `already_accepted`: false when this responder answered the event before with a
+     * different body (for example recomputed after a lost acknowledgement). The first answer stands. */
+    bodyMatched?: boolean;
 };
 export type Status = "queued" | "running" | "client_processing" | "completed" | "failed" | "cancelled";
 export type RequestRecord = {
@@ -349,6 +359,14 @@ export type ModelResolution = ModelScope & {
     headers?: Record<string, string>;
     nativeContentPolicy?: "require" | "omitCompleted" | "omitIncompatible";
 };
+/** Whose request a boundary event belongs to, so a receiver never reads the request back. */
+export type EventRequest = {
+    id: string;
+    externalId: string | null;
+    workspaceId: string;
+    userId: string | null;
+    idempotencyKey: string;
+};
 export type ModelCompletedEvent = {
     id: string;
     requestId: string;
@@ -357,10 +375,14 @@ export type ModelCompletedEvent = {
     type: "model.completed";
     data: {
         message: Message;
+        /** Step result: `stepId`, `stepInput` (`model`, `parameters`, `maxOutputTokens`, `timeoutMs`, `toolsBytes`),
+         * `usage`, costs, `finishReason` and the step's `attempts`. */
         result: Record<string, Json>;
+        /** 1-based number of the step this event completes. */
+        stepNumber: number;
+        request: EventRequest;
         events?: Json[];
         stepId?: string;
-        stepNumber?: number;
         attemptId?: string;
     };
 };
@@ -370,7 +392,10 @@ export type WebhookEvent = ModelCompletedEvent | {
     sequence: number;
     createdAt: string;
     type: "model.failed" | "model.cancelled";
-    data: Record<string, Json>;
+    data: Record<string, Json> & {
+        stepNumber: number;
+        request: EventRequest;
+    };
 };
 export declare class ConstellationError extends Error {
     status: number;

package/dist/types/sdk/src/index.d.ts

@@ -1,7 +1,7 @@
 /** External server SDK. Operator APIs and cookie sessions are internal-only clients. */
 import { RequestClient, type WebhookEvent, type Json, type Page, type PageQuery, type ModelResolution, type RequestRecord, type RequestEvent, type RecordTypes, type Timeline, type TimelineRow, type RequestListQuery, type RequestRecordsQuery, type RequestTimings, type TokenUsage } from "../../client-core/src/index";
 export { ConstellationError, uuid7, normalizeTimestamp, parseOpenerSentence } from "../../client-core/src/index";
-export type { InferenceParameters, Json, Tool, ToolCall, Message, RequestEventModes, RequestInput, Admission, AdmissionReceipt, ResponseOutcome, ObservationResponse, Observations, EventReference, ModelCompletedEvent, ModelDeltaEvent, ModelStreamFrame, RequestTimings, TimingInterval, TokenUsage, TokenPrice, CustomerPricing, BillingFacts, Status, RequestRecord, Step, Attempt, Span, TimelineRow, Timeline, Log, Inspection, RecordTypes, RequestEvent, Page, PageQuery, ModelScope, ModelResolution, WebhookEvent, RequestRecordsQuery, RequestListQuery, OpenerInput, OpenerTurn } from "../../client-core/src/index";
+export type { InferenceParameters, Json, Tool, ToolCall, InvalidToolInput, EventRequest, Message, RequestEventModes, RequestInput, Admission, AdmissionReceipt, ResponseOutcome, ObservationResponse, Observations, EventReference, ModelCompletedEvent, ModelDeltaEvent, ModelStreamFrame, RequestTimings, TimingInterval, TokenUsage, TokenPrice, CustomerPricing, BillingFacts, Status, RequestRecord, Step, Attempt, Span, TimelineRow, Timeline, Log, Inspection, RecordTypes, RequestEvent, Page, PageQuery, ModelScope, ModelResolution, WebhookEvent, RequestRecordsQuery, RequestListQuery, OpenerInput, OpenerTurn } from "../../client-core/src/index";
 /** OpenAPI contract filtered to this key's permissions and scope. */
 export type DiscoveryDocument = {
     openapi: string;
@@ -263,5 +263,5 @@ export declare class Constellation extends RequestClient {
         verify: (request: Request, expectedToken: string) => Promise<WebhookEvent>;
     };
 }
-export { PermanentEventError, HookConfigurationError, EventReceiver, prepareResponse, processEvent, ClientTelemetry, PERMANENT_RESPONSE_CODES } from "../../client-core/src/agent";
-export type { EventStore, AgentHooks, HookContext, ToolResult } from "../../client-core/src/agent";
+export { PermanentEventError, HookConfigurationError, EventReceiver, MemoryEventLock, redisEventLock, prepareResponse, processEvent, ClientTelemetry, PERMANENT_RESPONSE_CODES, invalidToolInputResult } from "../../client-core/src/agent";
+export type { EventLock, EventOutcome, EventOptions, AgentHooks, HookContext, ToolResult } from "../../client-core/src/agent";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lionhummer/constellation",
-  "version": "0.9.4",
+  "version": "0.11.0",
   "description": "TypeScript client for durable model requests and model events",
   "type": "module",
   "license": "UNLICENSED",