@lionhummer/constellation 0.9.2 → 0.10.0

package/README.md

@@ -70,12 +70,11 @@ console.log(request.status, request.output?.text);
 
 ## Build an agent with tools
 
-A tool-using agent needs four pieces:
+A tool-using agent needs three pieces:
 
-1. **A webhook** that receives events and stores them.
+1. **A webhook** that receives events.
 2. **Hooks** that run your tools and build the next input.
-3. **An `EventStore`**: a table in your database that remembers each event and the response you prepared for it.
-4. **`processEvent`**, which ties the other three together.
+3. **An `EventReceiver`**, which takes each event, runs your hooks in the background and sends the response. You don't store events or responses: Constellation re-sends an event that stays unanswered, and deduplicates your answers.
 
 ### 1. Turn on events
 
@@ -142,112 +141,19 @@ const hooks: AgentHooks = {
 
 `conversation.settings` must repeat the first request's `workspaceId`, `userId`, `externalId`, `model`, `planId`, `events` and `timeoutMs`. A continuation that changes any of them is rejected. You may change `messages`, `instructions`, `tools`, `parameters` and `cache`.
 
-### 3. Implement the `EventStore`
-
-The store is a small table in your own database. It guarantees two things:
-
-- an event you acknowledged is never lost;
-- a response you prepared is sent unchanged, even after a crash.
-
-Do not use an in-memory map in production. Here is a complete PostgreSQL version using [`pg`](https://node-postgres.com/):
-
-```sql
-CREATE TABLE constellation_events (
-  event_id          text PRIMARY KEY,
-  payload           jsonb NOT NULL,
-  prepared          boolean NOT NULL DEFAULT false,
-  response          jsonb,                         -- the exact body we send (or resend)
-  state             text NOT NULL DEFAULT 'pending', -- pending | done | rejected
-  rejection         jsonb,
-  failures          integer NOT NULL DEFAULT 0,
-  last_error        jsonb,
-  first_failure_at  timestamptz,
-  last_failure_at   timestamptz,
-  received_at       timestamptz NOT NULL DEFAULT now()
-);
-```
-
-```ts
-import pg from "pg";
-import { PermanentEventError, type EventStore, type WebhookEvent } from "@lionhummer/constellation";
-
-const db = new pg.Pool();
-
-export const store: EventStore & { pending(): Promise<WebhookEvent[]> } = {
-  // Save the event before acknowledging the webhook. Duplicates must match.
-  async accept(event) {
-    await db.query(
-      `INSERT INTO constellation_events (event_id, payload) VALUES ($1, $2)
-       ON CONFLICT (event_id) DO NOTHING`, [event.id, event]);
-    const { rows } = await db.query(
-      `SELECT payload = $2::jsonb AS same FROM constellation_events WHERE event_id = $1`, [event.id, event]);
-    if (!rows[0]?.same) throw new Error(`event ${event.id} redelivered with a different payload`);
-  },
-
-  // Run `prepare` at most once per event at a time, and save its result before returning it.
-  async prepare(event, prepare) {
-    const tx = await db.connect();
-    try {
-      await tx.query("BEGIN");
-      const { rows } = await tx.query(
-        `SELECT prepared, response, state, rejection FROM constellation_events
-         WHERE event_id = $1 FOR UPDATE`, [event.id]); // serializes concurrent processing
-      const row = rows[0];
-      if (row.state === "rejected") {
-        const r = row.rejection;
-        throw new PermanentEventError(r.status, r.code, r.message);
-      }
-      if (row.prepared) { await tx.query("COMMIT"); return row.response; }
-      const response = await prepare(); // runs your tools and hooks
-      await tx.query(
-        `UPDATE constellation_events SET prepared = true, response = $2 WHERE event_id = $1`,
-        [event.id, response]);
-      await tx.query("COMMIT");
-      return response;
-    } catch (error) {
-      await tx.query("ROLLBACK");
-      throw error;
-    } finally {
-      tx.release();
-    }
-  },
-
-  async complete(eventId) {
-    await db.query(`UPDATE constellation_events SET state = 'done' WHERE event_id = $1`, [eventId]);
-  },
-
-  // A permanent rejection: never resend this response.
-  async reject(eventId, error) {
-    await db.query(
-      `UPDATE constellation_events SET state = 'rejected', rejection = $2 WHERE event_id = $1`,
-      [eventId, { status: error.status, code: error.code, message: error.message }]);
-  },
-
-  // Optional: a retryable failure. Alert when failures pile up or get old.
-  async retry(eventId, error) {
-    await db.query(
-      `UPDATE constellation_events SET failures = failures + 1, last_error = $2,
-         first_failure_at = coalesce(first_failure_at, now()), last_failure_at = now()
-       WHERE event_id = $1`, [eventId, error]);
-  },
-
-  // Not part of EventStore: used by the recovery sweep below.
-  async pending() {
-    const { rows } = await db.query(
-      `SELECT payload FROM constellation_events WHERE state = 'pending' ORDER BY received_at LIMIT 100`);
-    return rows.map((row) => row.payload);
-  },
-};
-```
-
-`prepare` keeps a row lock while your tools run. That is acceptable for tools that finish in seconds or minutes. For very long work, replace the lock with a lease column.
+### 3. Receive webhooks
 
-### 4. Receive webhooks and process events
-
-Acknowledge quickly, then process in the background. The webhook handler below works with any framework that uses the web `Request`/`Response` types (Hono, Next.js route handlers, Bun, Deno):
+Acknowledge quickly, then process in the background. The handler below works with any framework that uses the web `Request`/`Response` types (Hono, Next.js route handlers, Bun, Deno):
 
 ```ts
-import { processEvent, PermanentEventError } from "@lionhummer/constellation";
+import { EventReceiver, redisEventLock } from "@lionhummer/constellation";
+
+const receiver = new EventReceiver(client, hooks, {
+  // One process: leave this out (an in-memory lock is used).
+  // Several processes or pods: share the lock so only one works on an event at a time.
+  lock: redisEventLock((args) => redis.call(...args)), // ioredis; node-redis: (args) => redis.sendCommand(args)
+  onFailure: (eventId, error) => logger.warn({ eventId, error }, "event will be re-sent"),
+});
 
 export async function webhook(request: Request): Promise<Response> {
   let event;
@@ -256,68 +162,60 @@ export async function webhook(request: Request): Promise<Response> {
   } catch {
     return new Response(null, { status: 401 });
   }
-  await store.accept(event); // if this throws, the 5xx makes Constellation redeliver
-  void handle(event);        // don't make the webhook wait for your tools
-  return new Response(null, { status: 202 });
-}
-
-async function handle(event: WebhookEvent) {
-  try {
-    await processEvent(client, store, event, hooks);
-  } catch (error) {
-    if (error instanceof PermanentEventError) {
-      await markConversationFailed(event.requestId, error); // will not be retried
-    } else {
-      logger.warn({ eventId: event.id, error }, "event processing will be retried");
-    }
-  }
+  const run = await receiver.claim(event);
+  if (!run) return new Response(null, { status: 202 }); // another process is working on it
+  void run().then((outcome) => {
+    if (outcome.status === "answered" && outcome.ended) markConversationEnded(event.requestId, outcome.ended);
+  }, () => {}); // a failure leaves the event unanswered; Constellation re-sends it
+  return new Response(null, { status: 204 });
 }
-
-// Optional: resume unfinished events right after a restart.
-for (const event of await store.pending()) void handle(event);
 ```
 
 Don't run tools inside the webhook call. Constellation waits 5 seconds for the acknowledgement by default, and nothing in Constellation waits for your tools.
 
-If your process dies while handling an event, you don't need a poller to recover it. Constellation sends an unanswered `model.completed` event again after the response window (10 minutes by default), up to 3 times, and stops as soon as your response arrives. A re-sent event has the same `id`, so `store.accept` and `store.prepare` resume it from your saved checkpoint. If the event is still being processed when the re-send arrives, acknowledge it anyway. Keep your processing lease shorter than the response window and renew it while work continues, so a re-send after a crash can take the event over.
+`outcome.ended` tells you when the request is over: `completed` or `failed` after your final answer, `closed` when it had already been cancelled or had expired. Requests that fail or are cancelled on the Constellation side arrive as `model.failed` and `model.cancelled` events. You never need to read the request back.
 
-## What `processEvent` does
+### Recovery
 
-For each event:
+If your process dies while handling an event, you don't need a poller or a store to recover it. Constellation sends an unanswered `model.completed` event again after the response window (10 minutes by default; operators can lower it, for example to 2 minutes), up to 3 times. A re-sent event has the same `id`:
 
-1. **Prepare the response, at most once.** `store.prepare` returns the saved response if one exists. Otherwise the SDK builds one:
-   - For `model.completed`, it runs every tool call in parallel and calls `onStepComplete`. To continue, it calls `prepareRequest`, which produces a continuation. To stop, it produces an `observe` response.
-   - For `model.failed` and `model.cancelled`, it calls `onError(event.data)` and produces an `observe` response.
+- while another process still holds the event's lock, `claim` returns `null` and you answer `202`. Constellation then checks again after another window without counting a re-send;
+- once the holder is gone, its lock expires (60 s by default, renewed while it works) and the re-send is processed from the start.
 
-   The response carries the idempotency key `event.id`, `inReplyTo: { requestId, eventId }` and the telemetry recorded while preparing. The store saves it before anything is sent.
-2. **Submit it** through `requests.submit`. Retries send the saved body unchanged.
-3. **Record the outcome**:
+If you answer an event twice, for example because the reply to your first answer was lost, Constellation keeps the first answer and reports the second as `already_accepted` with `bodyMatched: false`. Nothing continues twice.
 
-| Server answer | Meaning | `processEvent` |
-| --- | --- | --- |
-| `200`, `outcome: "accepted"` | Your response was accepted. A continuation starts the next step (`scheduled: true`). | `store.complete`, resolves |
-| `200`, `outcome: "already_accepted"` | A retry of a response that was already accepted, for example after a lost reply. Nothing runs twice. | `store.complete`, resolves |
-| `200`, `outcome: "event_closed"` | The request was cancelled or expired before your continuation arrived. Nothing continues; your telemetry is still recorded (`observationsRecorded`). | `store.complete`, resolves |
-| `400 bad_request` | The response itself is invalid, for example a changed model or a bad message. | `store.reject`, throws `PermanentEventError` |
-| `409 response_conflict` | A *different* response to this event was already accepted. | `store.reject`, throws `PermanentEventError` |
-| `410 observations_expired` | The request ended more than 24 hours ago. | `store.reject`, throws `PermanentEventError` |
-| Anything else: network error, timeout, `5xx`, `429`, `409 conflict`, `401`, `403` | Temporary. | `store.retry`, throws. The next attempt resends the same body. |
+## What happens to each event
+
+1. **Take the lock** for the event. If another process holds it: `busy` (answer `202`).
+2. **Prepare the response:**
+   - For `model.completed`, run every tool call in parallel and call `onStepComplete`. To continue, call `prepareRequest`, which produces a continuation. To stop, produce an `observe` response.
+   - A tool call marked `invalid` (its arguments were not valid JSON) is never run. Its result is an error from `invalidToolInputResult` with `isError: true`, so the model can correct the call on the next step, as the AI SDK does.
+   - For `model.failed` and `model.cancelled`, call `onError(event.data)` and produce an `observe` response.
 
-The permanent codes are exported as `PERMANENT_RESPONSE_CODES`. The receipt passed to `store.complete` includes `outcome`, so your store can record that a request ended without continuing.
+   The response carries the idempotency key `event.id`, `inReplyTo: { requestId, eventId }` and the telemetry recorded while preparing.
+3. **Submit it** and release the lock:
 
-Fix `401` and `403` by correcting the key: the saved response is still sent unchanged afterwards.
+| Server answer | Meaning | Result |
+| --- | --- | --- |
+| `200`, `outcome: "accepted"` | Your response was accepted. A continuation starts the next step (`scheduled: true`). | `answered` |
+| `200`, `outcome: "already_accepted"` | This event was already answered by you. `bodyMatched: false` means the earlier answer differed and stands. | `answered` |
+| `200`, `outcome: "event_closed"` | The request was cancelled or expired before your continuation arrived. Your telemetry is still recorded (`observationsRecorded`). | `answered`, `ended: "closed"` |
+| A permanent rejection (`PERMANENT_RESPONSE_CODES`, for example `400 bad_request`) of a continuation | The continuation can never be accepted. | The SDK sends an `observe` response with the error instead, which ends the request as failed: `ended: "failed"` |
+| Anything else: network error, timeout, `5xx`, `429`, `409 conflict`, `401`, `403` | Temporary. | `onFailure` runs and the call throws. The event stays unanswered and is re-sent. |
+
+While processing, the SDK renews the lock every third of its lease. If renewal fails, it aborts its own work through `signal`, so a re-send taken over elsewhere doesn't run alongside it.
 
 ## When your code fails
 
 | What happened | Without `onError` | With `onError` |
 | --- | --- | --- |
-| A tool or hook throws | `processEvent` throws and the event is retried later. Tools may run again, so use `effectId`. | `onError` runs. If it returns, the request stops and the error is recorded in telemetry. If it throws, the event is retried. |
-| No handler for a tool the model called, or `prepareRequest` is missing when continuing | The request is marked **failed** and `HookConfigurationError` (a `PermanentEventError`) is thrown. | `onError` runs, as above. |
+| A tool or hook throws | The event stays unanswered and is re-sent. Tools may run again, so use `effectId`. | `onError` runs. If it returns, the request stops and the error is recorded in telemetry. If it throws, the event is re-sent. |
+| No handler for a tool the model called, or `prepareRequest` is missing when continuing | The request ends as **failed** (`invalid_hook_configuration`). | `onError` runs, as above. |
 | The model step failed or was cancelled (`model.failed`, `model.cancelled`) | An `observe` response records your telemetry. | `onError(event.data)` runs first. |
 
 ## Side effects and duplicates
 
-Hooks can run more than once. A crash after a tool ran, but before the prepared response was saved, means the tool runs again. Constellation cannot make your side effects exactly-once; your code must:
+Delivery is at least once. A crash after a tool ran but before the answer was accepted means the tool runs again when the event is re-sent. Constellation cannot make your side effects exactly-once; your code must:
 
 - pass `effectId` to anything with side effects (payments, emails, writes). It is stable for one tool call of one event;
 - settle per-step billing or usage once per `event.id`;
@@ -341,6 +239,8 @@ tools: {
 },
 ```
 
+Work you do before the first model step, such as loading context and building the prompt, can travel with the request that starts the run: pass `telemetry: { spans, logs }` to `requests.submit`. Those spans belong to the request rather than a step: they sit at the top of the timeline at their own timestamps, before the first step (set `stepId` or `parentSpanId` to nest them elsewhere).
+
 The SDK records `client.processing` and one span per tool automatically. A response carries at most 512 spans, 128 logs and 1 MiB of telemetry; the SDK trims larger batches and records what it omitted. Do not put secrets or personal data in span names, attributes or log messages.
 
 ## Opening lines
@@ -361,6 +261,45 @@ if (opener) void client.openers.sentence(opener.id).then((sentence) => {
 
 Start the real request without waiting for the opener. `parseOpenerSentence(text)` applies the same checks to text you already have.
 
+## Telemetry queries
+
+Ask aggregate questions without SQL. Fields, filters and metrics are typed per dataset.
+
+```ts
+// How many fireworks-direct attempts failed yesterday, and why?
+const failures = await client.telemetry.query({
+  dataset: "attempts",
+  from: "2026-10-15T00:00:00Z", to: "2026-10-16T00:00:00Z",
+  where: { offeringId: { startsWith: "fireworks-direct:" }, status: "failed" },
+  groupBy: ["errorCode", "httpStatus"],
+  metrics: ["count", "uniq:requestId"],
+})
+
+// How often did a provider say "Response too long", per day?
+const mentions = await client.telemetry.query({
+  dataset: "attempts",
+  from: "2026-10-01T00:00:00Z", to: "2026-10-16T00:00:00Z",
+  where: { errorMessage: { contains: "Response too long" } },
+  groupBy: ["day"],
+})
+```
+
+Usage and costs come from the permanent billing record (`usage`, one row per settled request, or `usageAttempts`, one row per provider attempt for provider/model splits). Any range works:
+
+```ts
+// Customer charges, supplier cost and margin per workspace and provider last month, without QA
+const byProvider = await client.telemetry.query({
+  dataset: "usageAttempts",
+  from: "2026-09-01T00:00:00Z", to: "2026-10-01T00:00:00Z",
+  where: { workspaceId: { notIn: ["qa"] } },
+  groupBy: ["workspaceId", "providerId"],
+  metrics: ["uniq:requestId", "sum:customerCost", "sum:supplierCost", "sum:margin"],
+  limit: 10000,
+})
+```
+
+A scoped key only ever sees its workspace and user. Logs and usage need an unscoped key. Telemetry ranges are at most 92 days and results at most 1,000 rows; usage allows any range and 10,000 rows. Group by `minute`, `hour`, `day`, `week` or `month` for time series.
+
 ## Limits and timeouts
 
 | Setting | Default | Notes |
@@ -386,7 +325,7 @@ A failed provider call falls back to the next provider only before any output ha
 | `requests.preview(input)` | Validate a complete input and show routing, without calling a provider. |
 | `requests.cancel(id)` | Cancel a request. |
 | `openers.create(input)`, `openers.sentence(id)` | Start an opening line for a just-sent message (null when openers are turned off), then wait for its sentence (null when skipped or failed). |
-| `requests.getState(id)` | Committed state: status, steps, attempt outcomes, usage and cost. |
+| `requests.getState(id)` | Committed state: status, steps, attempt outcomes, usage and cost. For inspection and support tools; event processing never needs it, because boundary events carry the step number, step input and request identity. |
 | `requests.getSummary(id)`, `requests.get(id)` | Request detail, without or with spans and logs. |
 | `requests.list(query)` | Page through requests, e.g. by `externalId` or `status`. |
 | `requests.listEvents(id, after)`, `requests.streamEvents(id)` | Replay events after a sequence number, or stream them live. |
@@ -394,11 +333,12 @@ A failed provider call falls back to the next provider only before any output ha
 | `requests.getTimeline(id)`, `requests.getTimings(id)` | Trace and timing breakdown. |
 | `requests.getHttpExchange(id, attemptId)`, `requests.getHttpBody(...)` | The provider HTTP call captured for one attempt. |
 | `webhooks.verify(request, token)` | Authenticate and parse an incoming webhook. |
+| `telemetry.query(query)` | Count, group and aggregate attempts, requests, logs or permanent usage, e.g. failed attempts per offering and error code, or cost per workspace and provider. Needs telemetry permission; the key's scope always applies. |
 | `telemetry.*` | Read-only reporting. Returns metadata only unless you pass `include: "content"`. Needs telemetry permission. |
 | `management.registry.*` | Edit the model registry. Needs management permission. |
 | `discover()` | The OpenAPI document for exactly what this key may call. |
 
-Exports for event processing: `processEvent`, `prepareResponse` (build a response without submitting it), `EventReceiver`, `ClientTelemetry`, `PermanentEventError`, `HookConfigurationError`, `PERMANENT_RESPONSE_CODES`, `ConstellationError` and `uuid7`.
+Exports for event processing: `EventReceiver`, `processEvent` (inline: lock, prepare, submit), `prepareResponse` (build a response without submitting it), `MemoryEventLock`, `redisEventLock`, `invalidToolInputResult`, `ClientTelemetry`, `PermanentEventError`, `HookConfigurationError`, `PERMANENT_RESPONSE_CODES`, `ConstellationError` and `uuid7`.
 
 ## Troubleshooting
 
@@ -406,7 +346,7 @@ Exports for event processing: `processEvent`, `prepareResponse` (build a respons
 | --- | --- |
 | `models.list` is empty | The key's workspace/user scope, an access rule for that workspace, and an enabled model and offering. |
 | `terminal: "notify"` is rejected | The execution key has no webhook URL configured. |
-| The same webhook keeps arriving | Your endpoint returned non-2xx or took longer than the delivery timeout (5 s by default). Return 202 after `store.accept` and process in the background. |
+| The same webhook keeps arriving | Your endpoint returned non-2xx or took longer than the delivery timeout (5 s by default). Answer after `receiver.claim` and process in the background. |
 | `400` on a continuation | Model, workspace, user, `externalId`, plan, events or `timeoutMs` differ from the first request, or the message history is malformed. |
 | `409 response_conflict` | Your code produced a different response for an event that was already answered. Answers must come from the saved response. |
 | Tools ran twice | Expected after a crash. Make side effects idempotent with `effectId`. |

package/dist/index.js

@@ -424,15 +424,88 @@ class HookConfigurationError extends PermanentEventError {
   }
 }
 
-class EventReceiver {
-  store;
-  constructor(store) {
-    this.store = store;
+class MemoryEventLock {
+  #held = new Map;
+  async acquire(eventId, ttlMs) {
+    const until = this.#held.get(eventId);
+    if (until !== undefined && until > Date.now())
+      return false;
+    this.#held.set(eventId, Date.now() + ttlMs);
+    return true;
+  }
+  async renew(eventId, ttlMs) {
+    if (!this.#held.has(eventId))
+      return false;
+    this.#held.set(eventId, Date.now() + ttlMs);
+    return true;
+  }
+  async release(eventId) {
+    this.#held.delete(eventId);
   }
-  async receive(event) {
-    await this.store.accept(event);
+}
+var RENEW_SCRIPT = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end";
+var RELEASE_SCRIPT = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end";
+function redisEventLock(command, options = {}) {
+  const prefix = options.prefix ?? "constellation:event-lock:";
+  const tokens = new Map;
+  return {
+    async acquire(eventId, ttlMs) {
+      const token = uuid7();
+      const reply = await command(["SET", prefix + eventId, token, "NX", "PX", String(ttlMs)]);
+      if (reply !== "OK")
+        return false;
+      tokens.set(eventId, token);
+      return true;
+    },
+    async renew(eventId, ttlMs) {
+      const token = tokens.get(eventId);
+      if (!token)
+        return false;
+      return Number(await command(["EVAL", RENEW_SCRIPT, "1", prefix + eventId, token, String(ttlMs)])) === 1;
+    },
+    async release(eventId) {
+      const token = tokens.get(eventId);
+      tokens.delete(eventId);
+      if (token)
+        await command(["EVAL", RELEASE_SCRIPT, "1", prefix + eventId, token]);
+    }
+  };
+}
+function invalidToolInputResult(call) {
+  return {
+    error: "invalid_tool_input",
+    message: `The arguments for ${call.name} were not valid JSON (${call.invalid?.error ?? "unparseable"}). Call ${call.name} again with complete, valid JSON arguments.`
+  };
+}
+
+class EventReceiver {
+  client;
+  hooks;
+  options;
+  #lock;
+  constructor(client, hooks, options = {}) {
+    this.client = client;
+    this.hooks = hooks;
+    this.options = options;
+    this.#lock = options.lock ?? new MemoryEventLock;
+  }
+  async claim(event) {
+    const leaseMs = this.options.leaseMs ?? DEFAULT_LEASE_MS;
+    if (!await this.#lock.acquire(event.id, leaseMs))
+      return null;
+    return async (signal) => {
+      let hooks;
+      try {
+        hooks = typeof this.hooks === "function" ? await this.hooks(event) : this.hooks;
+      } catch (error) {
+        await this.#lock.release(event.id).catch(() => {});
+        throw error;
+      }
+      return runClaimed(this.client, this.#lock, event, hooks, { ...this.options, signal: signal ?? this.options.signal });
+    };
   }
 }
+var DEFAULT_LEASE_MS = 60000;
 
 class ClientTelemetry {
   parentSpanId;
@@ -563,6 +636,10 @@ async function prepareResponseBody(event, hooks, telemetry, signal) {
           return;
         signal?.throwIfAborted();
         const call = calls[index];
+        if (call.invalid) {
+          results[index] = { id: call.id, result: invalidToolInputResult(call), isError: true };
+          continue;
+        }
         const handler = hooks.tools?.[call.name];
         if (!handler)
           throw new HookConfigurationError(`No handler for tool ${call.name}`);
@@ -619,42 +696,61 @@ async function prepareResponse(event, hooks, signal) {
     response.telemetry = telemetry.snapshot();
   return response;
 }
-async function processEvent(client, store, event, hooks, signal) {
+async function processEvent(client, lock, event, hooks, options = {}) {
+  if (!await lock.acquire(event.id, options.leaseMs ?? DEFAULT_LEASE_MS))
+    return { status: "busy" };
+  return runClaimed(client, lock, event, hooks, options);
+}
+async function runClaimed(client, lock, event, hooks, options) {
+  const leaseMs = options.leaseMs ?? DEFAULT_LEASE_MS;
+  const fence = new AbortController;
+  const signal = options.signal ? AbortSignal.any([options.signal, fence.signal]) : fence.signal;
+  const renewal = setInterval(() => {
+    lock.renew(event.id, leaseMs).then((held) => {
+      if (!held)
+        fence.abort(new Error("Event lock lost"));
+    }, () => fence.abort(new Error("Event lock unavailable")));
+  }, Math.max(1000, Math.floor(leaseMs / 3)));
   let stage = "prepare";
   try {
-    const response = await store.prepare(event, () => prepareResponse(event, hooks, signal));
-    let receipt = null;
-    if (response) {
-      stage = "submit";
-      try {
-        receipt = await client.requests.submit(response, signal);
-      } catch (error) {
-        if (error instanceof ConstellationError && PERMANENT_RESPONSE_CODES.has(error.code)) {
-          const rejection = new PermanentEventError(error.status, error.code, error.message);
-          await store.reject(event.id, rejection);
-          throw rejection;
-        }
+    const response = await prepareResponse(event, hooks, signal);
+    if (!response)
+      return { status: "answered", action: "stop", receipt: null, ended: null };
+    stage = "submit";
+    signal.throwIfAborted();
+    const observe = "action" in response && response.action === "observe";
+    let receipt;
+    try {
+      receipt = await client.requests.submit(response, signal);
+    } catch (error) {
+      if (!(error instanceof ConstellationError) || !PERMANENT_RESPONSE_CODES.has(error.code) || observe)
         throw error;
-      }
-      if ("action" in response && response.action === "observe" && response.error) {
-        const failure = new HookConfigurationError(response.error.message);
-        await store.reject(event.id, failure);
-        throw failure;
-      }
+      receipt = await client.requests.submit({
+        action: "observe",
+        idempotencyKey: event.id,
+        inReplyTo: { requestId: event.requestId, eventId: event.id },
+        telemetry: response.telemetry ?? { spans: [], logs: [] },
+        error: { code: error.code, message: error.message.slice(0, 8192) || error.code }
+      }, signal);
+      return { status: "answered", action: "stop", receipt, ended: receipt.outcome === "event_closed" ? "closed" : "failed" };
     }
-    stage = "checkpoint";
-    await store.complete(event.id, receipt);
+    const failed = observe && "error" in response && !!response.error;
+    const ended = receipt.outcome === "event_closed" ? "closed" : observe ? failed ? "failed" : "completed" : null;
+    return { status: "answered", action: observe ? "stop" : "continue", receipt, ended };
   } catch (error) {
-    if (!(error instanceof PermanentEventError) && store.retry) {
-      const detail = error instanceof ConstellationError ? { status: error.status, code: error.code, message: error.message } : { status: 0, code: error instanceof Error && error.name === "TimeoutError" ? "timeout" : stage === "submit" ? "transport_error" : `event_${stage}_failed`, message: error instanceof Error ? error.message : "Event processing failed" };
+    if (options.onFailure) {
+      const detail = error instanceof ConstellationError ? { status: error.status, code: error.code, message: error.message } : { status: 0, code: error instanceof Error && error.name === "TimeoutError" ? "timeout" : stage === "submit" ? "transport_error" : "event_prepare_failed", message: error instanceof Error ? error.message : "Event processing failed" };
       try {
-        await store.retry(event.id, detail);
+        await options.onFailure(event.id, detail);
       } catch (monitorError) {
         if (error && typeof error === "object")
-          Object.assign(error, { retryHookError: monitorError });
+          Object.assign(error, { failureHookError: monitorError });
       }
     }
     throw error;
+  } finally {
+    clearInterval(renewal);
+    await lock.release(event.id).catch(() => {});
   }
 }
 
@@ -764,6 +860,10 @@ class Constellation extends RequestClient {
         get: (id, query = {}, signal) => this.#call(`/v1/telemetry/notifications/${encodeURIComponent(id)}${queryString2(query)}`, "GET", undefined, signal),
         listEvents: (query = {}, signal) => this.#call(`/v1/telemetry/notifications/events${queryString2(query)}`, "GET", undefined, signal)
       },
+      query: (query, signal) => {
+        const iso = (value) => value instanceof Date ? value.toISOString() : value;
+        return this.#call("/v1/telemetry/query", "POST", { ...query, from: iso(query.from), to: iso(query.to) }, signal);
+      },
       logs: {
         searchSystem: (query = {}, signal) => this.#call(`/v1/telemetry/logs/system${queryString2(query)}`, "GET", undefined, signal),
         searchRequests: (query = {}, signal) => this.#call(`/v1/telemetry/logs/requests${queryString2(query)}`, "GET", undefined, signal),
@@ -803,12 +903,15 @@ class Constellation extends RequestClient {
 }
 export {
   uuid7,
+  redisEventLock,
   processEvent,
   prepareResponse,
   parseOpenerSentence,
   normalizeTimestamp,
+  invalidToolInputResult,
   PermanentEventError,
   PERMANENT_RESPONSE_CODES,
+  MemoryEventLock,
   HookConfigurationError,
   EventReceiver,
   ConstellationError,

package/dist/types/client-core/src/agent.d.ts

@@ -10,34 +10,69 @@ export declare class PermanentEventError extends Error {
 export declare class HookConfigurationError extends PermanentEventError {
     constructor(message: string);
 }
-/** Application-owned durable event checkpoints. All methods must survive process restart. */
-export interface EventStore {
-    /** Atomically retain the event before processing it.
-     * Duplicate IDs must compare equal payloads. Resolve only after durable receipt. */
-    accept(event: WebhookEvent): Promise<void>;
-    /** Serialize processing per event. Return the saved response if present; otherwise
-     * run prepare and durably store its exact result before returning. A crash can
-     * rerun prepare: tool/billing effects must use the supplied stable identities. */
-    prepare(event: WebhookEvent, prepare: () => Promise<Admission | null>): Promise<Admission | null>;
-    /** Atomically mark the event processed after its saved response settled. `receipt.outcome` is
-     * `accepted`, `already_accepted` or `event_closed`; for `event_closed`, `receipt.observationsRecorded`
-     * says whether the continuation's telemetry was kept. `null` when no response was submitted. */
-    complete(eventId: string, receipt: AdmissionReceipt | null): Promise<void>;
-    /** Durably stop retries of a rejected saved response. Replays must rethrow this rejection. */
-    reject(eventId: string, error: PermanentEventError): Promise<void>;
-    /** Persist retry count, first/last failure timestamps and latest error in the existing inbox. Non-HTTP failures use status 0; alert on age/repeated failures.
-     * If this hook throws, processEvent still rethrows the original failure, with the hook error as its `retryHookError` property. */
-    retry?(eventId: string, error: {
+/** Exclusive, expiring ownership of one event across the application's processes.
+ * Constellation re-sends an event that stays unanswered; whoever holds the lock
+ * answers that re-send with 202 instead of starting a second run. */
+export interface EventLock {
+    /** Take the event for `ttlMs`. False while another holder's lease is live. */
+    acquire(eventId: string, ttlMs: number): Promise<boolean>;
+    /** Extend this holder's lease. False if it expired or another holder took it. */
+    renew(eventId: string, ttlMs: number): Promise<boolean>;
+    release(eventId: string): Promise<void>;
+}
+/** Correct for a single process. Use `redisEventLock` when several processes receive events. */
+export declare class MemoryEventLock implements EventLock {
+    #private;
+    acquire(eventId: string, ttlMs: number): Promise<boolean>;
+    renew(eventId: string, ttlMs: number): Promise<boolean>;
+    release(eventId: string): Promise<void>;
+}
+/** A lock shared through Redis. `command` sends one raw command and resolves its reply,
+ * for example `(args) => redis.call(...args)` (ioredis) or `(args) => client.sendCommand(args)` (node-redis). */
+export declare function redisEventLock(command: (args: string[]) => Promise<unknown>, options?: {
+    prefix?: string;
+}): EventLock;
+/** What processing an event did. `ended` says the request is over after this response:
+ * `completed` or `failed` from this answer, `closed` when the request had already ended. */
+export type EventOutcome = {
+    status: "busy";
+} | {
+    status: "answered";
+    action: "continue" | "stop";
+    receipt: AdmissionReceipt | null;
+    ended: "completed" | "failed" | "closed" | null;
+};
+export type EventOptions = {
+    signal?: AbortSignal;
+    /** Lock lease, renewed every third of it while processing. Keep it well below the response window. */
+    leaseMs?: number;
+    /** Reports a failure that leaves the event unanswered; Constellation will re-send it. */
+    onFailure?: (eventId: string, error: {
         status: number;
         code: string;
         message: string;
-    }): Promise<void>;
-}
+    }) => void | Promise<void>;
+};
+/** The error result returned to the model for a call whose arguments were not valid JSON. */
+export declare function invalidToolInputResult(call: {
+    name: string;
+    invalid?: {
+        error: string;
+    };
+}): ToolResult["result"];
+/** Webhook-side processing. After authenticating an event, call `claim`: `null` means another
+ * process holds it (answer 202); otherwise answer 204 and run the returned function in the
+ * background. Tools never run inside the webhook call. */
 export declare class EventReceiver {
-    private readonly store;
-    constructor(store: EventStore);
-    /** Call after webhook authentication. Return a successful HTTP response only after this resolves. */
-    receive(event: WebhookEvent): Promise<void>;
+    #private;
+    private readonly client;
+    private readonly hooks;
+    private readonly options;
+    /** `hooks` may be loaded per event, for example to bind them to the request's own records. */
+    constructor(client: RequestClient, hooks: AgentHooks | ((event: WebhookEvent) => Promise<AgentHooks>), options?: EventOptions & {
+        lock?: EventLock;
+    });
+    claim(event: WebhookEvent): Promise<((signal?: AbortSignal) => Promise<EventOutcome>) | null>;
 }
 export declare class ClientTelemetry {
     readonly parentSpanId?: string | undefined;
@@ -75,7 +110,6 @@ export interface AgentHooks {
 }
 /** Record the client processing interval and its actual child-span hierarchy. */
 export declare function prepareResponse(event: WebhookEvent, hooks: AgentHooks, signal?: AbortSignal): Promise<Admission | null>;
-/** Process an authenticated event using an application-owned durable checkpoint.
- * Any receipt (`accepted`, `already_accepted`, `event_closed`) completes the checkpoint; a permanent
- * error code rejects it with PermanentEventError; anything else reports through `store.retry` and throws. */
-export declare function processEvent(client: RequestClient, store: EventStore, event: WebhookEvent, hooks: AgentHooks, signal?: AbortSignal): Promise<void>;
+/** Process an authenticated event inline: take its lock, prepare and submit the response, release.
+ * Returns `busy` without doing anything while another process holds the event. */
+export declare function processEvent(client: RequestClient, lock: EventLock, event: WebhookEvent, hooks: AgentHooks, options?: EventOptions): Promise<EventOutcome>;

package/dist/types/client-core/src/index.d.ts

@@ -7,10 +7,17 @@ export type Tool = {
     description?: string;
     inputSchema: Json;
 };
+/** The provider's tool arguments were not valid JSON. `raw` is truncated to 16 KiB. */
+export type InvalidToolInput = {
+    raw: string;
+    error: string;
+};
+/** With `invalid` set, `arguments` is `{}`: answer with an error result instead of running the tool. */
 export type ToolCall = {
     id: string;
     name: string;
     arguments: Json;
+    invalid?: InvalidToolInput;
 };
 export type Message = {
     role: "system" | "user" | "assistant" | "tool";
@@ -122,6 +129,9 @@ export type AdmissionReceipt = {
     outcome?: ResponseOutcome;
     /** With `event_closed`: whether the continuation's telemetry was kept as late observations. */
     observationsRecorded?: boolean;
+    /** With `already_accepted`: false when this responder answered the event before with a
+     * different body (for example recomputed after a lost acknowledgement). The first answer stands. */
+    bodyMatched?: boolean;
 };
 export type Status = "queued" | "running" | "client_processing" | "completed" | "failed" | "cancelled";
 export type RequestRecord = {
@@ -349,6 +359,14 @@ export type ModelResolution = ModelScope & {
     headers?: Record<string, string>;
     nativeContentPolicy?: "require" | "omitCompleted" | "omitIncompatible";
 };
+/** Whose request a boundary event belongs to, so a receiver never reads the request back. */
+export type EventRequest = {
+    id: string;
+    externalId: string | null;
+    workspaceId: string;
+    userId: string | null;
+    idempotencyKey: string;
+};
 export type ModelCompletedEvent = {
     id: string;
     requestId: string;
@@ -357,10 +375,14 @@ export type ModelCompletedEvent = {
     type: "model.completed";
     data: {
         message: Message;
+        /** Step result: `stepId`, `stepInput` (`model`, `parameters`, `maxOutputTokens`, `timeoutMs`, `toolsBytes`),
+         * `usage`, costs, `finishReason` and the step's `attempts`. */
         result: Record<string, Json>;
+        /** 1-based number of the step this event completes. */
+        stepNumber: number;
+        request: EventRequest;
         events?: Json[];
         stepId?: string;
-        stepNumber?: number;
         attemptId?: string;
     };
 };
@@ -370,7 +392,10 @@ export type WebhookEvent = ModelCompletedEvent | {
     sequence: number;
     createdAt: string;
     type: "model.failed" | "model.cancelled";
-    data: Record<string, Json>;
+    data: Record<string, Json> & {
+        stepNumber: number;
+        request: EventRequest;
+    };
 };
 export declare class ConstellationError extends Error {
     status: number;

package/dist/types/sdk/src/index.d.ts

@@ -1,7 +1,7 @@
 /** External server SDK. Operator APIs and cookie sessions are internal-only clients. */
 import { RequestClient, type WebhookEvent, type Json, type Page, type PageQuery, type ModelResolution, type RequestRecord, type RequestEvent, type RecordTypes, type Timeline, type TimelineRow, type RequestListQuery, type RequestRecordsQuery, type RequestTimings, type TokenUsage } from "../../client-core/src/index";
 export { ConstellationError, uuid7, normalizeTimestamp, parseOpenerSentence } from "../../client-core/src/index";
-export type { InferenceParameters, Json, Tool, ToolCall, Message, RequestEventModes, RequestInput, Admission, AdmissionReceipt, ResponseOutcome, ObservationResponse, Observations, EventReference, ModelCompletedEvent, ModelDeltaEvent, ModelStreamFrame, RequestTimings, TimingInterval, TokenUsage, TokenPrice, CustomerPricing, BillingFacts, Status, RequestRecord, Step, Attempt, Span, TimelineRow, Timeline, Log, Inspection, RecordTypes, RequestEvent, Page, PageQuery, ModelScope, ModelResolution, WebhookEvent, RequestRecordsQuery, RequestListQuery, OpenerInput, OpenerTurn } from "../../client-core/src/index";
+export type { InferenceParameters, Json, Tool, ToolCall, InvalidToolInput, EventRequest, Message, RequestEventModes, RequestInput, Admission, AdmissionReceipt, ResponseOutcome, ObservationResponse, Observations, EventReference, ModelCompletedEvent, ModelDeltaEvent, ModelStreamFrame, RequestTimings, TimingInterval, TokenUsage, TokenPrice, CustomerPricing, BillingFacts, Status, RequestRecord, Step, Attempt, Span, TimelineRow, Timeline, Log, Inspection, RecordTypes, RequestEvent, Page, PageQuery, ModelScope, ModelResolution, WebhookEvent, RequestRecordsQuery, RequestListQuery, OpenerInput, OpenerTurn } from "../../client-core/src/index";
 /** OpenAPI contract filtered to this key's permissions and scope. */
 export type DiscoveryDocument = {
     openapi: string;
@@ -28,6 +28,63 @@ export type TelemetryPage<T> = Page<T> & {
 export type TelemetryQuery = {
     include?: "content";
 };
+/** Fields of each aggregate dataset, as POST /v1/telemetry/query accepts them. */
+export type AttemptField = "workspaceId" | "userId" | "externalId" | "requestId" | "providerId" | "offeringId" | "model" | "requestedModel" | "status" | "fallback" | "dispatched" | "errorCode" | "faultClass" | "httpStatus" | "errorMessage";
+export type AttemptNumber = "ttftMs" | "latencyMs" | "outputTokens" | "supplierCost" | "customerCost" | "httpStatus";
+export type RequestField = "workspaceId" | "userId" | "externalId" | "requestId" | "requestedModel" | "status";
+export type RequestNumber = "durationMs" | "supplierCost" | "customerCost";
+export type LogField = "level" | "name" | "status" | "requestId" | "message";
+/** Permanent billing record per settled request (unscoped telemetry keys only). */
+export type UsageField = "workspaceId" | "userId" | "externalId" | "requestId" | "requestedModel" | "planId" | "status";
+type UsageTotals = "supplierCost" | "customerCost" | "margin" | "inputTokens" | "cacheReadTokens" | "cacheWriteTokens" | "cacheWriteOneHourTokens" | "outputTokens" | "reasoningTokens";
+export type UsageNumber = UsageTotals | "durationMs";
+/** Permanent billing record per provider attempt; `status` is the attempt's, `requestStatus` the request's. */
+export type UsageAttemptField = Exclude<UsageField, "status"> | "requestStatus" | "providerId" | "model" | "providerModel" | "status";
+export type UsageAttemptNumber = UsageTotals;
+export type AggregateTimeGroup = "minute" | "hour" | "day" | "week" | "month";
+export type AggregateScalar = string | number | boolean;
+/** A value, any of several values, or operators combined with AND. `contains` is case-insensitive. */
+export type AggregateFilter = AggregateScalar | AggregateScalar[] | {
+    eq?: AggregateScalar;
+    in?: AggregateScalar[];
+    not?: AggregateScalar;
+    notIn?: AggregateScalar[];
+    contains?: string;
+    startsWith?: string;
+    gte?: number;
+    gt?: number;
+    lte?: number;
+    lt?: number;
+    /** true: the field has a value; false: it is empty. */
+    exists?: boolean;
+};
+export type AggregateMetric<N extends string, F extends string> = "count" | `uniq:${F}` | `${"sum" | "avg" | "min" | "max" | "p50" | "p90" | "p95" | "p99"}:${N}`;
+type AggregateShape<D extends string, F extends string, N extends string> = {
+    dataset: D;
+    /** Inclusive start and exclusive end; at most 92 days apart (usage datasets: any range). */
+    from: string | Date;
+    to: string | Date;
+    where?: Partial<Record<F | N, AggregateFilter>>;
+    /** At most four. Numeric fields other than httpStatus are aggregated, not grouped. */
+    groupBy?: (F | AggregateTimeGroup)[];
+    /** One to eight; default ["count"]. */
+    metrics?: AggregateMetric<N, F>[];
+    orderBy?: {
+        by: string;
+        descending?: boolean;
+    };
+    /** Default 100, at most 1,000 rows (usage datasets: 10,000). */
+    limit?: number;
+};
+/** A structured aggregate over telemetry facts or permanent usage. The key's workspace/user scope always applies; logs and usage need an unscoped key. */
+export type AggregateQuery = AggregateShape<"attempts", AttemptField, AttemptNumber> | AggregateShape<"requests", RequestField, RequestNumber> | AggregateShape<"logs", LogField, never> | AggregateShape<"usage", UsageField, UsageNumber> | AggregateShape<"usageAttempts", UsageAttemptField, UsageAttemptNumber>;
+export type AggregateResult = {
+    /** Groupings first, then metrics, in request order. */
+    columns: string[];
+    rows: Record<string, Json>[];
+    /** More groups matched than `limit`. */
+    truncated: boolean;
+};
 export type TelemetryReadOptions = TelemetryQuery & {
     signal?: AbortSignal;
 };
@@ -191,6 +248,8 @@ export declare class Constellation extends RequestClient {
                 after?: number;
             }, signal?: AbortSignal) => Promise<Json>;
         };
+        /** Count, group and aggregate attempts, requests or logs, e.g. failed attempts per offering and error code. */
+        query: (query: AggregateQuery, signal?: AbortSignal) => Promise<AggregateResult>;
         logs: {
             searchSystem: (query?: LogSearchQuery & TelemetryQuery, signal?: AbortSignal) => Promise<Page<Json>>;
             searchRequests: (query?: Omit<LogSearchQuery, "level" | "levelNot" | "target" | "targetNot"> & {
@@ -204,5 +263,5 @@ export declare class Constellation extends RequestClient {
         verify: (request: Request, expectedToken: string) => Promise<WebhookEvent>;
     };
 }
-export { PermanentEventError, HookConfigurationError, EventReceiver, prepareResponse, processEvent, ClientTelemetry, PERMANENT_RESPONSE_CODES } from "../../client-core/src/agent";
-export type { EventStore, AgentHooks, HookContext, ToolResult } from "../../client-core/src/agent";
+export { PermanentEventError, HookConfigurationError, EventReceiver, MemoryEventLock, redisEventLock, prepareResponse, processEvent, ClientTelemetry, PERMANENT_RESPONSE_CODES, invalidToolInputResult } from "../../client-core/src/agent";
+export type { EventLock, EventOutcome, EventOptions, AgentHooks, HookContext, ToolResult } from "../../client-core/src/agent";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lionhummer/constellation",
-  "version": "0.9.2",
+  "version": "0.10.0",
   "description": "TypeScript client for durable model requests and model events",
   "type": "module",
   "license": "UNLICENSED",