@relaymessenger/sdk 0.3.6-staging.3 → 0.3.6-staging.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +490 -20
  2. package/dist/a2ui.d.ts +79 -0
  3. package/dist/a2ui.d.ts.map +1 -0
  4. package/dist/a2ui.js +99 -0
  5. package/dist/a2ui.js.map +1 -0
  6. package/dist/buttons.d.ts +69 -0
  7. package/dist/buttons.d.ts.map +1 -0
  8. package/dist/buttons.js +141 -0
  9. package/dist/buttons.js.map +1 -0
  10. package/dist/calls/call-room.d.ts +110 -0
  11. package/dist/calls/call-room.d.ts.map +1 -0
  12. package/dist/calls/call-room.js +616 -0
  13. package/dist/calls/call-room.js.map +1 -0
  14. package/dist/calls/engine-werift-video.d.ts +70 -0
  15. package/dist/calls/engine-werift-video.d.ts.map +1 -0
  16. package/dist/calls/engine-werift-video.js +594 -0
  17. package/dist/calls/engine-werift-video.js.map +1 -0
  18. package/dist/calls/engine-werift.d.ts +48 -0
  19. package/dist/calls/engine-werift.d.ts.map +1 -0
  20. package/dist/calls/engine-werift.js +457 -0
  21. package/dist/calls/engine-werift.js.map +1 -0
  22. package/dist/calls/index.d.ts +11 -0
  23. package/dist/calls/index.d.ts.map +1 -0
  24. package/dist/calls/index.js +11 -0
  25. package/dist/calls/index.js.map +1 -0
  26. package/dist/calls/transport.d.ts +466 -0
  27. package/dist/calls/transport.d.ts.map +1 -0
  28. package/dist/calls/transport.js +1409 -0
  29. package/dist/calls/transport.js.map +1 -0
  30. package/dist/calls/video-frame.d.ts +56 -0
  31. package/dist/calls/video-frame.d.ts.map +1 -0
  32. package/dist/calls/video-frame.js +245 -0
  33. package/dist/calls/video-frame.js.map +1 -0
  34. package/dist/calls/video-rtp.d.ts +51 -0
  35. package/dist/calls/video-rtp.d.ts.map +1 -0
  36. package/dist/calls/video-rtp.js +294 -0
  37. package/dist/calls/video-rtp.js.map +1 -0
  38. package/dist/calls/video.d.ts +188 -0
  39. package/dist/calls/video.d.ts.map +1 -0
  40. package/dist/calls/video.js +228 -0
  41. package/dist/calls/video.js.map +1 -0
  42. package/dist/client.d.ts +134 -24
  43. package/dist/client.d.ts.map +1 -1
  44. package/dist/client.js +321 -33
  45. package/dist/client.js.map +1 -1
  46. package/dist/errors.d.ts +11 -0
  47. package/dist/errors.d.ts.map +1 -1
  48. package/dist/errors.js +16 -0
  49. package/dist/errors.js.map +1 -1
  50. package/dist/index.d.ts +8 -2
  51. package/dist/index.d.ts.map +1 -1
  52. package/dist/index.js +8 -2
  53. package/dist/index.js.map +1 -1
  54. package/dist/links.d.ts +57 -0
  55. package/dist/links.d.ts.map +1 -0
  56. package/dist/links.js +126 -0
  57. package/dist/links.js.map +1 -0
  58. package/dist/operations.d.ts +69 -25
  59. package/dist/operations.d.ts.map +1 -1
  60. package/dist/operations.js +94 -30
  61. package/dist/operations.js.map +1 -1
  62. package/dist/payment.d.ts +67 -0
  63. package/dist/payment.d.ts.map +1 -0
  64. package/dist/payment.js +147 -0
  65. package/dist/payment.js.map +1 -0
  66. package/dist/selection.d.ts +38 -0
  67. package/dist/selection.d.ts.map +1 -0
  68. package/dist/selection.js +132 -0
  69. package/dist/selection.js.map +1 -0
  70. package/dist/types.d.ts +960 -56
  71. package/dist/types.d.ts.map +1 -1
  72. package/dist/websocket.d.ts +8 -2
  73. package/dist/websocket.d.ts.map +1 -1
  74. package/dist/websocket.js +29 -10
  75. package/dist/websocket.js.map +1 -1
  76. package/package.json +38 -3
package/README.md CHANGED
@@ -2,6 +2,218 @@
2
2
 
3
3
  TypeScript client for Relay v1.
4
4
 
5
+ ## Selection
6
+
7
+ The SDK exposes `SelectionPart`, `SelectionPartResponse` (including read-only
8
+ viewer-relative `has_responded`), and `SelectionResponsePart` through the
9
+ existing message and event unions.
10
+
11
+ ```ts
12
+ import { partsWithSelection } from "@relaymessenger/sdk";
13
+
14
+ const parts = partsWithSelection("Which topics interest you?", {
15
+ type: "selection",
16
+ options: [
17
+ { value: "research", label: "Research" },
18
+ { value: "design", label: "Design" },
19
+ ],
20
+ });
21
+ // Send with relay.chats.messages.send(chatId, { message: { parts, idempotency_key } }).
22
+ ```
23
+
24
+ `selectionPart` validates an options array or complete part, returning a
25
+ normalized part or an error string. Options are limited to 25, trimmed labels
26
+ to 80 characters, and explicit case-sensitive ASCII token values to 100.
27
+ Unknown fields, duplicate values, and blank labels are rejected.
28
+ `partsWithSelection` also requires nonblank question text.
29
+
30
+ Selection inherits existing Chat membership rules: at most one human user,
31
+ with multiple agents allowed. Only the human user can respond; agents cannot.
32
+ The durable per-user response claim applies across that user's devices and
33
+ idempotency keys, without expanding group membership.
34
+
35
+ `answerMessages` accepts a `selection` fenced JSON block containing the options
36
+ array. It keeps invalid blocks as text with an error and never combines a
37
+ selection with buttons. Existing buttons retain their behavior.
38
+
39
+ The person opens the prompt, checks any number of options and submits them
40
+ once; checking sends nothing and only the submit does. A person answers a given
41
+ selection once, and reopening it afterwards shows what they chose without
42
+ letting them change it.
43
+
44
+ The user's new reply is exactly literal `• ` + each selected source label
45
+ joined with `\n`, then
46
+ `{ type: "selection_response", selected_values: ["research", "design"] }`,
47
+ in source-option order, with explicit `reply_to.message_id` and `part_index`.
48
+ The user client keeps its existing outgoing idempotency identity for retries.
49
+ The server also accepts exact legacy comma-joined source labels only for
50
+ compatibility. iOS may draw a checkmark in place of each bullet and repeat the
51
+ prompt's title, as presentation only, but portable text stays bulleted.
52
+ Metadata has no additional display text. Use `selected_values` and the source
53
+ reply target to dispatch your own handler, rather than splitting labels.
54
+ Signed webhook `unwrap` and WebSocket `onEvent` default types expose this
55
+ metadata after narrowing to `message.received`.
56
+
57
+ The CLI, Pi, OpenClaw, the Claude Code channel, MCP, and the Chat SDK adapter
58
+ include selection guidance and structured inbound discovery.
59
+ `selectionReply(parts, replyTo)` discovers values and the explicit source target.
60
+
61
+ ## Payment
62
+
63
+ Ask a person to pay in two steps. Create a payment request on your
64
+ organization's connected Stripe account, then send its `checkout_url` as a
65
+ `payment` part, the only part of its Message. The card reads its amount and
66
+ title from the request. The money settles to your Stripe account.
67
+
68
+ ```ts
69
+ const request = await relay.paymentRequests.create(
70
+ {
71
+ amount: 2400, // minor units
72
+ currency: "usd",
73
+ description: "House blend, 250 g",
74
+ category: "physical_goods", // or "digital_goods", "donation"
75
+ },
76
+ { idempotencyKey: `order-${orderId}` },
77
+ );
78
+ await relay.chats.messages.send(chatId, {
79
+ message: {
80
+ parts: [{ type: "payment", checkout_url: request.checkout_url }],
81
+ idempotency_key: crypto.randomUUID(),
82
+ },
83
+ });
84
+ ```
85
+
86
+ For a subscription, pass `mode: "subscription"` and a recurring `price_id`
87
+ instead of `amount` and `currency`. `relay.paymentRequests.list`, `retrieve`
88
+ and `cancel` read and cancel requests. The status moves only on Stripe's word
89
+ or your cancel, and arrives as `payment.succeeded`, `payment.canceled` or
90
+ `payment.expired`; a paid request also adds a `payment_receipt` message from
91
+ the payer, which arrives as `message.received`.
92
+
93
+ A model inside a bridge never holds the Relay token, so it gives the request's
94
+ fields and the bridge creates it. `answerMessages` lifts one `payment` fenced
95
+ JSON block (`description`, `category`, `amount` and `currency`, or
96
+ `mode: "subscription"` with `price_id`) out of the words and returns it as
97
+ `payment`, checked by `paymentRequestFields`; `createPaymentPart` creates the
98
+ request on the card Message's own idempotency key and returns the `payment`
99
+ part to send last. A second payment, or a payment beside buttons or a
100
+ selection, stays text with an error. `PAYMENT_GUIDANCE` is the text the CLI,
101
+ Pi, OpenClaw, MCP and the Claude Code channel carry; the text bridges also
102
+ carry `PAYMENT_BLOCK_INSTRUCTION`.
103
+
104
+ ## Location
105
+
106
+ Ask the person in a one-to-one chat to share their location, then read it.
107
+ The request puts a `location_request` Message from your agent in the chat;
108
+ the person chooses whether to share and for how long.
109
+
110
+ ```ts
111
+ await relay.chats.location.request(chatId);
112
+
113
+ // After `location.sharing.started` arrives:
114
+ const { data } = await relay.chats.location.retrieve(chatId);
115
+ for (const feature of data.features) {
116
+ const [longitude, latitude] = feature.geometry.coordinates;
117
+ console.log(feature.properties.handle, latitude, longitude, feature.properties.updated_at);
118
+ }
119
+ ```
120
+
121
+ `data` is a GeoJSON FeatureCollection, longitude first; `features` is empty
122
+ when nobody is sharing. `location.sharing.started` and
123
+ `location.sharing.stopped` fire when a share begins or ends, never when the
124
+ position moves, so read again when you need the latest position. A request
125
+ returns 409 in a group chat, in a chat with no person, or while the person is
126
+ already sharing, and 429 with `Retry-After` after one request in the same chat
127
+ in the last 60 seconds. The person's card arrives as a `location` part that
128
+ carries the share's state, never its position.
129
+
130
+ ## Cards (A2UI)
131
+
132
+ A card is an [A2UI v0.9.1](https://a2ui.org) surface, sent as a `data` part:
133
+ `{ type: "data", media_type: "application/a2ui+json", data: [A2UI messages] }`.
134
+ Send the card, read the tap, then change the same card in place.
135
+
136
+ ```ts
137
+ import Relay, {
138
+ A2UI_BASIC_CATALOG_ID,
139
+ readA2uiAction,
140
+ sendA2uiSurface,
141
+ updateA2uiSurface,
142
+ } from "@relaymessenger/sdk";
143
+
144
+ const relay = new Relay({
145
+ apiKey: process.env.RELAY_AGENT_TOKEN!,
146
+ webhookSecret: process.env.RELAY_WEBHOOK_SECRET!,
147
+ });
148
+
149
+ // 1. Send a card with a button.
150
+ await sendA2uiSurface(relay, chatId, {
151
+ surfaceId: "order-1042",
152
+ catalogId: A2UI_BASIC_CATALOG_ID,
153
+ components: [
154
+ { id: "root", component: "Card", child: "body" },
155
+ { id: "body", component: "Column", children: ["title", "status", "confirm"] },
156
+ { id: "title", component: "Text", text: "Oat latte, large", variant: "h3" },
157
+ { id: "status", component: "Text", text: { path: "/status" } },
158
+ { id: "confirm_label", component: "Text", text: "Confirm order" },
159
+ {
160
+ id: "confirm",
161
+ component: "Button",
162
+ child: "confirm_label",
163
+ variant: "primary",
164
+ action: { event: { name: "confirm_order", context: { order: "1042" } } },
165
+ },
166
+ ],
167
+ dataModel: { status: "Waiting for you" },
168
+ }, { idempotency_key: "order-1042-card" });
169
+
170
+ // 2. The tap arrives as message.received.
171
+ const event = relay.webhooks.unwrap(rawBody, { headers });
172
+ const tap = readA2uiAction(event);
173
+ if (tap?.action.name === "confirm_order" && event.event_type === "message.received") {
174
+ // 3. Change the same card to its done state. No new Message is added.
175
+ await updateA2uiSurface(relay, event.data.chat.id, tap.action.surfaceId, {
176
+ components: [
177
+ { id: "body", component: "Column", children: ["title", "status"] },
178
+ ],
179
+ dataModel: { path: "/status", value: `Confirmed, order ${tap.action.context.order}` },
180
+ });
181
+ }
182
+ ```
183
+
184
+ `sendA2uiSurface` sends `createSurface`, `updateComponents` and, with
185
+ `dataModel`, `updateDataModel` in one data part. `updateA2uiSurface` sends
186
+ `updateComponents` and `updateDataModel` for a surface already in the chat;
187
+ components replace their namesakes by `id`. A surface's first
188
+ `updateComponents` holds the component with the id `root`. `deleteA2uiSurface` sends
189
+ `deleteSurface`; when every surface of a Message is deleted, the Message reads
190
+ back with no parts and a non-null `unsent_at`. Each takes the rest of the
191
+ Message as its fourth argument: `text` becomes a text part before the card,
192
+ and `reply_to`, `idempotency_key`, `silent` and `metadata` pass through.
193
+ `a2uiPart` wraps any list of A2UI messages in a data part for
194
+ `relay.chats.messages.send`.
195
+
196
+ Relay applies each A2UI message on its own, checked against A2UI's schemas and
197
+ the catalog the surface names; a component, property or value the catalog does
198
+ not define is refused. The messages it could not apply come back in the
199
+ response's `a2ui_errors`: each gives `part_index` and `data_index` in your
200
+ request and `a2ui_message`, A2UI's own `error` message, whose `path` points
201
+ inside the failing message's body. A send that applies nothing throws a
202
+ `RelayAPIError` (404 unknown or deleted surface, 409 a `surfaceId` already live
203
+ in the chat, 422 anything else) whose `body.a2ui_errors` lists each.
204
+
205
+ Every `message.received` carries `metadata.a2uiClientCapabilities`: the
206
+ catalogs Relay's app draws, in order of preference, `RELAY_A2UI_CATALOG_ID`
207
+ (every basic catalog component and function, plus `PaymentRequest`) and
208
+ `A2UI_BASIC_CATALOG_ID`. When a surface sets `sendDataModel: true`, each tap
209
+ also carries that surface's data model; `readA2uiAction` returns it as
210
+ `dataModel`. A tap reaches only the person who tapped and the agent that
211
+ created the surface. Only an agent sends `createSurface`, `updateComponents`,
212
+ `updateDataModel` and `deleteSurface`; any agent in the chat may update any
213
+ surface in it.
214
+
215
+ ## Chat permissions
216
+
5
217
  Relay Chats support one human user with one or more agents. Contacts, Handles,
6
218
  and Participants remain generic API types, but selectable participants are
7
219
  agents. Agent-to-agent Chats remain supported. This Agent SDK does not expose phone address-book syncing, mutual
@@ -65,6 +277,46 @@ await relay.chats.messages.send(chatId, {
65
277
  });
66
278
  ```
67
279
 
280
+ ## Show task activity
281
+
282
+ Start activity when the work starts. Image generation uses `🖼️ Generating image`;
283
+ voice-note generation uses `🎙️ Generating voice note`.
284
+
285
+ ```ts
286
+ const state = await relay.chats.setActivity(chatId, {
287
+ text: "Generating image",
288
+ emoji: "🖼️",
289
+ });
290
+ const activityId = state.activity!.id;
291
+ ```
292
+
293
+ Renew every 60 seconds only while that task remains active. The same ID guards
294
+ updates against replacing a newer task; a replaced or cleared ID returns 409.
295
+ Each successful update renews the 90-second safety lease:
296
+
297
+ ```ts
298
+ await relay.chats.setActivity(chatId, {
299
+ text: "Generating image",
300
+ emoji: "🖼️",
301
+ activity_id: activityId,
302
+ });
303
+ ```
304
+
305
+ Clear when work completes, fails, or is cancelled. Guard cleanup with the task
306
+ ID so an old task cannot clear a newer one. Missing or replaced activity is a
307
+ successful no-op:
308
+
309
+ ```ts
310
+ await relay.chats.clearActivity(chatId, { activity_id: activityId });
311
+ const current = await relay.chats.getActivity(chatId);
312
+ ```
313
+
314
+ GET reads your own state. Text allows 1–21 visible characters and at most
315
+ 1024 UTF-8 bytes; `emoji` is one Unicode emoji or `null`. Response `version`
316
+ is a string and `activity` is an object or `null`. Chat handles may also carry
317
+ `activity_version` and `activity`. Typing remains `chats.startTyping` and
318
+ `chats.stopTyping`; activity does not add an agent event or polling transport.
319
+
68
320
  ## Share your Contact Card
69
321
 
70
322
  ```ts
@@ -107,6 +359,7 @@ Available resource methods:
107
359
  - `webhookEvents.list`
108
360
  - `webhookSubscriptions.create`, `retrieve`, `update`, `list`, `delete`
109
361
  - `contactCard.create`, `retrieve`, `update`
362
+ - `contacts.lookup({ handle })`
110
363
  - `blockedHandles.list`, `block`, `unblock`
111
364
  - `websocket.run`
112
365
 
@@ -179,28 +432,245 @@ attachment rules are at <https://docs.relayapp.im>.
179
432
  ## Individual audio Calls (staging)
180
433
 
181
434
  Calls join one user and one agent in an existing individual Chat. Use
182
- `relay.calls.create(chatId, { to: [handle], mode: "audio" }, { idempotencyKey })`;
435
+ `relay.calls.create(chatId, { to: [handle] }, { idempotencyKey })`;
183
436
  keep the same key and body when retrying an uncertain create response.
184
437
 
185
- `relay.calls` also exposes `retrieve`, `list`, `accept`, `decline`, `end`, and
186
- `connected`. Receive typed `call.created`, `call.updated`, and `call.ended`
187
- events through the existing signed Webhook or Agent WebSocket.
188
-
189
- After accepting a Call, an agent creates its audio connection with
190
- `relay.calls.connections.create(callId, { transport: "websocket" })`.
191
- The returned `connection.url` and short-lived `connection.token` belong to
192
- that Call's media socket. Connect using `Authorization: Bearer <token>`.
193
- They do not replace the Agent Token used for REST requests.
194
-
195
- Media uses raw PCM16 little-endian, 48 kHz stereo binary frames. The server
196
- sends JSON `start` with the format, then `ready`. Send `{"type":"clear"}` to
197
- discard unsent speech. An `ended` frame terminates the media socket. This
198
- socket is separate from `relay.websocket.run`, which carries durable events.
199
-
200
- WebRTC clients use the same `connections.create` resource with an SDP offer
201
- and microphone MID, then `connections.subscribe` and
202
- `connections.renegotiate`. The SDK exposes no media-provider credentials,
203
- session IDs, model configuration, or audio generation.
438
+ `relay.calls` also exposes `retrieve`, `list`, `room`, and `end`. Receive typed
439
+ `call.created`, `call.updated`, and `call.ended` events through the existing
440
+ signed Webhook or acknowledged Agent WebSocket. Agents do not configure a call
441
+ URL; receiving `call.created` and joining that Call's room is the answer path.
442
+
443
+ Every participant holds one signaling socket to the Call room.
444
+ `relay.calls.room(callId)` targets `GET /v1/calls/{callId}/room` with the same
445
+ bearer as every REST route. Register handlers, call `connect()`, and the SDK
446
+ sends `join`; `room.state` then tracks the latest `roomState`. Nothing is polled
447
+ and no audio bytes pass through this WebSocket.
448
+
449
+ ```ts
450
+ const room = relay.calls.room(event.data.call.id);
451
+ room.on("roomState", ({ call, participants }) => {
452
+ // Each participant has track: "audio" | null, muted and connected state.
453
+ });
454
+ room.on("answer", ({ session_description }) => {
455
+ // Apply Relay's answer to the local WebRTC publication.
456
+ });
457
+ room.on("offer", ({ session_description, track }) => {
458
+ // Relay subscribed this participant to the other side's `audio` track.
459
+ // Apply the offer, create a WebRTC answer, gather ICE, then send it:
460
+ room.send({ type: "answer", session_description: { type: "answer", sdp } });
461
+ });
462
+ room.on("ended", ({ reason }) => { /* the socket closes right after */ });
463
+ await room.connect(); // sends { type: "join" }; a callee answers here
464
+ ```
465
+
466
+ The stable client frames are `join`, `offer`, `answer`, `connected`,
467
+ `userUpdate`, `end`, and `heartbeat`. A publishing `offer` has exactly one
468
+ `tracks` entry, `{ mid, name: "audio" }`. Stable server frames are `roomState`,
469
+ `answer`, `offer` with `track: "audio"`, `ended`, and `error`; every
470
+ `roomState.participants[]` item reports `track: "audio" | null`. The SDK sends
471
+ `heartbeat` every 15 seconds until `close()`. `connected()`, `userUpdate()` and
472
+ `end()` are convenience methods for those frames, and `reconnect()` replaces
473
+ the signaling socket without first dropping the old one.
474
+
475
+ WebRTC carries the audio. Relay's server owns SFU/provider details, so clients
476
+ only exchange SDP in the room protocol and never receive Cloudflare
477
+ credentials, session IDs, or ICE-provider configuration. Node agents that want
478
+ this SDP/ICE and PCM plumbing handled for them use `RelayCallTransport` from
479
+ `@relaymessenger/sdk/calls`, below; `@relaymessenger/livekit` plugs it into
480
+ LiveKit Agents audio input and output.
481
+
482
+ ### Join a Call as the agent
483
+
484
+ `@relaymessenger/sdk/calls` owns the agent's WebRTC peer, so your code never
485
+ handles SDP, ICE, or SFU credentials. It needs WebRTC packages that
486
+ `@relaymessenger/sdk` lists as optional peer dependencies; install them only if
487
+ your agent joins Calls (`node-webcodecs` only for video):
488
+
489
+ ```sh
490
+ npm install @relaymessenger/sdk werift @evan/opus rtp-packet node-webcodecs
491
+ ```
492
+
493
+ ```ts
494
+ import Relay from "@relaymessenger/sdk";
495
+ import { RelayCallTransport } from "@relaymessenger/sdk/calls";
496
+
497
+ const relay = new Relay({ apiKey: process.env.RELAY_AGENT_TOKEN! });
498
+ const transport = new RelayCallTransport({ relay, callId });
499
+
500
+ transport.on("audio", ({ samples, sampleRate, channelCount }) => {
501
+ // Interleaved signed PCM16 from the remote Relay participant.
502
+ });
503
+ // Inbound audio is 48 kHz stereo unless you pass
504
+ // `inboundAudio: { sampleRate: 8000 | 12000 | 16000 | 24000 | 48000, channelCount: 1 | 2 }`;
505
+ // the `"wrtc"` engine accepts only the default.
506
+
507
+ await transport.connect();
508
+ await transport.writeAudio({
509
+ samples,
510
+ sampleRate: 48_000,
511
+ channelCount: 1,
512
+ });
513
+ await transport.waitForPlayout();
514
+ ```
515
+
516
+ `writeAudio()` resolves once its 10 ms slices are queued, `queuedAudioMs()` is
517
+ what has not left yet, and `waitForPlayout()` resolves when the queue is empty
518
+ and the pump is idle (early on `clearAudio()`). Until the other participant is
519
+ receiving the agent's audio (`subscribed`: its `roomState` entry lists `audio`
520
+ in `receiving`), written audio is held, in order, and the track keeps sending
521
+ silence; then it plays from the start, so a greeting that starts early is not
522
+ cut. A restart holds again until the new session is received. `clearAudio()`
523
+ drops held audio too.
524
+
525
+ ### Video
526
+
527
+ Relay calls carry video whenever a camera is on. The video API copies
528
+ LiveKit's `@livekit/rtc-node` names: `VideoSource`, `LocalVideoTrack`,
529
+ `VideoFrame`, `VideoBufferType`, `VideoStream`. Video needs the default
530
+ `"werift"` engine and the optional dependency `node-webcodecs` (prebuilt for
531
+ macOS arm64 and Linux x64/arm64).
532
+
533
+ Send a video feed:
534
+
535
+ ```ts
536
+ import {
537
+ LocalVideoTrack,
538
+ VideoBufferType,
539
+ VideoFrame,
540
+ VideoSource,
541
+ } from "@relaymessenger/sdk/calls";
542
+
543
+ await transport.connect();
544
+
545
+ const source = new VideoSource(640, 480);
546
+ const track = LocalVideoTrack.createVideoTrack("camera", source);
547
+ await transport.publishTrack(track, {
548
+ videoEncoding: { maxBitrate: 800_000, maxFramerate: 15 },
549
+ });
550
+
551
+ // RGBA, BGRA or I420 bytes, tightly packed.
552
+ source.captureFrame(new VideoFrame(rgba, 640, 480, VideoBufferType.RGBA));
553
+
554
+ // Camera off, then on again; the track stays negotiated.
555
+ await transport.unpublishTrack(track);
556
+ await transport.publishTrack(track);
557
+ ```
558
+
559
+ Receive the other participant's video:
560
+
561
+ ```ts
562
+ import { VideoBufferType, VideoStream } from "@relaymessenger/sdk/calls";
563
+
564
+ transport.on("trackSubscribed", async (track) => {
565
+ const stream = new VideoStream(track, { format: VideoBufferType.RGBA, capacity: 2 });
566
+ for await (const { frame, timestampUs } of stream) {
567
+ // frame.data is width x height x 4 bytes of RGBA.
568
+ }
569
+ });
570
+ transport.on("remoteVideo", (on) => {
571
+ // The other participant's camera started or stopped sending.
572
+ });
573
+ ```
574
+
575
+ Frames are decoded only while a `VideoStream` is open. `transport.videoStats()`
576
+ reports frames, packets, keyframes and decode errors in both directions.
577
+
578
+ ### ICE servers, TURN, and diagnostics
579
+
580
+ By default the peer uses the servers the Call room sends after it joins:
581
+ Cloudflare's STUN server and TURN credentials that Relay mints for the call.
582
+ An agent in a container or behind a firewall that blocks outbound UDP connects
583
+ through TURN with no setup. Each restart uses the room's latest servers. A room
584
+ that sends none falls back to Cloudflare's STUN server
585
+ (`stun:stun.cloudflare.com:3478`), as Cloudflare's own Realtime echo example
586
+ does.
587
+
588
+ The offer leaves as soon as the first local candidate exists, without waiting
589
+ for ICE gathering to finish: the SFU is ICE-lite and learns the agent's address
590
+ from its connectivity checks, TURN relay checks included.
591
+
592
+ To use your own servers instead, pass `iceServers` in the standard
593
+ `RTCIceServer` shape and, if every path must go through TURN,
594
+ `iceTransportPolicy: "relay"`. Your value replaces the room's. Both options are
595
+ accepted by `RelayCallTransport` and by `RelayLiveKitCall.connect()` in
596
+ `@relaymessenger/livekit`:
597
+
598
+ ```ts
599
+ const transport = new RelayCallTransport({
600
+ relay,
601
+ callId,
602
+ iceServers: [
603
+ {
604
+ urls: [
605
+ "turn:turn.example.com:3478?transport=udp",
606
+ "turn:turn.example.com:3478?transport=tcp",
607
+ "turns:turn.example.com:5349?transport=tcp",
608
+ ],
609
+ username: process.env.TURN_USERNAME!,
610
+ credential: process.env.TURN_CREDENTIAL!,
611
+ },
612
+ ],
613
+ iceTransportPolicy: "all",
614
+ });
615
+ ```
616
+
617
+ `iceServers` may also be a function; it is called before every peer
618
+ connection, so it can mint fresh TURN credentials for each restart. The room's
619
+ servers are on `room.iceServers` and its `iceServers` event.
620
+
621
+ `connect()` resolves when media first reaches `connected`. It has no overall
622
+ deadline: when an SFU session is not `connected` within
623
+ `sessionConnectTimeoutMs` (5 seconds by default) of its answer, becomes
624
+ `failed`, or stays `disconnected` for 7 seconds, the transport closes that
625
+ peer, waits 250 ms (x1.1 per further attempt, at most 10 s), and publishes
626
+ from a new peer on a new session, for as long as the Call is ringing or in
627
+ progress. Outgoing audio keeps flowing into the new peer. `connect()` rejects
628
+ only when the Call ends, the room or transport closes, the room reports an
629
+ error, or the `signal` passed to it aborts. Each replacement emits
630
+ `restarted` with the reason and the replaced session's summary, for example
631
+ `local: host 2, srflx 0, relay 0; remote: udp 1473; states: new→complete 0.2s, connecting 0.3s, no connected; …`,
632
+ and `diagnostics().restarts` counts them. `mediaConnectTimeoutMs` and
633
+ `connectionTimeoutMs` are deprecated names for `sessionConnectTimeoutMs`.
634
+
635
+ `transport.waitForPeerAudio(timeoutMs)` resolves once the person's audio has
636
+ arrived and the room shows them connected (the transport's `peerAudio` event);
637
+ start the agent's session after it so the first words are heard.
638
+
639
+ The same facts are available at any time from `transport.diagnostics()`: local candidate counts by type, the remote
640
+ candidates' transport and port (never their address), the ICE gathering, ICE
641
+ connection and peer connection state changes with their offsets from
642
+ `connect()`, packet counts in both directions (`inbound`: RTP received, Opus
643
+ decode failures, PCM frames delivered, first and last packet offsets, packets
644
+ in the last 5 s; `outbound`: PCM frames accepted, Opus packets, RTP written
645
+ with the caller's audio, RTP written with silence, first and last packet
646
+ offsets, packets in the last 5 s, paced queue size, pacer state), the room frames seen (`roomState` count, pull `offer` count,
647
+ `ended` reason, `error` messages), and the one-line `summary`, for example
648
+ `…; in: 1234 rtp, 0 bad, 1234 frames, first 0.9s last 41.2s, 250/5s; out: 2600 frames, 1300 opus, 1300 rtp, silence 700, first 1.1s last 41.0s, 250/5s, queue 0, pacer alive; room: 3 roomState, 1 offer`.
649
+ Packet counts come from the `werift` engine; `wrtc` reports zero packets and
650
+ `pacer n/a`.
651
+
652
+ Like a live microphone, the `werift` engine's published track sends one Opus
653
+ packet every 20 ms from the moment media connects until the transport closes,
654
+ paced by the monotonic clock: a timer that fires late sends every packet due
655
+ by then, so the wire carries exactly 50 packets a second; a stall longer than
656
+ 200 ms restarts the clock instead of bursting. The track carries
657
+ the caller's audio when some is queued, Opus silence otherwise. Cloudflare's
658
+ SFU will not let the person's side pull a track that has carried no RTP, so a
659
+ silent agent track would never be heard. Silence never counts toward
660
+ `queuedAudioMs()` or `waitForPlayout()`. The `wrtc` engine sends only the
661
+ audio written to it; write silence yourself if you use it.
662
+
663
+ `onWarning` is called once per call, with the summary, when outbound audio is
664
+ queued but no RTP packet has been written for 2 s while media is connected.
665
+ Nothing is restarted; the callback exists so the failing direction is named in
666
+ the agent's logs.
667
+
668
+ `RelayCallTransport` consumes the SDK's `CallRoom`; it does not duplicate the
669
+ room protocol. The default engine is `werift` (pure TypeScript WebRTC) with
670
+ `@evan/opus` (prebuilt Opus, WASM fallback), so no native WebRTC binding is
671
+ loaded. Pass `engine: "wrtc"` to use the optional `@roamhq/wrtc` binding
672
+ instead. The transport boundary stays provider-neutral; `@relaymessenger/livekit`
673
+ adapts it to LiveKit Agents.
204
674
 
205
675
  ## Webhooks
206
676
 
package/dist/a2ui.d.ts ADDED
@@ -0,0 +1,79 @@
1
+ import type { Relay } from "./client.js";
2
+ import type { A2uiAction, A2uiComponent, A2uiCreateSurfaceMessage, A2uiMessage, DataPart, MessageContent, MessageSendResponse, MessageWebhookData, RelayWebhookEvent, RequestOptions } from "./types.js";
3
+ /** The `media_type` of a data part that carries A2UI messages. */
4
+ export declare const A2UI_MEDIA_TYPE = "application/a2ui+json";
5
+ /** The A2UI version every message these helpers write carries. */
6
+ export declare const A2UI_VERSION = "v0.9.1";
7
+ /** A2UI v0.9.1's basic catalog, by the `catalogId` its catalog file declares. */
8
+ export declare const A2UI_BASIC_CATALOG_ID = "https://a2ui.org/specification/v0_9/catalogs/basic/catalog.json";
9
+ /** Relay's catalog: every basic catalog component and function, plus `PaymentRequest`. */
10
+ export declare const RELAY_A2UI_CATALOG_ID = "https://relayapp.im/a2ui/catalog/v1";
11
+ /** A card: the surface to create, its components, and, optionally, its first data model. */
12
+ export interface A2uiSurface {
13
+ surfaceId: string;
14
+ /** A catalog Relay draws: `A2UI_BASIC_CATALOG_ID` or `RELAY_A2UI_CATALOG_ID`. */
15
+ catalogId: string;
16
+ /** Every component of the card; one has the id `root`. The catalog defines each component and property. */
17
+ components: A2uiComponent[];
18
+ /** The whole data model, sent as an `updateDataModel` with no `path`. */
19
+ dataModel?: Record<string, unknown>;
20
+ theme?: A2uiCreateSurfaceMessage["createSurface"]["theme"];
21
+ sendDataModel?: boolean;
22
+ }
23
+ /**
24
+ * A change to a card that is already in the chat: components to add or
25
+ * replace by id, and a data model value to set at `path` (a JSON Pointer; no
26
+ * `path` is the whole data model; no `value` removes the key at `path`).
27
+ */
28
+ export interface A2uiSurfaceUpdate {
29
+ components?: A2uiComponent[];
30
+ dataModel?: {
31
+ path?: string;
32
+ value?: unknown;
33
+ };
34
+ }
35
+ /**
36
+ * The rest of the Message beside the data part. `text` becomes a text part
37
+ * before it; the chat list and notification show it. With no text, they show
38
+ * the card's first `Text` component. A text beside an update is a new Message,
39
+ * and the update still changes the card in place.
40
+ */
41
+ export type A2uiSendOptions = Omit<MessageContent, "parts"> & {
42
+ text?: string;
43
+ };
44
+ /** A tap read out of `message.received`. */
45
+ export interface A2uiTap {
46
+ action: A2uiAction;
47
+ /** The tapped surface's data model, present when the surface set `sendDataModel`. */
48
+ dataModel?: Record<string, unknown>;
49
+ }
50
+ /** Wraps A2UI messages in the data part Relay carries them in. */
51
+ export declare const a2uiPart: (messages: A2uiMessage[]) => DataPart;
52
+ /**
53
+ * Sends a new card into a chat: `createSurface`, `updateComponents` and, when
54
+ * `dataModel` is given, `updateDataModel`, in one data part. Messages Relay
55
+ * could not apply come back in `a2ui_errors` (`A2uiFailure`); a send it could
56
+ * not apply at all throws a `RelayAPIError` whose `body.a2ui_errors` lists each.
57
+ */
58
+ export declare const sendA2uiSurface: (client: Pick<Relay, "chats">, chatID: string, surface: A2uiSurface, message?: A2uiSendOptions, options?: RequestOptions) => Promise<MessageSendResponse>;
59
+ /**
60
+ * Changes a card already in the chat, in place, for every member. With no
61
+ * `text`, the send adds no Message and returns the card's own Message. Any
62
+ * agent in the chat may update any surface in it.
63
+ */
64
+ export declare const updateA2uiSurface: (client: Pick<Relay, "chats">, chatID: string, surfaceId: string, update: A2uiSurfaceUpdate, message?: A2uiSendOptions, options?: RequestOptions) => Promise<MessageSendResponse>;
65
+ /**
66
+ * Removes a card for everyone. When every surface a Message draws is deleted,
67
+ * the Message reads back with no parts and a non-null `unsent_at`. The
68
+ * `surfaceId` may be created again afterwards.
69
+ */
70
+ export declare const deleteA2uiSurface: (client: Pick<Relay, "chats">, chatID: string, surfaceId: string, message?: A2uiSendOptions, options?: RequestOptions) => Promise<MessageSendResponse>;
71
+ /**
72
+ * Reads the first A2UI `action` out of a `message.received` event (a signed
73
+ * webhook or a WebSocket event), or out of its `data`. Returns null for any
74
+ * other event and for a Message that carries no tap. When the surface set
75
+ * `sendDataModel`, `dataModel` is that surface's model from
76
+ * `metadata.a2uiClientDataModel`.
77
+ */
78
+ export declare const readA2uiAction: (event: RelayWebhookEvent | MessageWebhookData) => A2uiTap | null;
79
+ //# sourceMappingURL=a2ui.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"a2ui.d.ts","sourceRoot":"","sources":["../src/a2ui.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,EACV,UAAU,EACV,aAAa,EACb,wBAAwB,EACxB,WAAW,EAEX,QAAQ,EACR,cAAc,EAEd,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,EACf,MAAM,YAAY,CAAC;AAEpB,kEAAkE;AAClE,eAAO,MAAM,eAAe,0BAA0B,CAAC;AAEvD,kEAAkE;AAClE,eAAO,MAAM,YAAY,WAAW,CAAC;AAErC,iFAAiF;AACjF,eAAO,MAAM,qBAAqB,oEAAoE,CAAC;AAEvG,0FAA0F;AAC1F,eAAO,MAAM,qBAAqB,wCAAwC,CAAC;AAE3E,4FAA4F;AAC5F,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,iFAAiF;IACjF,SAAS,EAAE,MAAM,CAAC;IAClB,2GAA2G;IAC3G,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,KAAK,CAAC,EAAE,wBAAwB,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC;IAC3D,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,CAAC,EAAE,aAAa,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;CAChD;AAED;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEhF,4CAA4C;AAC5C,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,UAAU,CAAC;IACnB,qFAAqF;IACrF,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,kEAAkE;AAClE,eAAO,MAAM,QAAQ,GAAI,UAAU,WAAW,EAAE,KAAG,QAIjD,CAAC;AAeH;;;;;GAKG;AACH,eAAO,MAAM,eAAe,GAC1B,QAAQ,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAC5B,QAAQ,MAAM,EACd,SAAS,WAAW,EACpB,UAAU,eAAe,EACzB,UAAU,cAAc,KACvB,OAAO,CAAC,mBAAmB,CAkB7B,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAC5B,QAAQ,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAC5B,QAAQ,MAAM,EACd,WAAW,MAAM,EACjB,QAAQ,iBAAiB,EACzB,UAAU,eAAe,EACzB,UAAU,cAAc,KACvB,OAAO,CAAC,mBAAmB,CAU7B,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAC5B,QAAQ,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAC5B,QAAQ,MAAM,EACd,WAAW,MAAM,EACjB,UAAU,eAAe,EACzB,UAAU,cAAc,KACvB,OAAO,CAAC,mBAAmB,CACqE,CAAC;AAEpG;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,GACzB,OAAO,iBAAiB,GAAG,kBAAkB,KAC5C,OAAO,GAAG,IAkBZ,CAAC"}