openai 0.90.0 → 0.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -0
- data/README.md +1 -1
- data/examples/realtime/README.md +8 -0
- data/examples/realtime/browser/README.md +167 -0
- data/examples/realtime/browser/index.html +19 -0
- data/examples/realtime/browser/page.js +21 -0
- data/examples/realtime/browser/peer.js +180 -0
- data/examples/realtime/browser/server.rb +281 -0
- data/examples/responses/websocket_workflows.rb +66 -6
- data/lib/openai/auth/workload_identity.rb +5 -1
- data/lib/openai/auth/workload_identity_auth.rb +2 -2
- data/lib/openai/auth/x509_token_exchange.rb +1 -1
- data/lib/openai/helpers/local_audio/process.rb +1 -1
- data/lib/openai/helpers/responses_websocket/connection.rb +2 -1
- data/lib/openai/helpers/responses_websocket/session.rb +551 -0
- data/lib/openai/helpers/responses_websocket.rb +1 -0
- data/lib/openai/helpers/sorbet.rb +1 -1
- data/lib/openai/helpers/websocket/async_websocket_transport.rb +6 -1
- data/lib/openai/internal/poller.rb +2 -2
- data/lib/openai/internal/transport/base_client.rb +1 -1
- data/lib/openai/internal/util.rb +1 -1
- data/lib/openai/models/beta/agents/session_update_params.rb +108 -1
- data/lib/openai/models/beta/beta_response.rb +9 -3
- data/lib/openai/models/beta/beta_response_compaction_compacting_event.rb +75 -0
- data/lib/openai/models/beta/beta_response_stream_event.rb +4 -1
- data/lib/openai/models/beta/beta_responses_client_event.rb +25 -4
- data/lib/openai/models/beta/beta_responses_server_event.rb +39 -1
- data/lib/openai/models/beta/response_compact_params.rb +9 -3
- data/lib/openai/models/beta/response_create_params.rb +21 -4
- data/lib/openai/models/beta/session_turn_error.rb +3 -0
- data/lib/openai/models/chat_model.rb +3 -1
- data/lib/openai/models/responses/response_compact_params.rb +9 -3
- data/lib/openai/models/responses/response_compaction_compacting_event.rb +49 -0
- data/lib/openai/models/responses/response_create_params.rb +12 -1
- data/lib/openai/models/responses/response_stream_event.rb +4 -1
- data/lib/openai/models/responses/responses_client_event.rb +12 -1
- data/lib/openai/models/responses/responses_server_event.rb +36 -1
- data/lib/openai/models/webhooks/realtime_call_incoming_webhook_event.rb +6 -8
- data/lib/openai/resources/beta/agents/sessions.rb +6 -2
- data/lib/openai/resources/beta/responses.rb +2 -2
- data/lib/openai/resources/responses.rb +2 -2
- data/lib/openai/version.rb +1 -1
- data/lib/openai.rb +2 -0
- data/rbi/openai/helpers/responses_websocket/connection.rbi +5 -0
- data/rbi/openai/helpers/responses_websocket/session.rbi +206 -0
- data/rbi/openai/models/beta/agents/session_update_params.rbi +178 -0
- data/rbi/openai/models/beta/beta_response.rbi +6 -1
- data/rbi/openai/models/beta/beta_response_compaction_compacting_event.rbi +136 -0
- data/rbi/openai/models/beta/beta_response_stream_event.rbi +1 -0
- data/rbi/openai/models/beta/beta_responses_client_event.rbi +27 -4
- data/rbi/openai/models/beta/beta_responses_server_event.rbi +77 -0
- data/rbi/openai/models/beta/response_compact_params.rbi +6 -1
- data/rbi/openai/models/beta/response_create_params.rbi +21 -1
- data/rbi/openai/models/beta/session_turn_error.rbi +6 -0
- data/rbi/openai/models/chat_model.rbi +3 -1
- data/rbi/openai/models/responses/response_compact_params.rbi +6 -1
- data/rbi/openai/models/responses/response_compaction_compacting_event.rbi +78 -0
- data/rbi/openai/models/responses/response_create_params.rbi +15 -0
- data/rbi/openai/models/responses/response_stream_event.rbi +1 -0
- data/rbi/openai/models/responses/responses_client_event.rbi +15 -0
- data/rbi/openai/models/responses/responses_server_event.rbi +71 -0
- data/rbi/openai/models/webhooks/realtime_call_incoming_webhook_event.rbi +6 -8
- data/rbi/openai/resources/beta/agents/sessions.rbi +5 -1
- data/responses-websocket.md +117 -10
- data/sig/openai/helpers/responses_websocket/connection.rbs +5 -0
- data/sig/openai/helpers/responses_websocket/session.rbs +123 -0
- data/sig/openai/models/beta/agents/session_update_params.rbs +99 -0
- data/sig/openai/models/beta/beta_response.rbs +6 -2
- data/sig/openai/models/beta/beta_response_compaction_compacting_event.rbs +58 -0
- data/sig/openai/models/beta/beta_response_stream_event.rbs +1 -0
- data/sig/openai/models/beta/beta_responses_client_event.rbs +13 -2
- data/sig/openai/models/beta/beta_responses_server_event.rbs +35 -0
- data/sig/openai/models/beta/response_compact_params.rbs +6 -2
- data/sig/openai/models/beta/response_create_params.rbs +13 -2
- data/sig/openai/models/beta/session_turn_error.rbs +4 -0
- data/sig/openai/models/chat_model.rbs +6 -2
- data/sig/openai/models/responses/response_compact_params.rbs +6 -2
- data/sig/openai/models/responses/response_compaction_compacting_event.rbs +37 -0
- data/sig/openai/models/responses/response_create_params.rbs +7 -0
- data/sig/openai/models/responses/response_stream_event.rbs +1 -0
- data/sig/openai/models/responses/responses_client_event.rbs +7 -0
- data/sig/openai/models/responses/responses_server_event.rbs +32 -0
- data/sig/openai/resources/beta/agents/sessions.rbs +1 -0
- metadata +16 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5a61cd08e625c8aba5c20a9b541efa3deed121cf8ffdd5fb297e4326de359fc7
|
|
4
|
+
data.tar.gz: 64bf26a74023158218b6b64e113231a3dffece2b104ea7c8802ca774bb092173
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 37ae5e3821eaa44ddccf1727c913aa6551dd1d3a432e7b3be44ba98b8fa2497e5c87b9a0ac58ef44c3e1a3f338e3dff6d5e1f81ee86ef1a0f035b8c0152c5806
|
|
7
|
+
data.tar.gz: 5162f83225a3a33fe9e7937cf43294f4bfc141963e52ba2d886dd6a642dc16d76dbd43235faf5157bfaa9bb3aed556836b44215649d05178127142bf9a2f80ed
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.91.0](https://github.com/openai/openai-ruby/compare/v0.90.0...v0.91.0) (2026-09-18)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* **api:** add agent session model settings ([#708](https://github.com/openai/openai-ruby/issues/708)) ([244a98e](https://github.com/openai/openai-ruby/commit/244a98e6e2eba0aa30f094116249e84433d8794d))
|
|
9
|
+
* **api:** add audio-mini model choices ([#710](https://github.com/openai/openai-ruby/issues/710)) ([3cdfe3f](https://github.com/openai/openai-ruby/commit/3cdfe3f275eb0c3678803b1037b64271ace69135))
|
|
10
|
+
* **api:** add compaction progress events ([#706](https://github.com/openai/openai-ruby/issues/706)) ([47e2dc7](https://github.com/openai/openai-ruby/commit/47e2dc7e8cd22c1d949cfaeae32531d7dd23f9b5))
|
|
11
|
+
* **api:** add managed Responses WebSocket sessions ([#711](https://github.com/openai/openai-ruby/issues/711)) ([a619653](https://github.com/openai/openai-ruby/commit/a61965302b90800fbdb2b888a0a666695cfbf825))
|
|
12
|
+
* **api:** add prompt-cache prewarming ([#712](https://github.com/openai/openai-ruby/issues/712)) ([61b5692](https://github.com/openai/openai-ruby/commit/61b56920641a4b241ded57b289b587b5698db55a))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
### Chores
|
|
16
|
+
|
|
17
|
+
* **api:** clarify incoming SIP call ID usage ([#709](https://github.com/openai/openai-ruby/issues/709)) ([cedfada](https://github.com/openai/openai-ruby/commit/cedfada1f4bbdd4e106c84b6564c03271d5bf133))
|
|
18
|
+
* **deps-dev:** bump sorbet-runtime from 0.6.13455 to 0.6.13486 in the development-dependencies group ([#698](https://github.com/openai/openai-ruby/issues/698)) ([ef1dc85](https://github.com/openai/openai-ruby/commit/ef1dc859af816214dbd244a2da431e64193ac6e1))
|
|
19
|
+
* **deps:** bump ruby/setup-ruby from 1.321.0 to 1.322.0 ([#700](https://github.com/openai/openai-ruby/issues/700)) ([5bdc477](https://github.com/openai/openai-ruby/commit/5bdc477abc7e9e040debca40dd14e530b1c80ff1))
|
|
20
|
+
* **deps:** bump ruby/setup-ruby from 1.322.0 to 1.323.0 ([#707](https://github.com/openai/openai-ruby/issues/707)) ([460148c](https://github.com/openai/openai-ruby/commit/460148cd775fc588d1b20ccd6a4ef35f9985ad10))
|
|
21
|
+
|
|
3
22
|
## [0.90.0](https://github.com/openai/openai-ruby/compare/v0.89.0...v0.90.0) (2026-09-10)
|
|
4
23
|
|
|
5
24
|
|
data/README.md
CHANGED
data/examples/realtime/README.md
CHANGED
|
@@ -10,6 +10,8 @@ This directory contains runnable examples for the Realtime WebSocket surface:
|
|
|
10
10
|
- `websocket_voice_turn.rb` uploads one raw 24 kHz mono PCM16 turn, explicitly
|
|
11
11
|
commits it, streams the assistant's PCM response to standard output, returns
|
|
12
12
|
its transcript to embedded callers, verifies a completed response, and exits.
|
|
13
|
+
- [`sip_webhook.rb`](SIP.md) verifies signed SIP incoming calls, applies a fixed
|
|
14
|
+
tenant/admission policy, deduplicates attempts, and owns accepted-call cleanup.
|
|
13
15
|
- `sideband.rb` attaches to an existing, application-authorized WebRTC or SIP
|
|
14
16
|
call, updates its session policy, verifies the typed update event, and exits.
|
|
15
17
|
- `function_calling.rb` forces one local function call, validates and executes
|
|
@@ -218,3 +220,9 @@ require a separately reviewed concurrency and device-lifecycle boundary.
|
|
|
218
220
|
|
|
219
221
|
See the repository's [Realtime WebSocket guide](../../realtime.md) for the
|
|
220
222
|
public connection API, custom transports, proxy behavior, and TLS setup.
|
|
223
|
+
|
|
224
|
+
## Native browser reference workflows
|
|
225
|
+
|
|
226
|
+
See [browser/README.md](browser/README.md) for the runnable short-lived-secret
|
|
227
|
+
and backend-owned SDP workflows, local application authentication, native
|
|
228
|
+
browser Start/Stop, optional trusted sideband, and failed-handoff cleanup.
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# Native browser peers with a Ruby signaling application
|
|
2
|
+
|
|
3
|
+
Two **ordinary Realtime API** examples share a small, single-operator local app:
|
|
4
|
+
|
|
5
|
+
1. `/direct`: Ruby issues a short-lived client secret. The browser sends its
|
|
6
|
+
native SDP directly to OpenAI and owns the peer lifetime.
|
|
7
|
+
2. `/backend`: Ruby forwards the browser's opaque SDP to the shipped
|
|
8
|
+
`client.realtime.calls.create`. Ruby retains the known call, returns the
|
|
9
|
+
answer, optionally confirms trusted sideband policy, and explicitly hangs up.
|
|
10
|
+
|
|
11
|
+
These do not use the distinct [Live API example](../../live/webrtc.rb).
|
|
12
|
+
`RTCPeerConnection`, `getUserMedia`, browser data channels and browser audio own
|
|
13
|
+
all negotiation, ICE, DTLS, RTP, codecs, echo cancellation and device permissions.
|
|
14
|
+
Ruby implements only application authentication, signaling and call ownership.
|
|
15
|
+
|
|
16
|
+
## Run
|
|
17
|
+
|
|
18
|
+
Use Ruby 3.3+, the repository's installed dependencies (`./scripts/bootstrap`),
|
|
19
|
+
and a separately installed WEBrick. WEBrick is an example server, not a gem
|
|
20
|
+
runtime dependency. No JavaScript package installation or build is needed.
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
gem install webrick -v '~> 1.9'
|
|
24
|
+
# Supply OPENAI_API_KEY through your environment/secret manager.
|
|
25
|
+
export BROWSER_APP_TOKEN="$(ruby -rsecurerandom -e 'puts SecureRandom.hex(32)')"
|
|
26
|
+
ruby examples/realtime/browser/server.rb
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Open `http://127.0.0.1:9292/direct` or `http://127.0.0.1:9292/backend` in a browser
|
|
30
|
+
with microphone and native WebRTC support. Paste **BROWSER_APP_TOKEN**, not the
|
|
31
|
+
OpenAI key, into the password field. Start, allow the microphone, wait for
|
|
32
|
+
`Connected`, and speak. Stop releases microphone tracks, remote tracks, the data
|
|
33
|
+
channel, peer and audio element. Stop works even during a permission prompt;
|
|
34
|
+
if permission later resolves, those new tracks are immediately stopped.
|
|
35
|
+
Start stays disabled through backend cleanup acknowledgment and a subsequent
|
|
36
|
+
five-second creation cooldown. If cleanup fails, Start remains disabled; reload
|
|
37
|
+
only after confirming that the backend released the call. This example does not
|
|
38
|
+
poll for readiness or automatically retry startup.
|
|
39
|
+
|
|
40
|
+
The app uses the standard OpenAI endpoint and requires project access to
|
|
41
|
+
`gpt-realtime-2.1` (or `OPENAI_REALTIME_MODEL`). Real requests incur API usage.
|
|
42
|
+
`PORT` defaults to 9292. Use the exact numeric loopback URL, not `localhost`.
|
|
43
|
+
Browsers allow microphone capture on loopback; remote deployment requires HTTPS.
|
|
44
|
+
Autoplay may require using the audio controls. Permission and local offer
|
|
45
|
+
preparation have a 30-second deadline. Signaling and peer establishment then have
|
|
46
|
+
a separate 60-second deadline: up to 15 seconds for creation, 15 for optional
|
|
47
|
+
sideband setup, 20 for the server handoff lease, plus delivery time. The server
|
|
48
|
+
lease still expires 20 seconds after answer preparation if no acknowledgment
|
|
49
|
+
arrives; the longer browser deadline does not extend it.
|
|
50
|
+
|
|
51
|
+
For the backend's optional existing Ruby sideband example, install
|
|
52
|
+
`async-websocket` and launch with `BROWSER_SIDEBAND=1`. Before handing back the
|
|
53
|
+
answer, it waits (up to 15 seconds) for a typed `session.updated` acknowledging
|
|
54
|
+
its policy. It closes the control socket after the acknowledgement; closing a
|
|
55
|
+
sideband alone does not hang up a call. Tools are empty here. Real tool execution
|
|
56
|
+
must independently authorize every operation against the authenticated user.
|
|
57
|
+
|
|
58
|
+
## Authentication and authorization boundary
|
|
59
|
+
|
|
60
|
+
This is a loopback-only, single-operator example, not a multi-user hosted service.
|
|
61
|
+
The random application bearer token grants this operator permission to spend
|
|
62
|
+
this project's Realtime quota and control this app's single backend call.
|
|
63
|
+
Static assets are public on loopback. Every API operation requires the token
|
|
64
|
+
and an exact Origin; every request requires the exact Host. Cross-origin
|
|
65
|
+
preflights are rejected. There are no ambient authentication cookies: the
|
|
66
|
+
explicit bearer header plus Origin check provides the CSRF boundary. Responses
|
|
67
|
+
use `no-store`, a restrictive CSP, no-referrer and same-origin resource policy.
|
|
68
|
+
The application token stays in page memory; no credentials enter URLs or web
|
|
69
|
+
storage. The standard API key stays on Ruby's backend.
|
|
70
|
+
|
|
71
|
+
The app limits creation/issuance to one request per five seconds and one known
|
|
72
|
+
backend call at a time. It supplies a trusted `OpenAI-Safety-Identifier` derived
|
|
73
|
+
from the operator token. For a deployed application, replace this local auth
|
|
74
|
+
with your existing login and user entitlements, use a stable pseudonymous user
|
|
75
|
+
identifier, add per-user quotas, HTTPS and shared durable call ownership. Never
|
|
76
|
+
accept a browser-supplied OpenAI call ID as proof of ownership.
|
|
77
|
+
|
|
78
|
+
The direct secret expires after 30 seconds, **not** the resulting session.
|
|
79
|
+
Secrets can establish multiple sessions before expiration and clients can
|
|
80
|
+
change session configuration. They are delegated capabilities, not immutable
|
|
81
|
+
policy or single-use grants. A failed token response allocates no backend call;
|
|
82
|
+
the unused secret expires. Ruby cannot revoke that secret through an invented
|
|
83
|
+
endpoint. Closing a browser peer is best effort remote termination; if a direct
|
|
84
|
+
SDP request is accepted but its response is lost, Ruby has no known call to
|
|
85
|
+
reconcile. Choose backend ownership when explicit server hangup is needed.
|
|
86
|
+
|
|
87
|
+
## Backend ownership and failure handling
|
|
88
|
+
|
|
89
|
+
The shipped SDK owns a known allocation during creation-response buffering and
|
|
90
|
+
already cleans up failed delivery to Ruby. This app does not replace that code.
|
|
91
|
+
Once `calls.create` returns, the app validates Location against the fixed OpenAI
|
|
92
|
+
origin and call path, records the call immediately, and then consumes the answer.
|
|
93
|
+
It never returns the call ID to the browser.
|
|
94
|
+
|
|
95
|
+
The browser supplies a random **application operation ID** before allocation;
|
|
96
|
+
Stop can therefore request cleanup even if the answer response is lost. Stopped
|
|
97
|
+
operation IDs are retained in memory until process exit, so a Stop that overtakes
|
|
98
|
+
its create prevents that later allocation. This single-operator example is
|
|
99
|
+
intended for bounded local runs; a deployed service needs durable operation
|
|
100
|
+
records and a retention policy tied to a defined request-expiry contract. The
|
|
101
|
+
app remains the owner after successful HTTP delivery. It requires `/api/ack`
|
|
102
|
+
after native remote-description application, peer connection and channel open.
|
|
103
|
+
A 20-second handoff lease is refreshed after answer preparation and optional
|
|
104
|
+
sideband setup, before delivery, and expires without that acknowledgement. An acknowledged
|
|
105
|
+
call has a 60-second lease, renewed by the browser every ten seconds. A paused
|
|
106
|
+
background tab can lose its lease and must Start again. Expired leases cannot
|
|
107
|
+
be revived. Page exit sends best effort Stop; the timer covers lost requests.
|
|
108
|
+
|
|
109
|
+
Answer-write failures, sideband failures and explicit Stop immediately attempt
|
|
110
|
+
hangup. Each cleanup attempt has a five-second deadline and no SDK retry. A
|
|
111
|
+
failed attempt retains ownership, blocks new allocations, and is retried by the
|
|
112
|
+
application timer after five seconds. Never retry uncertain creation requests.
|
|
113
|
+
INT/TERM drains HTTP work and attempts one final bounded cleanup.
|
|
114
|
+
|
|
115
|
+
This small in-memory example cannot recover registry state after SIGKILL or a
|
|
116
|
+
machine crash. A missing/invalid Location causes a sanitized 502 and **cannot be
|
|
117
|
+
reclaimed without an identifiable allocation**. Neither case is claimed as
|
|
118
|
+
reliable cleanup; a production deployment needs a platform-approved durable
|
|
119
|
+
reconciliation/TTL contract. HTTP success does not prove browser handoff, and
|
|
120
|
+
best effort hangup does not guarantee remote termination during an outage.
|
|
121
|
+
|
|
122
|
+
## Stable launch and verification contract
|
|
123
|
+
|
|
124
|
+
A protected live harness can launch this exact server with the environment above,
|
|
125
|
+
plus `BROWSER_SIDEBAND=1 BROWSER_MANUAL_TURNS=1`. Manual-turn mode configures
|
|
126
|
+
`audio.input.turn_detection: null` **before** media attaches; the harness must
|
|
127
|
+
explicitly commit audio/create a response on the native data channel. All modes
|
|
128
|
+
start with `max_output_tokens: 128`. Do not log credentials, SDP, audio or events.
|
|
129
|
+
|
|
130
|
+
The UI has `#token`, `#start`, `#stop`, `#status` and an `audio` element. For more
|
|
131
|
+
precise harness control, import `/peer.js` and construct
|
|
132
|
+
`new BrowserPeer({mode: 'backend', audio, status})`. `await peer.start(token)`
|
|
133
|
+
finishes startup; success requires status `Connected` (failure is handled and
|
|
134
|
+
reported through status). Failed startup waits for its bounded backend cleanup
|
|
135
|
+
attempt. Autoplay denial leaves `Connected` intact; playback help appears beside
|
|
136
|
+
the audio controls. `peer.current.pc` and `.dc` are the native browser
|
|
137
|
+
objects. Attach event observers without replacing its lifecycle handlers.
|
|
138
|
+
`await peer.stop()` returns `true` only after the backend acknowledges cleanup,
|
|
139
|
+
`false` on failed cleanup. Repeated Stop calls share the retained cleanup promise
|
|
140
|
+
and result, including when an event callback initiated cleanup. A new run clears
|
|
141
|
+
that result; a run stopped before backend submission or a direct peer has no
|
|
142
|
+
server acknowledgment.
|
|
143
|
+
Local media stops immediately. The cleanup request can wait up to 40 seconds for
|
|
144
|
+
serialized creation and sideband setup (up to 30 seconds), hangup (five seconds),
|
|
145
|
+
and delivery. Failed startup can wait this additional cleanup interval before
|
|
146
|
+
settling. Expired requests remain best effort; server leases and retries still
|
|
147
|
+
cover unsuccessful delivery.
|
|
148
|
+
The cleanup result does not include the subsequent five-second restart cooldown;
|
|
149
|
+
wait until `peer.stopping` is false before starting another run. Failed cleanup
|
|
150
|
+
keeps that flag set until reload and displays the recovery guidance above.
|
|
151
|
+
Server diagnostics include only fixed lifecycle metadata, including
|
|
152
|
+
`[browser] call released` and the existing sideband success marker. The harness
|
|
153
|
+
must require successful cleanup and stop the server after its bounded run.
|
|
154
|
+
|
|
155
|
+
Offline tests (Node 22.7+ for the browser-controller tests):
|
|
156
|
+
|
|
157
|
+
```sh
|
|
158
|
+
bundle exec ruby test/openai/realtime/browser/server_test.rb
|
|
159
|
+
node --test test/openai/realtime/browser/peer_test.mjs
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
The required Ruby CI test job also runs the controller suite. The controller
|
|
163
|
+
tests simulate browser boundaries; they are not evidence of a
|
|
164
|
+
real ICE/media exchange. Ruby tests exercise real SDK serialization with fake
|
|
165
|
+
HTTP responses. Live API/audio evidence must be reported separately by the
|
|
166
|
+
protected harness; do not equate a rendered page or fake SDP test with a paid
|
|
167
|
+
end-to-end call.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>Ruby Realtime browser reference</title></head>
|
|
4
|
+
<body>
|
|
5
|
+
<h1>Ruby Realtime browser reference</h1>
|
|
6
|
+
<nav><a href="/direct">Browser-owned</a> | <a href="/backend">Backend-owned</a></nav>
|
|
7
|
+
<h2 id="mode"></h2>
|
|
8
|
+
<p>Ordinary Realtime API. Your browser owns the microphone, media and peer connection.</p>
|
|
9
|
+
<p>Paste the local application's BROWSER_APP_TOKEN. Never paste your OpenAI API key.</p>
|
|
10
|
+
<label>Application token <input id="token" type="password" autocomplete="off" spellcheck="false"></label>
|
|
11
|
+
<button id="start" type="button">Start</button>
|
|
12
|
+
<button id="stop" type="button" disabled>Stop</button>
|
|
13
|
+
<p id="status" role="status">Stopped</p>
|
|
14
|
+
<p>Allow microphone access, then speak after Connected. Stop releases microphone and peer resources.</p>
|
|
15
|
+
<p>If audio does not play automatically, use the Play control below.</p>
|
|
16
|
+
<audio controls autoplay></audio>
|
|
17
|
+
<script type="module" src="/page.js"></script>
|
|
18
|
+
</body>
|
|
19
|
+
</html>
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import {BrowserPeer} from './peer.js';
|
|
2
|
+
|
|
3
|
+
const mode = location.pathname === '/backend' ? 'backend' : 'direct';
|
|
4
|
+
const start = document.querySelector('#start');
|
|
5
|
+
const stop = document.querySelector('#stop');
|
|
6
|
+
const status = document.querySelector('#status');
|
|
7
|
+
const token = document.querySelector('#token');
|
|
8
|
+
document.querySelector('#mode').textContent = mode === 'direct'
|
|
9
|
+
? '1. Browser-owned secret and direct SDP exchange'
|
|
10
|
+
: '2. Backend-owned SDP forwarding and call lifetime';
|
|
11
|
+
const peer = new BrowserPeer({mode, audio: document.querySelector('audio'), status(message) {
|
|
12
|
+
status.textContent = message;
|
|
13
|
+
start.disabled = Boolean(peer.current) || peer.stopping;
|
|
14
|
+
stop.disabled = !peer.current;
|
|
15
|
+
}});
|
|
16
|
+
start.addEventListener('click', () => {
|
|
17
|
+
if (token.value.length < 32) { status.textContent = 'Enter the application token (not an OpenAI API key).'; return; }
|
|
18
|
+
peer.start(token.value);
|
|
19
|
+
});
|
|
20
|
+
stop.addEventListener('click', () => peer.stop());
|
|
21
|
+
window.addEventListener('pagehide', () => peer.stop());
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
// Native browser APIs own negotiation, ICE, media and data channels.
|
|
2
|
+
export class BrowserPeer {
|
|
3
|
+
constructor({mode, audio, status, environment = globalThis}) {
|
|
4
|
+
this.mode = mode;
|
|
5
|
+
this.audio = audio;
|
|
6
|
+
this.status = status;
|
|
7
|
+
this.env = environment;
|
|
8
|
+
this.current = null;
|
|
9
|
+
this.stopping = false;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
async request(run, path, body = '', type = 'text/plain', cleanup = false) {
|
|
13
|
+
const response = await this.env.fetch(path, {
|
|
14
|
+
method: 'POST', body, cache: 'no-store', redirect: 'error',
|
|
15
|
+
headers: {Authorization: `Bearer ${run.token}`, 'Content-Type': type,
|
|
16
|
+
'X-Operation-Id': run.id},
|
|
17
|
+
// Cleanup may wait for serialized creation (15s), sideband (15s), then hangup (5s).
|
|
18
|
+
signal: cleanup ? this.env.AbortSignal.timeout(40000) : run.abort.signal,
|
|
19
|
+
keepalive: cleanup,
|
|
20
|
+
});
|
|
21
|
+
if (!response.ok) throw new Error('Application request failed.');
|
|
22
|
+
return response;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
check(run) {
|
|
26
|
+
if (this.current !== run) throw new DOMException('Stopped', 'AbortError');
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async wait(run, operation) {
|
|
30
|
+
let cancel;
|
|
31
|
+
const cancelled = new Promise((_, reject) => {
|
|
32
|
+
cancel = () => reject(new DOMException('Stopped', 'AbortError'));
|
|
33
|
+
run.abort.signal.addEventListener('abort', cancel, {once: true});
|
|
34
|
+
if (run.abort.signal.aborted) cancel();
|
|
35
|
+
});
|
|
36
|
+
try { return await Promise.race([operation, cancelled]); }
|
|
37
|
+
finally { run.abort.signal.removeEventListener('abort', cancel); }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async start(token) {
|
|
41
|
+
if (this.current || this.stopping) return;
|
|
42
|
+
this.cleanup = undefined;
|
|
43
|
+
const run = {token, id: this.env.crypto.randomUUID(), abort: new this.env.AbortController()};
|
|
44
|
+
this.current = run;
|
|
45
|
+
this.status('Starting');
|
|
46
|
+
run.deadline = this.env.setTimeout(() => this.stop('Startup timed out'), 30000);
|
|
47
|
+
try {
|
|
48
|
+
// The permission prompt itself cannot be aborted. Settle start on Stop,
|
|
49
|
+
// and separately release any tracks that arrive after cancellation.
|
|
50
|
+
const media = this.env.navigator.mediaDevices.getUserMedia({audio: true}).then(stream => {
|
|
51
|
+
if (this.current !== run) stream.getTracks().forEach(track => track.stop());
|
|
52
|
+
else run.stream = stream;
|
|
53
|
+
return stream;
|
|
54
|
+
});
|
|
55
|
+
const stream = await this.wait(run, media);
|
|
56
|
+
this.check(run);
|
|
57
|
+
run.stream = stream;
|
|
58
|
+
run.pc = new this.env.RTCPeerConnection();
|
|
59
|
+
stream.getTracks().forEach(track => {
|
|
60
|
+
track.addEventListener('ended', () => { if (this.current === run) this.stop('Microphone ended'); });
|
|
61
|
+
run.pc.addTrack(track, stream);
|
|
62
|
+
});
|
|
63
|
+
run.pc.ontrack = event => {
|
|
64
|
+
if (this.current === run) {
|
|
65
|
+
this.audio.srcObject = event.streams[0] || new this.env.MediaStream([event.track]);
|
|
66
|
+
// Playback help remains beside the controls, independent of connection status.
|
|
67
|
+
this.audio.play().catch(() => {});
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
run.dc = run.pc.createDataChannel('oai-events');
|
|
71
|
+
const ready = new Promise((resolve, reject) => {
|
|
72
|
+
const checkReady = () => {
|
|
73
|
+
if (run.pc.connectionState === 'connected' && run.dc.readyState === 'open') resolve();
|
|
74
|
+
};
|
|
75
|
+
run.abort.signal.addEventListener('abort', () => reject(new DOMException('Stopped', 'AbortError')), {once: true});
|
|
76
|
+
run.pc.onconnectionstatechange = () => {
|
|
77
|
+
if (this.current !== run) return;
|
|
78
|
+
if (['failed', 'closed'].includes(run.pc.connectionState)) this.stop('Peer failed');
|
|
79
|
+
else checkReady(); // disconnected is transient; backend lease still bounds abandonment.
|
|
80
|
+
};
|
|
81
|
+
run.dc.onopen = checkReady;
|
|
82
|
+
run.dc.onclose = () => { if (this.current === run) this.stop('Data channel closed'); };
|
|
83
|
+
run.dc.onerror = () => { if (this.current === run) this.stop('Data channel failed'); };
|
|
84
|
+
});
|
|
85
|
+
// Attach a rejection handler now; negotiation can fail before awaiting ready.
|
|
86
|
+
ready.catch(() => {});
|
|
87
|
+
run.dc.onmessage = event => {
|
|
88
|
+
if (this.current !== run) return;
|
|
89
|
+
try {
|
|
90
|
+
const message = JSON.parse(event.data);
|
|
91
|
+
if (message.type === 'error') this.stop('Realtime error');
|
|
92
|
+
} catch { this.stop('Invalid Realtime event'); }
|
|
93
|
+
};
|
|
94
|
+
const offer = await this.wait(run, run.pc.createOffer());
|
|
95
|
+
this.check(run);
|
|
96
|
+
await this.wait(run, run.pc.setLocalDescription(offer));
|
|
97
|
+
this.check(run);
|
|
98
|
+
// Give signaling its own budget after permission/local offer preparation:
|
|
99
|
+
// creation (15s), optional sideband (15s), handoff (20s), and delivery time.
|
|
100
|
+
this.env.clearTimeout(run.deadline);
|
|
101
|
+
run.deadline = this.env.setTimeout(() => this.stop('Startup timed out'), 60000);
|
|
102
|
+
let answer;
|
|
103
|
+
if (this.mode === 'direct') {
|
|
104
|
+
const secretResponse = await this.request(run, '/api/secret');
|
|
105
|
+
const secret = await secretResponse.json();
|
|
106
|
+
this.check(run);
|
|
107
|
+
const response = await this.env.fetch('https://api.openai.com/v1/realtime/calls', {
|
|
108
|
+
method: 'POST', body: run.pc.localDescription.sdp, signal: run.abort.signal,
|
|
109
|
+
headers: {Authorization: `Bearer ${secret.value}`, 'Content-Type': 'application/sdp'},
|
|
110
|
+
cache: 'no-store', redirect: 'error',
|
|
111
|
+
});
|
|
112
|
+
if (!response.ok) throw new Error('Direct SDP exchange failed.');
|
|
113
|
+
answer = await response.text();
|
|
114
|
+
} else {
|
|
115
|
+
run.submitted = true;
|
|
116
|
+
const response = await this.request(run, '/api/calls', run.pc.localDescription.sdp, 'application/sdp');
|
|
117
|
+
answer = (await response.json()).sdp;
|
|
118
|
+
}
|
|
119
|
+
this.check(run);
|
|
120
|
+
await this.wait(run, run.pc.setRemoteDescription({type: 'answer', sdp: answer}));
|
|
121
|
+
this.check(run);
|
|
122
|
+
await ready;
|
|
123
|
+
this.check(run);
|
|
124
|
+
if (this.mode === 'backend') {
|
|
125
|
+
await this.request(run, '/api/ack');
|
|
126
|
+
this.check(run);
|
|
127
|
+
// No overlapping renewals. A stalled request times out and stops locally.
|
|
128
|
+
const renew = async () => {
|
|
129
|
+
if (this.current !== run) return;
|
|
130
|
+
run.renewDeadline = this.env.setTimeout(() => this.stop('Lease renewal timed out'), 10000);
|
|
131
|
+
try {
|
|
132
|
+
await this.request(run, '/api/renew');
|
|
133
|
+
this.env.clearTimeout(run.renewDeadline);
|
|
134
|
+
if (this.current === run) run.heartbeat = this.env.setTimeout(renew, 10000);
|
|
135
|
+
} catch { if (this.current === run) this.stop('Lease renewal failed'); }
|
|
136
|
+
};
|
|
137
|
+
run.heartbeat = this.env.setTimeout(renew, 10000);
|
|
138
|
+
}
|
|
139
|
+
this.env.clearTimeout(run.deadline);
|
|
140
|
+
this.status('Connected');
|
|
141
|
+
} catch {
|
|
142
|
+
if (this.current === run) this.stop('Start failed');
|
|
143
|
+
// A timeout or peer callback may already have stopped this run.
|
|
144
|
+
await run.cleanup;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
stop(message = 'Stopped') {
|
|
149
|
+
const run = this.current;
|
|
150
|
+
if (!run) return this.cleanup;
|
|
151
|
+
this.stopping = true;
|
|
152
|
+
this.current = null; // Invalidate callbacks before closing owned resources.
|
|
153
|
+
run.abort.abort();
|
|
154
|
+
this.env.clearTimeout(run.deadline);
|
|
155
|
+
this.env.clearTimeout(run.heartbeat);
|
|
156
|
+
this.env.clearTimeout(run.renewDeadline);
|
|
157
|
+
run.stream?.getTracks().forEach(track => track.stop());
|
|
158
|
+
run.dc?.close();
|
|
159
|
+
run.pc?.getReceivers().forEach(receiver => receiver.track?.stop());
|
|
160
|
+
run.pc?.close();
|
|
161
|
+
this.audio.pause();
|
|
162
|
+
this.audio.srcObject = null;
|
|
163
|
+
const allowRestart = () => this.env.setTimeout(() => {
|
|
164
|
+
this.stopping = false;
|
|
165
|
+
this.status(message);
|
|
166
|
+
}, 5000);
|
|
167
|
+
if (run.submitted) {
|
|
168
|
+
// Independent of the aborted startup request. If delivery fails, the
|
|
169
|
+
// application lease reclaims the known call; pagehide is only best effort.
|
|
170
|
+
run.cleanup = this.request(run, '/api/stop', '', 'text/plain', true).then(() => true).catch(() => {
|
|
171
|
+
this.status('Stopped; server cleanup pending. Reload only after backend release is confirmed.');
|
|
172
|
+
return false;
|
|
173
|
+
});
|
|
174
|
+
this.cleanup = run.cleanup;
|
|
175
|
+
run.cleanup.then(released => { if (released) allowRestart(); });
|
|
176
|
+
} else allowRestart();
|
|
177
|
+
this.status(message);
|
|
178
|
+
return run.cleanup;
|
|
179
|
+
}
|
|
180
|
+
}
|