ai 7.0.100 → 7.0.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/dist/index.d.ts +82 -20
  3. package/dist/index.js +2017 -330
  4. package/dist/index.js.map +1 -1
  5. package/dist/internal/index.js +12 -5
  6. package/dist/internal/index.js.map +1 -1
  7. package/docs/03-agents/07-workflow-agent.mdx +1 -1
  8. package/docs/03-ai-sdk-core/36-realtime.mdx +41 -18
  9. package/docs/04-ai-sdk-ui/21-transport.mdx +1 -1
  10. package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +364 -48
  11. package/docs/07-reference/04-ai-sdk-workflow/02-workflow-chat-transport.mdx +4 -4
  12. package/docs/07-reference/05-ai-sdk-errors/index.mdx +131 -36
  13. package/package.json +12 -12
  14. package/src/generate-text/execute-tools-from-stream.ts +7 -0
  15. package/src/generate-text/stream-text.ts +3 -1
  16. package/src/prompt/create-tool-model-output.ts +16 -5
  17. package/src/realtime/__fixtures__/fake-live-websocket.ts +71 -0
  18. package/src/realtime/__fixtures__/fake-realtime.ts +36 -0
  19. package/src/realtime/__fixtures__/fake-webrtc.ts +133 -0
  20. package/src/realtime/browser-realtime-audio.ts +107 -10
  21. package/src/realtime/browser-realtime-live-websocket.ts +247 -0
  22. package/src/realtime/browser-realtime-transport.ts +235 -69
  23. package/src/realtime/browser-realtime-webrtc.ts +582 -0
  24. package/src/realtime/encode-realtime-frame.ts +33 -0
  25. package/src/realtime/index.ts +1 -0
  26. package/src/realtime/realtime-attempt.ts +45 -0
  27. package/src/realtime/realtime-command-tracker.ts +81 -0
  28. package/src/realtime/realtime-event-channel.ts +170 -0
  29. package/src/realtime/realtime-event-reducer.ts +3 -0
  30. package/src/realtime/realtime-session-state.ts +65 -0
  31. package/src/realtime/realtime-session.ts +768 -218
  32. package/src/realtime/realtime-types.ts +1 -1
  33. package/src/realtime/validate-realtime-setup.ts +37 -0
package/dist/index.js CHANGED
@@ -14,7 +14,7 @@ import {
14
14
  createIdGenerator as createIdGenerator9,
15
15
  dynamicTool,
16
16
  experimental_toolCaller,
17
- generateId as generateId3,
17
+ generateId as generateId4,
18
18
  jsonSchema,
19
19
  parseJsonEventStream as parseJsonEventStream3,
20
20
  tool,
@@ -1178,7 +1178,7 @@ import {
1178
1178
  } from "@ai-sdk/provider-utils";
1179
1179
 
1180
1180
  // src/version.ts
1181
- var VERSION = true ? "7.0.100" : "0.0.0-test";
1181
+ var VERSION = true ? "7.0.102" : "0.0.0-test";
1182
1182
 
1183
1183
  // src/util/download/download.ts
1184
1184
  var download = async ({
@@ -2007,6 +2007,9 @@ function getMediaTypeFromUrl(url, fallbackMediaType = "application/octet-stream"
2007
2007
 
2008
2008
  // src/prompt/create-tool-model-output.ts
2009
2009
  import { getErrorMessage as getErrorMessage3 } from "@ai-sdk/provider";
2010
+ import {
2011
+ parseJSON
2012
+ } from "@ai-sdk/provider-utils";
2010
2013
  async function createToolModelOutput({
2011
2014
  toolCallId,
2012
2015
  input,
@@ -2017,15 +2020,19 @@ async function createToolModelOutput({
2017
2020
  if (errorMode === "text") {
2018
2021
  return { type: "error-text", value: getErrorMessage3(output) };
2019
2022
  } else if (errorMode === "json") {
2020
- return { type: "error-json", value: toJSONValue(output) };
2023
+ return { type: "error-json", value: await toJSONValue(output) };
2021
2024
  }
2022
2025
  if (tool2 == null ? void 0 : tool2.toModelOutput) {
2023
2026
  return await tool2.toModelOutput({ toolCallId, input, output });
2024
2027
  }
2025
- return typeof output === "string" ? { type: "text", value: output } : { type: "json", value: toJSONValue(output) };
2028
+ return typeof output === "string" ? { type: "text", value: output } : { type: "json", value: await toJSONValue(output) };
2026
2029
  }
2027
- function toJSONValue(value) {
2028
- return value === void 0 ? null : value;
2030
+ async function toJSONValue(value) {
2031
+ if (value === void 0) {
2032
+ return null;
2033
+ }
2034
+ const serialized = JSON.stringify(value);
2035
+ return serialized === void 0 ? null : await parseJSON({ text: serialized });
2029
2036
  }
2030
2037
 
2031
2038
  // src/prompt/prepare-language-model-call-options.ts
@@ -5549,7 +5556,7 @@ async function generateText({
5549
5556
  experimental_include,
5550
5557
  include = experimental_include,
5551
5558
  _internal: {
5552
- generateId: generateId4 = originalGenerateId,
5559
+ generateId: generateId5 = originalGenerateId,
5553
5560
  generateCallId = originalGenerateCallId,
5554
5561
  now: now2 = now
5555
5562
  } = {},
@@ -5949,7 +5956,7 @@ async function generateText({
5949
5956
  }
5950
5957
  );
5951
5958
  const responseData = {
5952
- id: (_b27 = (_a28 = result.response) == null ? void 0 : _a28.id) != null ? _b27 : generateId4(),
5959
+ id: (_b27 = (_a28 = result.response) == null ? void 0 : _a28.id) != null ? _b27 : generateId5(),
5953
5960
  timestamp: (_d3 = (_c3 = result.response) == null ? void 0 : _c3.timestamp) != null ? _d3 : /* @__PURE__ */ new Date(),
5954
5961
  modelId: (_f2 = (_e2 = result.response) == null ? void 0 : _e2.modelId) != null ? _f2 : stepModel.modelId,
5955
5962
  headers: (_g2 = result.response) == null ? void 0 : _g2.headers,
@@ -6069,7 +6076,7 @@ async function generateText({
6069
6076
  if (toolApprovalStatus.type === "not-applicable") {
6070
6077
  continue;
6071
6078
  }
6072
- const approvalId = generateId4();
6079
+ const approvalId = generateId5();
6073
6080
  const signature = await maybeSignApproval({
6074
6081
  secret: experimental_toolApprovalSecret,
6075
6082
  approvalId,
@@ -8393,7 +8400,7 @@ function executeToolsFromStream({
8393
8400
  toolApproval,
8394
8401
  runtimeContext,
8395
8402
  toolApprovalSecret,
8396
- generateId: generateId4,
8403
+ generateId: generateId5,
8397
8404
  onToolExecutionStart,
8398
8405
  onToolExecutionEnd,
8399
8406
  executeToolInTelemetryContext,
@@ -8432,7 +8439,7 @@ function executeToolsFromStream({
8432
8439
  }
8433
8440
  return;
8434
8441
  }
8435
- const approvalId = generateId4();
8442
+ const approvalId = generateId5();
8436
8443
  const signature = await maybeSignApproval({
8437
8444
  secret: toolApprovalSecret,
8438
8445
  approvalId,
@@ -8467,6 +8474,11 @@ function executeToolsFromStream({
8467
8474
  reason: toolApprovalStatus.reason,
8468
8475
  providerExecuted: chunk.providerExecuted
8469
8476
  });
8477
+ controller.enqueue({
8478
+ type: "tool-output-denied",
8479
+ toolCallId: chunk.toolCallId,
8480
+ toolName: chunk.toolName
8481
+ });
8470
8482
  return;
8471
8483
  }
8472
8484
  case "approved": {
@@ -8715,7 +8727,7 @@ async function streamLanguageModelCall({
8715
8727
  toolsContext,
8716
8728
  experimental_sandbox: sandbox,
8717
8729
  _internal: {
8718
- generateId: generateId4 = originalGenerateId2,
8730
+ generateId: generateId5 = originalGenerateId2,
8719
8731
  generateCallId = originalGenerateCallId2,
8720
8732
  now: now2 = now
8721
8733
  } = {},
@@ -8801,7 +8813,7 @@ async function streamLanguageModelCall({
8801
8813
  provider: resolvedModel.provider,
8802
8814
  modelId: resolvedModel.modelId,
8803
8815
  toolChoice: stepToolChoice,
8804
- generateId: generateId4,
8816
+ generateId: generateId5,
8805
8817
  now: now2,
8806
8818
  callStartTimestampMs,
8807
8819
  onLanguageModelCallEnd
@@ -8823,7 +8835,7 @@ function createLanguageModelV4StreamPartToLanguageModelStreamPartTransform({
8823
8835
  provider,
8824
8836
  modelId,
8825
8837
  toolChoice,
8826
- generateId: generateId4,
8838
+ generateId: generateId5,
8827
8839
  now: now2,
8828
8840
  callStartTimestampMs,
8829
8841
  onLanguageModelCallEnd
@@ -8835,7 +8847,7 @@ function createLanguageModelV4StreamPartToLanguageModelStreamPartTransform({
8835
8847
  const reasoningPartIndexes = /* @__PURE__ */ new Map();
8836
8848
  const rawTextPartIndexes = /* @__PURE__ */ new Map();
8837
8849
  const rawReasoningPartIndexes = /* @__PURE__ */ new Map();
8838
- let responseId = generateId4();
8850
+ let responseId = generateId5();
8839
8851
  let responseModelId = modelId;
8840
8852
  let timeToFirstOutputMs;
8841
8853
  let previousOutputChunkTimestampMs;
@@ -9242,6 +9254,7 @@ var isOutputChunkType = {
9242
9254
  "tool-call": true,
9243
9255
  "tool-result": false,
9244
9256
  "tool-error": false,
9257
+ "tool-output-denied": false,
9245
9258
  "tool-execution-end": false,
9246
9259
  "model-call-start": false,
9247
9260
  "model-call-response-metadata": false,
@@ -9324,7 +9337,7 @@ function streamText({
9324
9337
  include = experimental_include,
9325
9338
  _internal: {
9326
9339
  now: now2 = now,
9327
- generateId: generateId4 = originalGenerateId3,
9340
+ generateId: generateId5 = originalGenerateId3,
9328
9341
  generateCallId = originalGenerateCallId3
9329
9342
  } = {},
9330
9343
  ...settings
@@ -9403,7 +9416,7 @@ function streamText({
9403
9416
  onToolExecutionStart: resolvedOnToolExecutionStart,
9404
9417
  onToolExecutionEnd: resolvedOnToolExecutionEnd,
9405
9418
  now: now2,
9406
- generateId: generateId4,
9419
+ generateId: generateId5,
9407
9420
  generateCallId,
9408
9421
  download: download2,
9409
9422
  // assign default values to include:
@@ -9631,7 +9644,7 @@ var DefaultStreamTextResult = class {
9631
9644
  providerOptions,
9632
9645
  prepareStep,
9633
9646
  now: now2,
9634
- generateId: generateId4,
9647
+ generateId: generateId5,
9635
9648
  generateCallId,
9636
9649
  timeout,
9637
9650
  onChunk,
@@ -9686,7 +9699,7 @@ var DefaultStreamTextResult = class {
9686
9699
  usedIds.add(id);
9687
9700
  return id;
9688
9701
  }
9689
- const generatedId = generateId4();
9702
+ const generatedId = generateId5();
9690
9703
  let uniqueId = generatedId;
9691
9704
  let suffix = 0;
9692
9705
  while (usedIds.has(uniqueId)) {
@@ -10621,7 +10634,7 @@ var DefaultStreamTextResult = class {
10621
10634
  toolApproval,
10622
10635
  runtimeContext,
10623
10636
  toolApprovalSecret: experimental_toolApprovalSecret,
10624
- generateId: generateId4,
10637
+ generateId: generateId5,
10625
10638
  // the callbacks need to be passed down and handled by executeToolCall
10626
10639
  // to guarantee that the onToolExecutionStart callback is invoked before the tool execute function
10627
10640
  onToolExecutionStart: filterNullable2(
@@ -10664,7 +10677,7 @@ var DefaultStreamTextResult = class {
10664
10677
  let modelCallPerformance = createModelCallPerformance();
10665
10678
  const toolExecutionMs = {};
10666
10679
  const createStepResponse = () => ({
10667
- id: generateId4(),
10680
+ id: generateId5(),
10668
10681
  timestamp: /* @__PURE__ */ new Date(),
10669
10682
  modelId: model.modelId
10670
10683
  });
@@ -10730,7 +10743,8 @@ var DefaultStreamTextResult = class {
10730
10743
  case "tool-input-start":
10731
10744
  case "tool-input-end":
10732
10745
  case "tool-input-delta":
10733
- case "tool-approval-request": {
10746
+ case "tool-approval-request":
10747
+ case "tool-output-denied": {
10734
10748
  enqueueStepPart(controller, chunk);
10735
10749
  break;
10736
10750
  }
@@ -11494,7 +11508,7 @@ function createUIMessageStream({
11494
11508
  onStepFinish,
11495
11509
  onEnd,
11496
11510
  onFinish,
11497
- generateId: generateId4 = generateIdFunc
11511
+ generateId: generateId5 = generateIdFunc
11498
11512
  }) {
11499
11513
  let controller;
11500
11514
  const ongoingStreamPromises = [];
@@ -11586,7 +11600,7 @@ function createUIMessageStream({
11586
11600
  });
11587
11601
  return handleUIMessageStreamFinish({
11588
11602
  stream,
11589
- messageId: generateId4(),
11603
+ messageId: generateId5(),
11590
11604
  originalMessages,
11591
11605
  onStepEnd: onStepEnd != null ? onStepEnd : onStepFinish,
11592
11606
  onEnd: onEnd != null ? onEnd : onFinish,
@@ -14481,7 +14495,7 @@ async function generateObject(options) {
14481
14495
  onStepFinish,
14482
14496
  onFinish,
14483
14497
  _internal: {
14484
- generateId: generateId4 = originalGenerateId4,
14498
+ generateId: generateId5 = originalGenerateId4,
14485
14499
  currentDate = () => /* @__PURE__ */ new Date()
14486
14500
  } = {},
14487
14501
  ...settings
@@ -14521,7 +14535,7 @@ async function generateObject(options) {
14521
14535
  const resolvedOnStepStart = onStepStart != null ? onStepStart : experimental_onStepStart;
14522
14536
  const resolvedOnStepEnd = onStepEnd != null ? onStepEnd : onStepFinish;
14523
14537
  const jsonSchema2 = await outputStrategy.jsonSchema();
14524
- const callId = generateId4();
14538
+ const callId = generateId5();
14525
14539
  await notify({
14526
14540
  event: {
14527
14541
  callId,
@@ -14590,7 +14604,7 @@ async function generateObject(options) {
14590
14604
  })
14591
14605
  );
14592
14606
  const responseData = {
14593
- id: (_b25 = (_a25 = generateResult.response) == null ? void 0 : _a25.id) != null ? _b25 : generateId4(),
14607
+ id: (_b25 = (_a25 = generateResult.response) == null ? void 0 : _a25.id) != null ? _b25 : generateId5(),
14594
14608
  timestamp: (_d = (_c = generateResult.response) == null ? void 0 : _c.timestamp) != null ? _d : currentDate(),
14595
14609
  modelId: (_f = (_e = generateResult.response) == null ? void 0 : _e.modelId) != null ? _f : model.modelId,
14596
14610
  headers: (_g = generateResult.response) == null ? void 0 : _g.headers,
@@ -14850,7 +14864,7 @@ function streamObject(options) {
14850
14864
  },
14851
14865
  onFinish,
14852
14866
  _internal: {
14853
- generateId: generateId4 = originalGenerateId5,
14867
+ generateId: generateId5 = originalGenerateId5,
14854
14868
  currentDate = () => /* @__PURE__ */ new Date(),
14855
14869
  now: now2 = now
14856
14870
  } = {},
@@ -14897,7 +14911,7 @@ function streamObject(options) {
14897
14911
  onError,
14898
14912
  onFinish,
14899
14913
  download: download2,
14900
- generateId: generateId4,
14914
+ generateId: generateId5,
14901
14915
  currentDate,
14902
14916
  now: now2
14903
14917
  });
@@ -14926,7 +14940,7 @@ var DefaultStreamObjectResult = class {
14926
14940
  onError,
14927
14941
  onFinish,
14928
14942
  download: download2,
14929
- generateId: generateId4,
14943
+ generateId: generateId5,
14930
14944
  currentDate,
14931
14945
  now: now2
14932
14946
  }) {
@@ -14960,7 +14974,7 @@ var DefaultStreamObjectResult = class {
14960
14974
  }
14961
14975
  });
14962
14976
  this.baseStream = stitchableStream.stream.pipeThrough(eventProcessor);
14963
- const callId = generateId4();
14977
+ const callId = generateId5();
14964
14978
  (async () => {
14965
14979
  const jsonSchema2 = await outputStrategy.jsonSchema();
14966
14980
  await notify({
@@ -15058,7 +15072,7 @@ var DefaultStreamObjectResult = class {
15058
15072
  let accumulatedText = "";
15059
15073
  let textDelta = "";
15060
15074
  let fullResponse = {
15061
- id: generateId4(),
15075
+ id: generateId5(),
15062
15076
  timestamp: currentDate(),
15063
15077
  modelId: model.modelId
15064
15078
  };
@@ -17022,6 +17036,9 @@ function resolveRealtimeToolDescription({
17022
17036
  });
17023
17037
  }
17024
17038
 
17039
+ // src/realtime/realtime-session.ts
17040
+ import { UnsupportedFunctionalityError as UnsupportedFunctionalityError6 } from "@ai-sdk/provider";
17041
+
17025
17042
  // src/realtime/browser-realtime-audio.ts
17026
17043
  var BrowserRealtimeAudio = class {
17027
17044
  constructor(options) {
@@ -17029,16 +17046,23 @@ var BrowserRealtimeAudio = class {
17029
17046
  this.captureProcessor = null;
17030
17047
  this.captureSource = null;
17031
17048
  this.captureStream = null;
17049
+ this.ownsCaptureStream = true;
17050
+ this.captureCleanup = [];
17051
+ this.captureGeneration = 0;
17032
17052
  this.playbackContext = null;
17033
17053
  this.playbackQueue = [];
17034
17054
  this.playbackTime = 0;
17035
17055
  this.playbackStartTime = 0;
17036
17056
  this.activeSources = /* @__PURE__ */ new Set();
17037
17057
  this.isPlaying = false;
17058
+ this.playbackPaused = false;
17059
+ var _a25;
17038
17060
  this.captureSampleRate = options.captureSampleRate;
17039
17061
  this.playbackSampleRate = options.playbackSampleRate;
17040
17062
  this.onAudio = options.onAudio;
17041
17063
  this.onPlayingChange = options.onPlayingChange;
17064
+ this.maxPlaybackSeconds = (_a25 = options.maxPlaybackSeconds) != null ? _a25 : Infinity;
17065
+ this.onError = options.onError;
17042
17066
  this.onCapturingChange = options.onCapturingChange;
17043
17067
  }
17044
17068
  ensurePlaybackContext() {
@@ -17046,35 +17070,97 @@ var BrowserRealtimeAudio = class {
17046
17070
  this.playbackContext = new AudioContext({
17047
17071
  sampleRate: this.playbackSampleRate
17048
17072
  });
17073
+ const context = this.playbackContext;
17074
+ context.onstatechange = () => {
17075
+ if (this.playbackContext === context)
17076
+ this.setPlaying(
17077
+ context.state === "running" && this.activeSources.size > 0
17078
+ );
17079
+ };
17049
17080
  }
17050
17081
  }
17051
- startCapture(stream) {
17082
+ async resumePlayback() {
17083
+ var _a25;
17084
+ this.ensurePlaybackContext();
17085
+ const context = this.playbackContext;
17086
+ if (this.playbackPaused) this.stopPlayback();
17087
+ if (this.playbackContext !== context) return;
17088
+ await (context == null ? void 0 : context.resume());
17089
+ if (this.playbackContext !== context) return;
17090
+ this.playbackPaused = false;
17091
+ this.setPlaying(
17092
+ ((_a25 = this.playbackContext) == null ? void 0 : _a25.state) === "running" && this.activeSources.size > 0
17093
+ );
17094
+ }
17095
+ startCapture(stream, options) {
17096
+ var _a25, _b25;
17097
+ const generation = this.captureGeneration + 1;
17098
+ this.stopCapture();
17099
+ if (generation !== this.captureGeneration) {
17100
+ if ((_a25 = options == null ? void 0 : options.ownsStream) != null ? _a25 : true)
17101
+ stream.getTracks().forEach((track) => track.stop());
17102
+ return;
17103
+ }
17104
+ this.captureStream = stream;
17105
+ this.ownsCaptureStream = (_b25 = options == null ? void 0 : options.ownsStream) != null ? _b25 : true;
17052
17106
  const ctx = new AudioContext({ sampleRate: this.captureSampleRate });
17053
17107
  this.captureContext = ctx;
17054
- this.captureStream = stream;
17108
+ void ctx.resume().catch((error) => {
17109
+ var _a26;
17110
+ if (this.captureContext === ctx) (_a26 = this.onError) == null ? void 0 : _a26.call(this, error);
17111
+ });
17055
17112
  const source = ctx.createMediaStreamSource(stream);
17056
17113
  this.captureSource = source;
17057
- const processor = ctx.createScriptProcessor(4096, 1, 1);
17114
+ const processor = ctx.createScriptProcessor(1024, 1, 1);
17058
17115
  this.captureProcessor = processor;
17059
17116
  processor.onaudioprocess = (event) => {
17117
+ var _a26;
17118
+ if (this.captureContext !== ctx) return;
17060
17119
  const inputData = event.inputBuffer.getChannelData(0);
17061
17120
  const samples = resampleAudio(
17062
17121
  new Float32Array(inputData),
17063
17122
  ctx.sampleRate,
17064
17123
  this.captureSampleRate
17065
17124
  );
17066
- this.onAudio(encodeRealtimeAudio(samples));
17125
+ try {
17126
+ this.onAudio(encodeRealtimeAudio(samples));
17127
+ } catch (error) {
17128
+ (_a26 = this.onError) == null ? void 0 : _a26.call(
17129
+ this,
17130
+ error instanceof Error ? error : new Error(String(error))
17131
+ );
17132
+ }
17067
17133
  };
17068
17134
  source.connect(processor);
17069
17135
  processor.connect(ctx.destination);
17070
- this.onCapturingChange(true);
17136
+ const updateCapturing = () => this.onCapturingChange(
17137
+ stream.getAudioTracks().some(
17138
+ (track) => track.enabled && !track.muted && track.readyState === "live"
17139
+ )
17140
+ );
17141
+ for (const track of stream.getAudioTracks()) {
17142
+ for (const event of ["ended", "mute", "unmute"]) {
17143
+ track.addEventListener(event, updateCapturing);
17144
+ this.captureCleanup.push(
17145
+ () => track.removeEventListener(event, updateCapturing)
17146
+ );
17147
+ }
17148
+ }
17149
+ updateCapturing();
17071
17150
  }
17072
17151
  stopCapture() {
17073
17152
  var _a25, _b25, _c, _d;
17153
+ this.captureGeneration++;
17154
+ if (this.captureProcessor != null)
17155
+ this.captureProcessor.onaudioprocess = null;
17156
+ for (const cleanup of this.captureCleanup) cleanup();
17157
+ this.captureCleanup = [];
17074
17158
  (_a25 = this.captureProcessor) == null ? void 0 : _a25.disconnect();
17075
17159
  (_b25 = this.captureSource) == null ? void 0 : _b25.disconnect();
17076
- void ((_c = this.captureContext) == null ? void 0 : _c.close());
17077
- (_d = this.captureStream) == null ? void 0 : _d.getTracks().forEach((track) => track.stop());
17160
+ void ((_c = this.captureContext) == null ? void 0 : _c.close().catch(() => {
17161
+ }));
17162
+ if (this.ownsCaptureStream)
17163
+ (_d = this.captureStream) == null ? void 0 : _d.getTracks().forEach((track) => track.stop());
17078
17164
  this.captureProcessor = null;
17079
17165
  this.captureSource = null;
17080
17166
  this.captureContext = null;
@@ -17082,15 +17168,34 @@ var BrowserRealtimeAudio = class {
17082
17168
  this.onCapturingChange(false);
17083
17169
  }
17084
17170
  playAudio(base64Audio) {
17171
+ var _a25, _b25, _c;
17172
+ if (this.playbackPaused) return;
17085
17173
  this.ensurePlaybackContext();
17086
- this.playbackQueue.push(decodeRealtimeAudio(base64Audio));
17174
+ const samples = decodeRealtimeAudio(base64Audio);
17175
+ if (Math.max(
17176
+ 0,
17177
+ this.playbackTime - ((_b25 = (_a25 = this.playbackContext) == null ? void 0 : _a25.currentTime) != null ? _b25 : 0)
17178
+ ) + samples.length / this.playbackSampleRate > this.maxPlaybackSeconds) {
17179
+ this.playbackPaused = true;
17180
+ this.stopPlayback();
17181
+ (_c = this.onError) == null ? void 0 : _c.call(
17182
+ this,
17183
+ new Error(
17184
+ "Realtime audio playback buffer is full; playback paused, call resumePlayback() to resume at the live edge"
17185
+ )
17186
+ );
17187
+ return;
17188
+ }
17189
+ this.playbackQueue.push(samples);
17087
17190
  this.schedulePlayback();
17088
17191
  }
17089
17192
  stopPlayback() {
17090
17193
  this.playbackQueue = [];
17091
17194
  for (const source of this.activeSources) {
17092
17195
  try {
17196
+ source.onended = null;
17093
17197
  source.stop();
17198
+ source.disconnect();
17094
17199
  } catch (e) {
17095
17200
  }
17096
17201
  }
@@ -17109,8 +17214,12 @@ var BrowserRealtimeAudio = class {
17109
17214
  var _a25;
17110
17215
  this.stopCapture();
17111
17216
  this.stopPlayback();
17112
- void ((_a25 = this.playbackContext) == null ? void 0 : _a25.close());
17217
+ if (this.playbackContext != null) this.playbackContext.onstatechange = null;
17218
+ void ((_a25 = this.playbackContext) == null ? void 0 : _a25.close().catch(() => {
17219
+ }));
17113
17220
  this.playbackContext = null;
17221
+ this.playbackTime = 0;
17222
+ this.playbackPaused = false;
17114
17223
  }
17115
17224
  setPlaying(isPlaying) {
17116
17225
  var _a25, _b25;
@@ -17140,8 +17249,10 @@ var BrowserRealtimeAudio = class {
17140
17249
  source.start(startTime);
17141
17250
  this.playbackTime = startTime + buffer.duration;
17142
17251
  this.activeSources.add(source);
17143
- this.setPlaying(true);
17252
+ this.setPlaying(ctx.state === "running");
17253
+ if (this.playbackContext !== ctx) return;
17144
17254
  source.onended = () => {
17255
+ source.disconnect();
17145
17256
  this.activeSources.delete(source);
17146
17257
  if (this.playbackQueue.length === 0 && this.activeSources.size === 0) {
17147
17258
  this.setPlaying(false);
@@ -17151,168 +17262,1323 @@ var BrowserRealtimeAudio = class {
17151
17262
  }
17152
17263
  };
17153
17264
 
17154
- // src/realtime/browser-realtime-transport.ts
17265
+ // src/realtime/realtime-event-channel.ts
17155
17266
  import { safeParseJSON as safeParseJSON5 } from "@ai-sdk/provider-utils";
17267
+ var RealtimeEventChannel = class {
17268
+ constructor(options) {
17269
+ this.options = options;
17270
+ this.active = true;
17271
+ this.finishing = false;
17272
+ this.incoming = Promise.resolve();
17273
+ this.outgoing = Promise.resolve();
17274
+ this.incomingCount = 0;
17275
+ this.incomingBytes = 0;
17276
+ this.outgoingCount = 0;
17277
+ var _a25, _b25, _c;
17278
+ this.parse = (_c = (_b25 = (_a25 = options.model).createServerEventParser) == null ? void 0 : _b25.call(_a25)) != null ? _c : options.model.parseServerEvent.bind(options.model);
17279
+ }
17280
+ dispose() {
17281
+ this.active = false;
17282
+ }
17283
+ /** Stop accepting messages, but deliver already received terminal events. */
17284
+ async finish() {
17285
+ this.stopWriting();
17286
+ await this.incoming;
17287
+ this.dispose();
17288
+ }
17289
+ stopWriting() {
17290
+ this.finishing = true;
17291
+ }
17292
+ send(event, shouldSend) {
17293
+ var _a25;
17294
+ if (!this.active || this.finishing)
17295
+ throw new Error("Realtime connection is closed");
17296
+ if (this.outgoingCount >= ((_a25 = this.options.maxPending) != null ? _a25 : 512)) {
17297
+ throw new Error("Realtime outgoing queue is full");
17298
+ }
17299
+ this.outgoingCount++;
17300
+ const operation = this.outgoing.then(async () => {
17301
+ if (!this.active || this.finishing)
17302
+ throw new Error("Realtime connection is closed");
17303
+ if ((shouldSend == null ? void 0 : shouldSend()) === false) return;
17304
+ if (!this.active || this.finishing)
17305
+ throw new Error("Realtime connection is closed");
17306
+ const data = await this.options.model.serializeClientEvent(event);
17307
+ if (!this.active || this.finishing)
17308
+ throw new Error("Realtime connection is closed");
17309
+ if ((shouldSend == null ? void 0 : shouldSend()) === false) return;
17310
+ if (!this.active || this.finishing)
17311
+ throw new Error("Realtime connection is closed");
17312
+ if (data != null) this.options.send(data);
17313
+ });
17314
+ this.outgoing = operation.catch((error) => {
17315
+ if (!this.finishing) {
17316
+ if (event.type !== "input-audio-append" && event.type !== "session-close")
17317
+ this.report(error);
17318
+ }
17319
+ }).finally(() => {
17320
+ this.outgoingCount--;
17321
+ });
17322
+ return operation;
17323
+ }
17324
+ receive(data) {
17325
+ var _a25, _b25, _c;
17326
+ if (!this.active || this.finishing) return;
17327
+ const bytes = typeof data === "string" ? data.length * 2 : data instanceof Blob ? data.size : (_a25 = data.byteLength) != null ? _a25 : 0;
17328
+ if (this.incomingCount >= ((_b25 = this.options.maxPending) != null ? _b25 : 512) || this.incomingBytes + bytes > ((_c = this.options.maxPendingBytes) != null ? _c : 8 * 1024 * 1024)) {
17329
+ this.finishing = true;
17330
+ this.fatal(
17331
+ new Error(
17332
+ "Realtime incoming queue is full; input protocol continuity was lost"
17333
+ )
17334
+ );
17335
+ return;
17336
+ }
17337
+ this.incomingCount++;
17338
+ this.incomingBytes += bytes;
17339
+ this.incoming = this.incoming.then(async () => {
17340
+ var _a26, _b26;
17341
+ if (!this.active) return;
17342
+ const text2 = typeof data === "string" ? data : data instanceof Blob ? await data.text() : new TextDecoder().decode(data);
17343
+ const parsed = await safeParseJSON5({ text: text2 });
17344
+ if (!this.active) return;
17345
+ if (!parsed.success)
17346
+ throw new Error("Invalid JSON in realtime server message", {
17347
+ cause: parsed.error
17348
+ });
17349
+ const health = (_b26 = (_a26 = this.options.model).getHealthCheckResponse) == null ? void 0 : _b26.call(
17350
+ _a26,
17351
+ parsed.value
17352
+ );
17353
+ if (!this.active) return;
17354
+ if (health != null && !this.finishing) this.options.send(health);
17355
+ if (!this.active) return;
17356
+ const result = this.parse(parsed.value);
17357
+ for (const event of Array.isArray(result) ? result : [result]) {
17358
+ if (!this.active) return;
17359
+ try {
17360
+ await this.options.onEvent(event);
17361
+ } catch (error) {
17362
+ this.report(error);
17363
+ }
17364
+ }
17365
+ }).catch((error) => {
17366
+ this.finishing = true;
17367
+ this.fatal(error instanceof Error ? error : new Error(String(error)));
17368
+ }).finally(() => {
17369
+ this.incomingCount--;
17370
+ this.incomingBytes -= bytes;
17371
+ });
17372
+ }
17373
+ fatal(error) {
17374
+ if (!this.active) return;
17375
+ if (this.options.onFatalError != null) this.options.onFatalError(error);
17376
+ else this.report(error);
17377
+ }
17378
+ report(error) {
17379
+ if (!this.active) return;
17380
+ try {
17381
+ this.options.onError(
17382
+ error instanceof Error ? error : new Error(String(error))
17383
+ );
17384
+ } catch (e) {
17385
+ }
17386
+ }
17387
+ };
17388
+
17389
+ // src/realtime/encode-realtime-frame.ts
17390
+ var REALTIME_MAX_FRAME_BYTES = 128 * 1024;
17391
+ var REALTIME_MAX_BUFFERED_BYTES = 128 * 1024;
17392
+ function encodeRealtimeFrame(value) {
17393
+ if (value instanceof ArrayBuffer || ArrayBuffer.isView(value))
17394
+ return { data: value, byteLength: value.byteLength };
17395
+ if (value instanceof Blob) return { data: value, byteLength: value.size };
17396
+ const data = typeof value === "string" ? value : JSON.stringify(value);
17397
+ if (data === void 0)
17398
+ throw new Error("Realtime event could not be encoded as JSON");
17399
+ return { data, byteLength: new TextEncoder().encode(data).byteLength };
17400
+ }
17401
+ function assertRealtimeFrameBudget(frame, bufferedAmount) {
17402
+ if (frame.byteLength > REALTIME_MAX_FRAME_BYTES)
17403
+ throw new Error(
17404
+ `Realtime frame exceeds the ${REALTIME_MAX_FRAME_BYTES}-byte limit; reduce the event payload`
17405
+ );
17406
+ if (bufferedAmount + frame.byteLength > REALTIME_MAX_BUFFERED_BYTES)
17407
+ throw new Error(
17408
+ `Realtime send buffer is full (${REALTIME_MAX_BUFFERED_BYTES}-byte limit); wait before retrying`
17409
+ );
17410
+ }
17411
+
17412
+ // src/realtime/browser-realtime-transport.ts
17413
+ function getCloseError(event) {
17414
+ if (event.code === 1e3 && event.wasClean) return void 0;
17415
+ return new Error(
17416
+ `Realtime WebSocket closed unexpectedly (code ${event.code}${event.reason === "" ? "" : `: ${event.reason}`})`
17417
+ );
17418
+ }
17156
17419
  var BrowserRealtimeTransport = class {
17157
17420
  constructor(options) {
17421
+ this.options = options;
17158
17422
  this.ws = null;
17159
- this.sendQueue = Promise.resolve();
17423
+ this.epoch = 0;
17424
+ this.failing = false;
17425
+ this.closing = false;
17426
+ var _a25;
17160
17427
  this.model = options.model;
17428
+ this.continuous = ((_a25 = options.model.capabilities) == null ? void 0 : _a25.conversation) === "continuous";
17161
17429
  this.onServerEvent = options.onServerEvent;
17162
17430
  this.onError = options.onError;
17163
17431
  this.onClose = options.onClose;
17164
17432
  }
17433
+ get isOpen() {
17434
+ var _a25;
17435
+ return !this.closing && !this.failing && ((_a25 = this.ws) == null ? void 0 : _a25.readyState) === WebSocket.OPEN;
17436
+ }
17165
17437
  connect({
17438
+ mode,
17166
17439
  token,
17167
17440
  url,
17168
- onOpen
17441
+ onOpen,
17442
+ protocols
17169
17443
  }) {
17170
- var _a25;
17171
- (_a25 = this.ws) == null ? void 0 : _a25.close();
17172
- this.ws = null;
17173
- const wsConfig = this.model.getWebSocketConfig({ token, url });
17174
- const ws = new WebSocket(wsConfig.url, wsConfig.protocols);
17444
+ var _a25, _b25;
17445
+ this.disconnect();
17446
+ const epoch = this.epoch;
17447
+ if (mode !== "relay" && (mode !== "client-secret" || typeof token !== "string" || token.trim() === ""))
17448
+ throw new Error(
17449
+ "Realtime client-secret connection requires a nonempty token"
17450
+ );
17451
+ const wsConfig = mode === "relay" ? { url, protocols } : (_b25 = (_a25 = this.model).getWebSocketConfig) == null ? void 0 : _b25.call(_a25, { token, url });
17452
+ if (this.epoch !== epoch) return;
17453
+ if (wsConfig == null)
17454
+ throw new Error("Model does not support client-secret WebSockets");
17455
+ const finalUrl = wsConfig.url;
17456
+ const finalProtocols = wsConfig.protocols;
17457
+ if (this.epoch !== epoch) return;
17458
+ try {
17459
+ if (typeof finalUrl !== "string" || !/^wss?:\/\//i.test(finalUrl))
17460
+ throw new Error("Invalid realtime WebSocket URL");
17461
+ const parsed = new URL(finalUrl);
17462
+ if (!["ws:", "wss:"].includes(parsed.protocol) || parsed.hostname === "" || finalUrl.includes("#"))
17463
+ throw new Error("Invalid realtime WebSocket URL");
17464
+ } catch (e) {
17465
+ throw new Error(
17466
+ "Invalid realtime WebSocket URL; expected an absolute ws:// or wss:// URL with a host and no fragment"
17467
+ );
17468
+ }
17469
+ if (this.epoch !== epoch) return;
17470
+ const ws = new WebSocket(finalUrl, finalProtocols);
17175
17471
  this.ws = ws;
17472
+ let starting = false;
17473
+ let connectionError;
17474
+ const codec = new RealtimeEventChannel({
17475
+ model: this.model,
17476
+ send: (data) => {
17477
+ if (this.epoch !== epoch || this.ws !== ws || !this.isOpen)
17478
+ throw new Error("Realtime WebSocket is not open");
17479
+ this.sendRaw(data);
17480
+ },
17481
+ onEvent: this.onServerEvent,
17482
+ onError: (error) => {
17483
+ if (!starting) this.onError(error);
17484
+ },
17485
+ onFatalError: (error) => this.fail(error)
17486
+ });
17487
+ if (this.epoch !== epoch) {
17488
+ codec.dispose();
17489
+ return;
17490
+ }
17491
+ this.codec = codec;
17176
17492
  ws.onopen = () => {
17177
17493
  if (this.ws !== ws) return;
17178
- onOpen();
17494
+ starting = true;
17495
+ try {
17496
+ void Promise.resolve(onOpen()).catch((error) => {
17497
+ if (this.ws === ws)
17498
+ this.fail(
17499
+ error instanceof Error ? error : new Error(String(error))
17500
+ );
17501
+ }).finally(() => {
17502
+ starting = false;
17503
+ });
17504
+ } catch (error) {
17505
+ if (this.epoch === epoch)
17506
+ this.fail(error instanceof Error ? error : new Error(String(error)));
17507
+ }
17179
17508
  };
17180
17509
  ws.onmessage = (messageEvent) => {
17181
- void this.handleMessage(messageEvent);
17510
+ var _a26;
17511
+ if (this.ws === ws) (_a26 = this.codec) == null ? void 0 : _a26.receive(messageEvent.data);
17182
17512
  };
17183
17513
  ws.onerror = () => {
17184
- this.onError(new Error("WebSocket connection error"));
17514
+ if (this.ws === ws) {
17515
+ connectionError = new Error("WebSocket connection error");
17516
+ this.failing = true;
17517
+ }
17185
17518
  };
17186
- ws.onclose = () => {
17519
+ ws.onclose = (event) => {
17520
+ var _a26;
17187
17521
  if (this.ws === ws) {
17188
17522
  this.ws = null;
17189
- this.onClose();
17523
+ const closeError = (_a26 = getCloseError(event)) != null ? _a26 : connectionError;
17524
+ codec.stopWriting();
17525
+ this.notifyClosing();
17526
+ if (this.epoch !== epoch) return;
17527
+ const complete = () => {
17528
+ if (this.epoch !== epoch) return;
17529
+ clearTimeout(this.drainTimer);
17530
+ this.epoch++;
17531
+ codec.dispose();
17532
+ try {
17533
+ this.onClose(closeError);
17534
+ } catch (error) {
17535
+ this.reportCallbackError(error);
17536
+ }
17537
+ };
17538
+ this.drainTimer = setTimeout(complete, 1e3);
17539
+ void this.awaitDrain(codec.finish(), complete);
17190
17540
  }
17191
17541
  };
17192
17542
  }
17193
17543
  disconnect() {
17194
17544
  var _a25;
17195
- (_a25 = this.ws) == null ? void 0 : _a25.close();
17545
+ this.epoch++;
17546
+ clearTimeout(this.drainTimer);
17547
+ const ws = this.ws;
17196
17548
  this.ws = null;
17549
+ (_a25 = this.codec) == null ? void 0 : _a25.dispose();
17550
+ this.codec = void 0;
17551
+ this.failing = false;
17552
+ this.closing = false;
17553
+ ws == null ? void 0 : ws.close();
17554
+ }
17555
+ sendEvent(event, shouldSend, automaticAudio = false) {
17556
+ if (this.failing || this.closing)
17557
+ throw new Error("Realtime connection is closed");
17558
+ if (this.codec == null) return Promise.resolve();
17559
+ const epoch = this.epoch;
17560
+ const failed = (error) => {
17561
+ if (automaticAudio && this.epoch === epoch && this.isOpen)
17562
+ this.fail(error instanceof Error ? error : new Error(String(error)));
17563
+ };
17564
+ try {
17565
+ const sent = this.codec.send(event, shouldSend);
17566
+ void sent.catch(failed);
17567
+ return sent;
17568
+ } catch (error) {
17569
+ failed(error);
17570
+ throw error;
17571
+ }
17197
17572
  }
17198
- sendEvent(event) {
17199
- this.sendQueue = this.sendQueue.then(async () => {
17200
- const serialized = await this.model.serializeClientEvent(event);
17201
- if (serialized != null) {
17202
- this.sendRaw(serialized);
17203
- }
17204
- }).catch((error) => {
17205
- this.onError(
17206
- error instanceof Error ? error : new Error(`Failed to send realtime event: ${String(error)}`)
17207
- );
17208
- });
17573
+ finish() {
17574
+ var _a25, _b25;
17575
+ return (_b25 = (_a25 = this.codec) == null ? void 0 : _a25.finish()) != null ? _b25 : Promise.resolve();
17209
17576
  }
17210
- sendRaw(data) {
17577
+ /** Drain accepted events when media or protocol continuity is lost. */
17578
+ fail(error) {
17211
17579
  var _a25;
17212
- if (((_a25 = this.ws) == null ? void 0 : _a25.readyState) === WebSocket.OPEN) {
17213
- if (typeof data === "string" || data instanceof ArrayBuffer || ArrayBuffer.isView(data) || data instanceof Blob) {
17214
- this.ws.send(data);
17215
- } else {
17216
- this.ws.send(JSON.stringify(data));
17217
- }
17580
+ if (this.failing) return;
17581
+ this.failing = true;
17582
+ const epoch = this.epoch;
17583
+ const ws = this.ws;
17584
+ this.ws = null;
17585
+ (_a25 = this.codec) == null ? void 0 : _a25.stopWriting();
17586
+ ws == null ? void 0 : ws.close();
17587
+ this.notifyClosing();
17588
+ if (this.epoch !== epoch) return;
17589
+ const drain = this.finish();
17590
+ if (this.options.onFatalError != null)
17591
+ this.options.onFatalError(error, drain);
17592
+ else {
17593
+ const complete = () => {
17594
+ if (epoch !== this.epoch) return;
17595
+ this.disconnect();
17596
+ try {
17597
+ this.onClose();
17598
+ } catch (cause) {
17599
+ this.reportCallbackError(cause);
17600
+ }
17601
+ };
17602
+ this.drainTimer = setTimeout(complete, 1e3);
17603
+ void this.awaitDrain(drain, complete);
17604
+ this.reportCallbackError(error);
17218
17605
  }
17219
17606
  }
17220
- dispose() {
17221
- this.disconnect();
17607
+ notifyClosing() {
17608
+ var _a25, _b25;
17609
+ if (this.closing) return;
17610
+ this.closing = true;
17611
+ try {
17612
+ (_b25 = (_a25 = this.options).onClosing) == null ? void 0 : _b25.call(_a25);
17613
+ } catch (error) {
17614
+ this.reportCallbackError(error);
17615
+ }
17222
17616
  }
17223
- async handleMessage(messageEvent) {
17224
- let text2;
17225
- if (typeof messageEvent.data === "string") {
17226
- text2 = messageEvent.data;
17227
- } else if (messageEvent.data instanceof Blob) {
17228
- text2 = await messageEvent.data.text();
17229
- } else {
17230
- text2 = new TextDecoder().decode(messageEvent.data);
17617
+ async awaitDrain(drain, complete) {
17618
+ try {
17619
+ await drain;
17620
+ } catch (e) {
17231
17621
  }
17232
- const parseResult = await safeParseJSON5({ text: text2 });
17233
- if (!parseResult.success) return;
17234
- const rawEvent = parseResult.value;
17235
- if (this.model.getHealthCheckResponse != null) {
17236
- const autoResponse = this.model.getHealthCheckResponse(rawEvent);
17237
- if (autoResponse != null) {
17238
- this.sendRaw(autoResponse);
17239
- }
17622
+ try {
17623
+ complete();
17624
+ } catch (error) {
17625
+ this.reportCallbackError(error);
17240
17626
  }
17241
- const result = this.model.parseServerEvent(rawEvent);
17242
- const events = Array.isArray(result) ? result : [result];
17243
- for (const event of events) {
17244
- await this.onServerEvent(event);
17627
+ }
17628
+ reportCallbackError(error) {
17629
+ try {
17630
+ this.onError(error instanceof Error ? error : new Error(String(error)));
17631
+ } catch (e) {
17245
17632
  }
17246
17633
  }
17634
+ sendRaw(data) {
17635
+ const ws = this.ws;
17636
+ const epoch = this.epoch;
17637
+ if (ws == null || !this.isOpen) return;
17638
+ const frame = encodeRealtimeFrame(data);
17639
+ if (this.epoch !== epoch || this.ws !== ws || !this.isOpen)
17640
+ throw new Error("Realtime connection is closed");
17641
+ if (this.continuous) assertRealtimeFrameBudget(frame, ws.bufferedAmount);
17642
+ ws.send(frame.data);
17643
+ }
17644
+ dispose() {
17645
+ this.disconnect();
17646
+ }
17247
17647
  };
17248
17648
 
17249
- // src/realtime/realtime-event-reducer.ts
17250
- import { safeParseJSON as safeParseJSON6 } from "@ai-sdk/provider-utils";
17251
- function createInitialRealtimeState() {
17252
- return {
17253
- status: "disconnected",
17254
- messages: [],
17255
- events: [],
17256
- isCapturing: false,
17257
- isPlaying: false
17258
- };
17259
- }
17260
- var RealtimeEventReducer = class {
17261
- constructor(maxEvents = 500) {
17262
- this.maxEvents = maxEvents;
17263
- this.currentAssistantMessageId = null;
17264
- this.textAccumulators = /* @__PURE__ */ new Map();
17265
- this.toolArgAccumulators = /* @__PURE__ */ new Map();
17266
- this.toolCallIdToMessageId = /* @__PURE__ */ new Map();
17267
- this.toolCallIdToName = /* @__PURE__ */ new Map();
17268
- this.inputAudioMessageInsertIndex = /* @__PURE__ */ new Map();
17269
- this.itemIdToPartLocation = /* @__PURE__ */ new Map();
17270
- }
17271
- setStatus(state, status) {
17272
- return { ...state, status };
17649
+ // src/realtime/browser-realtime-live-websocket.ts
17650
+ import { UnsupportedFunctionalityError as UnsupportedFunctionalityError5 } from "@ai-sdk/provider";
17651
+ var BrowserRealtimeLiveWebSocket = class {
17652
+ constructor(options) {
17653
+ this.options = options;
17654
+ this.generation = 0;
17655
+ this.ready = false;
17656
+ this.capturingStarted = false;
17657
+ this.pendingAudio = 0;
17658
+ this.captureGeneration = 0;
17659
+ this.captureEnabled = true;
17660
+ var _a25, _b25, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l;
17661
+ const input = (_a25 = options.sessionConfig) == null ? void 0 : _a25.inputAudioFormat;
17662
+ const output = (_b25 = options.sessionConfig) == null ? void 0 : _b25.outputAudioFormat;
17663
+ const sampleRate = (_e = (_d = (_c = input == null ? void 0 : input.rate) != null ? _c : output == null ? void 0 : output.rate) != null ? _d : options.sampleRate) != null ? _e : 24e3;
17664
+ const format = {
17665
+ type: (_g = (_f = input == null ? void 0 : input.type) != null ? _f : output == null ? void 0 : output.type) != null ? _g : "audio/pcm",
17666
+ rate: sampleRate
17667
+ };
17668
+ this.config = {
17669
+ ...options.sessionConfig,
17670
+ inputAudioFormat: {
17671
+ ...format,
17672
+ ...input
17673
+ },
17674
+ outputAudioFormat: {
17675
+ ...format,
17676
+ ...output
17677
+ }
17678
+ };
17679
+ this.transport = new BrowserRealtimeTransport({
17680
+ model: options.model,
17681
+ onServerEvent: options.onEvent,
17682
+ onError: options.onError,
17683
+ onFatalError: options.onFatalError,
17684
+ onClosing: () => {
17685
+ var _a26;
17686
+ const generation = this.generation;
17687
+ (_a26 = options.onClosing) == null ? void 0 : _a26.call(options);
17688
+ if (generation === this.generation) this.stopCapture();
17689
+ },
17690
+ onClose: options.onClose
17691
+ });
17692
+ this.audio = new BrowserRealtimeAudio({
17693
+ captureSampleRate: (_i = (_h = this.config.inputAudioFormat) == null ? void 0 : _h.rate) != null ? _i : sampleRate,
17694
+ playbackSampleRate: (_k = (_j = this.config.outputAudioFormat) == null ? void 0 : _j.rate) != null ? _k : sampleRate,
17695
+ maxPlaybackSeconds: (_l = options.maxPlaybackBufferSeconds) != null ? _l : 2,
17696
+ onAudio: (audio) => this.sendAudio(audio),
17697
+ onError: options.onError,
17698
+ onCapturingChange: options.onCapturing,
17699
+ onPlayingChange: options.onPlaying
17700
+ });
17273
17701
  }
17274
- setCapturing(state, isCapturing) {
17275
- return { ...state, isCapturing };
17702
+ connect(options) {
17703
+ const generation = this.generation + 1;
17704
+ this.dispose();
17705
+ if (generation !== this.generation) return;
17706
+ for (const format of [
17707
+ this.config.inputAudioFormat,
17708
+ this.config.outputAudioFormat
17709
+ ]) {
17710
+ if ((format == null ? void 0 : format.type) !== "audio/pcm")
17711
+ throw new UnsupportedFunctionalityError5({
17712
+ functionality: "Continuous browser WebSocket audio supports PCM16 only"
17713
+ });
17714
+ if (format.rate == null || !Number.isFinite(format.rate) || format.rate < 8e3 || format.rate > 96e3)
17715
+ throw new Error("Invalid realtime PCM sample rate");
17716
+ }
17717
+ this.stream = options.stream;
17718
+ this.captureEnabled = options.capture !== false;
17719
+ this.audio.ensurePlaybackContext();
17720
+ void this.audio.resumePlayback().catch((error) => {
17721
+ if (generation === this.generation) this.options.onError(error);
17722
+ });
17723
+ this.transport.connect({
17724
+ mode: "relay",
17725
+ url: options.url,
17726
+ protocols: options.protocols,
17727
+ onOpen: () => {
17728
+ var _a25, _b25;
17729
+ return this.transport.sendEvent({
17730
+ type: (_b25 = (_a25 = this.options.model.capabilities) == null ? void 0 : _a25.startup) != null ? _b25 : "session-update",
17731
+ config: this.config
17732
+ });
17733
+ }
17734
+ });
17276
17735
  }
17277
- setPlaying(state, isPlaying) {
17278
- return { ...state, isPlaying };
17736
+ startCapture() {
17737
+ this.ready = true;
17738
+ if (!this.captureEnabled || this.capturingStarted) return;
17739
+ void this.resumeCapture().catch((error) => this.options.onError(error));
17279
17740
  }
17280
- addUserTextMessage(state, text2) {
17281
- return {
17282
- ...state,
17283
- messages: [
17284
- ...state.messages,
17285
- {
17286
- id: `user-${Date.now()}`,
17287
- role: "user",
17288
- parts: [{ type: "text", text: text2, state: "done" }]
17289
- }
17290
- ]
17291
- };
17741
+ async resumeCapture(suppliedStream) {
17742
+ const generation = this.captureGeneration + 1;
17743
+ this.stopCapture();
17744
+ if (generation !== this.captureGeneration) return;
17745
+ this.ready = true;
17746
+ this.capturingStarted = true;
17747
+ this.captureEnabled = true;
17748
+ if (suppliedStream != null) this.stream = suppliedStream;
17749
+ const supplied = this.stream;
17750
+ const stream = supplied != null ? supplied : await navigator.mediaDevices.getUserMedia({ audio: true });
17751
+ if (generation !== this.captureGeneration || !this.ready) {
17752
+ if (supplied == null) stream.getTracks().forEach((track) => track.stop());
17753
+ return;
17754
+ }
17755
+ if (!stream.getAudioTracks().some((track) => track.readyState === "live")) {
17756
+ if (supplied == null) stream.getTracks().forEach((track) => track.stop());
17757
+ throw new Error("Realtime requires a live audio track");
17758
+ }
17759
+ this.audio.startCapture(stream, { ownsStream: supplied == null });
17292
17760
  }
17293
- addToolOutput(state, callId, result) {
17294
- return {
17295
- state: this.updateToolPartState(state, callId, result),
17296
- output: {
17297
- callId,
17298
- name: this.toolCallIdToName.get(callId),
17299
- output: JSON.stringify(result)
17761
+ sendAudio(audio) {
17762
+ if (!this.ready || !this.transport.isOpen) return;
17763
+ if (this.pendingAudio >= 8) {
17764
+ this.transport.fail(new Error("Realtime audio send queue is full"));
17765
+ return;
17766
+ }
17767
+ const generation = this.generation;
17768
+ let sent;
17769
+ try {
17770
+ sent = this.transport.sendEvent(
17771
+ { type: "input-audio-append", audio },
17772
+ void 0,
17773
+ true
17774
+ );
17775
+ } catch (error) {
17776
+ if (!this.ready || !this.transport.isOpen) return;
17777
+ throw error;
17778
+ }
17779
+ this.pendingAudio++;
17780
+ void sent.then(
17781
+ () => {
17782
+ if (generation === this.generation) this.pendingAudio--;
17783
+ },
17784
+ (error) => {
17785
+ if (generation === this.generation && this.ready && this.transport.isOpen)
17786
+ this.options.onFatalError(error);
17300
17787
  }
17301
- };
17788
+ );
17302
17789
  }
17303
- async reduceServerEvent(state, event) {
17304
- var _a25;
17305
- let nextState = this.pushEvent(state, event);
17306
- const effects = [];
17307
- switch (event.type) {
17308
- case "session-created":
17309
- case "session-updated": {
17310
- if (nextState.status === "connecting") {
17311
- nextState = this.setStatus(nextState, "connected");
17790
+ sendEvent(event, shouldSend) {
17791
+ var _a25, _b25;
17792
+ if (event.type === "session-update") {
17793
+ for (const key of ["inputAudioFormat", "outputAudioFormat"]) {
17794
+ const update = event.config[key];
17795
+ if (update != null && (update.type !== ((_a25 = this.config[key]) == null ? void 0 : _a25.type) || update.rate != null && update.rate !== ((_b25 = this.config[key]) == null ? void 0 : _b25.rate))) {
17796
+ throw new Error(
17797
+ "Changing Live WebSocket audio format requires reconnecting"
17798
+ );
17312
17799
  }
17313
- break;
17314
17800
  }
17315
- case "audio-delta": {
17801
+ }
17802
+ return this.transport.sendEvent(event, shouldSend);
17803
+ }
17804
+ playAudio(audio) {
17805
+ try {
17806
+ this.audio.playAudio(audio);
17807
+ } catch (error) {
17808
+ this.options.onError(
17809
+ error instanceof Error ? error : new Error(String(error))
17810
+ );
17811
+ }
17812
+ }
17813
+ stopPlayback() {
17814
+ this.audio.stopPlayback();
17815
+ }
17816
+ resumePlayback() {
17817
+ return this.audio.resumePlayback();
17818
+ }
17819
+ stopCapture() {
17820
+ this.captureGeneration++;
17821
+ this.ready = false;
17822
+ this.capturingStarted = false;
17823
+ this.captureEnabled = false;
17824
+ this.audio.stopCapture();
17825
+ }
17826
+ finish() {
17827
+ return this.transport.finish();
17828
+ }
17829
+ dispose() {
17830
+ this.generation++;
17831
+ this.stopCapture();
17832
+ this.ready = false;
17833
+ this.capturingStarted = false;
17834
+ this.pendingAudio = 0;
17835
+ this.stream = void 0;
17836
+ this.transport.disconnect();
17837
+ this.audio.dispose();
17838
+ }
17839
+ };
17840
+
17841
+ // src/realtime/browser-realtime-webrtc.ts
17842
+ import { safeParseJSON as safeParseJSON6 } from "@ai-sdk/provider-utils";
17843
+ var MAX_SESSION_ANSWER_BYTES = 1024 * 1024;
17844
+ async function readSessionAnswer(response) {
17845
+ var _a25;
17846
+ const reader = (_a25 = response.body) == null ? void 0 : _a25.getReader();
17847
+ if (reader == null) throw new Error("Invalid realtime session answer");
17848
+ const decoder = new TextDecoder();
17849
+ let bytes = 0;
17850
+ let text2 = "";
17851
+ try {
17852
+ if (Number(response.headers.get("content-length")) > MAX_SESSION_ANSWER_BYTES)
17853
+ throw new Error("Realtime session answer exceeds the 1 MiB limit");
17854
+ while (true) {
17855
+ const { done, value } = await reader.read();
17856
+ if (done) return text2 + decoder.decode();
17857
+ bytes += value.byteLength;
17858
+ if (bytes > MAX_SESSION_ANSWER_BYTES)
17859
+ throw new Error("Realtime session answer exceeds the 1 MiB limit");
17860
+ text2 += decoder.decode(value, { stream: true });
17861
+ }
17862
+ } catch (error) {
17863
+ void reader.cancel(error).catch(() => {
17864
+ });
17865
+ throw error;
17866
+ } finally {
17867
+ reader.releaseLock();
17868
+ }
17869
+ }
17870
+ function selectAudioTrack(stream) {
17871
+ var _a25;
17872
+ const live = stream.getAudioTracks().filter((track) => track.readyState === "live");
17873
+ return (_a25 = live.find((track) => track.enabled && !track.muted)) != null ? _a25 : live[0];
17874
+ }
17875
+ var BrowserRealtimeWebRTC = class {
17876
+ constructor(options) {
17877
+ this.options = options;
17878
+ this.ownsStream = false;
17879
+ this.generation = 0;
17880
+ this.closingNotified = false;
17881
+ this.captureGeneration = 0;
17882
+ this.senderOperations = Promise.resolve();
17883
+ this.trackCleanups = [];
17884
+ }
17885
+ async connect({
17886
+ api,
17887
+ sessionConfig,
17888
+ stream,
17889
+ timeoutMs,
17890
+ capture = true
17891
+ }) {
17892
+ var _a25, _b25;
17893
+ const generation = this.generation + 1;
17894
+ this.dispose();
17895
+ const current = () => generation === this.generation;
17896
+ if (!current()) return;
17897
+ const config = (_b25 = (_a25 = this.options.model).getWebRTCConfig) == null ? void 0 : _b25.call(_a25);
17898
+ if (!current()) return;
17899
+ if (config == null)
17900
+ throw new Error("This model does not support WebRTC configuration");
17901
+ const abort = new AbortController();
17902
+ this.abort = abort;
17903
+ const timeout = setTimeout(
17904
+ () => abort.abort(new Error("Realtime startup timed out")),
17905
+ timeoutMs
17906
+ );
17907
+ const cancelled = new Promise((_, reject) => {
17908
+ abort.signal.addEventListener(
17909
+ "abort",
17910
+ () => reject(abort.signal.reason),
17911
+ { once: true }
17912
+ );
17913
+ });
17914
+ const start = async () => {
17915
+ var _a26;
17916
+ const captureGeneration = this.captureGeneration;
17917
+ let media = capture ? stream != null ? stream : await navigator.mediaDevices.getUserMedia({ audio: true }) : void 0;
17918
+ if (!current() || abort.signal.aborted) {
17919
+ if (stream == null) media == null ? void 0 : media.getTracks().forEach((track2) => track2.stop());
17920
+ throw new Error("Realtime connection cancelled");
17921
+ }
17922
+ if (captureGeneration !== this.captureGeneration) {
17923
+ if (stream == null) media == null ? void 0 : media.getTracks().forEach((track2) => track2.stop());
17924
+ media = void 0;
17925
+ }
17926
+ this.stream = media;
17927
+ this.ownsStream = stream == null;
17928
+ const track = media == null ? void 0 : selectAudioTrack(media);
17929
+ if (media != null && track == null) {
17930
+ throw new Error("Realtime requires a live audio track");
17931
+ }
17932
+ const pc = new RTCPeerConnection();
17933
+ this.pc = pc;
17934
+ const audio = document.createElement("audio");
17935
+ audio.autoplay = true;
17936
+ this.audio = audio;
17937
+ audio.onplaying = () => {
17938
+ if (current()) this.options.onPlaying(true);
17939
+ };
17940
+ const stopped = () => {
17941
+ if (current()) this.options.onPlaying(false);
17942
+ };
17943
+ audio.onpause = stopped;
17944
+ audio.onended = stopped;
17945
+ audio.onwaiting = stopped;
17946
+ pc.ontrack = (event) => {
17947
+ var _a27;
17948
+ if (!current()) return;
17949
+ audio.srcObject = (_a27 = event.streams[0]) != null ? _a27 : new MediaStream([event.track]);
17950
+ void audio.play().catch((error) => {
17951
+ if (current()) this.options.onError(error);
17952
+ });
17953
+ };
17954
+ pc.onconnectionstatechange = () => {
17955
+ var _a27;
17956
+ if (!current()) return;
17957
+ if (pc.connectionState === "connected")
17958
+ clearTimeout(this.disconnectTimer);
17959
+ else if (pc.connectionState === "disconnected") {
17960
+ clearTimeout(this.disconnectTimer);
17961
+ this.disconnectTimer = setTimeout(() => {
17962
+ if (current() && pc.connectionState === "disconnected")
17963
+ this.fail(
17964
+ new Error(
17965
+ "Realtime peer disconnected beyond recovery grace period"
17966
+ )
17967
+ );
17968
+ }, (_a27 = this.options.disconnectTimeoutMs) != null ? _a27 : 5e3);
17969
+ } else if (pc.connectionState === "closed") {
17970
+ this.drainClose(new Error("Realtime peer connection closed"));
17971
+ } else if (pc.connectionState === "failed") {
17972
+ this.fail(
17973
+ new Error(`Realtime peer connection ${pc.connectionState}`)
17974
+ );
17975
+ }
17976
+ };
17977
+ pc.oniceconnectionstatechange = () => {
17978
+ if (current() && pc.iceConnectionState === "failed")
17979
+ this.fail(new Error("Realtime ICE connection failed"));
17980
+ };
17981
+ const dc = pc.createDataChannel(config.dataChannelLabel);
17982
+ this.dc = dc;
17983
+ const codec = new RealtimeEventChannel({
17984
+ model: this.options.model,
17985
+ onEvent: this.options.onEvent,
17986
+ onError: this.options.onError,
17987
+ onFatalError: (error) => {
17988
+ if (current()) this.fail(error);
17989
+ },
17990
+ send: (data) => {
17991
+ if (!current() || this.finishing || dc.readyState !== "open")
17992
+ throw new Error("Realtime data channel is not open");
17993
+ if (dc.bufferedAmount > 1024 * 1024)
17994
+ throw new Error("Realtime data channel buffer is full");
17995
+ const encoded = typeof data === "string" ? data : JSON.stringify(data);
17996
+ if (!current() || this.finishing || dc.readyState !== "open")
17997
+ throw new Error("Realtime data channel is not open");
17998
+ dc.send(encoded);
17999
+ }
18000
+ });
18001
+ if (!current()) {
18002
+ codec.dispose();
18003
+ return;
18004
+ }
18005
+ this.codec = codec;
18006
+ dc.onmessage = (event) => {
18007
+ var _a27;
18008
+ if (current()) (_a27 = this.codec) == null ? void 0 : _a27.receive(event.data);
18009
+ };
18010
+ dc.onclose = () => {
18011
+ if (current())
18012
+ this.drainClose(new Error("Realtime data channel closed"));
18013
+ };
18014
+ dc.onerror = () => {
18015
+ if (current()) this.fail(new Error("Realtime data channel error"));
18016
+ };
18017
+ const opened = new Promise((resolve3, reject) => {
18018
+ this.rejectStartup = reject;
18019
+ dc.onopen = () => {
18020
+ this.rejectStartup = void 0;
18021
+ resolve3();
18022
+ };
18023
+ });
18024
+ void opened.catch(() => {
18025
+ });
18026
+ const gathered = new Promise((resolve3) => {
18027
+ pc.onicegatheringstatechange = () => {
18028
+ if (pc.iceGatheringState === "complete") resolve3();
18029
+ };
18030
+ });
18031
+ this.sender = media != null && track != null ? pc.addTrack(track, media) : pc.addTransceiver("audio", { direction: "sendrecv" }).sender;
18032
+ this.observeCapture();
18033
+ if (!current() || abort.signal.aborted) return;
18034
+ const offer = await pc.createOffer();
18035
+ if (!current() || abort.signal.aborted) return;
18036
+ await pc.setLocalDescription(offer);
18037
+ if (pc.iceGatheringState !== "complete")
18038
+ await Promise.race([gathered, cancelled]);
18039
+ if (!current() || abort.signal.aborted) return;
18040
+ const sdp = (_a26 = pc.localDescription) == null ? void 0 : _a26.sdp;
18041
+ if (sdp == null) throw new Error("Realtime offer has no SDP");
18042
+ const response = await fetch(api, {
18043
+ method: "POST",
18044
+ headers: { "Content-Type": "application/json" },
18045
+ body: JSON.stringify({ sdp, sessionConfig }),
18046
+ signal: abort.signal
18047
+ });
18048
+ if (!response.ok)
18049
+ throw new Error(
18050
+ `Failed to create realtime session: ${response.status}`
18051
+ );
18052
+ const parsed = await safeParseJSON6({
18053
+ text: await readSessionAnswer(response)
18054
+ });
18055
+ if (!current() || abort.signal.aborted) return;
18056
+ if (!parsed.success || typeof parsed.value !== "object" || parsed.value == null || !("sdp" in parsed.value) || typeof parsed.value.sdp !== "string" || parsed.value.sdp.trim().length === 0 || !("sessionId" in parsed.value) || typeof parsed.value.sessionId !== "string" || parsed.value.sessionId.trim().length === 0) {
18057
+ throw new Error("Invalid realtime session answer");
18058
+ }
18059
+ await pc.setRemoteDescription({ type: "answer", sdp: parsed.value.sdp });
18060
+ await Promise.race([opened, cancelled]);
18061
+ };
18062
+ try {
18063
+ await Promise.race([start(), cancelled]);
18064
+ } catch (error) {
18065
+ if (current()) this.dispose();
18066
+ throw error;
18067
+ } finally {
18068
+ clearTimeout(timeout);
18069
+ this.rejectStartup = void 0;
18070
+ }
18071
+ }
18072
+ sendEvent(event, shouldSend) {
18073
+ var _a25, _b25, _c;
18074
+ if (this.finishing || ((_a25 = this.dc) == null ? void 0 : _a25.readyState) !== "open")
18075
+ throw new Error("Realtime data channel is not open");
18076
+ return (_c = (_b25 = this.codec) == null ? void 0 : _b25.send(event, shouldSend)) != null ? _c : Promise.resolve();
18077
+ }
18078
+ async resumePlayback() {
18079
+ if (this.audio == null)
18080
+ throw new Error("Realtime playback is not connected");
18081
+ await this.audio.play();
18082
+ }
18083
+ stopPlayback() {
18084
+ var _a25;
18085
+ (_a25 = this.audio) == null ? void 0 : _a25.pause();
18086
+ }
18087
+ finish() {
18088
+ var _a25, _b25;
18089
+ return (_b25 = (_a25 = this.codec) == null ? void 0 : _a25.finish()) != null ? _b25 : Promise.resolve();
18090
+ }
18091
+ fail(error) {
18092
+ var _a25;
18093
+ if (this.finishing === "failure") return;
18094
+ this.finishing = "failure";
18095
+ clearTimeout(this.finishTimer);
18096
+ if (this.rejectStartup != null) {
18097
+ this.rejectStartup(error);
18098
+ (_a25 = this.abort) == null ? void 0 : _a25.abort(error);
18099
+ return;
18100
+ }
18101
+ const generation = this.generation;
18102
+ const drain = this.finish();
18103
+ this.notifyClosing();
18104
+ if (generation !== this.generation) return;
18105
+ void this.stopCaptureForShutdown(generation);
18106
+ if (generation !== this.generation) return;
18107
+ if (this.options.onFatalError != null) {
18108
+ this.options.onFatalError(error, drain);
18109
+ return;
18110
+ }
18111
+ const complete = () => {
18112
+ if (generation !== this.generation) return;
18113
+ this.dispose();
18114
+ try {
18115
+ this.options.onClose();
18116
+ } catch (cause) {
18117
+ this.reportCallbackError(cause);
18118
+ }
18119
+ };
18120
+ this.finishTimer = setTimeout(complete, 1e3);
18121
+ void this.awaitDrain(drain, complete);
18122
+ try {
18123
+ this.options.onError(error);
18124
+ } catch (e) {
18125
+ }
18126
+ }
18127
+ drainClose(error) {
18128
+ if (this.finishing) return;
18129
+ if (this.rejectStartup != null) {
18130
+ this.fail(error);
18131
+ return;
18132
+ }
18133
+ this.finishing = "close";
18134
+ const generation = this.generation;
18135
+ const complete = () => {
18136
+ if (generation !== this.generation || this.finishing !== "close") return;
18137
+ clearTimeout(this.finishTimer);
18138
+ try {
18139
+ this.options.onClose(error);
18140
+ } catch (cause) {
18141
+ this.reportCallbackError(cause);
18142
+ } finally {
18143
+ if (generation === this.generation) this.dispose();
18144
+ }
18145
+ };
18146
+ this.finishTimer = setTimeout(complete, 1e3);
18147
+ void this.awaitDrain(this.finish(), complete);
18148
+ this.notifyClosing();
18149
+ if (generation !== this.generation) return;
18150
+ void this.stopCaptureForShutdown(generation);
18151
+ }
18152
+ notifyClosing() {
18153
+ var _a25, _b25;
18154
+ if (this.closingNotified) return;
18155
+ this.closingNotified = true;
18156
+ try {
18157
+ (_b25 = (_a25 = this.options).onClosing) == null ? void 0 : _b25.call(_a25);
18158
+ } catch (error) {
18159
+ this.reportCallbackError(error);
18160
+ }
18161
+ }
18162
+ async stopCaptureForShutdown(generation) {
18163
+ try {
18164
+ await this.stopCapture();
18165
+ } catch (cause) {
18166
+ if (generation !== this.generation || this.finishing !== "close") return;
18167
+ try {
18168
+ this.fail(cause instanceof Error ? cause : new Error(String(cause)));
18169
+ } catch (error) {
18170
+ this.reportCallbackError(error);
18171
+ }
18172
+ }
18173
+ }
18174
+ async awaitDrain(drain, complete) {
18175
+ try {
18176
+ await drain;
18177
+ } catch (e) {
18178
+ }
18179
+ try {
18180
+ complete();
18181
+ } catch (error) {
18182
+ this.reportCallbackError(error);
18183
+ }
18184
+ }
18185
+ reportCallbackError(error) {
18186
+ try {
18187
+ this.options.onError(
18188
+ error instanceof Error ? error : new Error(String(error))
18189
+ );
18190
+ } catch (e) {
18191
+ }
18192
+ }
18193
+ async stopCapture() {
18194
+ var _a25;
18195
+ const generation = ++this.captureGeneration;
18196
+ for (const cleanup of this.trackCleanups) cleanup();
18197
+ this.trackCleanups = [];
18198
+ if (this.ownsStream)
18199
+ (_a25 = this.stream) == null ? void 0 : _a25.getTracks().forEach((track) => track.stop());
18200
+ this.stream = void 0;
18201
+ const pc = this.pc;
18202
+ const sender = this.sender;
18203
+ await this.queueSenderOperation(async () => {
18204
+ if (pc !== this.pc || sender !== this.sender) return;
18205
+ if (pc != null && sender != null) await this.detachSender(pc, sender);
18206
+ if (generation === this.captureGeneration && pc === this.pc)
18207
+ this.options.onCapturing(false);
18208
+ });
18209
+ }
18210
+ async startCapture(supplied) {
18211
+ const generation = this.captureGeneration + 1;
18212
+ await this.stopCapture();
18213
+ if (generation !== this.captureGeneration || this.pc == null || this.finishing)
18214
+ return;
18215
+ const media = supplied != null ? supplied : await navigator.mediaDevices.getUserMedia({ audio: true });
18216
+ if (generation !== this.captureGeneration || this.pc == null || this.finishing) {
18217
+ if (supplied == null) media.getTracks().forEach((track2) => track2.stop());
18218
+ return;
18219
+ }
18220
+ const track = selectAudioTrack(media);
18221
+ if (track == null) {
18222
+ if (supplied == null) media.getTracks().forEach((track2) => track2.stop());
18223
+ throw new Error("Realtime requires a live audio track");
18224
+ }
18225
+ const pc = this.pc;
18226
+ const sender = this.sender;
18227
+ await this.queueSenderOperation(async () => {
18228
+ const current = () => generation === this.captureGeneration && pc === this.pc && sender === this.sender && pc.connectionState !== "closed" && !this.finishing;
18229
+ let attached = false;
18230
+ try {
18231
+ if (!current() || sender == null) return;
18232
+ await sender.replaceTrack(track);
18233
+ if (!current()) {
18234
+ if (pc === this.pc) await this.detachSender(pc, sender);
18235
+ return;
18236
+ }
18237
+ this.stream = media;
18238
+ this.ownsStream = supplied == null;
18239
+ attached = true;
18240
+ this.observeCapture();
18241
+ } finally {
18242
+ if (!attached && supplied == null)
18243
+ media.getTracks().forEach((track2) => track2.stop());
18244
+ }
18245
+ });
18246
+ }
18247
+ queueSenderOperation(operation) {
18248
+ const pending = this.senderOperations.then(operation);
18249
+ this.senderOperations = pending.catch(() => {
18250
+ });
18251
+ return pending;
18252
+ }
18253
+ async detachSender(pc, sender) {
18254
+ if (pc.connectionState === "closed") return;
18255
+ try {
18256
+ await sender.replaceTrack(null);
18257
+ } catch (error) {
18258
+ if (pc !== this.pc) return;
18259
+ this.closePeer(pc);
18260
+ this.fail(error instanceof Error ? error : new Error(String(error)));
18261
+ }
18262
+ }
18263
+ closePeer(pc) {
18264
+ pc.ontrack = null;
18265
+ pc.onconnectionstatechange = null;
18266
+ pc.oniceconnectionstatechange = null;
18267
+ pc.onicegatheringstatechange = null;
18268
+ if (pc.connectionState !== "closed") pc.close();
18269
+ }
18270
+ observeCapture() {
18271
+ const generation = this.captureGeneration;
18272
+ const sender = this.sender;
18273
+ const track = sender == null ? void 0 : sender.track;
18274
+ const update = () => {
18275
+ if (generation === this.captureGeneration && this.sender === sender)
18276
+ this.options.onCapturing(
18277
+ (sender == null ? void 0 : sender.track) != null && sender.track.readyState === "live" && sender.track.enabled && !sender.track.muted
18278
+ );
18279
+ };
18280
+ if (track != null) {
18281
+ for (const event of ["ended", "mute", "unmute"]) {
18282
+ track.addEventListener(event, update);
18283
+ this.trackCleanups.push(() => track.removeEventListener(event, update));
18284
+ }
18285
+ }
18286
+ update();
18287
+ }
18288
+ dispose() {
18289
+ var _a25, _b25, _c;
18290
+ this.generation++;
18291
+ this.captureGeneration++;
18292
+ clearTimeout(this.finishTimer);
18293
+ clearTimeout(this.disconnectTimer);
18294
+ this.finishing = void 0;
18295
+ this.closingNotified = false;
18296
+ this.senderOperations = Promise.resolve();
18297
+ (_a25 = this.abort) == null ? void 0 : _a25.abort(new Error("Realtime connection cancelled"));
18298
+ this.abort = void 0;
18299
+ (_b25 = this.codec) == null ? void 0 : _b25.dispose();
18300
+ this.codec = void 0;
18301
+ if (this.dc != null) {
18302
+ this.dc.onopen = null;
18303
+ this.dc.onclose = null;
18304
+ this.dc.onerror = null;
18305
+ this.dc.onmessage = null;
18306
+ this.dc.close();
18307
+ this.dc = void 0;
18308
+ }
18309
+ if (this.pc != null) {
18310
+ this.closePeer(this.pc);
18311
+ this.pc = void 0;
18312
+ }
18313
+ for (const cleanup of this.trackCleanups) cleanup();
18314
+ this.trackCleanups = [];
18315
+ if (this.ownsStream)
18316
+ (_c = this.stream) == null ? void 0 : _c.getTracks().forEach((track) => track.stop());
18317
+ this.stream = void 0;
18318
+ this.sender = void 0;
18319
+ if (this.audio != null) {
18320
+ this.audio.onplaying = null;
18321
+ this.audio.onpause = null;
18322
+ this.audio.onended = null;
18323
+ this.audio.onwaiting = null;
18324
+ this.audio.pause();
18325
+ this.audio.srcObject = null;
18326
+ this.audio = void 0;
18327
+ }
18328
+ this.options.onCapturing(false);
18329
+ this.options.onPlaying(false);
18330
+ }
18331
+ };
18332
+
18333
+ // src/realtime/realtime-attempt.ts
18334
+ var RealtimeAttempt = class {
18335
+ constructor() {
18336
+ this.abort = new AbortController();
18337
+ this.timers = /* @__PURE__ */ new Map();
18338
+ this.active = true;
18339
+ this.ready = false;
18340
+ this.closing = false;
18341
+ this.transportClosing = false;
18342
+ }
18343
+ timer(name25, duration, callback) {
18344
+ this.clearTimer(name25);
18345
+ this.timers.set(
18346
+ name25,
18347
+ setTimeout(() => {
18348
+ this.timers.delete(name25);
18349
+ if (this.active) callback();
18350
+ }, duration)
18351
+ );
18352
+ }
18353
+ clearTimer(name25) {
18354
+ clearTimeout(this.timers.get(name25));
18355
+ this.timers.delete(name25);
18356
+ }
18357
+ beginClose() {
18358
+ var _a25;
18359
+ this.closing = true;
18360
+ return (_a25 = this.closePromise) != null ? _a25 : this.closePromise = new Promise((resolve3) => {
18361
+ this.settleClose = resolve3;
18362
+ });
18363
+ }
18364
+ retire() {
18365
+ var _a25;
18366
+ if (!this.active) return;
18367
+ this.active = false;
18368
+ this.abort.abort();
18369
+ for (const timer of this.timers.values()) clearTimeout(timer);
18370
+ this.timers.clear();
18371
+ (_a25 = this.settleClose) == null ? void 0 : _a25.call(this);
18372
+ this.settleClose = void 0;
18373
+ }
18374
+ };
18375
+
18376
+ // src/realtime/realtime-command-tracker.ts
18377
+ import { generateId as generateId3 } from "@ai-sdk/provider-utils";
18378
+ var RealtimeCommandTracker = class {
18379
+ constructor(sendCommand) {
18380
+ this.sendCommand = sendCommand;
18381
+ this.pending = /* @__PURE__ */ new Map();
18382
+ this.recent = /* @__PURE__ */ new Set();
18383
+ }
18384
+ validateId(id) {
18385
+ if (this.pending.has(id) || this.recent.has(id))
18386
+ throw new Error(
18387
+ "Realtime command ID was already used; retry with a fresh eventId"
18388
+ );
18389
+ if (this.pending.size >= 512)
18390
+ throw new Error("Too many pending realtime commands");
18391
+ }
18392
+ complete(id) {
18393
+ const command = this.pending.get(id);
18394
+ if (command == null) return;
18395
+ this.pending.delete(id);
18396
+ this.recent.add(id);
18397
+ if (this.recent.size > 4096) {
18398
+ const oldest = this.recent.values().next().value;
18399
+ if (oldest != null) this.recent.delete(oldest);
18400
+ }
18401
+ return command;
18402
+ }
18403
+ send(event) {
18404
+ var _a25;
18405
+ const muted = event.type === "input-audio-mute" ? true : event.type === "input-audio-unmute" ? false : void 0;
18406
+ if (event.type === "input-audio-mute" || event.type === "input-audio-unmute")
18407
+ event = { ...event, eventId: (_a25 = event.eventId) != null ? _a25 : generateId3() };
18408
+ const id = "eventId" in event ? event.eventId : void 0;
18409
+ if (id != null) {
18410
+ this.validateId(id);
18411
+ this.pending.set(id, { muted });
18412
+ }
18413
+ const awaitsAcknowledgement = event.type === "context-append" || event.type === "session-update" || muted != null;
18414
+ try {
18415
+ const sent = this.sendCommand(event);
18416
+ void sent.then(
18417
+ () => {
18418
+ if (id != null && !awaitsAcknowledgement) this.complete(id);
18419
+ },
18420
+ () => {
18421
+ if (id != null) this.complete(id);
18422
+ }
18423
+ );
18424
+ return sent;
18425
+ } catch (error) {
18426
+ if (id != null) this.complete(id);
18427
+ throw error;
18428
+ }
18429
+ }
18430
+ receive(event) {
18431
+ if (event.type === "command-acknowledged" && event.clientEventId != null)
18432
+ return this.complete(event.clientEventId);
18433
+ if (event.type === "error" && event.clientEventId != null)
18434
+ this.complete(event.clientEventId);
18435
+ }
18436
+ };
18437
+
18438
+ // src/realtime/validate-realtime-setup.ts
18439
+ var setupSchema = z.object({
18440
+ token: z.string().refine((value) => value.trim().length > 0),
18441
+ url: z.string().refine((value) => {
18442
+ try {
18443
+ const url = new URL(value);
18444
+ return (url.protocol === "ws:" || url.protocol === "wss:") && url.hostname !== "";
18445
+ } catch (e) {
18446
+ return false;
18447
+ }
18448
+ }),
18449
+ expiresAt: z.number().positive().max(Number.MAX_SAFE_INTEGER).optional(),
18450
+ tools: z.array(
18451
+ z.object({
18452
+ type: z.literal("function"),
18453
+ name: z.string().min(1),
18454
+ description: z.string().optional(),
18455
+ parameters: z.record(z.string(), z.unknown())
18456
+ })
18457
+ ).optional()
18458
+ });
18459
+ function validateRealtimeSetup(payload) {
18460
+ const result = setupSchema.safeParse(payload);
18461
+ if (!result.success)
18462
+ throw new Error(
18463
+ "Invalid realtime setup: expected a nonempty token, a ws(s) URL, and valid optional expiresAt/tools"
18464
+ );
18465
+ return result.data;
18466
+ }
18467
+
18468
+ // src/realtime/realtime-session-state.ts
18469
+ function createSessionState() {
18470
+ return {
18471
+ transcripts: [],
18472
+ delegations: [],
18473
+ isInputMuted: false,
18474
+ finalization: "pending"
18475
+ };
18476
+ }
18477
+ function reduceSessionState(state, event, limit, muted) {
18478
+ var _a25;
18479
+ switch (event.type) {
18480
+ case "session-started":
18481
+ return {
18482
+ ...state,
18483
+ sessionId: event.sessionId,
18484
+ delegationMode: event.delegationMode
18485
+ };
18486
+ case "session-usage":
18487
+ return { ...state, usage: event.usage };
18488
+ case "session-closed":
18489
+ return {
18490
+ ...state,
18491
+ sessionId: (_a25 = event.sessionId) != null ? _a25 : state.sessionId,
18492
+ usage: event.usage,
18493
+ terminationReason: event.reason,
18494
+ finalization: "confirmed"
18495
+ };
18496
+ case "transcript-fragment":
18497
+ return {
18498
+ ...state,
18499
+ transcripts: [...state.transcripts, event].slice(-limit)
18500
+ };
18501
+ case "delegation-created":
18502
+ return state.delegations.some(
18503
+ (item) => item.delegationId === event.delegationId
18504
+ ) ? state : {
18505
+ ...state,
18506
+ delegations: [...state.delegations, event].slice(-limit)
18507
+ };
18508
+ case "command-acknowledged":
18509
+ return muted == null ? state : { ...state, isInputMuted: muted };
18510
+ default:
18511
+ return state;
18512
+ }
18513
+ }
18514
+
18515
+ // src/realtime/realtime-event-reducer.ts
18516
+ import { safeParseJSON as safeParseJSON7 } from "@ai-sdk/provider-utils";
18517
+ function createInitialRealtimeState() {
18518
+ return {
18519
+ status: "disconnected",
18520
+ messages: [],
18521
+ events: [],
18522
+ isCapturing: false,
18523
+ isPlaying: false
18524
+ };
18525
+ }
18526
+ var RealtimeEventReducer = class {
18527
+ constructor(maxEvents = 500) {
18528
+ this.maxEvents = maxEvents;
18529
+ this.currentAssistantMessageId = null;
18530
+ this.textAccumulators = /* @__PURE__ */ new Map();
18531
+ this.toolArgAccumulators = /* @__PURE__ */ new Map();
18532
+ this.toolCallIdToMessageId = /* @__PURE__ */ new Map();
18533
+ this.toolCallIdToName = /* @__PURE__ */ new Map();
18534
+ this.inputAudioMessageInsertIndex = /* @__PURE__ */ new Map();
18535
+ this.itemIdToPartLocation = /* @__PURE__ */ new Map();
18536
+ }
18537
+ setStatus(state, status) {
18538
+ return { ...state, status };
18539
+ }
18540
+ setCapturing(state, isCapturing) {
18541
+ return { ...state, isCapturing };
18542
+ }
18543
+ setPlaying(state, isPlaying) {
18544
+ return { ...state, isPlaying };
18545
+ }
18546
+ addUserTextMessage(state, text2) {
18547
+ return {
18548
+ ...state,
18549
+ messages: [
18550
+ ...state.messages,
18551
+ {
18552
+ id: `user-${Date.now()}`,
18553
+ role: "user",
18554
+ parts: [{ type: "text", text: text2, state: "done" }]
18555
+ }
18556
+ ]
18557
+ };
18558
+ }
18559
+ addToolOutput(state, callId, result) {
18560
+ return {
18561
+ state: this.updateToolPartState(state, callId, result),
18562
+ output: {
18563
+ callId,
18564
+ name: this.toolCallIdToName.get(callId),
18565
+ output: JSON.stringify(result)
18566
+ }
18567
+ };
18568
+ }
18569
+ async reduceServerEvent(state, event) {
18570
+ var _a25;
18571
+ let nextState = this.pushEvent(state, event);
18572
+ const effects = [];
18573
+ switch (event.type) {
18574
+ case "session-created":
18575
+ case "session-updated": {
18576
+ if (nextState.status === "connecting") {
18577
+ nextState = this.setStatus(nextState, "connected");
18578
+ }
18579
+ break;
18580
+ }
18581
+ case "audio-delta": {
17316
18582
  effects.push({
17317
18583
  type: "play-audio",
17318
18584
  itemId: event.itemId,
@@ -17376,7 +18642,7 @@ var RealtimeEventReducer = class {
17376
18642
  case "function-call-arguments-done": {
17377
18643
  this.toolArgAccumulators.delete(event.callId);
17378
18644
  this.toolCallIdToName.set(event.callId, event.name);
17379
- const parseResult = await safeParseJSON6({ text: event.arguments });
18645
+ const parseResult = await safeParseJSON7({ text: event.arguments });
17380
18646
  const parsedInput = parseResult.success ? parseResult.value : {};
17381
18647
  const messageId = this.toolCallIdToMessageId.get(event.callId);
17382
18648
  if (messageId != null) {
@@ -17616,98 +18882,375 @@ var RealtimeEventReducer = class {
17616
18882
  // src/realtime/realtime-session.ts
17617
18883
  var AbstractRealtimeSession = class {
17618
18884
  constructor(options) {
18885
+ this.options = options;
17619
18886
  this.state = createInitialRealtimeState();
18887
+ this.publication = 0;
18888
+ this.captureGeneration = 0;
18889
+ this.captureRequested = false;
17620
18890
  this.currentResponseItemId = null;
17621
- // Tool calls requested by the current (tool-bearing) response, the outputs
17622
- // that have been submitted for them, and whether that response has finished
17623
- // delivering its tool calls. Used to request a single response only once
17624
- // every tool output for the turn has been submitted.
17625
18891
  this.toolCallsInResponse = /* @__PURE__ */ new Set();
17626
18892
  this.submittedToolOutputs = /* @__PURE__ */ new Set();
17627
18893
  this.responseToolCallsClosed = false;
17628
- var _a25, _b25, _c, _d, _e, _f, _g, _h;
17629
- this.model = options.model;
17630
- this.api = options.api;
17631
- this.sessionConfig = options.sessionConfig;
18894
+ var _a25, _b25, _c, _d, _e;
18895
+ const capabilities = options.model.capabilities;
18896
+ this.sessionLifecycle = (capabilities == null ? void 0 : capabilities.startup) === "session-start" || (capabilities == null ? void 0 : capabilities.finalization) === "session-close";
18897
+ this.continuous = (capabilities == null ? void 0 : capabilities.conversation) === "continuous";
17632
18898
  this.maxEvents = (_a25 = options.maxEvents) != null ? _a25 : 500;
18899
+ if (!Number.isSafeInteger(this.maxEvents) || this.maxEvents < 1)
18900
+ throw new Error("maxEvents must be a positive integer");
18901
+ for (const timeout of [
18902
+ (_b25 = options.startupTimeoutMs) != null ? _b25 : 3e4,
18903
+ (_c = options.closeTimeoutMs) != null ? _c : 15e3,
18904
+ (_d = options.rtcDisconnectTimeoutMs) != null ? _d : 5e3
18905
+ ]) {
18906
+ if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647)
18907
+ throw new Error(
18908
+ "Realtime timeouts must be positive finite timer durations"
18909
+ );
18910
+ }
18911
+ const budget = (_e = options.maxPlaybackBufferSeconds) != null ? _e : 2;
18912
+ if (!Number.isFinite(budget) || budget <= 0)
18913
+ throw new Error("maxPlaybackBufferSeconds must be positive and finite");
17633
18914
  this.reducer = new RealtimeEventReducer(this.maxEvents);
17634
18915
  this.onToolCall = options.onToolCall;
17635
18916
  this.onEvent = options.onEvent;
17636
18917
  this.onError = options.onError;
17637
- const sampleRate = (_b25 = options.sampleRate) != null ? _b25 : 24e3;
17638
- const captureSampleRate = (_e = (_d = (_c = options.sessionConfig) == null ? void 0 : _c.inputAudioFormat) == null ? void 0 : _d.rate) != null ? _e : sampleRate;
17639
- const playbackSampleRate = (_h = (_g = (_f = options.sessionConfig) == null ? void 0 : _f.outputAudioFormat) == null ? void 0 : _g.rate) != null ? _h : sampleRate;
17640
- this.transport = new BrowserRealtimeTransport({
17641
- model: this.model,
17642
- onServerEvent: (event) => this.handleServerEvent(event),
17643
- onError: (error) => {
17644
- var _a26;
17645
- this.applyState(this.reducer.setStatus(this.state, "error"));
17646
- (_a26 = this.onError) == null ? void 0 : _a26.call(this, error);
17647
- },
17648
- onClose: () => {
17649
- this.applyState(this.reducer.setStatus(this.state, "disconnected"));
17650
- }
17651
- });
17652
- this.audio = new BrowserRealtimeAudio({
17653
- captureSampleRate,
17654
- playbackSampleRate,
17655
- onAudio: (audio) => this.sendAudio(audio),
17656
- onCapturingChange: (isCapturing) => {
17657
- this.applyState(this.reducer.setCapturing(this.state, isCapturing));
17658
- },
17659
- onPlayingChange: (isPlaying) => {
17660
- this.applyState(this.reducer.setPlaying(this.state, isPlaying));
17661
- }
17662
- });
17663
18918
  }
17664
- // ── Connection ─────────────────────────────────────────────────────
17665
- async connect() {
17666
- var _a25;
17667
- this.applyState(this.reducer.setStatus(this.state, "connecting"));
18919
+ validateConnection() {
18920
+ var _a25, _b25, _c;
18921
+ const { model, api } = this.options;
18922
+ if ((!this.continuous || api.session != null) && this.options.maxPlaybackBufferSeconds != null)
18923
+ throw new Error(
18924
+ "maxPlaybackBufferSeconds is supported only for continuous PCM sessions"
18925
+ );
18926
+ const connection = api.session != null ? "webrtc" : api.token != null ? "client-secret-websocket" : "server-websocket";
18927
+ if (!((_b25 = (_a25 = model.capabilities) == null ? void 0 : _a25.connections) != null ? _b25 : ["client-secret-websocket"]).includes(connection) || ((_c = model.capabilities) == null ? void 0 : _c.transports) != null && !model.capabilities.transports.includes(
18928
+ api.session != null ? "webrtc" : "websocket"
18929
+ ))
18930
+ throw new Error(`Realtime model does not support ${connection}`);
18931
+ if (api.session != null) {
18932
+ if (model.getWebRTCConfig == null)
18933
+ throw new Error("Realtime model does not support WebRTC configuration");
18934
+ return;
18935
+ }
18936
+ if (this.continuous && api.websocket == null)
18937
+ throw new Error(
18938
+ "Continuous PCM sessions require an application WebSocket relay"
18939
+ );
18940
+ if (api.token != null && model.getWebSocketConfig == null)
18941
+ throw new Error(
18942
+ "Realtime model does not support client-secret WebSocket configuration"
18943
+ );
18944
+ }
18945
+ async connect(connectOptions) {
18946
+ var _a25, _b25, _c, _d;
18947
+ if ((_a25 = this.attempt) == null ? void 0 : _a25.active)
18948
+ throw new Error("Realtime session is already active");
18949
+ const attempt = new RealtimeAttempt();
18950
+ this.attempt = attempt;
18951
+ const current = () => this.attempt === attempt && attempt.active;
17668
18952
  try {
17669
- const response = await fetch(this.api.token, {
17670
- method: "POST",
17671
- headers: { "Content-Type": "application/json" },
17672
- body: JSON.stringify({ sessionConfig: this.sessionConfig })
18953
+ this.applyState({ ...this.state, status: "connecting" });
18954
+ if (!current()) return;
18955
+ this.validateConnection();
18956
+ const { model, api, sessionConfig } = this.options;
18957
+ if ((connectOptions == null ? void 0 : connectOptions.capture) === false) this.stopAudioCapture();
18958
+ if (!current()) return;
18959
+ if ((connectOptions == null ? void 0 : connectOptions.stream) != null && connectOptions.stream !== this.suppliedStream) {
18960
+ if (this.captureRequested && !this.sessionLifecycle && !this.continuous && api.session == null)
18961
+ this.startAudioCapture(connectOptions.stream);
18962
+ else this.suppliedStream = connectOptions.stream;
18963
+ }
18964
+ if (!current()) return;
18965
+ this.reducer = new RealtimeEventReducer(this.maxEvents);
18966
+ this.currentResponseItemId = null;
18967
+ this.toolCallsInResponse.clear();
18968
+ this.submittedToolOutputs.clear();
18969
+ this.responseToolCallsClosed = false;
18970
+ if (this.sessionLifecycle)
18971
+ this.applyState({ ...this.state, session: createSessionState() });
18972
+ if (!current()) return;
18973
+ this.commands = new RealtimeCommandTracker((event) => {
18974
+ const writable = () => current() && !attempt.transportClosing && attempt.cause == null && (!attempt.closing || event.type === "session-close");
18975
+ if (!writable()) throw new Error("Realtime connection is closed");
18976
+ return this.sendTransport(event, writable);
17673
18977
  });
17674
- if (!response.ok) {
17675
- throw new Error(`Failed to fetch realtime setup: ${response.status}`);
17676
- }
17677
- const setupData = await response.json();
17678
- const { token, url, tools: toolDefinitions } = setupData;
17679
- const config = {
17680
- ...this.sessionConfig,
17681
- tools: toolDefinitions
18978
+ attempt.timer("startup", (_b25 = this.options.startupTimeoutMs) != null ? _b25 : 3e4, () => {
18979
+ if (current())
18980
+ this.fail(new Error("Realtime session startup timed out"));
18981
+ });
18982
+ const callbacks = {
18983
+ model,
18984
+ onEvent: async (event) => {
18985
+ var _a26, _b26, _c2;
18986
+ if (!current()) return;
18987
+ try {
18988
+ await this.handleServerEvent(event, attempt);
18989
+ } catch (error) {
18990
+ if (current())
18991
+ this.fail(
18992
+ error,
18993
+ (_c2 = (_b26 = (_a26 = this.rtc) != null ? _a26 : this.pcm) != null ? _b26 : this.transport) == null ? void 0 : _c2.finish()
18994
+ );
18995
+ }
18996
+ },
18997
+ onError: (error) => {
18998
+ if (current()) void this.reportError(error, attempt);
18999
+ },
19000
+ onFatalError: (error, drain) => {
19001
+ if (current()) this.fail(error, drain);
19002
+ },
19003
+ onClosing: () => {
19004
+ if (!current()) return;
19005
+ attempt.beginClose();
19006
+ attempt.transportClosing = true;
19007
+ attempt.clearTimer("startup");
19008
+ attempt.clearTimer("close");
19009
+ this.stopAudioCapture();
19010
+ if (!current()) return;
19011
+ this.applyState({
19012
+ ...this.state,
19013
+ status: attempt.cause == null ? "closing" : "error"
19014
+ });
19015
+ },
19016
+ onClose: (error) => {
19017
+ var _a26, _b26;
19018
+ if (!current()) return;
19019
+ const finalizationConfirmed = ((_a26 = model.capabilities) == null ? void 0 : _a26.finalization) === "session-close" && ((_b26 = this.state.session) == null ? void 0 : _b26.finalization) === "confirmed";
19020
+ if (error != null && !finalizationConfirmed) this.fail(error);
19021
+ else if (!attempt.ready && !finalizationConfirmed)
19022
+ this.fail(
19023
+ new Error("Realtime connection closed before becoming ready")
19024
+ );
19025
+ else this.disconnect();
19026
+ },
19027
+ onCapturing: (isCapturing) => {
19028
+ if (current()) this.applyState({ ...this.state, isCapturing });
19029
+ },
19030
+ onPlaying: (isPlaying) => {
19031
+ if (current()) this.applyState({ ...this.state, isPlaying });
19032
+ }
17682
19033
  };
17683
- this.audio.ensurePlaybackContext();
17684
- this.transport.connect({
17685
- token,
17686
- url,
17687
- onOpen: () => {
17688
- this.sendEvent({
17689
- type: "session-update",
17690
- config
19034
+ if (api.session != null) {
19035
+ this.rtc = new BrowserRealtimeWebRTC({
19036
+ ...callbacks,
19037
+ disconnectTimeoutMs: this.options.rtcDisconnectTimeoutMs
19038
+ });
19039
+ await this.rtc.connect({
19040
+ api: api.session,
19041
+ sessionConfig,
19042
+ stream: connectOptions == null ? void 0 : connectOptions.stream,
19043
+ capture: connectOptions == null ? void 0 : connectOptions.capture,
19044
+ timeoutMs: (_c = this.options.startupTimeoutMs) != null ? _c : 3e4
19045
+ });
19046
+ } else if (api.websocket != null && ((_d = model.capabilities) == null ? void 0 : _d.conversation) === "continuous") {
19047
+ this.pcm = new BrowserRealtimeLiveWebSocket({
19048
+ ...callbacks,
19049
+ sessionConfig,
19050
+ sampleRate: this.options.sampleRate,
19051
+ maxPlaybackBufferSeconds: this.options.maxPlaybackBufferSeconds
19052
+ });
19053
+ this.pcm.connect({
19054
+ url: api.websocket,
19055
+ protocols: api.protocols,
19056
+ stream: connectOptions == null ? void 0 : connectOptions.stream,
19057
+ capture: connectOptions == null ? void 0 : connectOptions.capture
19058
+ });
19059
+ } else {
19060
+ this.ensureAudio();
19061
+ let config = sessionConfig != null ? sessionConfig : {};
19062
+ let token;
19063
+ let url = api.websocket;
19064
+ if (api.token != null) {
19065
+ const response = await fetch(api.token, {
19066
+ method: "POST",
19067
+ headers: { "Content-Type": "application/json" },
19068
+ body: JSON.stringify({ sessionConfig }),
19069
+ signal: attempt.abort.signal
19070
+ });
19071
+ if (!current()) return;
19072
+ if (!response.ok)
19073
+ throw new Error(
19074
+ `Failed to fetch realtime setup: ${response.status}`
19075
+ );
19076
+ const payload = await response.json().catch(() => {
19077
+ throw new Error("Invalid realtime setup response");
17691
19078
  });
19079
+ if (!current() || attempt.closing) return;
19080
+ const setup = validateRealtimeSetup(payload);
19081
+ token = setup.token;
19082
+ url = setup.url;
19083
+ config = {
19084
+ ...sessionConfig,
19085
+ ...setup.tools == null ? {} : { tools: setup.tools }
19086
+ };
17692
19087
  }
17693
- });
19088
+ if (url == null) throw new Error("Realtime WebSocket URL is missing");
19089
+ if (api.token != null && token == null)
19090
+ throw new Error("Realtime client-secret connection requires a token");
19091
+ this.ensureAudio().ensurePlaybackContext();
19092
+ if (!current()) return;
19093
+ this.transport = new BrowserRealtimeTransport({
19094
+ ...callbacks,
19095
+ onServerEvent: callbacks.onEvent
19096
+ });
19097
+ this.transport.connect({
19098
+ ...token != null ? { mode: "client-secret", token } : { mode: "relay", protocols: api.protocols },
19099
+ url,
19100
+ onOpen: () => {
19101
+ var _a26, _b26;
19102
+ const writable = () => current() && !attempt.closing && attempt.cause == null;
19103
+ if (!writable()) return;
19104
+ return this.sendTransport(
19105
+ {
19106
+ type: (_b26 = (_a26 = model.capabilities) == null ? void 0 : _a26.startup) != null ? _b26 : "session-update",
19107
+ config
19108
+ },
19109
+ writable
19110
+ );
19111
+ }
19112
+ });
19113
+ }
17694
19114
  } catch (error) {
17695
- this.applyState(this.reducer.setStatus(this.state, "error"));
17696
- (_a25 = this.onError) == null ? void 0 : _a25.call(
17697
- this,
17698
- error instanceof Error ? error : new Error(`Connection failed: ${String(error)}`)
17699
- );
19115
+ if (current()) this.fail(error);
19116
+ }
19117
+ }
19118
+ fail(error, drain) {
19119
+ const attempt = this.attempt;
19120
+ if (attempt == null || !attempt.active || attempt.cause != null) return;
19121
+ attempt.cause = error instanceof Error ? error : new Error(String(error));
19122
+ if (drain != null) attempt.beginClose();
19123
+ this.applyState({ ...this.state, status: "error" });
19124
+ if (this.attempt !== attempt || !attempt.active) return;
19125
+ this.stopAudioCapture();
19126
+ if (this.attempt !== attempt || !attempt.active) return;
19127
+ if (drain == null) this.disconnect();
19128
+ else void this.drainAttempt(attempt, drain);
19129
+ void this.reportError(attempt.cause, attempt);
19130
+ }
19131
+ async drainAttempt(attempt, drain) {
19132
+ if (this.attempt === attempt && attempt.active)
19133
+ attempt.timer("drain", 1e3, () => this.finishAttempt(attempt));
19134
+ try {
19135
+ await drain;
19136
+ } catch (e) {
19137
+ }
19138
+ this.finishAttempt(attempt);
19139
+ }
19140
+ finishAttempt(attempt) {
19141
+ if (this.attempt !== attempt || !attempt.active) return;
19142
+ try {
19143
+ this.disconnect();
19144
+ } catch (error) {
19145
+ void this.reportError(error, attempt);
17700
19146
  }
17701
19147
  }
19148
+ closeFailed(attempt, error) {
19149
+ var _a25, _b25;
19150
+ if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
19151
+ return;
19152
+ attempt.clearTimer("close");
19153
+ const transport = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.transport;
19154
+ void this.drainAttempt(attempt, transport == null ? void 0 : transport.finish());
19155
+ void this.reportError(error, attempt);
19156
+ }
17702
19157
  disconnect() {
17703
- this.transport.disconnect();
17704
- this.applyState(this.reducer.setStatus(this.state, "disconnected"));
19158
+ const attempt = this.attempt;
19159
+ this.captureGeneration++;
19160
+ this.captureRequested = false;
19161
+ this.suppliedStream = void 0;
19162
+ const transport = this.transport;
19163
+ const audio = this.audio;
19164
+ const pcm = this.pcm;
19165
+ const rtc = this.rtc;
19166
+ this.transport = void 0;
19167
+ this.audio = void 0;
19168
+ this.pcm = void 0;
19169
+ this.rtc = void 0;
19170
+ this.commands = void 0;
19171
+ attempt == null ? void 0 : attempt.retire();
19172
+ transport == null ? void 0 : transport.dispose();
19173
+ audio == null ? void 0 : audio.dispose();
19174
+ pcm == null ? void 0 : pcm.dispose();
19175
+ rtc == null ? void 0 : rtc.dispose();
19176
+ if (this.attempt !== attempt) return;
19177
+ const session = this.state.session;
19178
+ this.applyState({
19179
+ ...this.state,
19180
+ status: (attempt == null ? void 0 : attempt.cause) != null ? "error" : "disconnected",
19181
+ isCapturing: false,
19182
+ isPlaying: false,
19183
+ ...session != null ? {
19184
+ session: session.finalization === "confirmed" ? session : { ...session, finalization: "unconfirmed" }
19185
+ } : {}
19186
+ });
19187
+ }
19188
+ /** Wait for final usage when the model supports a session-close acknowledgement. */
19189
+ close(options) {
19190
+ var _a25, _b25, _c, _d;
19191
+ const attempt = this.attempt;
19192
+ if ((attempt == null ? void 0 : attempt.closePromise) != null) return attempt.closePromise;
19193
+ if (this.state.status !== "connected" || ((_a25 = this.options.model.capabilities) == null ? void 0 : _a25.finalization) !== "session-close" || attempt == null) {
19194
+ this.disconnect();
19195
+ return Promise.resolve();
19196
+ }
19197
+ if ((options == null ? void 0 : options.eventId) != null) (_b25 = this.commands) == null ? void 0 : _b25.validateId(options.eventId);
19198
+ const promise = attempt.beginClose();
19199
+ this.applyState({ ...this.state, status: "closing" });
19200
+ if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
19201
+ return promise;
19202
+ this.stopAudioCapture();
19203
+ if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
19204
+ return promise;
19205
+ attempt.timer(
19206
+ "close",
19207
+ (_c = this.options.closeTimeoutMs) != null ? _c : 15e3,
19208
+ () => this.finishAttempt(attempt)
19209
+ );
19210
+ const failed = (error) => this.closeFailed(attempt, error);
19211
+ try {
19212
+ void ((_d = this.commands) == null ? void 0 : _d.send({ type: "session-close", eventId: options == null ? void 0 : options.eventId }).catch(failed));
19213
+ } catch (error) {
19214
+ failed(error);
19215
+ }
19216
+ return promise;
19217
+ }
19218
+ sendTransport(event, guard, automaticAudio = false) {
19219
+ var _a25, _b25;
19220
+ const transport = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.transport;
19221
+ if (transport == null) throw new Error("Realtime connection is not open");
19222
+ if (transport === this.transport)
19223
+ return this.transport.sendEvent(event, guard, automaticAudio);
19224
+ return transport.sendEvent(event, guard);
17705
19225
  }
17706
- // ── Sending events ─────────────────────────────────────────────────
17707
19226
  sendEvent(event) {
17708
- this.transport.sendEvent(event);
19227
+ var _a25, _b25, _c, _d;
19228
+ if (this.state.status === "error" || this.state.status === "closing" || ((_a25 = this.attempt) == null ? void 0 : _a25.closing) || !((_b25 = this.attempt) == null ? void 0 : _b25.active))
19229
+ throw new Error("Realtime session is not accepting submissions");
19230
+ if (this.sessionLifecycle && this.state.status !== "connected")
19231
+ throw new Error("Realtime session is not accepting submissions");
19232
+ if (this.rtc != null && event.type === "input-audio-append" || (this.continuous || this.rtc != null) && (event.type === "input-audio-commit" || event.type === "input-audio-clear"))
19233
+ throw new UnsupportedFunctionalityError6({
19234
+ functionality: "JSON audio commands unsupported for this session transport"
19235
+ });
19236
+ if (event.type === "session-close")
19237
+ return this.close({ eventId: event.eventId });
19238
+ if (event.type === "session-start" && this.sessionLifecycle)
19239
+ throw new Error("Realtime session has already started");
19240
+ if (!this.sessionLifecycle && this.state.session == null) {
19241
+ const attempt = this.attempt;
19242
+ return this.sendTransport(
19243
+ event,
19244
+ () => this.attempt === attempt && attempt.active && !attempt.closing && attempt.cause == null
19245
+ );
19246
+ }
19247
+ return (_d = (_c = this.commands) == null ? void 0 : _c.send(event)) != null ? _d : this.sendTransport(event);
17709
19248
  }
17710
19249
  sendTextMessage(text2) {
19250
+ if (this.continuous)
19251
+ throw new UnsupportedFunctionalityError6({
19252
+ functionality: "sendTextMessage for continuous sessions; client-delegation text is application-owned, use context-append for context"
19253
+ });
17711
19254
  this.sendEvent({
17712
19255
  type: "conversation-item-create",
17713
19256
  item: { type: "text-message", role: "user", text: text2 }
@@ -17715,8 +19258,12 @@ var AbstractRealtimeSession = class {
17715
19258
  this.sendEvent({ type: "response-create" });
17716
19259
  this.applyState(this.reducer.addUserTextMessage(this.state, text2));
17717
19260
  }
17718
- sendAudio(base64Audio) {
17719
- this.sendEvent({ type: "input-audio-append", audio: base64Audio });
19261
+ sendAudio(audio) {
19262
+ const attempt = this.attempt;
19263
+ void this.sendEvent({ type: "input-audio-append", audio }).catch((error) => {
19264
+ if ((attempt == null ? void 0 : attempt.active) && this.attempt === attempt)
19265
+ void this.reportError(error, attempt);
19266
+ });
17720
19267
  }
17721
19268
  commitAudio() {
17722
19269
  this.sendEvent({ type: "input-audio-commit" });
@@ -17733,161 +19280,301 @@ var AbstractRealtimeSession = class {
17733
19280
  cancelResponse() {
17734
19281
  this.sendEvent({ type: "response-cancel" });
17735
19282
  }
17736
- // ── Tool output ───────────────────────────────────────────────────
19283
+ sendAutomaticAudio(audio) {
19284
+ var _a25;
19285
+ if (this.state.status !== "connected" || !((_a25 = this.attempt) == null ? void 0 : _a25.active) || this.attempt.closing)
19286
+ return;
19287
+ const attempt = this.attempt;
19288
+ const failed = (error) => {
19289
+ if ((attempt == null ? void 0 : attempt.active) && this.attempt === attempt && !attempt.closing)
19290
+ this.fail(error);
19291
+ };
19292
+ try {
19293
+ void this.sendTransport(
19294
+ { type: "input-audio-append", audio },
19295
+ () => this.attempt === attempt && attempt.active && !attempt.closing,
19296
+ true
19297
+ ).catch(failed);
19298
+ } catch (error) {
19299
+ failed(error);
19300
+ }
19301
+ }
17737
19302
  addToolOutput(callId, result) {
19303
+ if (this.continuous)
19304
+ throw new UnsupportedFunctionalityError6({
19305
+ functionality: "addToolOutput for continuous sessions; client delegation is application-owned"
19306
+ });
19307
+ const attempt = this.attempt;
19308
+ if (!(attempt == null ? void 0 : attempt.active) || attempt.closing || attempt.cause != null)
19309
+ throw new Error("Realtime session is not accepting submissions");
17738
19310
  const { state, output } = this.reducer.addToolOutput(
17739
19311
  this.state,
17740
19312
  callId,
17741
19313
  result
17742
19314
  );
17743
19315
  this.applyState(state);
19316
+ if (this.attempt !== attempt || !(attempt == null ? void 0 : attempt.active)) return;
17744
19317
  this.sendEvent({
17745
19318
  type: "conversation-item-create",
17746
- item: {
17747
- type: "function-call-output",
17748
- callId: output.callId,
17749
- name: output.name,
17750
- output: output.output
17751
- }
19319
+ item: { type: "function-call-output", ...output }
17752
19320
  });
17753
19321
  this.submittedToolOutputs.add(callId);
17754
19322
  this.maybeRequestToolResponse();
17755
19323
  }
17756
- /**
17757
- * Requests a single response once the tool-bearing response has finished
17758
- * delivering its tool calls and every one of them has an output. Requesting a
17759
- * response after each individual output can cause the model to continue
17760
- * without the full tool context on multi-tool turns.
17761
- */
17762
19324
  maybeRequestToolResponse() {
17763
- if (!this.responseToolCallsClosed) return;
17764
- if (this.toolCallsInResponse.size === 0) return;
17765
- for (const callId of this.toolCallsInResponse) {
17766
- if (!this.submittedToolOutputs.has(callId)) return;
17767
- }
19325
+ var _a25;
19326
+ if (this.state.status !== "connected" || ((_a25 = this.attempt) == null ? void 0 : _a25.closing) || !this.responseToolCallsClosed || this.toolCallsInResponse.size === 0 || [...this.toolCallsInResponse].some(
19327
+ (id) => !this.submittedToolOutputs.has(id)
19328
+ ))
19329
+ return;
17768
19330
  this.sendEvent({ type: "response-create" });
17769
19331
  this.toolCallsInResponse.clear();
17770
19332
  this.submittedToolOutputs.clear();
17771
19333
  this.responseToolCallsClosed = false;
17772
19334
  }
17773
- // ── Audio capture ──────────────────────────────────────────────────
17774
19335
  startAudioCapture(stream) {
17775
- this.audio.startCapture(stream);
19336
+ var _a25;
19337
+ const legacy = !this.sessionLifecycle && !this.continuous && this.options.api.session == null;
19338
+ if (!legacy && this.state.status !== "connected")
19339
+ throw new Error("Realtime session is not accepting capture");
19340
+ if (((_a25 = this.attempt) == null ? void 0 : _a25.active) && (this.attempt.closing || this.attempt.cause != null))
19341
+ throw new Error("Realtime session is not accepting capture");
19342
+ this.suppliedStream = stream;
19343
+ if (legacy) {
19344
+ this.captureGeneration++;
19345
+ this.captureRequested = true;
19346
+ this.ensureAudio().startCapture(stream);
19347
+ return;
19348
+ }
19349
+ const attempt = this.attempt;
19350
+ void this.resumeAudioCapture().catch((error) => {
19351
+ if (attempt == null ? void 0 : attempt.active) void this.reportError(error, attempt);
19352
+ });
19353
+ }
19354
+ async resumeAudioCapture() {
19355
+ const accepting = () => {
19356
+ var _a25, _b25;
19357
+ return !((_a25 = this.attempt) == null ? void 0 : _a25.closing) && ((_b25 = this.attempt) == null ? void 0 : _b25.cause) == null && (this.state.status === "connected" || !this.sessionLifecycle && this.state.status === "connecting");
19358
+ };
19359
+ if (!accepting())
19360
+ throw new Error("Realtime session is not accepting capture");
19361
+ this.captureRequested = true;
19362
+ if (this.rtc != null) return this.rtc.startCapture(this.suppliedStream);
19363
+ if (this.pcm != null) return this.pcm.resumeCapture(this.suppliedStream);
19364
+ const audio = this.audio;
19365
+ if (audio == null)
19366
+ throw new Error("Realtime capture transport is not ready");
19367
+ const captureGeneration = ++this.captureGeneration;
19368
+ const attempt = this.attempt;
19369
+ const supplied = this.suppliedStream;
19370
+ const stream = supplied != null ? supplied : await navigator.mediaDevices.getUserMedia({ audio: true });
19371
+ if (!(attempt == null ? void 0 : attempt.active) || this.attempt !== attempt || this.audio !== audio || !accepting() || captureGeneration !== this.captureGeneration) {
19372
+ if (supplied == null) stream.getTracks().forEach((track) => track.stop());
19373
+ return;
19374
+ }
19375
+ audio.startCapture(stream, {
19376
+ ownsStream: supplied == null || !this.sessionLifecycle || this.options.api.token != null
19377
+ });
17776
19378
  }
17777
19379
  stopAudioCapture() {
17778
- this.audio.stopCapture();
19380
+ this.captureGeneration++;
19381
+ this.captureRequested = false;
19382
+ if (this.audio != null && (!this.sessionLifecycle || this.options.api.token != null))
19383
+ this.suppliedStream = void 0;
19384
+ const pcm = this.pcm;
19385
+ const audio = this.audio;
19386
+ const rtc = this.rtc;
19387
+ const attempt = this.attempt;
19388
+ if (rtc != null)
19389
+ void rtc.stopCapture().catch((error) => {
19390
+ if ((attempt == null ? void 0 : attempt.active) && this.attempt === attempt)
19391
+ void this.reportError(error, attempt);
19392
+ });
19393
+ pcm == null ? void 0 : pcm.stopCapture();
19394
+ audio == null ? void 0 : audio.stopCapture();
17779
19395
  }
17780
- // ── Playback ───────────────────────────────────────────────────────
17781
19396
  stopPlayback() {
17782
- this.audio.stopPlayback();
19397
+ var _a25, _b25, _c;
19398
+ (_c = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.audio) == null ? void 0 : _c.stopPlayback();
19399
+ }
19400
+ async resumePlayback() {
19401
+ var _a25, _b25, _c;
19402
+ await ((_c = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.audio) == null ? void 0 : _c.resumePlayback());
17783
19403
  }
17784
- // ── Cleanup ────────────────────────────────────────────────────────
17785
19404
  dispose() {
17786
- this.transport.dispose();
17787
- this.audio.dispose();
17788
- this.applyState(
17789
- this.reducer.setStatus(
17790
- this.reducer.setPlaying(
17791
- this.reducer.setCapturing(this.state, false),
17792
- false
17793
- ),
17794
- "disconnected"
17795
- )
17796
- );
19405
+ this.disconnect();
19406
+ }
19407
+ ensureAudio() {
19408
+ var _a25, _b25, _c, _d, _e, _f;
19409
+ if (this.audio != null) return this.audio;
19410
+ const { sessionConfig, sampleRate } = this.options;
19411
+ const audio = new BrowserRealtimeAudio({
19412
+ captureSampleRate: (_c = (_b25 = (_a25 = sessionConfig == null ? void 0 : sessionConfig.inputAudioFormat) == null ? void 0 : _a25.rate) != null ? _b25 : sampleRate) != null ? _c : 24e3,
19413
+ playbackSampleRate: (_f = (_e = (_d = sessionConfig == null ? void 0 : sessionConfig.outputAudioFormat) == null ? void 0 : _d.rate) != null ? _e : sampleRate) != null ? _f : 24e3,
19414
+ onAudio: (value) => {
19415
+ if (this.audio === audio) this.sendAutomaticAudio(value);
19416
+ },
19417
+ onError: (error) => {
19418
+ if (this.audio !== audio) return;
19419
+ void this.reportError(error, this.attempt);
19420
+ },
19421
+ onCapturingChange: (isCapturing) => {
19422
+ if (this.audio === audio)
19423
+ this.applyState({ ...this.state, isCapturing });
19424
+ },
19425
+ onPlayingChange: (isPlaying) => {
19426
+ if (this.audio === audio) this.applyState({ ...this.state, isPlaying });
19427
+ }
19428
+ });
19429
+ this.audio = audio;
19430
+ return audio;
17797
19431
  }
17798
- // ── Private helpers ────────────────────────────────────────────────
17799
19432
  applyState(nextState) {
17800
- const previousState = this.state;
19433
+ const publication = ++this.publication;
19434
+ const previous = this.state;
17801
19435
  this.state = nextState;
17802
- if (previousState.status !== nextState.status) {
17803
- this.setState("status", nextState.status);
17804
- }
17805
- if (previousState.messages !== nextState.messages) {
17806
- this.setState("messages", nextState.messages);
17807
- }
17808
- if (previousState.events !== nextState.events) {
17809
- this.setState("events", nextState.events);
17810
- }
17811
- if (previousState.isCapturing !== nextState.isCapturing) {
17812
- this.setState("isCapturing", nextState.isCapturing);
17813
- }
17814
- if (previousState.isPlaying !== nextState.isPlaying) {
17815
- this.setState("isPlaying", nextState.isPlaying);
17816
- }
17817
- }
17818
- async executeToolCall({
17819
- name: name25,
17820
- args,
17821
- callId
17822
- }) {
17823
- var _a25, _b25;
17824
- if (this.onToolCall == null) {
17825
- (_a25 = this.onError) == null ? void 0 : _a25.call(this, new Error(`No handler provided for tool "${name25}"`));
19436
+ const update = (key) => {
19437
+ if (publication === this.publication && previous[key] !== nextState[key])
19438
+ this.setState(key, nextState[key]);
19439
+ };
19440
+ update("status");
19441
+ update("messages");
19442
+ update("events");
19443
+ update("isCapturing");
19444
+ update("isPlaying");
19445
+ update("session");
19446
+ }
19447
+ async executeTool(callId, name25, args, attempt) {
19448
+ if (this.continuous || this.attempt !== attempt || !attempt.active || attempt.closing || attempt.cause != null)
17826
19449
  return;
17827
- }
17828
19450
  try {
19451
+ if (this.onToolCall == null) {
19452
+ void this.reportError(
19453
+ new Error(`No handler provided for tool "${name25}"`),
19454
+ attempt
19455
+ );
19456
+ return;
19457
+ }
17829
19458
  const result = await this.onToolCall({
17830
19459
  toolCall: { toolCallId: callId, toolName: name25, args }
17831
19460
  });
17832
- if (result !== void 0) {
19461
+ if (result !== void 0 && this.attempt === attempt && (attempt == null ? void 0 : attempt.active) && this.state.status === "connected")
17833
19462
  this.addToolOutput(callId, result);
17834
- }
17835
19463
  } catch (error) {
17836
- (_b25 = this.onError) == null ? void 0 : _b25.call(
17837
- this,
17838
- error instanceof Error ? error : new Error(`Client tool execution failed: ${String(error)}`)
17839
- );
19464
+ if (attempt.active) void this.reportError(error, attempt);
17840
19465
  }
17841
19466
  }
17842
- async handleServerEvent(event) {
17843
- var _a25;
19467
+ async handleServerEvent(event, attempt) {
19468
+ var _a25, _b25, _c, _d, _e, _f;
19469
+ const current = () => this.attempt === attempt && attempt.active;
19470
+ if (!current()) return;
19471
+ if (event.type === "session-started" && event.delegationMode === "provider")
19472
+ throw new UnsupportedFunctionalityError6({
19473
+ functionality: "Provider delegation mode; this realtime runtime supports client delegation only"
19474
+ });
19475
+ if (this.continuous && event.type === "audio-delta")
19476
+ throw new UnsupportedFunctionalityError6({
19477
+ functionality: "Turn-based audio-delta in a continuous PCM session; use audio-chunk"
19478
+ });
19479
+ const command = (_a25 = this.commands) == null ? void 0 : _a25.receive(event);
17844
19480
  const result = await this.reducer.reduceServerEvent(this.state, event);
17845
- this.applyState(result.state);
17846
- (_a25 = this.onEvent) == null ? void 0 : _a25.call(this, event);
19481
+ if (!current()) return;
19482
+ const session = (_b25 = this.state.session) != null ? _b25 : event.type === "session-started" ? createSessionState() : void 0;
19483
+ this.applyState({
19484
+ ...result.state,
19485
+ status: this.state.status,
19486
+ ...session == null ? {} : {
19487
+ session: reduceSessionState(
19488
+ session,
19489
+ event,
19490
+ this.maxEvents,
19491
+ command == null ? void 0 : command.muted
19492
+ )
19493
+ }
19494
+ });
19495
+ if (!current()) return;
17847
19496
  for (const effect of result.effects) {
17848
- this.handleReducerEffect(effect);
19497
+ this.handleReducerEffect(effect, attempt);
19498
+ if (!current()) return;
17849
19499
  }
19500
+ if (event.type === "audio-chunk")
19501
+ (_d = (_c = this.pcm) != null ? _c : this.audio) == null ? void 0 : _d.playAudio(event.delta);
19502
+ if (!current()) return;
17850
19503
  if (event.type === "response-done" && this.toolCallsInResponse.size > 0) {
17851
19504
  this.responseToolCallsClosed = true;
17852
19505
  this.maybeRequestToolResponse();
17853
19506
  }
19507
+ if (!current()) return;
19508
+ if (event.type === "session-closed" && session != null) {
19509
+ this.disconnect();
19510
+ this.notifyEvent(event, attempt, true);
19511
+ return;
19512
+ }
19513
+ if ((event.type === "session-started" || event.type === "session-created" || event.type === "session-updated") && attempt.active && !attempt.closing && attempt.cause == null) {
19514
+ const ready = ((_e = this.options.model.capabilities) == null ? void 0 : _e.startup) === "session-start" ? event.type === "session-started" : event.type !== "session-started";
19515
+ if (ready) {
19516
+ attempt.ready = true;
19517
+ attempt.clearTimer("startup");
19518
+ this.applyState({ ...this.state, status: "connected" });
19519
+ if (!current() || attempt.closing) return;
19520
+ (_f = this.pcm) == null ? void 0 : _f.startCapture();
19521
+ }
19522
+ }
19523
+ if (current()) this.notifyEvent(event, attempt);
17854
19524
  }
17855
- handleReducerEffect(effect) {
17856
- var _a25;
19525
+ handleReducerEffect(effect, attempt) {
19526
+ var _a25, _b25, _c, _d;
17857
19527
  switch (effect.type) {
17858
- case "play-audio": {
19528
+ case "play-audio":
17859
19529
  this.currentResponseItemId = effect.itemId;
17860
- this.audio.playAudio(effect.delta);
19530
+ (_a25 = this.audio) == null ? void 0 : _a25.playAudio(effect.delta);
17861
19531
  break;
17862
- }
17863
- case "speech-started": {
17864
- if (this.state.isPlaying) {
17865
- const playedMs = this.audio.getPlaybackOffsetMs();
17866
- this.audio.stopPlayback();
17867
- if (this.currentResponseItemId != null) {
19532
+ case "speech-started":
19533
+ if (!this.continuous && this.state.isPlaying) {
19534
+ const playedMs = (_c = (_b25 = this.audio) == null ? void 0 : _b25.getPlaybackOffsetMs()) != null ? _c : 0;
19535
+ const itemId = this.currentResponseItemId;
19536
+ (_d = this.audio) == null ? void 0 : _d.stopPlayback();
19537
+ if (this.attempt !== attempt || !attempt.active) return;
19538
+ if (itemId != null && !attempt.closing && attempt.cause == null)
17868
19539
  this.sendEvent({
17869
19540
  type: "conversation-item-truncate",
17870
- itemId: this.currentResponseItemId,
19541
+ itemId,
17871
19542
  contentIndex: 0,
17872
19543
  audioEndMs: Math.round(playedMs)
17873
19544
  });
17874
- }
17875
19545
  }
17876
19546
  break;
17877
- }
17878
- case "tool-call": {
19547
+ case "tool-call":
19548
+ if (this.continuous) break;
17879
19549
  this.toolCallsInResponse.add(effect.callId);
17880
- void this.executeToolCall({
17881
- name: effect.name,
17882
- args: effect.args,
17883
- callId: effect.callId
17884
- });
19550
+ void this.executeTool(effect.callId, effect.name, effect.args, attempt);
17885
19551
  break;
17886
- }
17887
- case "error": {
17888
- (_a25 = this.onError) == null ? void 0 : _a25.call(this, effect.error);
19552
+ case "error":
19553
+ void this.reportError(effect.error, attempt);
17889
19554
  break;
17890
- }
19555
+ }
19556
+ }
19557
+ async reportError(error, attempt) {
19558
+ var _a25;
19559
+ if (this.attempt !== attempt) return;
19560
+ try {
19561
+ await ((_a25 = this.onError) == null ? void 0 : _a25.call(
19562
+ this,
19563
+ error instanceof Error ? error : new Error(String(error))
19564
+ ));
19565
+ } catch (e) {
19566
+ }
19567
+ }
19568
+ notifyEvent(event, attempt, terminal = false) {
19569
+ var _a25;
19570
+ if (this.attempt !== attempt || !attempt.active && !terminal) return;
19571
+ const report = (error) => {
19572
+ if (attempt.active || terminal) void this.reportError(error, attempt);
19573
+ };
19574
+ try {
19575
+ void Promise.resolve((_a25 = this.onEvent) == null ? void 0 : _a25.call(this, event)).catch(report);
19576
+ } catch (error) {
19577
+ report(error);
17891
19578
  }
17892
19579
  }
17893
19580
  };
@@ -18544,7 +20231,7 @@ var DefaultTranscriptionResult = class {
18544
20231
 
18545
20232
  // src/transcribe/stream-transcribe.ts
18546
20233
  import {
18547
- UnsupportedFunctionalityError as UnsupportedFunctionalityError5
20234
+ UnsupportedFunctionalityError as UnsupportedFunctionalityError7
18548
20235
  } from "@ai-sdk/provider";
18549
20236
  import {
18550
20237
  DelayedPromise as DelayedPromise3,
@@ -18567,7 +20254,7 @@ function streamTranscribe({
18567
20254
  }
18568
20255
  const doStream = (_a25 = resolvedModel.doStream) == null ? void 0 : _a25.bind(resolvedModel);
18569
20256
  if (doStream == null) {
18570
- throw new UnsupportedFunctionalityError5({
20257
+ throw new UnsupportedFunctionalityError7({
18571
20258
  functionality: "streaming transcription",
18572
20259
  message: `The ${resolvedModel.provider} model "${resolvedModel.modelId}" does not support streaming transcription.` + (typeof model === "string" ? " String model IDs resolve through the global provider (AI Gateway by default). If that provider does not support streaming transcription, pass a provider model instance instead (e.g. openai.transcription('gpt-realtime-whisper')) or upgrade @ai-sdk/gateway to a version with streaming transcription support." : "")
18573
20260
  });
@@ -19143,13 +20830,13 @@ import {
19143
20830
  } from "@ai-sdk/provider-utils";
19144
20831
 
19145
20832
  // src/ui/convert-file-list-to-file-ui-parts.ts
19146
- import { UnsupportedFunctionalityError as UnsupportedFunctionalityError6 } from "@ai-sdk/provider";
20833
+ import { UnsupportedFunctionalityError as UnsupportedFunctionalityError8 } from "@ai-sdk/provider";
19147
20834
  async function convertFileListToFileUIParts(files) {
19148
20835
  if (files == null) {
19149
20836
  return [];
19150
20837
  }
19151
20838
  if (!globalThis.FileList || !(files instanceof globalThis.FileList)) {
19152
- throw new UnsupportedFunctionalityError6({
20839
+ throw new UnsupportedFunctionalityError8({
19153
20840
  functionality: "FileList",
19154
20841
  message: "FileList is not supported in the current environment"
19155
20842
  });
@@ -19333,8 +21020,8 @@ var DefaultChatTransport = class extends HttpChatTransport {
19333
21020
  // src/ui/chat.ts
19334
21021
  var AbstractChat = class {
19335
21022
  constructor({
19336
- generateId: generateId4 = generateIdFunc2,
19337
- id = generateId4(),
21023
+ generateId: generateId5 = generateIdFunc2,
21024
+ id = generateId5(),
19338
21025
  transport = new DefaultChatTransport(),
19339
21026
  messageMetadataSchema,
19340
21027
  dataPartSchemas,
@@ -19546,7 +21233,7 @@ var AbstractChat = class {
19546
21233
  };
19547
21234
  this.id = id;
19548
21235
  this.transport = transport;
19549
- this.generateId = generateId4;
21236
+ this.generateId = generateId5;
19550
21237
  this.messageMetadataSchema = messageMetadataSchema;
19551
21238
  this.dataPartSchemas = dataPartSchemas;
19552
21239
  this.state = state;
@@ -20132,7 +21819,7 @@ export {
20132
21819
  extractReasoningMiddleware,
20133
21820
  fingerprintTools,
20134
21821
  gateway3 as gateway,
20135
- generateId3 as generateId,
21822
+ generateId4 as generateId,
20136
21823
  generateImage,
20137
21824
  generateObject,
20138
21825
  generateSpeech,