ai 7.0.101 → 7.0.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/index.d.ts +82 -20
- package/dist/index.js +2010 -330
- package/dist/index.js.map +1 -1
- package/dist/internal/index.js +1 -1
- package/docs/03-agents/07-workflow-agent.mdx +1 -1
- package/docs/03-ai-sdk-core/36-realtime.mdx +41 -18
- package/docs/04-ai-sdk-ui/21-transport.mdx +1 -1
- package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +364 -48
- package/docs/07-reference/04-ai-sdk-workflow/02-workflow-chat-transport.mdx +4 -4
- package/docs/07-reference/05-ai-sdk-errors/index.mdx +131 -36
- package/package.json +12 -12
- package/src/generate-text/execute-tools-from-stream.ts +7 -0
- package/src/generate-text/stream-text.ts +3 -1
- package/src/realtime/__fixtures__/fake-live-websocket.ts +71 -0
- package/src/realtime/__fixtures__/fake-realtime.ts +36 -0
- package/src/realtime/__fixtures__/fake-webrtc.ts +133 -0
- package/src/realtime/browser-realtime-audio.ts +107 -10
- package/src/realtime/browser-realtime-live-websocket.ts +247 -0
- package/src/realtime/browser-realtime-transport.ts +235 -69
- package/src/realtime/browser-realtime-webrtc.ts +582 -0
- package/src/realtime/encode-realtime-frame.ts +33 -0
- package/src/realtime/index.ts +1 -0
- package/src/realtime/realtime-attempt.ts +45 -0
- package/src/realtime/realtime-command-tracker.ts +81 -0
- package/src/realtime/realtime-event-channel.ts +170 -0
- package/src/realtime/realtime-event-reducer.ts +3 -0
- package/src/realtime/realtime-session-state.ts +65 -0
- package/src/realtime/realtime-session.ts +768 -218
- package/src/realtime/realtime-types.ts +1 -1
- package/src/realtime/validate-realtime-setup.ts +37 -0
package/dist/index.js
CHANGED
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
createIdGenerator as createIdGenerator9,
|
|
15
15
|
dynamicTool,
|
|
16
16
|
experimental_toolCaller,
|
|
17
|
-
generateId as
|
|
17
|
+
generateId as generateId4,
|
|
18
18
|
jsonSchema,
|
|
19
19
|
parseJsonEventStream as parseJsonEventStream3,
|
|
20
20
|
tool,
|
|
@@ -1178,7 +1178,7 @@ import {
|
|
|
1178
1178
|
} from "@ai-sdk/provider-utils";
|
|
1179
1179
|
|
|
1180
1180
|
// src/version.ts
|
|
1181
|
-
var VERSION = true ? "7.0.
|
|
1181
|
+
var VERSION = true ? "7.0.102" : "0.0.0-test";
|
|
1182
1182
|
|
|
1183
1183
|
// src/util/download/download.ts
|
|
1184
1184
|
var download = async ({
|
|
@@ -5556,7 +5556,7 @@ async function generateText({
|
|
|
5556
5556
|
experimental_include,
|
|
5557
5557
|
include = experimental_include,
|
|
5558
5558
|
_internal: {
|
|
5559
|
-
generateId:
|
|
5559
|
+
generateId: generateId5 = originalGenerateId,
|
|
5560
5560
|
generateCallId = originalGenerateCallId,
|
|
5561
5561
|
now: now2 = now
|
|
5562
5562
|
} = {},
|
|
@@ -5956,7 +5956,7 @@ async function generateText({
|
|
|
5956
5956
|
}
|
|
5957
5957
|
);
|
|
5958
5958
|
const responseData = {
|
|
5959
|
-
id: (_b27 = (_a28 = result.response) == null ? void 0 : _a28.id) != null ? _b27 :
|
|
5959
|
+
id: (_b27 = (_a28 = result.response) == null ? void 0 : _a28.id) != null ? _b27 : generateId5(),
|
|
5960
5960
|
timestamp: (_d3 = (_c3 = result.response) == null ? void 0 : _c3.timestamp) != null ? _d3 : /* @__PURE__ */ new Date(),
|
|
5961
5961
|
modelId: (_f2 = (_e2 = result.response) == null ? void 0 : _e2.modelId) != null ? _f2 : stepModel.modelId,
|
|
5962
5962
|
headers: (_g2 = result.response) == null ? void 0 : _g2.headers,
|
|
@@ -6076,7 +6076,7 @@ async function generateText({
|
|
|
6076
6076
|
if (toolApprovalStatus.type === "not-applicable") {
|
|
6077
6077
|
continue;
|
|
6078
6078
|
}
|
|
6079
|
-
const approvalId =
|
|
6079
|
+
const approvalId = generateId5();
|
|
6080
6080
|
const signature = await maybeSignApproval({
|
|
6081
6081
|
secret: experimental_toolApprovalSecret,
|
|
6082
6082
|
approvalId,
|
|
@@ -8400,7 +8400,7 @@ function executeToolsFromStream({
|
|
|
8400
8400
|
toolApproval,
|
|
8401
8401
|
runtimeContext,
|
|
8402
8402
|
toolApprovalSecret,
|
|
8403
|
-
generateId:
|
|
8403
|
+
generateId: generateId5,
|
|
8404
8404
|
onToolExecutionStart,
|
|
8405
8405
|
onToolExecutionEnd,
|
|
8406
8406
|
executeToolInTelemetryContext,
|
|
@@ -8439,7 +8439,7 @@ function executeToolsFromStream({
|
|
|
8439
8439
|
}
|
|
8440
8440
|
return;
|
|
8441
8441
|
}
|
|
8442
|
-
const approvalId =
|
|
8442
|
+
const approvalId = generateId5();
|
|
8443
8443
|
const signature = await maybeSignApproval({
|
|
8444
8444
|
secret: toolApprovalSecret,
|
|
8445
8445
|
approvalId,
|
|
@@ -8474,6 +8474,11 @@ function executeToolsFromStream({
|
|
|
8474
8474
|
reason: toolApprovalStatus.reason,
|
|
8475
8475
|
providerExecuted: chunk.providerExecuted
|
|
8476
8476
|
});
|
|
8477
|
+
controller.enqueue({
|
|
8478
|
+
type: "tool-output-denied",
|
|
8479
|
+
toolCallId: chunk.toolCallId,
|
|
8480
|
+
toolName: chunk.toolName
|
|
8481
|
+
});
|
|
8477
8482
|
return;
|
|
8478
8483
|
}
|
|
8479
8484
|
case "approved": {
|
|
@@ -8722,7 +8727,7 @@ async function streamLanguageModelCall({
|
|
|
8722
8727
|
toolsContext,
|
|
8723
8728
|
experimental_sandbox: sandbox,
|
|
8724
8729
|
_internal: {
|
|
8725
|
-
generateId:
|
|
8730
|
+
generateId: generateId5 = originalGenerateId2,
|
|
8726
8731
|
generateCallId = originalGenerateCallId2,
|
|
8727
8732
|
now: now2 = now
|
|
8728
8733
|
} = {},
|
|
@@ -8808,7 +8813,7 @@ async function streamLanguageModelCall({
|
|
|
8808
8813
|
provider: resolvedModel.provider,
|
|
8809
8814
|
modelId: resolvedModel.modelId,
|
|
8810
8815
|
toolChoice: stepToolChoice,
|
|
8811
|
-
generateId:
|
|
8816
|
+
generateId: generateId5,
|
|
8812
8817
|
now: now2,
|
|
8813
8818
|
callStartTimestampMs,
|
|
8814
8819
|
onLanguageModelCallEnd
|
|
@@ -8830,7 +8835,7 @@ function createLanguageModelV4StreamPartToLanguageModelStreamPartTransform({
|
|
|
8830
8835
|
provider,
|
|
8831
8836
|
modelId,
|
|
8832
8837
|
toolChoice,
|
|
8833
|
-
generateId:
|
|
8838
|
+
generateId: generateId5,
|
|
8834
8839
|
now: now2,
|
|
8835
8840
|
callStartTimestampMs,
|
|
8836
8841
|
onLanguageModelCallEnd
|
|
@@ -8842,7 +8847,7 @@ function createLanguageModelV4StreamPartToLanguageModelStreamPartTransform({
|
|
|
8842
8847
|
const reasoningPartIndexes = /* @__PURE__ */ new Map();
|
|
8843
8848
|
const rawTextPartIndexes = /* @__PURE__ */ new Map();
|
|
8844
8849
|
const rawReasoningPartIndexes = /* @__PURE__ */ new Map();
|
|
8845
|
-
let responseId =
|
|
8850
|
+
let responseId = generateId5();
|
|
8846
8851
|
let responseModelId = modelId;
|
|
8847
8852
|
let timeToFirstOutputMs;
|
|
8848
8853
|
let previousOutputChunkTimestampMs;
|
|
@@ -9249,6 +9254,7 @@ var isOutputChunkType = {
|
|
|
9249
9254
|
"tool-call": true,
|
|
9250
9255
|
"tool-result": false,
|
|
9251
9256
|
"tool-error": false,
|
|
9257
|
+
"tool-output-denied": false,
|
|
9252
9258
|
"tool-execution-end": false,
|
|
9253
9259
|
"model-call-start": false,
|
|
9254
9260
|
"model-call-response-metadata": false,
|
|
@@ -9331,7 +9337,7 @@ function streamText({
|
|
|
9331
9337
|
include = experimental_include,
|
|
9332
9338
|
_internal: {
|
|
9333
9339
|
now: now2 = now,
|
|
9334
|
-
generateId:
|
|
9340
|
+
generateId: generateId5 = originalGenerateId3,
|
|
9335
9341
|
generateCallId = originalGenerateCallId3
|
|
9336
9342
|
} = {},
|
|
9337
9343
|
...settings
|
|
@@ -9410,7 +9416,7 @@ function streamText({
|
|
|
9410
9416
|
onToolExecutionStart: resolvedOnToolExecutionStart,
|
|
9411
9417
|
onToolExecutionEnd: resolvedOnToolExecutionEnd,
|
|
9412
9418
|
now: now2,
|
|
9413
|
-
generateId:
|
|
9419
|
+
generateId: generateId5,
|
|
9414
9420
|
generateCallId,
|
|
9415
9421
|
download: download2,
|
|
9416
9422
|
// assign default values to include:
|
|
@@ -9638,7 +9644,7 @@ var DefaultStreamTextResult = class {
|
|
|
9638
9644
|
providerOptions,
|
|
9639
9645
|
prepareStep,
|
|
9640
9646
|
now: now2,
|
|
9641
|
-
generateId:
|
|
9647
|
+
generateId: generateId5,
|
|
9642
9648
|
generateCallId,
|
|
9643
9649
|
timeout,
|
|
9644
9650
|
onChunk,
|
|
@@ -9693,7 +9699,7 @@ var DefaultStreamTextResult = class {
|
|
|
9693
9699
|
usedIds.add(id);
|
|
9694
9700
|
return id;
|
|
9695
9701
|
}
|
|
9696
|
-
const generatedId =
|
|
9702
|
+
const generatedId = generateId5();
|
|
9697
9703
|
let uniqueId = generatedId;
|
|
9698
9704
|
let suffix = 0;
|
|
9699
9705
|
while (usedIds.has(uniqueId)) {
|
|
@@ -10628,7 +10634,7 @@ var DefaultStreamTextResult = class {
|
|
|
10628
10634
|
toolApproval,
|
|
10629
10635
|
runtimeContext,
|
|
10630
10636
|
toolApprovalSecret: experimental_toolApprovalSecret,
|
|
10631
|
-
generateId:
|
|
10637
|
+
generateId: generateId5,
|
|
10632
10638
|
// the callbacks need to be passed down and handled by executeToolCall
|
|
10633
10639
|
// to guarantee that the onToolExecutionStart callback is invoked before the tool execute function
|
|
10634
10640
|
onToolExecutionStart: filterNullable2(
|
|
@@ -10671,7 +10677,7 @@ var DefaultStreamTextResult = class {
|
|
|
10671
10677
|
let modelCallPerformance = createModelCallPerformance();
|
|
10672
10678
|
const toolExecutionMs = {};
|
|
10673
10679
|
const createStepResponse = () => ({
|
|
10674
|
-
id:
|
|
10680
|
+
id: generateId5(),
|
|
10675
10681
|
timestamp: /* @__PURE__ */ new Date(),
|
|
10676
10682
|
modelId: model.modelId
|
|
10677
10683
|
});
|
|
@@ -10737,7 +10743,8 @@ var DefaultStreamTextResult = class {
|
|
|
10737
10743
|
case "tool-input-start":
|
|
10738
10744
|
case "tool-input-end":
|
|
10739
10745
|
case "tool-input-delta":
|
|
10740
|
-
case "tool-approval-request":
|
|
10746
|
+
case "tool-approval-request":
|
|
10747
|
+
case "tool-output-denied": {
|
|
10741
10748
|
enqueueStepPart(controller, chunk);
|
|
10742
10749
|
break;
|
|
10743
10750
|
}
|
|
@@ -11501,7 +11508,7 @@ function createUIMessageStream({
|
|
|
11501
11508
|
onStepFinish,
|
|
11502
11509
|
onEnd,
|
|
11503
11510
|
onFinish,
|
|
11504
|
-
generateId:
|
|
11511
|
+
generateId: generateId5 = generateIdFunc
|
|
11505
11512
|
}) {
|
|
11506
11513
|
let controller;
|
|
11507
11514
|
const ongoingStreamPromises = [];
|
|
@@ -11593,7 +11600,7 @@ function createUIMessageStream({
|
|
|
11593
11600
|
});
|
|
11594
11601
|
return handleUIMessageStreamFinish({
|
|
11595
11602
|
stream,
|
|
11596
|
-
messageId:
|
|
11603
|
+
messageId: generateId5(),
|
|
11597
11604
|
originalMessages,
|
|
11598
11605
|
onStepEnd: onStepEnd != null ? onStepEnd : onStepFinish,
|
|
11599
11606
|
onEnd: onEnd != null ? onEnd : onFinish,
|
|
@@ -14488,7 +14495,7 @@ async function generateObject(options) {
|
|
|
14488
14495
|
onStepFinish,
|
|
14489
14496
|
onFinish,
|
|
14490
14497
|
_internal: {
|
|
14491
|
-
generateId:
|
|
14498
|
+
generateId: generateId5 = originalGenerateId4,
|
|
14492
14499
|
currentDate = () => /* @__PURE__ */ new Date()
|
|
14493
14500
|
} = {},
|
|
14494
14501
|
...settings
|
|
@@ -14528,7 +14535,7 @@ async function generateObject(options) {
|
|
|
14528
14535
|
const resolvedOnStepStart = onStepStart != null ? onStepStart : experimental_onStepStart;
|
|
14529
14536
|
const resolvedOnStepEnd = onStepEnd != null ? onStepEnd : onStepFinish;
|
|
14530
14537
|
const jsonSchema2 = await outputStrategy.jsonSchema();
|
|
14531
|
-
const callId =
|
|
14538
|
+
const callId = generateId5();
|
|
14532
14539
|
await notify({
|
|
14533
14540
|
event: {
|
|
14534
14541
|
callId,
|
|
@@ -14597,7 +14604,7 @@ async function generateObject(options) {
|
|
|
14597
14604
|
})
|
|
14598
14605
|
);
|
|
14599
14606
|
const responseData = {
|
|
14600
|
-
id: (_b25 = (_a25 = generateResult.response) == null ? void 0 : _a25.id) != null ? _b25 :
|
|
14607
|
+
id: (_b25 = (_a25 = generateResult.response) == null ? void 0 : _a25.id) != null ? _b25 : generateId5(),
|
|
14601
14608
|
timestamp: (_d = (_c = generateResult.response) == null ? void 0 : _c.timestamp) != null ? _d : currentDate(),
|
|
14602
14609
|
modelId: (_f = (_e = generateResult.response) == null ? void 0 : _e.modelId) != null ? _f : model.modelId,
|
|
14603
14610
|
headers: (_g = generateResult.response) == null ? void 0 : _g.headers,
|
|
@@ -14857,7 +14864,7 @@ function streamObject(options) {
|
|
|
14857
14864
|
},
|
|
14858
14865
|
onFinish,
|
|
14859
14866
|
_internal: {
|
|
14860
|
-
generateId:
|
|
14867
|
+
generateId: generateId5 = originalGenerateId5,
|
|
14861
14868
|
currentDate = () => /* @__PURE__ */ new Date(),
|
|
14862
14869
|
now: now2 = now
|
|
14863
14870
|
} = {},
|
|
@@ -14904,7 +14911,7 @@ function streamObject(options) {
|
|
|
14904
14911
|
onError,
|
|
14905
14912
|
onFinish,
|
|
14906
14913
|
download: download2,
|
|
14907
|
-
generateId:
|
|
14914
|
+
generateId: generateId5,
|
|
14908
14915
|
currentDate,
|
|
14909
14916
|
now: now2
|
|
14910
14917
|
});
|
|
@@ -14933,7 +14940,7 @@ var DefaultStreamObjectResult = class {
|
|
|
14933
14940
|
onError,
|
|
14934
14941
|
onFinish,
|
|
14935
14942
|
download: download2,
|
|
14936
|
-
generateId:
|
|
14943
|
+
generateId: generateId5,
|
|
14937
14944
|
currentDate,
|
|
14938
14945
|
now: now2
|
|
14939
14946
|
}) {
|
|
@@ -14967,7 +14974,7 @@ var DefaultStreamObjectResult = class {
|
|
|
14967
14974
|
}
|
|
14968
14975
|
});
|
|
14969
14976
|
this.baseStream = stitchableStream.stream.pipeThrough(eventProcessor);
|
|
14970
|
-
const callId =
|
|
14977
|
+
const callId = generateId5();
|
|
14971
14978
|
(async () => {
|
|
14972
14979
|
const jsonSchema2 = await outputStrategy.jsonSchema();
|
|
14973
14980
|
await notify({
|
|
@@ -15065,7 +15072,7 @@ var DefaultStreamObjectResult = class {
|
|
|
15065
15072
|
let accumulatedText = "";
|
|
15066
15073
|
let textDelta = "";
|
|
15067
15074
|
let fullResponse = {
|
|
15068
|
-
id:
|
|
15075
|
+
id: generateId5(),
|
|
15069
15076
|
timestamp: currentDate(),
|
|
15070
15077
|
modelId: model.modelId
|
|
15071
15078
|
};
|
|
@@ -17029,6 +17036,9 @@ function resolveRealtimeToolDescription({
|
|
|
17029
17036
|
});
|
|
17030
17037
|
}
|
|
17031
17038
|
|
|
17039
|
+
// src/realtime/realtime-session.ts
|
|
17040
|
+
import { UnsupportedFunctionalityError as UnsupportedFunctionalityError6 } from "@ai-sdk/provider";
|
|
17041
|
+
|
|
17032
17042
|
// src/realtime/browser-realtime-audio.ts
|
|
17033
17043
|
var BrowserRealtimeAudio = class {
|
|
17034
17044
|
constructor(options) {
|
|
@@ -17036,16 +17046,23 @@ var BrowserRealtimeAudio = class {
|
|
|
17036
17046
|
this.captureProcessor = null;
|
|
17037
17047
|
this.captureSource = null;
|
|
17038
17048
|
this.captureStream = null;
|
|
17049
|
+
this.ownsCaptureStream = true;
|
|
17050
|
+
this.captureCleanup = [];
|
|
17051
|
+
this.captureGeneration = 0;
|
|
17039
17052
|
this.playbackContext = null;
|
|
17040
17053
|
this.playbackQueue = [];
|
|
17041
17054
|
this.playbackTime = 0;
|
|
17042
17055
|
this.playbackStartTime = 0;
|
|
17043
17056
|
this.activeSources = /* @__PURE__ */ new Set();
|
|
17044
17057
|
this.isPlaying = false;
|
|
17058
|
+
this.playbackPaused = false;
|
|
17059
|
+
var _a25;
|
|
17045
17060
|
this.captureSampleRate = options.captureSampleRate;
|
|
17046
17061
|
this.playbackSampleRate = options.playbackSampleRate;
|
|
17047
17062
|
this.onAudio = options.onAudio;
|
|
17048
17063
|
this.onPlayingChange = options.onPlayingChange;
|
|
17064
|
+
this.maxPlaybackSeconds = (_a25 = options.maxPlaybackSeconds) != null ? _a25 : Infinity;
|
|
17065
|
+
this.onError = options.onError;
|
|
17049
17066
|
this.onCapturingChange = options.onCapturingChange;
|
|
17050
17067
|
}
|
|
17051
17068
|
ensurePlaybackContext() {
|
|
@@ -17053,35 +17070,97 @@ var BrowserRealtimeAudio = class {
|
|
|
17053
17070
|
this.playbackContext = new AudioContext({
|
|
17054
17071
|
sampleRate: this.playbackSampleRate
|
|
17055
17072
|
});
|
|
17073
|
+
const context = this.playbackContext;
|
|
17074
|
+
context.onstatechange = () => {
|
|
17075
|
+
if (this.playbackContext === context)
|
|
17076
|
+
this.setPlaying(
|
|
17077
|
+
context.state === "running" && this.activeSources.size > 0
|
|
17078
|
+
);
|
|
17079
|
+
};
|
|
17056
17080
|
}
|
|
17057
17081
|
}
|
|
17058
|
-
|
|
17082
|
+
async resumePlayback() {
|
|
17083
|
+
var _a25;
|
|
17084
|
+
this.ensurePlaybackContext();
|
|
17085
|
+
const context = this.playbackContext;
|
|
17086
|
+
if (this.playbackPaused) this.stopPlayback();
|
|
17087
|
+
if (this.playbackContext !== context) return;
|
|
17088
|
+
await (context == null ? void 0 : context.resume());
|
|
17089
|
+
if (this.playbackContext !== context) return;
|
|
17090
|
+
this.playbackPaused = false;
|
|
17091
|
+
this.setPlaying(
|
|
17092
|
+
((_a25 = this.playbackContext) == null ? void 0 : _a25.state) === "running" && this.activeSources.size > 0
|
|
17093
|
+
);
|
|
17094
|
+
}
|
|
17095
|
+
startCapture(stream, options) {
|
|
17096
|
+
var _a25, _b25;
|
|
17097
|
+
const generation = this.captureGeneration + 1;
|
|
17098
|
+
this.stopCapture();
|
|
17099
|
+
if (generation !== this.captureGeneration) {
|
|
17100
|
+
if ((_a25 = options == null ? void 0 : options.ownsStream) != null ? _a25 : true)
|
|
17101
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
17102
|
+
return;
|
|
17103
|
+
}
|
|
17104
|
+
this.captureStream = stream;
|
|
17105
|
+
this.ownsCaptureStream = (_b25 = options == null ? void 0 : options.ownsStream) != null ? _b25 : true;
|
|
17059
17106
|
const ctx = new AudioContext({ sampleRate: this.captureSampleRate });
|
|
17060
17107
|
this.captureContext = ctx;
|
|
17061
|
-
|
|
17108
|
+
void ctx.resume().catch((error) => {
|
|
17109
|
+
var _a26;
|
|
17110
|
+
if (this.captureContext === ctx) (_a26 = this.onError) == null ? void 0 : _a26.call(this, error);
|
|
17111
|
+
});
|
|
17062
17112
|
const source = ctx.createMediaStreamSource(stream);
|
|
17063
17113
|
this.captureSource = source;
|
|
17064
|
-
const processor = ctx.createScriptProcessor(
|
|
17114
|
+
const processor = ctx.createScriptProcessor(1024, 1, 1);
|
|
17065
17115
|
this.captureProcessor = processor;
|
|
17066
17116
|
processor.onaudioprocess = (event) => {
|
|
17117
|
+
var _a26;
|
|
17118
|
+
if (this.captureContext !== ctx) return;
|
|
17067
17119
|
const inputData = event.inputBuffer.getChannelData(0);
|
|
17068
17120
|
const samples = resampleAudio(
|
|
17069
17121
|
new Float32Array(inputData),
|
|
17070
17122
|
ctx.sampleRate,
|
|
17071
17123
|
this.captureSampleRate
|
|
17072
17124
|
);
|
|
17073
|
-
|
|
17125
|
+
try {
|
|
17126
|
+
this.onAudio(encodeRealtimeAudio(samples));
|
|
17127
|
+
} catch (error) {
|
|
17128
|
+
(_a26 = this.onError) == null ? void 0 : _a26.call(
|
|
17129
|
+
this,
|
|
17130
|
+
error instanceof Error ? error : new Error(String(error))
|
|
17131
|
+
);
|
|
17132
|
+
}
|
|
17074
17133
|
};
|
|
17075
17134
|
source.connect(processor);
|
|
17076
17135
|
processor.connect(ctx.destination);
|
|
17077
|
-
this.onCapturingChange(
|
|
17136
|
+
const updateCapturing = () => this.onCapturingChange(
|
|
17137
|
+
stream.getAudioTracks().some(
|
|
17138
|
+
(track) => track.enabled && !track.muted && track.readyState === "live"
|
|
17139
|
+
)
|
|
17140
|
+
);
|
|
17141
|
+
for (const track of stream.getAudioTracks()) {
|
|
17142
|
+
for (const event of ["ended", "mute", "unmute"]) {
|
|
17143
|
+
track.addEventListener(event, updateCapturing);
|
|
17144
|
+
this.captureCleanup.push(
|
|
17145
|
+
() => track.removeEventListener(event, updateCapturing)
|
|
17146
|
+
);
|
|
17147
|
+
}
|
|
17148
|
+
}
|
|
17149
|
+
updateCapturing();
|
|
17078
17150
|
}
|
|
17079
17151
|
stopCapture() {
|
|
17080
17152
|
var _a25, _b25, _c, _d;
|
|
17153
|
+
this.captureGeneration++;
|
|
17154
|
+
if (this.captureProcessor != null)
|
|
17155
|
+
this.captureProcessor.onaudioprocess = null;
|
|
17156
|
+
for (const cleanup of this.captureCleanup) cleanup();
|
|
17157
|
+
this.captureCleanup = [];
|
|
17081
17158
|
(_a25 = this.captureProcessor) == null ? void 0 : _a25.disconnect();
|
|
17082
17159
|
(_b25 = this.captureSource) == null ? void 0 : _b25.disconnect();
|
|
17083
|
-
void ((_c = this.captureContext) == null ? void 0 : _c.close())
|
|
17084
|
-
|
|
17160
|
+
void ((_c = this.captureContext) == null ? void 0 : _c.close().catch(() => {
|
|
17161
|
+
}));
|
|
17162
|
+
if (this.ownsCaptureStream)
|
|
17163
|
+
(_d = this.captureStream) == null ? void 0 : _d.getTracks().forEach((track) => track.stop());
|
|
17085
17164
|
this.captureProcessor = null;
|
|
17086
17165
|
this.captureSource = null;
|
|
17087
17166
|
this.captureContext = null;
|
|
@@ -17089,15 +17168,34 @@ var BrowserRealtimeAudio = class {
|
|
|
17089
17168
|
this.onCapturingChange(false);
|
|
17090
17169
|
}
|
|
17091
17170
|
playAudio(base64Audio) {
|
|
17171
|
+
var _a25, _b25, _c;
|
|
17172
|
+
if (this.playbackPaused) return;
|
|
17092
17173
|
this.ensurePlaybackContext();
|
|
17093
|
-
|
|
17174
|
+
const samples = decodeRealtimeAudio(base64Audio);
|
|
17175
|
+
if (Math.max(
|
|
17176
|
+
0,
|
|
17177
|
+
this.playbackTime - ((_b25 = (_a25 = this.playbackContext) == null ? void 0 : _a25.currentTime) != null ? _b25 : 0)
|
|
17178
|
+
) + samples.length / this.playbackSampleRate > this.maxPlaybackSeconds) {
|
|
17179
|
+
this.playbackPaused = true;
|
|
17180
|
+
this.stopPlayback();
|
|
17181
|
+
(_c = this.onError) == null ? void 0 : _c.call(
|
|
17182
|
+
this,
|
|
17183
|
+
new Error(
|
|
17184
|
+
"Realtime audio playback buffer is full; playback paused, call resumePlayback() to resume at the live edge"
|
|
17185
|
+
)
|
|
17186
|
+
);
|
|
17187
|
+
return;
|
|
17188
|
+
}
|
|
17189
|
+
this.playbackQueue.push(samples);
|
|
17094
17190
|
this.schedulePlayback();
|
|
17095
17191
|
}
|
|
17096
17192
|
stopPlayback() {
|
|
17097
17193
|
this.playbackQueue = [];
|
|
17098
17194
|
for (const source of this.activeSources) {
|
|
17099
17195
|
try {
|
|
17196
|
+
source.onended = null;
|
|
17100
17197
|
source.stop();
|
|
17198
|
+
source.disconnect();
|
|
17101
17199
|
} catch (e) {
|
|
17102
17200
|
}
|
|
17103
17201
|
}
|
|
@@ -17116,8 +17214,12 @@ var BrowserRealtimeAudio = class {
|
|
|
17116
17214
|
var _a25;
|
|
17117
17215
|
this.stopCapture();
|
|
17118
17216
|
this.stopPlayback();
|
|
17119
|
-
|
|
17217
|
+
if (this.playbackContext != null) this.playbackContext.onstatechange = null;
|
|
17218
|
+
void ((_a25 = this.playbackContext) == null ? void 0 : _a25.close().catch(() => {
|
|
17219
|
+
}));
|
|
17120
17220
|
this.playbackContext = null;
|
|
17221
|
+
this.playbackTime = 0;
|
|
17222
|
+
this.playbackPaused = false;
|
|
17121
17223
|
}
|
|
17122
17224
|
setPlaying(isPlaying) {
|
|
17123
17225
|
var _a25, _b25;
|
|
@@ -17147,8 +17249,10 @@ var BrowserRealtimeAudio = class {
|
|
|
17147
17249
|
source.start(startTime);
|
|
17148
17250
|
this.playbackTime = startTime + buffer.duration;
|
|
17149
17251
|
this.activeSources.add(source);
|
|
17150
|
-
this.setPlaying(
|
|
17252
|
+
this.setPlaying(ctx.state === "running");
|
|
17253
|
+
if (this.playbackContext !== ctx) return;
|
|
17151
17254
|
source.onended = () => {
|
|
17255
|
+
source.disconnect();
|
|
17152
17256
|
this.activeSources.delete(source);
|
|
17153
17257
|
if (this.playbackQueue.length === 0 && this.activeSources.size === 0) {
|
|
17154
17258
|
this.setPlaying(false);
|
|
@@ -17158,172 +17262,1327 @@ var BrowserRealtimeAudio = class {
|
|
|
17158
17262
|
}
|
|
17159
17263
|
};
|
|
17160
17264
|
|
|
17161
|
-
// src/realtime/
|
|
17265
|
+
// src/realtime/realtime-event-channel.ts
|
|
17162
17266
|
import { safeParseJSON as safeParseJSON5 } from "@ai-sdk/provider-utils";
|
|
17267
|
+
var RealtimeEventChannel = class {
|
|
17268
|
+
constructor(options) {
|
|
17269
|
+
this.options = options;
|
|
17270
|
+
this.active = true;
|
|
17271
|
+
this.finishing = false;
|
|
17272
|
+
this.incoming = Promise.resolve();
|
|
17273
|
+
this.outgoing = Promise.resolve();
|
|
17274
|
+
this.incomingCount = 0;
|
|
17275
|
+
this.incomingBytes = 0;
|
|
17276
|
+
this.outgoingCount = 0;
|
|
17277
|
+
var _a25, _b25, _c;
|
|
17278
|
+
this.parse = (_c = (_b25 = (_a25 = options.model).createServerEventParser) == null ? void 0 : _b25.call(_a25)) != null ? _c : options.model.parseServerEvent.bind(options.model);
|
|
17279
|
+
}
|
|
17280
|
+
dispose() {
|
|
17281
|
+
this.active = false;
|
|
17282
|
+
}
|
|
17283
|
+
/** Stop accepting messages, but deliver already received terminal events. */
|
|
17284
|
+
async finish() {
|
|
17285
|
+
this.stopWriting();
|
|
17286
|
+
await this.incoming;
|
|
17287
|
+
this.dispose();
|
|
17288
|
+
}
|
|
17289
|
+
stopWriting() {
|
|
17290
|
+
this.finishing = true;
|
|
17291
|
+
}
|
|
17292
|
+
send(event, shouldSend) {
|
|
17293
|
+
var _a25;
|
|
17294
|
+
if (!this.active || this.finishing)
|
|
17295
|
+
throw new Error("Realtime connection is closed");
|
|
17296
|
+
if (this.outgoingCount >= ((_a25 = this.options.maxPending) != null ? _a25 : 512)) {
|
|
17297
|
+
throw new Error("Realtime outgoing queue is full");
|
|
17298
|
+
}
|
|
17299
|
+
this.outgoingCount++;
|
|
17300
|
+
const operation = this.outgoing.then(async () => {
|
|
17301
|
+
if (!this.active || this.finishing)
|
|
17302
|
+
throw new Error("Realtime connection is closed");
|
|
17303
|
+
if ((shouldSend == null ? void 0 : shouldSend()) === false) return;
|
|
17304
|
+
if (!this.active || this.finishing)
|
|
17305
|
+
throw new Error("Realtime connection is closed");
|
|
17306
|
+
const data = await this.options.model.serializeClientEvent(event);
|
|
17307
|
+
if (!this.active || this.finishing)
|
|
17308
|
+
throw new Error("Realtime connection is closed");
|
|
17309
|
+
if ((shouldSend == null ? void 0 : shouldSend()) === false) return;
|
|
17310
|
+
if (!this.active || this.finishing)
|
|
17311
|
+
throw new Error("Realtime connection is closed");
|
|
17312
|
+
if (data != null) this.options.send(data);
|
|
17313
|
+
});
|
|
17314
|
+
this.outgoing = operation.catch((error) => {
|
|
17315
|
+
if (!this.finishing) {
|
|
17316
|
+
if (event.type !== "input-audio-append" && event.type !== "session-close")
|
|
17317
|
+
this.report(error);
|
|
17318
|
+
}
|
|
17319
|
+
}).finally(() => {
|
|
17320
|
+
this.outgoingCount--;
|
|
17321
|
+
});
|
|
17322
|
+
return operation;
|
|
17323
|
+
}
|
|
17324
|
+
receive(data) {
|
|
17325
|
+
var _a25, _b25, _c;
|
|
17326
|
+
if (!this.active || this.finishing) return;
|
|
17327
|
+
const bytes = typeof data === "string" ? data.length * 2 : data instanceof Blob ? data.size : (_a25 = data.byteLength) != null ? _a25 : 0;
|
|
17328
|
+
if (this.incomingCount >= ((_b25 = this.options.maxPending) != null ? _b25 : 512) || this.incomingBytes + bytes > ((_c = this.options.maxPendingBytes) != null ? _c : 8 * 1024 * 1024)) {
|
|
17329
|
+
this.finishing = true;
|
|
17330
|
+
this.fatal(
|
|
17331
|
+
new Error(
|
|
17332
|
+
"Realtime incoming queue is full; input protocol continuity was lost"
|
|
17333
|
+
)
|
|
17334
|
+
);
|
|
17335
|
+
return;
|
|
17336
|
+
}
|
|
17337
|
+
this.incomingCount++;
|
|
17338
|
+
this.incomingBytes += bytes;
|
|
17339
|
+
this.incoming = this.incoming.then(async () => {
|
|
17340
|
+
var _a26, _b26;
|
|
17341
|
+
if (!this.active) return;
|
|
17342
|
+
const text2 = typeof data === "string" ? data : data instanceof Blob ? await data.text() : new TextDecoder().decode(data);
|
|
17343
|
+
const parsed = await safeParseJSON5({ text: text2 });
|
|
17344
|
+
if (!this.active) return;
|
|
17345
|
+
if (!parsed.success)
|
|
17346
|
+
throw new Error("Invalid JSON in realtime server message", {
|
|
17347
|
+
cause: parsed.error
|
|
17348
|
+
});
|
|
17349
|
+
const health = (_b26 = (_a26 = this.options.model).getHealthCheckResponse) == null ? void 0 : _b26.call(
|
|
17350
|
+
_a26,
|
|
17351
|
+
parsed.value
|
|
17352
|
+
);
|
|
17353
|
+
if (!this.active) return;
|
|
17354
|
+
if (health != null && !this.finishing) this.options.send(health);
|
|
17355
|
+
if (!this.active) return;
|
|
17356
|
+
const result = this.parse(parsed.value);
|
|
17357
|
+
for (const event of Array.isArray(result) ? result : [result]) {
|
|
17358
|
+
if (!this.active) return;
|
|
17359
|
+
try {
|
|
17360
|
+
await this.options.onEvent(event);
|
|
17361
|
+
} catch (error) {
|
|
17362
|
+
this.report(error);
|
|
17363
|
+
}
|
|
17364
|
+
}
|
|
17365
|
+
}).catch((error) => {
|
|
17366
|
+
this.finishing = true;
|
|
17367
|
+
this.fatal(error instanceof Error ? error : new Error(String(error)));
|
|
17368
|
+
}).finally(() => {
|
|
17369
|
+
this.incomingCount--;
|
|
17370
|
+
this.incomingBytes -= bytes;
|
|
17371
|
+
});
|
|
17372
|
+
}
|
|
17373
|
+
fatal(error) {
|
|
17374
|
+
if (!this.active) return;
|
|
17375
|
+
if (this.options.onFatalError != null) this.options.onFatalError(error);
|
|
17376
|
+
else this.report(error);
|
|
17377
|
+
}
|
|
17378
|
+
report(error) {
|
|
17379
|
+
if (!this.active) return;
|
|
17380
|
+
try {
|
|
17381
|
+
this.options.onError(
|
|
17382
|
+
error instanceof Error ? error : new Error(String(error))
|
|
17383
|
+
);
|
|
17384
|
+
} catch (e) {
|
|
17385
|
+
}
|
|
17386
|
+
}
|
|
17387
|
+
};
|
|
17388
|
+
|
|
17389
|
+
// src/realtime/encode-realtime-frame.ts
|
|
17390
|
+
var REALTIME_MAX_FRAME_BYTES = 128 * 1024;
|
|
17391
|
+
var REALTIME_MAX_BUFFERED_BYTES = 128 * 1024;
|
|
17392
|
+
function encodeRealtimeFrame(value) {
|
|
17393
|
+
if (value instanceof ArrayBuffer || ArrayBuffer.isView(value))
|
|
17394
|
+
return { data: value, byteLength: value.byteLength };
|
|
17395
|
+
if (value instanceof Blob) return { data: value, byteLength: value.size };
|
|
17396
|
+
const data = typeof value === "string" ? value : JSON.stringify(value);
|
|
17397
|
+
if (data === void 0)
|
|
17398
|
+
throw new Error("Realtime event could not be encoded as JSON");
|
|
17399
|
+
return { data, byteLength: new TextEncoder().encode(data).byteLength };
|
|
17400
|
+
}
|
|
17401
|
+
function assertRealtimeFrameBudget(frame, bufferedAmount) {
|
|
17402
|
+
if (frame.byteLength > REALTIME_MAX_FRAME_BYTES)
|
|
17403
|
+
throw new Error(
|
|
17404
|
+
`Realtime frame exceeds the ${REALTIME_MAX_FRAME_BYTES}-byte limit; reduce the event payload`
|
|
17405
|
+
);
|
|
17406
|
+
if (bufferedAmount + frame.byteLength > REALTIME_MAX_BUFFERED_BYTES)
|
|
17407
|
+
throw new Error(
|
|
17408
|
+
`Realtime send buffer is full (${REALTIME_MAX_BUFFERED_BYTES}-byte limit); wait before retrying`
|
|
17409
|
+
);
|
|
17410
|
+
}
|
|
17411
|
+
|
|
17412
|
+
// src/realtime/browser-realtime-transport.ts
|
|
17413
|
+
function getCloseError(event) {
|
|
17414
|
+
if (event.code === 1e3 && event.wasClean) return void 0;
|
|
17415
|
+
return new Error(
|
|
17416
|
+
`Realtime WebSocket closed unexpectedly (code ${event.code}${event.reason === "" ? "" : `: ${event.reason}`})`
|
|
17417
|
+
);
|
|
17418
|
+
}
|
|
17163
17419
|
var BrowserRealtimeTransport = class {
|
|
17164
17420
|
constructor(options) {
|
|
17421
|
+
this.options = options;
|
|
17165
17422
|
this.ws = null;
|
|
17166
|
-
this.
|
|
17423
|
+
this.epoch = 0;
|
|
17424
|
+
this.failing = false;
|
|
17425
|
+
this.closing = false;
|
|
17426
|
+
var _a25;
|
|
17167
17427
|
this.model = options.model;
|
|
17428
|
+
this.continuous = ((_a25 = options.model.capabilities) == null ? void 0 : _a25.conversation) === "continuous";
|
|
17168
17429
|
this.onServerEvent = options.onServerEvent;
|
|
17169
17430
|
this.onError = options.onError;
|
|
17170
17431
|
this.onClose = options.onClose;
|
|
17171
17432
|
}
|
|
17433
|
+
get isOpen() {
|
|
17434
|
+
var _a25;
|
|
17435
|
+
return !this.closing && !this.failing && ((_a25 = this.ws) == null ? void 0 : _a25.readyState) === WebSocket.OPEN;
|
|
17436
|
+
}
|
|
17172
17437
|
connect({
|
|
17438
|
+
mode,
|
|
17173
17439
|
token,
|
|
17174
17440
|
url,
|
|
17175
|
-
onOpen
|
|
17441
|
+
onOpen,
|
|
17442
|
+
protocols
|
|
17176
17443
|
}) {
|
|
17177
|
-
var _a25;
|
|
17178
|
-
|
|
17179
|
-
|
|
17180
|
-
|
|
17181
|
-
|
|
17444
|
+
var _a25, _b25;
|
|
17445
|
+
this.disconnect();
|
|
17446
|
+
const epoch = this.epoch;
|
|
17447
|
+
if (mode !== "relay" && (mode !== "client-secret" || typeof token !== "string" || token.trim() === ""))
|
|
17448
|
+
throw new Error(
|
|
17449
|
+
"Realtime client-secret connection requires a nonempty token"
|
|
17450
|
+
);
|
|
17451
|
+
const wsConfig = mode === "relay" ? { url, protocols } : (_b25 = (_a25 = this.model).getWebSocketConfig) == null ? void 0 : _b25.call(_a25, { token, url });
|
|
17452
|
+
if (this.epoch !== epoch) return;
|
|
17453
|
+
if (wsConfig == null)
|
|
17454
|
+
throw new Error("Model does not support client-secret WebSockets");
|
|
17455
|
+
const finalUrl = wsConfig.url;
|
|
17456
|
+
const finalProtocols = wsConfig.protocols;
|
|
17457
|
+
if (this.epoch !== epoch) return;
|
|
17458
|
+
try {
|
|
17459
|
+
if (typeof finalUrl !== "string" || !/^wss?:\/\//i.test(finalUrl))
|
|
17460
|
+
throw new Error("Invalid realtime WebSocket URL");
|
|
17461
|
+
const parsed = new URL(finalUrl);
|
|
17462
|
+
if (!["ws:", "wss:"].includes(parsed.protocol) || parsed.hostname === "" || finalUrl.includes("#"))
|
|
17463
|
+
throw new Error("Invalid realtime WebSocket URL");
|
|
17464
|
+
} catch (e) {
|
|
17465
|
+
throw new Error(
|
|
17466
|
+
"Invalid realtime WebSocket URL; expected an absolute ws:// or wss:// URL with a host and no fragment"
|
|
17467
|
+
);
|
|
17468
|
+
}
|
|
17469
|
+
if (this.epoch !== epoch) return;
|
|
17470
|
+
const ws = new WebSocket(finalUrl, finalProtocols);
|
|
17182
17471
|
this.ws = ws;
|
|
17472
|
+
let starting = false;
|
|
17473
|
+
let connectionError;
|
|
17474
|
+
const codec = new RealtimeEventChannel({
|
|
17475
|
+
model: this.model,
|
|
17476
|
+
send: (data) => {
|
|
17477
|
+
if (this.epoch !== epoch || this.ws !== ws || !this.isOpen)
|
|
17478
|
+
throw new Error("Realtime WebSocket is not open");
|
|
17479
|
+
this.sendRaw(data);
|
|
17480
|
+
},
|
|
17481
|
+
onEvent: this.onServerEvent,
|
|
17482
|
+
onError: (error) => {
|
|
17483
|
+
if (!starting) this.onError(error);
|
|
17484
|
+
},
|
|
17485
|
+
onFatalError: (error) => this.fail(error)
|
|
17486
|
+
});
|
|
17487
|
+
if (this.epoch !== epoch) {
|
|
17488
|
+
codec.dispose();
|
|
17489
|
+
return;
|
|
17490
|
+
}
|
|
17491
|
+
this.codec = codec;
|
|
17183
17492
|
ws.onopen = () => {
|
|
17184
17493
|
if (this.ws !== ws) return;
|
|
17185
|
-
|
|
17494
|
+
starting = true;
|
|
17495
|
+
try {
|
|
17496
|
+
void Promise.resolve(onOpen()).catch((error) => {
|
|
17497
|
+
if (this.ws === ws)
|
|
17498
|
+
this.fail(
|
|
17499
|
+
error instanceof Error ? error : new Error(String(error))
|
|
17500
|
+
);
|
|
17501
|
+
}).finally(() => {
|
|
17502
|
+
starting = false;
|
|
17503
|
+
});
|
|
17504
|
+
} catch (error) {
|
|
17505
|
+
if (this.epoch === epoch)
|
|
17506
|
+
this.fail(error instanceof Error ? error : new Error(String(error)));
|
|
17507
|
+
}
|
|
17186
17508
|
};
|
|
17187
17509
|
ws.onmessage = (messageEvent) => {
|
|
17188
|
-
|
|
17510
|
+
var _a26;
|
|
17511
|
+
if (this.ws === ws) (_a26 = this.codec) == null ? void 0 : _a26.receive(messageEvent.data);
|
|
17189
17512
|
};
|
|
17190
17513
|
ws.onerror = () => {
|
|
17191
|
-
this.
|
|
17514
|
+
if (this.ws === ws) {
|
|
17515
|
+
connectionError = new Error("WebSocket connection error");
|
|
17516
|
+
this.failing = true;
|
|
17517
|
+
}
|
|
17192
17518
|
};
|
|
17193
|
-
ws.onclose = () => {
|
|
17519
|
+
ws.onclose = (event) => {
|
|
17520
|
+
var _a26;
|
|
17194
17521
|
if (this.ws === ws) {
|
|
17195
17522
|
this.ws = null;
|
|
17196
|
-
|
|
17523
|
+
const closeError = (_a26 = getCloseError(event)) != null ? _a26 : connectionError;
|
|
17524
|
+
codec.stopWriting();
|
|
17525
|
+
this.notifyClosing();
|
|
17526
|
+
if (this.epoch !== epoch) return;
|
|
17527
|
+
const complete = () => {
|
|
17528
|
+
if (this.epoch !== epoch) return;
|
|
17529
|
+
clearTimeout(this.drainTimer);
|
|
17530
|
+
this.epoch++;
|
|
17531
|
+
codec.dispose();
|
|
17532
|
+
try {
|
|
17533
|
+
this.onClose(closeError);
|
|
17534
|
+
} catch (error) {
|
|
17535
|
+
this.reportCallbackError(error);
|
|
17536
|
+
}
|
|
17537
|
+
};
|
|
17538
|
+
this.drainTimer = setTimeout(complete, 1e3);
|
|
17539
|
+
void this.awaitDrain(codec.finish(), complete);
|
|
17197
17540
|
}
|
|
17198
17541
|
};
|
|
17199
17542
|
}
|
|
17200
17543
|
disconnect() {
|
|
17201
17544
|
var _a25;
|
|
17202
|
-
|
|
17545
|
+
this.epoch++;
|
|
17546
|
+
clearTimeout(this.drainTimer);
|
|
17547
|
+
const ws = this.ws;
|
|
17203
17548
|
this.ws = null;
|
|
17549
|
+
(_a25 = this.codec) == null ? void 0 : _a25.dispose();
|
|
17550
|
+
this.codec = void 0;
|
|
17551
|
+
this.failing = false;
|
|
17552
|
+
this.closing = false;
|
|
17553
|
+
ws == null ? void 0 : ws.close();
|
|
17554
|
+
}
|
|
17555
|
+
sendEvent(event, shouldSend, automaticAudio = false) {
|
|
17556
|
+
if (this.failing || this.closing)
|
|
17557
|
+
throw new Error("Realtime connection is closed");
|
|
17558
|
+
if (this.codec == null) return Promise.resolve();
|
|
17559
|
+
const epoch = this.epoch;
|
|
17560
|
+
const failed = (error) => {
|
|
17561
|
+
if (automaticAudio && this.epoch === epoch && this.isOpen)
|
|
17562
|
+
this.fail(error instanceof Error ? error : new Error(String(error)));
|
|
17563
|
+
};
|
|
17564
|
+
try {
|
|
17565
|
+
const sent = this.codec.send(event, shouldSend);
|
|
17566
|
+
void sent.catch(failed);
|
|
17567
|
+
return sent;
|
|
17568
|
+
} catch (error) {
|
|
17569
|
+
failed(error);
|
|
17570
|
+
throw error;
|
|
17571
|
+
}
|
|
17204
17572
|
}
|
|
17205
|
-
|
|
17206
|
-
|
|
17207
|
-
|
|
17208
|
-
if (serialized != null) {
|
|
17209
|
-
this.sendRaw(serialized);
|
|
17210
|
-
}
|
|
17211
|
-
}).catch((error) => {
|
|
17212
|
-
this.onError(
|
|
17213
|
-
error instanceof Error ? error : new Error(`Failed to send realtime event: ${String(error)}`)
|
|
17214
|
-
);
|
|
17215
|
-
});
|
|
17573
|
+
finish() {
|
|
17574
|
+
var _a25, _b25;
|
|
17575
|
+
return (_b25 = (_a25 = this.codec) == null ? void 0 : _a25.finish()) != null ? _b25 : Promise.resolve();
|
|
17216
17576
|
}
|
|
17217
|
-
|
|
17577
|
+
/** Drain accepted events when media or protocol continuity is lost. */
|
|
17578
|
+
fail(error) {
|
|
17218
17579
|
var _a25;
|
|
17219
|
-
if (
|
|
17220
|
-
|
|
17221
|
-
|
|
17222
|
-
|
|
17223
|
-
|
|
17224
|
-
|
|
17580
|
+
if (this.failing) return;
|
|
17581
|
+
this.failing = true;
|
|
17582
|
+
const epoch = this.epoch;
|
|
17583
|
+
const ws = this.ws;
|
|
17584
|
+
this.ws = null;
|
|
17585
|
+
(_a25 = this.codec) == null ? void 0 : _a25.stopWriting();
|
|
17586
|
+
ws == null ? void 0 : ws.close();
|
|
17587
|
+
this.notifyClosing();
|
|
17588
|
+
if (this.epoch !== epoch) return;
|
|
17589
|
+
const drain = this.finish();
|
|
17590
|
+
if (this.options.onFatalError != null)
|
|
17591
|
+
this.options.onFatalError(error, drain);
|
|
17592
|
+
else {
|
|
17593
|
+
const complete = () => {
|
|
17594
|
+
if (epoch !== this.epoch) return;
|
|
17595
|
+
this.disconnect();
|
|
17596
|
+
try {
|
|
17597
|
+
this.onClose();
|
|
17598
|
+
} catch (cause) {
|
|
17599
|
+
this.reportCallbackError(cause);
|
|
17600
|
+
}
|
|
17601
|
+
};
|
|
17602
|
+
this.drainTimer = setTimeout(complete, 1e3);
|
|
17603
|
+
void this.awaitDrain(drain, complete);
|
|
17604
|
+
this.reportCallbackError(error);
|
|
17225
17605
|
}
|
|
17226
17606
|
}
|
|
17227
|
-
|
|
17228
|
-
|
|
17607
|
+
notifyClosing() {
|
|
17608
|
+
var _a25, _b25;
|
|
17609
|
+
if (this.closing) return;
|
|
17610
|
+
this.closing = true;
|
|
17611
|
+
try {
|
|
17612
|
+
(_b25 = (_a25 = this.options).onClosing) == null ? void 0 : _b25.call(_a25);
|
|
17613
|
+
} catch (error) {
|
|
17614
|
+
this.reportCallbackError(error);
|
|
17615
|
+
}
|
|
17229
17616
|
}
|
|
17230
|
-
async
|
|
17231
|
-
|
|
17232
|
-
|
|
17233
|
-
|
|
17234
|
-
} else if (messageEvent.data instanceof Blob) {
|
|
17235
|
-
text2 = await messageEvent.data.text();
|
|
17236
|
-
} else {
|
|
17237
|
-
text2 = new TextDecoder().decode(messageEvent.data);
|
|
17617
|
+
async awaitDrain(drain, complete) {
|
|
17618
|
+
try {
|
|
17619
|
+
await drain;
|
|
17620
|
+
} catch (e) {
|
|
17238
17621
|
}
|
|
17239
|
-
|
|
17240
|
-
|
|
17241
|
-
|
|
17242
|
-
|
|
17243
|
-
const autoResponse = this.model.getHealthCheckResponse(rawEvent);
|
|
17244
|
-
if (autoResponse != null) {
|
|
17245
|
-
this.sendRaw(autoResponse);
|
|
17246
|
-
}
|
|
17622
|
+
try {
|
|
17623
|
+
complete();
|
|
17624
|
+
} catch (error) {
|
|
17625
|
+
this.reportCallbackError(error);
|
|
17247
17626
|
}
|
|
17248
|
-
|
|
17249
|
-
|
|
17250
|
-
|
|
17251
|
-
|
|
17627
|
+
}
|
|
17628
|
+
reportCallbackError(error) {
|
|
17629
|
+
try {
|
|
17630
|
+
this.onError(error instanceof Error ? error : new Error(String(error)));
|
|
17631
|
+
} catch (e) {
|
|
17252
17632
|
}
|
|
17253
17633
|
}
|
|
17634
|
+
sendRaw(data) {
|
|
17635
|
+
const ws = this.ws;
|
|
17636
|
+
const epoch = this.epoch;
|
|
17637
|
+
if (ws == null || !this.isOpen) return;
|
|
17638
|
+
const frame = encodeRealtimeFrame(data);
|
|
17639
|
+
if (this.epoch !== epoch || this.ws !== ws || !this.isOpen)
|
|
17640
|
+
throw new Error("Realtime connection is closed");
|
|
17641
|
+
if (this.continuous) assertRealtimeFrameBudget(frame, ws.bufferedAmount);
|
|
17642
|
+
ws.send(frame.data);
|
|
17643
|
+
}
|
|
17644
|
+
dispose() {
|
|
17645
|
+
this.disconnect();
|
|
17646
|
+
}
|
|
17254
17647
|
};
|
|
17255
17648
|
|
|
17256
|
-
// src/realtime/realtime-
|
|
17257
|
-
import {
|
|
17258
|
-
|
|
17259
|
-
|
|
17260
|
-
|
|
17261
|
-
|
|
17262
|
-
|
|
17263
|
-
|
|
17264
|
-
|
|
17265
|
-
|
|
17266
|
-
|
|
17267
|
-
var
|
|
17268
|
-
|
|
17269
|
-
|
|
17270
|
-
|
|
17271
|
-
|
|
17272
|
-
|
|
17273
|
-
|
|
17274
|
-
|
|
17275
|
-
this.
|
|
17276
|
-
|
|
17277
|
-
|
|
17278
|
-
|
|
17279
|
-
|
|
17649
|
+
// src/realtime/browser-realtime-live-websocket.ts
|
|
17650
|
+
import { UnsupportedFunctionalityError as UnsupportedFunctionalityError5 } from "@ai-sdk/provider";
|
|
17651
|
+
var BrowserRealtimeLiveWebSocket = class {
|
|
17652
|
+
constructor(options) {
|
|
17653
|
+
this.options = options;
|
|
17654
|
+
this.generation = 0;
|
|
17655
|
+
this.ready = false;
|
|
17656
|
+
this.capturingStarted = false;
|
|
17657
|
+
this.pendingAudio = 0;
|
|
17658
|
+
this.captureGeneration = 0;
|
|
17659
|
+
this.captureEnabled = true;
|
|
17660
|
+
var _a25, _b25, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l;
|
|
17661
|
+
const input = (_a25 = options.sessionConfig) == null ? void 0 : _a25.inputAudioFormat;
|
|
17662
|
+
const output = (_b25 = options.sessionConfig) == null ? void 0 : _b25.outputAudioFormat;
|
|
17663
|
+
const sampleRate = (_e = (_d = (_c = input == null ? void 0 : input.rate) != null ? _c : output == null ? void 0 : output.rate) != null ? _d : options.sampleRate) != null ? _e : 24e3;
|
|
17664
|
+
const format = {
|
|
17665
|
+
type: (_g = (_f = input == null ? void 0 : input.type) != null ? _f : output == null ? void 0 : output.type) != null ? _g : "audio/pcm",
|
|
17666
|
+
rate: sampleRate
|
|
17667
|
+
};
|
|
17668
|
+
this.config = {
|
|
17669
|
+
...options.sessionConfig,
|
|
17670
|
+
inputAudioFormat: {
|
|
17671
|
+
...format,
|
|
17672
|
+
...input
|
|
17673
|
+
},
|
|
17674
|
+
outputAudioFormat: {
|
|
17675
|
+
...format,
|
|
17676
|
+
...output
|
|
17677
|
+
}
|
|
17678
|
+
};
|
|
17679
|
+
this.transport = new BrowserRealtimeTransport({
|
|
17680
|
+
model: options.model,
|
|
17681
|
+
onServerEvent: options.onEvent,
|
|
17682
|
+
onError: options.onError,
|
|
17683
|
+
onFatalError: options.onFatalError,
|
|
17684
|
+
onClosing: () => {
|
|
17685
|
+
var _a26;
|
|
17686
|
+
const generation = this.generation;
|
|
17687
|
+
(_a26 = options.onClosing) == null ? void 0 : _a26.call(options);
|
|
17688
|
+
if (generation === this.generation) this.stopCapture();
|
|
17689
|
+
},
|
|
17690
|
+
onClose: options.onClose
|
|
17691
|
+
});
|
|
17692
|
+
this.audio = new BrowserRealtimeAudio({
|
|
17693
|
+
captureSampleRate: (_i = (_h = this.config.inputAudioFormat) == null ? void 0 : _h.rate) != null ? _i : sampleRate,
|
|
17694
|
+
playbackSampleRate: (_k = (_j = this.config.outputAudioFormat) == null ? void 0 : _j.rate) != null ? _k : sampleRate,
|
|
17695
|
+
maxPlaybackSeconds: (_l = options.maxPlaybackBufferSeconds) != null ? _l : 2,
|
|
17696
|
+
onAudio: (audio) => this.sendAudio(audio),
|
|
17697
|
+
onError: options.onError,
|
|
17698
|
+
onCapturingChange: options.onCapturing,
|
|
17699
|
+
onPlayingChange: options.onPlaying
|
|
17700
|
+
});
|
|
17280
17701
|
}
|
|
17281
|
-
|
|
17282
|
-
|
|
17702
|
+
connect(options) {
|
|
17703
|
+
const generation = this.generation + 1;
|
|
17704
|
+
this.dispose();
|
|
17705
|
+
if (generation !== this.generation) return;
|
|
17706
|
+
for (const format of [
|
|
17707
|
+
this.config.inputAudioFormat,
|
|
17708
|
+
this.config.outputAudioFormat
|
|
17709
|
+
]) {
|
|
17710
|
+
if ((format == null ? void 0 : format.type) !== "audio/pcm")
|
|
17711
|
+
throw new UnsupportedFunctionalityError5({
|
|
17712
|
+
functionality: "Continuous browser WebSocket audio supports PCM16 only"
|
|
17713
|
+
});
|
|
17714
|
+
if (format.rate == null || !Number.isFinite(format.rate) || format.rate < 8e3 || format.rate > 96e3)
|
|
17715
|
+
throw new Error("Invalid realtime PCM sample rate");
|
|
17716
|
+
}
|
|
17717
|
+
this.stream = options.stream;
|
|
17718
|
+
this.captureEnabled = options.capture !== false;
|
|
17719
|
+
this.audio.ensurePlaybackContext();
|
|
17720
|
+
void this.audio.resumePlayback().catch((error) => {
|
|
17721
|
+
if (generation === this.generation) this.options.onError(error);
|
|
17722
|
+
});
|
|
17723
|
+
this.transport.connect({
|
|
17724
|
+
mode: "relay",
|
|
17725
|
+
url: options.url,
|
|
17726
|
+
protocols: options.protocols,
|
|
17727
|
+
onOpen: () => {
|
|
17728
|
+
var _a25, _b25;
|
|
17729
|
+
return this.transport.sendEvent({
|
|
17730
|
+
type: (_b25 = (_a25 = this.options.model.capabilities) == null ? void 0 : _a25.startup) != null ? _b25 : "session-update",
|
|
17731
|
+
config: this.config
|
|
17732
|
+
});
|
|
17733
|
+
}
|
|
17734
|
+
});
|
|
17283
17735
|
}
|
|
17284
|
-
|
|
17285
|
-
|
|
17736
|
+
startCapture() {
|
|
17737
|
+
this.ready = true;
|
|
17738
|
+
if (!this.captureEnabled || this.capturingStarted) return;
|
|
17739
|
+
void this.resumeCapture().catch((error) => this.options.onError(error));
|
|
17286
17740
|
}
|
|
17287
|
-
|
|
17288
|
-
|
|
17289
|
-
|
|
17290
|
-
|
|
17291
|
-
|
|
17292
|
-
|
|
17293
|
-
|
|
17294
|
-
|
|
17295
|
-
|
|
17296
|
-
|
|
17297
|
-
|
|
17298
|
-
|
|
17741
|
+
async resumeCapture(suppliedStream) {
|
|
17742
|
+
const generation = this.captureGeneration + 1;
|
|
17743
|
+
this.stopCapture();
|
|
17744
|
+
if (generation !== this.captureGeneration) return;
|
|
17745
|
+
this.ready = true;
|
|
17746
|
+
this.capturingStarted = true;
|
|
17747
|
+
this.captureEnabled = true;
|
|
17748
|
+
if (suppliedStream != null) this.stream = suppliedStream;
|
|
17749
|
+
const supplied = this.stream;
|
|
17750
|
+
const stream = supplied != null ? supplied : await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
17751
|
+
if (generation !== this.captureGeneration || !this.ready) {
|
|
17752
|
+
if (supplied == null) stream.getTracks().forEach((track) => track.stop());
|
|
17753
|
+
return;
|
|
17754
|
+
}
|
|
17755
|
+
if (!stream.getAudioTracks().some((track) => track.readyState === "live")) {
|
|
17756
|
+
if (supplied == null) stream.getTracks().forEach((track) => track.stop());
|
|
17757
|
+
throw new Error("Realtime requires a live audio track");
|
|
17758
|
+
}
|
|
17759
|
+
this.audio.startCapture(stream, { ownsStream: supplied == null });
|
|
17299
17760
|
}
|
|
17300
|
-
|
|
17301
|
-
return
|
|
17302
|
-
|
|
17303
|
-
|
|
17304
|
-
|
|
17305
|
-
|
|
17306
|
-
|
|
17761
|
+
sendAudio(audio) {
|
|
17762
|
+
if (!this.ready || !this.transport.isOpen) return;
|
|
17763
|
+
if (this.pendingAudio >= 8) {
|
|
17764
|
+
this.transport.fail(new Error("Realtime audio send queue is full"));
|
|
17765
|
+
return;
|
|
17766
|
+
}
|
|
17767
|
+
const generation = this.generation;
|
|
17768
|
+
let sent;
|
|
17769
|
+
try {
|
|
17770
|
+
sent = this.transport.sendEvent(
|
|
17771
|
+
{ type: "input-audio-append", audio },
|
|
17772
|
+
void 0,
|
|
17773
|
+
true
|
|
17774
|
+
);
|
|
17775
|
+
} catch (error) {
|
|
17776
|
+
if (!this.ready || !this.transport.isOpen) return;
|
|
17777
|
+
throw error;
|
|
17778
|
+
}
|
|
17779
|
+
this.pendingAudio++;
|
|
17780
|
+
void sent.then(
|
|
17781
|
+
() => {
|
|
17782
|
+
if (generation === this.generation) this.pendingAudio--;
|
|
17783
|
+
},
|
|
17784
|
+
(error) => {
|
|
17785
|
+
if (generation === this.generation && this.ready && this.transport.isOpen)
|
|
17786
|
+
this.options.onFatalError(error);
|
|
17307
17787
|
}
|
|
17308
|
-
|
|
17788
|
+
);
|
|
17309
17789
|
}
|
|
17310
|
-
|
|
17311
|
-
var _a25;
|
|
17312
|
-
|
|
17313
|
-
|
|
17314
|
-
|
|
17315
|
-
|
|
17316
|
-
|
|
17317
|
-
|
|
17318
|
-
|
|
17790
|
+
sendEvent(event, shouldSend) {
|
|
17791
|
+
var _a25, _b25;
|
|
17792
|
+
if (event.type === "session-update") {
|
|
17793
|
+
for (const key of ["inputAudioFormat", "outputAudioFormat"]) {
|
|
17794
|
+
const update = event.config[key];
|
|
17795
|
+
if (update != null && (update.type !== ((_a25 = this.config[key]) == null ? void 0 : _a25.type) || update.rate != null && update.rate !== ((_b25 = this.config[key]) == null ? void 0 : _b25.rate))) {
|
|
17796
|
+
throw new Error(
|
|
17797
|
+
"Changing Live WebSocket audio format requires reconnecting"
|
|
17798
|
+
);
|
|
17319
17799
|
}
|
|
17320
|
-
break;
|
|
17321
17800
|
}
|
|
17322
|
-
|
|
17323
|
-
|
|
17324
|
-
|
|
17325
|
-
|
|
17326
|
-
|
|
17801
|
+
}
|
|
17802
|
+
return this.transport.sendEvent(event, shouldSend);
|
|
17803
|
+
}
|
|
17804
|
+
playAudio(audio) {
|
|
17805
|
+
try {
|
|
17806
|
+
this.audio.playAudio(audio);
|
|
17807
|
+
} catch (error) {
|
|
17808
|
+
this.options.onError(
|
|
17809
|
+
error instanceof Error ? error : new Error(String(error))
|
|
17810
|
+
);
|
|
17811
|
+
}
|
|
17812
|
+
}
|
|
17813
|
+
stopPlayback() {
|
|
17814
|
+
this.audio.stopPlayback();
|
|
17815
|
+
}
|
|
17816
|
+
resumePlayback() {
|
|
17817
|
+
return this.audio.resumePlayback();
|
|
17818
|
+
}
|
|
17819
|
+
stopCapture() {
|
|
17820
|
+
this.captureGeneration++;
|
|
17821
|
+
this.ready = false;
|
|
17822
|
+
this.capturingStarted = false;
|
|
17823
|
+
this.captureEnabled = false;
|
|
17824
|
+
this.audio.stopCapture();
|
|
17825
|
+
}
|
|
17826
|
+
finish() {
|
|
17827
|
+
return this.transport.finish();
|
|
17828
|
+
}
|
|
17829
|
+
dispose() {
|
|
17830
|
+
this.generation++;
|
|
17831
|
+
this.stopCapture();
|
|
17832
|
+
this.ready = false;
|
|
17833
|
+
this.capturingStarted = false;
|
|
17834
|
+
this.pendingAudio = 0;
|
|
17835
|
+
this.stream = void 0;
|
|
17836
|
+
this.transport.disconnect();
|
|
17837
|
+
this.audio.dispose();
|
|
17838
|
+
}
|
|
17839
|
+
};
|
|
17840
|
+
|
|
17841
|
+
// src/realtime/browser-realtime-webrtc.ts
|
|
17842
|
+
import { safeParseJSON as safeParseJSON6 } from "@ai-sdk/provider-utils";
|
|
17843
|
+
var MAX_SESSION_ANSWER_BYTES = 1024 * 1024;
|
|
17844
|
+
async function readSessionAnswer(response) {
|
|
17845
|
+
var _a25;
|
|
17846
|
+
const reader = (_a25 = response.body) == null ? void 0 : _a25.getReader();
|
|
17847
|
+
if (reader == null) throw new Error("Invalid realtime session answer");
|
|
17848
|
+
const decoder = new TextDecoder();
|
|
17849
|
+
let bytes = 0;
|
|
17850
|
+
let text2 = "";
|
|
17851
|
+
try {
|
|
17852
|
+
if (Number(response.headers.get("content-length")) > MAX_SESSION_ANSWER_BYTES)
|
|
17853
|
+
throw new Error("Realtime session answer exceeds the 1 MiB limit");
|
|
17854
|
+
while (true) {
|
|
17855
|
+
const { done, value } = await reader.read();
|
|
17856
|
+
if (done) return text2 + decoder.decode();
|
|
17857
|
+
bytes += value.byteLength;
|
|
17858
|
+
if (bytes > MAX_SESSION_ANSWER_BYTES)
|
|
17859
|
+
throw new Error("Realtime session answer exceeds the 1 MiB limit");
|
|
17860
|
+
text2 += decoder.decode(value, { stream: true });
|
|
17861
|
+
}
|
|
17862
|
+
} catch (error) {
|
|
17863
|
+
void reader.cancel(error).catch(() => {
|
|
17864
|
+
});
|
|
17865
|
+
throw error;
|
|
17866
|
+
} finally {
|
|
17867
|
+
reader.releaseLock();
|
|
17868
|
+
}
|
|
17869
|
+
}
|
|
17870
|
+
function selectAudioTrack(stream) {
|
|
17871
|
+
var _a25;
|
|
17872
|
+
const live = stream.getAudioTracks().filter((track) => track.readyState === "live");
|
|
17873
|
+
return (_a25 = live.find((track) => track.enabled && !track.muted)) != null ? _a25 : live[0];
|
|
17874
|
+
}
|
|
17875
|
+
var BrowserRealtimeWebRTC = class {
|
|
17876
|
+
constructor(options) {
|
|
17877
|
+
this.options = options;
|
|
17878
|
+
this.ownsStream = false;
|
|
17879
|
+
this.generation = 0;
|
|
17880
|
+
this.closingNotified = false;
|
|
17881
|
+
this.captureGeneration = 0;
|
|
17882
|
+
this.senderOperations = Promise.resolve();
|
|
17883
|
+
this.trackCleanups = [];
|
|
17884
|
+
}
|
|
17885
|
+
async connect({
|
|
17886
|
+
api,
|
|
17887
|
+
sessionConfig,
|
|
17888
|
+
stream,
|
|
17889
|
+
timeoutMs,
|
|
17890
|
+
capture = true
|
|
17891
|
+
}) {
|
|
17892
|
+
var _a25, _b25;
|
|
17893
|
+
const generation = this.generation + 1;
|
|
17894
|
+
this.dispose();
|
|
17895
|
+
const current = () => generation === this.generation;
|
|
17896
|
+
if (!current()) return;
|
|
17897
|
+
const config = (_b25 = (_a25 = this.options.model).getWebRTCConfig) == null ? void 0 : _b25.call(_a25);
|
|
17898
|
+
if (!current()) return;
|
|
17899
|
+
if (config == null)
|
|
17900
|
+
throw new Error("This model does not support WebRTC configuration");
|
|
17901
|
+
const abort = new AbortController();
|
|
17902
|
+
this.abort = abort;
|
|
17903
|
+
const timeout = setTimeout(
|
|
17904
|
+
() => abort.abort(new Error("Realtime startup timed out")),
|
|
17905
|
+
timeoutMs
|
|
17906
|
+
);
|
|
17907
|
+
const cancelled = new Promise((_, reject) => {
|
|
17908
|
+
abort.signal.addEventListener(
|
|
17909
|
+
"abort",
|
|
17910
|
+
() => reject(abort.signal.reason),
|
|
17911
|
+
{ once: true }
|
|
17912
|
+
);
|
|
17913
|
+
});
|
|
17914
|
+
const start = async () => {
|
|
17915
|
+
var _a26;
|
|
17916
|
+
const captureGeneration = this.captureGeneration;
|
|
17917
|
+
let media = capture ? stream != null ? stream : await navigator.mediaDevices.getUserMedia({ audio: true }) : void 0;
|
|
17918
|
+
if (!current() || abort.signal.aborted) {
|
|
17919
|
+
if (stream == null) media == null ? void 0 : media.getTracks().forEach((track2) => track2.stop());
|
|
17920
|
+
throw new Error("Realtime connection cancelled");
|
|
17921
|
+
}
|
|
17922
|
+
if (captureGeneration !== this.captureGeneration) {
|
|
17923
|
+
if (stream == null) media == null ? void 0 : media.getTracks().forEach((track2) => track2.stop());
|
|
17924
|
+
media = void 0;
|
|
17925
|
+
}
|
|
17926
|
+
this.stream = media;
|
|
17927
|
+
this.ownsStream = stream == null;
|
|
17928
|
+
const track = media == null ? void 0 : selectAudioTrack(media);
|
|
17929
|
+
if (media != null && track == null) {
|
|
17930
|
+
throw new Error("Realtime requires a live audio track");
|
|
17931
|
+
}
|
|
17932
|
+
const pc = new RTCPeerConnection();
|
|
17933
|
+
this.pc = pc;
|
|
17934
|
+
const audio = document.createElement("audio");
|
|
17935
|
+
audio.autoplay = true;
|
|
17936
|
+
this.audio = audio;
|
|
17937
|
+
audio.onplaying = () => {
|
|
17938
|
+
if (current()) this.options.onPlaying(true);
|
|
17939
|
+
};
|
|
17940
|
+
const stopped = () => {
|
|
17941
|
+
if (current()) this.options.onPlaying(false);
|
|
17942
|
+
};
|
|
17943
|
+
audio.onpause = stopped;
|
|
17944
|
+
audio.onended = stopped;
|
|
17945
|
+
audio.onwaiting = stopped;
|
|
17946
|
+
pc.ontrack = (event) => {
|
|
17947
|
+
var _a27;
|
|
17948
|
+
if (!current()) return;
|
|
17949
|
+
audio.srcObject = (_a27 = event.streams[0]) != null ? _a27 : new MediaStream([event.track]);
|
|
17950
|
+
void audio.play().catch((error) => {
|
|
17951
|
+
if (current()) this.options.onError(error);
|
|
17952
|
+
});
|
|
17953
|
+
};
|
|
17954
|
+
pc.onconnectionstatechange = () => {
|
|
17955
|
+
var _a27;
|
|
17956
|
+
if (!current()) return;
|
|
17957
|
+
if (pc.connectionState === "connected")
|
|
17958
|
+
clearTimeout(this.disconnectTimer);
|
|
17959
|
+
else if (pc.connectionState === "disconnected") {
|
|
17960
|
+
clearTimeout(this.disconnectTimer);
|
|
17961
|
+
this.disconnectTimer = setTimeout(() => {
|
|
17962
|
+
if (current() && pc.connectionState === "disconnected")
|
|
17963
|
+
this.fail(
|
|
17964
|
+
new Error(
|
|
17965
|
+
"Realtime peer disconnected beyond recovery grace period"
|
|
17966
|
+
)
|
|
17967
|
+
);
|
|
17968
|
+
}, (_a27 = this.options.disconnectTimeoutMs) != null ? _a27 : 5e3);
|
|
17969
|
+
} else if (pc.connectionState === "closed") {
|
|
17970
|
+
this.drainClose(new Error("Realtime peer connection closed"));
|
|
17971
|
+
} else if (pc.connectionState === "failed") {
|
|
17972
|
+
this.fail(
|
|
17973
|
+
new Error(`Realtime peer connection ${pc.connectionState}`)
|
|
17974
|
+
);
|
|
17975
|
+
}
|
|
17976
|
+
};
|
|
17977
|
+
pc.oniceconnectionstatechange = () => {
|
|
17978
|
+
if (current() && pc.iceConnectionState === "failed")
|
|
17979
|
+
this.fail(new Error("Realtime ICE connection failed"));
|
|
17980
|
+
};
|
|
17981
|
+
const dc = pc.createDataChannel(config.dataChannelLabel);
|
|
17982
|
+
this.dc = dc;
|
|
17983
|
+
const codec = new RealtimeEventChannel({
|
|
17984
|
+
model: this.options.model,
|
|
17985
|
+
onEvent: this.options.onEvent,
|
|
17986
|
+
onError: this.options.onError,
|
|
17987
|
+
onFatalError: (error) => {
|
|
17988
|
+
if (current()) this.fail(error);
|
|
17989
|
+
},
|
|
17990
|
+
send: (data) => {
|
|
17991
|
+
if (!current() || this.finishing || dc.readyState !== "open")
|
|
17992
|
+
throw new Error("Realtime data channel is not open");
|
|
17993
|
+
if (dc.bufferedAmount > 1024 * 1024)
|
|
17994
|
+
throw new Error("Realtime data channel buffer is full");
|
|
17995
|
+
const encoded = typeof data === "string" ? data : JSON.stringify(data);
|
|
17996
|
+
if (!current() || this.finishing || dc.readyState !== "open")
|
|
17997
|
+
throw new Error("Realtime data channel is not open");
|
|
17998
|
+
dc.send(encoded);
|
|
17999
|
+
}
|
|
18000
|
+
});
|
|
18001
|
+
if (!current()) {
|
|
18002
|
+
codec.dispose();
|
|
18003
|
+
return;
|
|
18004
|
+
}
|
|
18005
|
+
this.codec = codec;
|
|
18006
|
+
dc.onmessage = (event) => {
|
|
18007
|
+
var _a27;
|
|
18008
|
+
if (current()) (_a27 = this.codec) == null ? void 0 : _a27.receive(event.data);
|
|
18009
|
+
};
|
|
18010
|
+
dc.onclose = () => {
|
|
18011
|
+
if (current())
|
|
18012
|
+
this.drainClose(new Error("Realtime data channel closed"));
|
|
18013
|
+
};
|
|
18014
|
+
dc.onerror = () => {
|
|
18015
|
+
if (current()) this.fail(new Error("Realtime data channel error"));
|
|
18016
|
+
};
|
|
18017
|
+
const opened = new Promise((resolve3, reject) => {
|
|
18018
|
+
this.rejectStartup = reject;
|
|
18019
|
+
dc.onopen = () => {
|
|
18020
|
+
this.rejectStartup = void 0;
|
|
18021
|
+
resolve3();
|
|
18022
|
+
};
|
|
18023
|
+
});
|
|
18024
|
+
void opened.catch(() => {
|
|
18025
|
+
});
|
|
18026
|
+
const gathered = new Promise((resolve3) => {
|
|
18027
|
+
pc.onicegatheringstatechange = () => {
|
|
18028
|
+
if (pc.iceGatheringState === "complete") resolve3();
|
|
18029
|
+
};
|
|
18030
|
+
});
|
|
18031
|
+
this.sender = media != null && track != null ? pc.addTrack(track, media) : pc.addTransceiver("audio", { direction: "sendrecv" }).sender;
|
|
18032
|
+
this.observeCapture();
|
|
18033
|
+
if (!current() || abort.signal.aborted) return;
|
|
18034
|
+
const offer = await pc.createOffer();
|
|
18035
|
+
if (!current() || abort.signal.aborted) return;
|
|
18036
|
+
await pc.setLocalDescription(offer);
|
|
18037
|
+
if (pc.iceGatheringState !== "complete")
|
|
18038
|
+
await Promise.race([gathered, cancelled]);
|
|
18039
|
+
if (!current() || abort.signal.aborted) return;
|
|
18040
|
+
const sdp = (_a26 = pc.localDescription) == null ? void 0 : _a26.sdp;
|
|
18041
|
+
if (sdp == null) throw new Error("Realtime offer has no SDP");
|
|
18042
|
+
const response = await fetch(api, {
|
|
18043
|
+
method: "POST",
|
|
18044
|
+
headers: { "Content-Type": "application/json" },
|
|
18045
|
+
body: JSON.stringify({ sdp, sessionConfig }),
|
|
18046
|
+
signal: abort.signal
|
|
18047
|
+
});
|
|
18048
|
+
if (!response.ok)
|
|
18049
|
+
throw new Error(
|
|
18050
|
+
`Failed to create realtime session: ${response.status}`
|
|
18051
|
+
);
|
|
18052
|
+
const parsed = await safeParseJSON6({
|
|
18053
|
+
text: await readSessionAnswer(response)
|
|
18054
|
+
});
|
|
18055
|
+
if (!current() || abort.signal.aborted) return;
|
|
18056
|
+
if (!parsed.success || typeof parsed.value !== "object" || parsed.value == null || !("sdp" in parsed.value) || typeof parsed.value.sdp !== "string" || parsed.value.sdp.trim().length === 0 || !("sessionId" in parsed.value) || typeof parsed.value.sessionId !== "string" || parsed.value.sessionId.trim().length === 0) {
|
|
18057
|
+
throw new Error("Invalid realtime session answer");
|
|
18058
|
+
}
|
|
18059
|
+
await pc.setRemoteDescription({ type: "answer", sdp: parsed.value.sdp });
|
|
18060
|
+
await Promise.race([opened, cancelled]);
|
|
18061
|
+
};
|
|
18062
|
+
try {
|
|
18063
|
+
await Promise.race([start(), cancelled]);
|
|
18064
|
+
} catch (error) {
|
|
18065
|
+
if (current()) this.dispose();
|
|
18066
|
+
throw error;
|
|
18067
|
+
} finally {
|
|
18068
|
+
clearTimeout(timeout);
|
|
18069
|
+
this.rejectStartup = void 0;
|
|
18070
|
+
}
|
|
18071
|
+
}
|
|
18072
|
+
sendEvent(event, shouldSend) {
|
|
18073
|
+
var _a25, _b25, _c;
|
|
18074
|
+
if (this.finishing || ((_a25 = this.dc) == null ? void 0 : _a25.readyState) !== "open")
|
|
18075
|
+
throw new Error("Realtime data channel is not open");
|
|
18076
|
+
return (_c = (_b25 = this.codec) == null ? void 0 : _b25.send(event, shouldSend)) != null ? _c : Promise.resolve();
|
|
18077
|
+
}
|
|
18078
|
+
async resumePlayback() {
|
|
18079
|
+
if (this.audio == null)
|
|
18080
|
+
throw new Error("Realtime playback is not connected");
|
|
18081
|
+
await this.audio.play();
|
|
18082
|
+
}
|
|
18083
|
+
stopPlayback() {
|
|
18084
|
+
var _a25;
|
|
18085
|
+
(_a25 = this.audio) == null ? void 0 : _a25.pause();
|
|
18086
|
+
}
|
|
18087
|
+
finish() {
|
|
18088
|
+
var _a25, _b25;
|
|
18089
|
+
return (_b25 = (_a25 = this.codec) == null ? void 0 : _a25.finish()) != null ? _b25 : Promise.resolve();
|
|
18090
|
+
}
|
|
18091
|
+
fail(error) {
|
|
18092
|
+
var _a25;
|
|
18093
|
+
if (this.finishing === "failure") return;
|
|
18094
|
+
this.finishing = "failure";
|
|
18095
|
+
clearTimeout(this.finishTimer);
|
|
18096
|
+
if (this.rejectStartup != null) {
|
|
18097
|
+
this.rejectStartup(error);
|
|
18098
|
+
(_a25 = this.abort) == null ? void 0 : _a25.abort(error);
|
|
18099
|
+
return;
|
|
18100
|
+
}
|
|
18101
|
+
const generation = this.generation;
|
|
18102
|
+
const drain = this.finish();
|
|
18103
|
+
this.notifyClosing();
|
|
18104
|
+
if (generation !== this.generation) return;
|
|
18105
|
+
void this.stopCaptureForShutdown(generation);
|
|
18106
|
+
if (generation !== this.generation) return;
|
|
18107
|
+
if (this.options.onFatalError != null) {
|
|
18108
|
+
this.options.onFatalError(error, drain);
|
|
18109
|
+
return;
|
|
18110
|
+
}
|
|
18111
|
+
const complete = () => {
|
|
18112
|
+
if (generation !== this.generation) return;
|
|
18113
|
+
this.dispose();
|
|
18114
|
+
try {
|
|
18115
|
+
this.options.onClose();
|
|
18116
|
+
} catch (cause) {
|
|
18117
|
+
this.reportCallbackError(cause);
|
|
18118
|
+
}
|
|
18119
|
+
};
|
|
18120
|
+
this.finishTimer = setTimeout(complete, 1e3);
|
|
18121
|
+
void this.awaitDrain(drain, complete);
|
|
18122
|
+
try {
|
|
18123
|
+
this.options.onError(error);
|
|
18124
|
+
} catch (e) {
|
|
18125
|
+
}
|
|
18126
|
+
}
|
|
18127
|
+
drainClose(error) {
|
|
18128
|
+
if (this.finishing) return;
|
|
18129
|
+
if (this.rejectStartup != null) {
|
|
18130
|
+
this.fail(error);
|
|
18131
|
+
return;
|
|
18132
|
+
}
|
|
18133
|
+
this.finishing = "close";
|
|
18134
|
+
const generation = this.generation;
|
|
18135
|
+
const complete = () => {
|
|
18136
|
+
if (generation !== this.generation || this.finishing !== "close") return;
|
|
18137
|
+
clearTimeout(this.finishTimer);
|
|
18138
|
+
try {
|
|
18139
|
+
this.options.onClose(error);
|
|
18140
|
+
} catch (cause) {
|
|
18141
|
+
this.reportCallbackError(cause);
|
|
18142
|
+
} finally {
|
|
18143
|
+
if (generation === this.generation) this.dispose();
|
|
18144
|
+
}
|
|
18145
|
+
};
|
|
18146
|
+
this.finishTimer = setTimeout(complete, 1e3);
|
|
18147
|
+
void this.awaitDrain(this.finish(), complete);
|
|
18148
|
+
this.notifyClosing();
|
|
18149
|
+
if (generation !== this.generation) return;
|
|
18150
|
+
void this.stopCaptureForShutdown(generation);
|
|
18151
|
+
}
|
|
18152
|
+
notifyClosing() {
|
|
18153
|
+
var _a25, _b25;
|
|
18154
|
+
if (this.closingNotified) return;
|
|
18155
|
+
this.closingNotified = true;
|
|
18156
|
+
try {
|
|
18157
|
+
(_b25 = (_a25 = this.options).onClosing) == null ? void 0 : _b25.call(_a25);
|
|
18158
|
+
} catch (error) {
|
|
18159
|
+
this.reportCallbackError(error);
|
|
18160
|
+
}
|
|
18161
|
+
}
|
|
18162
|
+
async stopCaptureForShutdown(generation) {
|
|
18163
|
+
try {
|
|
18164
|
+
await this.stopCapture();
|
|
18165
|
+
} catch (cause) {
|
|
18166
|
+
if (generation !== this.generation || this.finishing !== "close") return;
|
|
18167
|
+
try {
|
|
18168
|
+
this.fail(cause instanceof Error ? cause : new Error(String(cause)));
|
|
18169
|
+
} catch (error) {
|
|
18170
|
+
this.reportCallbackError(error);
|
|
18171
|
+
}
|
|
18172
|
+
}
|
|
18173
|
+
}
|
|
18174
|
+
async awaitDrain(drain, complete) {
|
|
18175
|
+
try {
|
|
18176
|
+
await drain;
|
|
18177
|
+
} catch (e) {
|
|
18178
|
+
}
|
|
18179
|
+
try {
|
|
18180
|
+
complete();
|
|
18181
|
+
} catch (error) {
|
|
18182
|
+
this.reportCallbackError(error);
|
|
18183
|
+
}
|
|
18184
|
+
}
|
|
18185
|
+
reportCallbackError(error) {
|
|
18186
|
+
try {
|
|
18187
|
+
this.options.onError(
|
|
18188
|
+
error instanceof Error ? error : new Error(String(error))
|
|
18189
|
+
);
|
|
18190
|
+
} catch (e) {
|
|
18191
|
+
}
|
|
18192
|
+
}
|
|
18193
|
+
async stopCapture() {
|
|
18194
|
+
var _a25;
|
|
18195
|
+
const generation = ++this.captureGeneration;
|
|
18196
|
+
for (const cleanup of this.trackCleanups) cleanup();
|
|
18197
|
+
this.trackCleanups = [];
|
|
18198
|
+
if (this.ownsStream)
|
|
18199
|
+
(_a25 = this.stream) == null ? void 0 : _a25.getTracks().forEach((track) => track.stop());
|
|
18200
|
+
this.stream = void 0;
|
|
18201
|
+
const pc = this.pc;
|
|
18202
|
+
const sender = this.sender;
|
|
18203
|
+
await this.queueSenderOperation(async () => {
|
|
18204
|
+
if (pc !== this.pc || sender !== this.sender) return;
|
|
18205
|
+
if (pc != null && sender != null) await this.detachSender(pc, sender);
|
|
18206
|
+
if (generation === this.captureGeneration && pc === this.pc)
|
|
18207
|
+
this.options.onCapturing(false);
|
|
18208
|
+
});
|
|
18209
|
+
}
|
|
18210
|
+
async startCapture(supplied) {
|
|
18211
|
+
const generation = this.captureGeneration + 1;
|
|
18212
|
+
await this.stopCapture();
|
|
18213
|
+
if (generation !== this.captureGeneration || this.pc == null || this.finishing)
|
|
18214
|
+
return;
|
|
18215
|
+
const media = supplied != null ? supplied : await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
18216
|
+
if (generation !== this.captureGeneration || this.pc == null || this.finishing) {
|
|
18217
|
+
if (supplied == null) media.getTracks().forEach((track2) => track2.stop());
|
|
18218
|
+
return;
|
|
18219
|
+
}
|
|
18220
|
+
const track = selectAudioTrack(media);
|
|
18221
|
+
if (track == null) {
|
|
18222
|
+
if (supplied == null) media.getTracks().forEach((track2) => track2.stop());
|
|
18223
|
+
throw new Error("Realtime requires a live audio track");
|
|
18224
|
+
}
|
|
18225
|
+
const pc = this.pc;
|
|
18226
|
+
const sender = this.sender;
|
|
18227
|
+
await this.queueSenderOperation(async () => {
|
|
18228
|
+
const current = () => generation === this.captureGeneration && pc === this.pc && sender === this.sender && pc.connectionState !== "closed" && !this.finishing;
|
|
18229
|
+
let attached = false;
|
|
18230
|
+
try {
|
|
18231
|
+
if (!current() || sender == null) return;
|
|
18232
|
+
await sender.replaceTrack(track);
|
|
18233
|
+
if (!current()) {
|
|
18234
|
+
if (pc === this.pc) await this.detachSender(pc, sender);
|
|
18235
|
+
return;
|
|
18236
|
+
}
|
|
18237
|
+
this.stream = media;
|
|
18238
|
+
this.ownsStream = supplied == null;
|
|
18239
|
+
attached = true;
|
|
18240
|
+
this.observeCapture();
|
|
18241
|
+
} finally {
|
|
18242
|
+
if (!attached && supplied == null)
|
|
18243
|
+
media.getTracks().forEach((track2) => track2.stop());
|
|
18244
|
+
}
|
|
18245
|
+
});
|
|
18246
|
+
}
|
|
18247
|
+
queueSenderOperation(operation) {
|
|
18248
|
+
const pending = this.senderOperations.then(operation);
|
|
18249
|
+
this.senderOperations = pending.catch(() => {
|
|
18250
|
+
});
|
|
18251
|
+
return pending;
|
|
18252
|
+
}
|
|
18253
|
+
async detachSender(pc, sender) {
|
|
18254
|
+
if (pc.connectionState === "closed") return;
|
|
18255
|
+
try {
|
|
18256
|
+
await sender.replaceTrack(null);
|
|
18257
|
+
} catch (error) {
|
|
18258
|
+
if (pc !== this.pc) return;
|
|
18259
|
+
this.closePeer(pc);
|
|
18260
|
+
this.fail(error instanceof Error ? error : new Error(String(error)));
|
|
18261
|
+
}
|
|
18262
|
+
}
|
|
18263
|
+
closePeer(pc) {
|
|
18264
|
+
pc.ontrack = null;
|
|
18265
|
+
pc.onconnectionstatechange = null;
|
|
18266
|
+
pc.oniceconnectionstatechange = null;
|
|
18267
|
+
pc.onicegatheringstatechange = null;
|
|
18268
|
+
if (pc.connectionState !== "closed") pc.close();
|
|
18269
|
+
}
|
|
18270
|
+
observeCapture() {
|
|
18271
|
+
const generation = this.captureGeneration;
|
|
18272
|
+
const sender = this.sender;
|
|
18273
|
+
const track = sender == null ? void 0 : sender.track;
|
|
18274
|
+
const update = () => {
|
|
18275
|
+
if (generation === this.captureGeneration && this.sender === sender)
|
|
18276
|
+
this.options.onCapturing(
|
|
18277
|
+
(sender == null ? void 0 : sender.track) != null && sender.track.readyState === "live" && sender.track.enabled && !sender.track.muted
|
|
18278
|
+
);
|
|
18279
|
+
};
|
|
18280
|
+
if (track != null) {
|
|
18281
|
+
for (const event of ["ended", "mute", "unmute"]) {
|
|
18282
|
+
track.addEventListener(event, update);
|
|
18283
|
+
this.trackCleanups.push(() => track.removeEventListener(event, update));
|
|
18284
|
+
}
|
|
18285
|
+
}
|
|
18286
|
+
update();
|
|
18287
|
+
}
|
|
18288
|
+
dispose() {
|
|
18289
|
+
var _a25, _b25, _c;
|
|
18290
|
+
this.generation++;
|
|
18291
|
+
this.captureGeneration++;
|
|
18292
|
+
clearTimeout(this.finishTimer);
|
|
18293
|
+
clearTimeout(this.disconnectTimer);
|
|
18294
|
+
this.finishing = void 0;
|
|
18295
|
+
this.closingNotified = false;
|
|
18296
|
+
this.senderOperations = Promise.resolve();
|
|
18297
|
+
(_a25 = this.abort) == null ? void 0 : _a25.abort(new Error("Realtime connection cancelled"));
|
|
18298
|
+
this.abort = void 0;
|
|
18299
|
+
(_b25 = this.codec) == null ? void 0 : _b25.dispose();
|
|
18300
|
+
this.codec = void 0;
|
|
18301
|
+
if (this.dc != null) {
|
|
18302
|
+
this.dc.onopen = null;
|
|
18303
|
+
this.dc.onclose = null;
|
|
18304
|
+
this.dc.onerror = null;
|
|
18305
|
+
this.dc.onmessage = null;
|
|
18306
|
+
this.dc.close();
|
|
18307
|
+
this.dc = void 0;
|
|
18308
|
+
}
|
|
18309
|
+
if (this.pc != null) {
|
|
18310
|
+
this.closePeer(this.pc);
|
|
18311
|
+
this.pc = void 0;
|
|
18312
|
+
}
|
|
18313
|
+
for (const cleanup of this.trackCleanups) cleanup();
|
|
18314
|
+
this.trackCleanups = [];
|
|
18315
|
+
if (this.ownsStream)
|
|
18316
|
+
(_c = this.stream) == null ? void 0 : _c.getTracks().forEach((track) => track.stop());
|
|
18317
|
+
this.stream = void 0;
|
|
18318
|
+
this.sender = void 0;
|
|
18319
|
+
if (this.audio != null) {
|
|
18320
|
+
this.audio.onplaying = null;
|
|
18321
|
+
this.audio.onpause = null;
|
|
18322
|
+
this.audio.onended = null;
|
|
18323
|
+
this.audio.onwaiting = null;
|
|
18324
|
+
this.audio.pause();
|
|
18325
|
+
this.audio.srcObject = null;
|
|
18326
|
+
this.audio = void 0;
|
|
18327
|
+
}
|
|
18328
|
+
this.options.onCapturing(false);
|
|
18329
|
+
this.options.onPlaying(false);
|
|
18330
|
+
}
|
|
18331
|
+
};
|
|
18332
|
+
|
|
18333
|
+
// src/realtime/realtime-attempt.ts
|
|
18334
|
+
var RealtimeAttempt = class {
|
|
18335
|
+
constructor() {
|
|
18336
|
+
this.abort = new AbortController();
|
|
18337
|
+
this.timers = /* @__PURE__ */ new Map();
|
|
18338
|
+
this.active = true;
|
|
18339
|
+
this.ready = false;
|
|
18340
|
+
this.closing = false;
|
|
18341
|
+
this.transportClosing = false;
|
|
18342
|
+
}
|
|
18343
|
+
timer(name25, duration, callback) {
|
|
18344
|
+
this.clearTimer(name25);
|
|
18345
|
+
this.timers.set(
|
|
18346
|
+
name25,
|
|
18347
|
+
setTimeout(() => {
|
|
18348
|
+
this.timers.delete(name25);
|
|
18349
|
+
if (this.active) callback();
|
|
18350
|
+
}, duration)
|
|
18351
|
+
);
|
|
18352
|
+
}
|
|
18353
|
+
clearTimer(name25) {
|
|
18354
|
+
clearTimeout(this.timers.get(name25));
|
|
18355
|
+
this.timers.delete(name25);
|
|
18356
|
+
}
|
|
18357
|
+
beginClose() {
|
|
18358
|
+
var _a25;
|
|
18359
|
+
this.closing = true;
|
|
18360
|
+
return (_a25 = this.closePromise) != null ? _a25 : this.closePromise = new Promise((resolve3) => {
|
|
18361
|
+
this.settleClose = resolve3;
|
|
18362
|
+
});
|
|
18363
|
+
}
|
|
18364
|
+
retire() {
|
|
18365
|
+
var _a25;
|
|
18366
|
+
if (!this.active) return;
|
|
18367
|
+
this.active = false;
|
|
18368
|
+
this.abort.abort();
|
|
18369
|
+
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
18370
|
+
this.timers.clear();
|
|
18371
|
+
(_a25 = this.settleClose) == null ? void 0 : _a25.call(this);
|
|
18372
|
+
this.settleClose = void 0;
|
|
18373
|
+
}
|
|
18374
|
+
};
|
|
18375
|
+
|
|
18376
|
+
// src/realtime/realtime-command-tracker.ts
|
|
18377
|
+
import { generateId as generateId3 } from "@ai-sdk/provider-utils";
|
|
18378
|
+
var RealtimeCommandTracker = class {
|
|
18379
|
+
constructor(sendCommand) {
|
|
18380
|
+
this.sendCommand = sendCommand;
|
|
18381
|
+
this.pending = /* @__PURE__ */ new Map();
|
|
18382
|
+
this.recent = /* @__PURE__ */ new Set();
|
|
18383
|
+
}
|
|
18384
|
+
validateId(id) {
|
|
18385
|
+
if (this.pending.has(id) || this.recent.has(id))
|
|
18386
|
+
throw new Error(
|
|
18387
|
+
"Realtime command ID was already used; retry with a fresh eventId"
|
|
18388
|
+
);
|
|
18389
|
+
if (this.pending.size >= 512)
|
|
18390
|
+
throw new Error("Too many pending realtime commands");
|
|
18391
|
+
}
|
|
18392
|
+
complete(id) {
|
|
18393
|
+
const command = this.pending.get(id);
|
|
18394
|
+
if (command == null) return;
|
|
18395
|
+
this.pending.delete(id);
|
|
18396
|
+
this.recent.add(id);
|
|
18397
|
+
if (this.recent.size > 4096) {
|
|
18398
|
+
const oldest = this.recent.values().next().value;
|
|
18399
|
+
if (oldest != null) this.recent.delete(oldest);
|
|
18400
|
+
}
|
|
18401
|
+
return command;
|
|
18402
|
+
}
|
|
18403
|
+
send(event) {
|
|
18404
|
+
var _a25;
|
|
18405
|
+
const muted = event.type === "input-audio-mute" ? true : event.type === "input-audio-unmute" ? false : void 0;
|
|
18406
|
+
if (event.type === "input-audio-mute" || event.type === "input-audio-unmute")
|
|
18407
|
+
event = { ...event, eventId: (_a25 = event.eventId) != null ? _a25 : generateId3() };
|
|
18408
|
+
const id = "eventId" in event ? event.eventId : void 0;
|
|
18409
|
+
if (id != null) {
|
|
18410
|
+
this.validateId(id);
|
|
18411
|
+
this.pending.set(id, { muted });
|
|
18412
|
+
}
|
|
18413
|
+
const awaitsAcknowledgement = event.type === "context-append" || event.type === "session-update" || muted != null;
|
|
18414
|
+
try {
|
|
18415
|
+
const sent = this.sendCommand(event);
|
|
18416
|
+
void sent.then(
|
|
18417
|
+
() => {
|
|
18418
|
+
if (id != null && !awaitsAcknowledgement) this.complete(id);
|
|
18419
|
+
},
|
|
18420
|
+
() => {
|
|
18421
|
+
if (id != null) this.complete(id);
|
|
18422
|
+
}
|
|
18423
|
+
);
|
|
18424
|
+
return sent;
|
|
18425
|
+
} catch (error) {
|
|
18426
|
+
if (id != null) this.complete(id);
|
|
18427
|
+
throw error;
|
|
18428
|
+
}
|
|
18429
|
+
}
|
|
18430
|
+
receive(event) {
|
|
18431
|
+
if (event.type === "command-acknowledged" && event.clientEventId != null)
|
|
18432
|
+
return this.complete(event.clientEventId);
|
|
18433
|
+
if (event.type === "error" && event.clientEventId != null)
|
|
18434
|
+
this.complete(event.clientEventId);
|
|
18435
|
+
}
|
|
18436
|
+
};
|
|
18437
|
+
|
|
18438
|
+
// src/realtime/validate-realtime-setup.ts
|
|
18439
|
+
var setupSchema = z.object({
|
|
18440
|
+
token: z.string().refine((value) => value.trim().length > 0),
|
|
18441
|
+
url: z.string().refine((value) => {
|
|
18442
|
+
try {
|
|
18443
|
+
const url = new URL(value);
|
|
18444
|
+
return (url.protocol === "ws:" || url.protocol === "wss:") && url.hostname !== "";
|
|
18445
|
+
} catch (e) {
|
|
18446
|
+
return false;
|
|
18447
|
+
}
|
|
18448
|
+
}),
|
|
18449
|
+
expiresAt: z.number().positive().max(Number.MAX_SAFE_INTEGER).optional(),
|
|
18450
|
+
tools: z.array(
|
|
18451
|
+
z.object({
|
|
18452
|
+
type: z.literal("function"),
|
|
18453
|
+
name: z.string().min(1),
|
|
18454
|
+
description: z.string().optional(),
|
|
18455
|
+
parameters: z.record(z.string(), z.unknown())
|
|
18456
|
+
})
|
|
18457
|
+
).optional()
|
|
18458
|
+
});
|
|
18459
|
+
function validateRealtimeSetup(payload) {
|
|
18460
|
+
const result = setupSchema.safeParse(payload);
|
|
18461
|
+
if (!result.success)
|
|
18462
|
+
throw new Error(
|
|
18463
|
+
"Invalid realtime setup: expected a nonempty token, a ws(s) URL, and valid optional expiresAt/tools"
|
|
18464
|
+
);
|
|
18465
|
+
return result.data;
|
|
18466
|
+
}
|
|
18467
|
+
|
|
18468
|
+
// src/realtime/realtime-session-state.ts
|
|
18469
|
+
function createSessionState() {
|
|
18470
|
+
return {
|
|
18471
|
+
transcripts: [],
|
|
18472
|
+
delegations: [],
|
|
18473
|
+
isInputMuted: false,
|
|
18474
|
+
finalization: "pending"
|
|
18475
|
+
};
|
|
18476
|
+
}
|
|
18477
|
+
function reduceSessionState(state, event, limit, muted) {
|
|
18478
|
+
var _a25;
|
|
18479
|
+
switch (event.type) {
|
|
18480
|
+
case "session-started":
|
|
18481
|
+
return {
|
|
18482
|
+
...state,
|
|
18483
|
+
sessionId: event.sessionId,
|
|
18484
|
+
delegationMode: event.delegationMode
|
|
18485
|
+
};
|
|
18486
|
+
case "session-usage":
|
|
18487
|
+
return { ...state, usage: event.usage };
|
|
18488
|
+
case "session-closed":
|
|
18489
|
+
return {
|
|
18490
|
+
...state,
|
|
18491
|
+
sessionId: (_a25 = event.sessionId) != null ? _a25 : state.sessionId,
|
|
18492
|
+
usage: event.usage,
|
|
18493
|
+
terminationReason: event.reason,
|
|
18494
|
+
finalization: "confirmed"
|
|
18495
|
+
};
|
|
18496
|
+
case "transcript-fragment":
|
|
18497
|
+
return {
|
|
18498
|
+
...state,
|
|
18499
|
+
transcripts: [...state.transcripts, event].slice(-limit)
|
|
18500
|
+
};
|
|
18501
|
+
case "delegation-created":
|
|
18502
|
+
return state.delegations.some(
|
|
18503
|
+
(item) => item.delegationId === event.delegationId
|
|
18504
|
+
) ? state : {
|
|
18505
|
+
...state,
|
|
18506
|
+
delegations: [...state.delegations, event].slice(-limit)
|
|
18507
|
+
};
|
|
18508
|
+
case "command-acknowledged":
|
|
18509
|
+
return muted == null ? state : { ...state, isInputMuted: muted };
|
|
18510
|
+
default:
|
|
18511
|
+
return state;
|
|
18512
|
+
}
|
|
18513
|
+
}
|
|
18514
|
+
|
|
18515
|
+
// src/realtime/realtime-event-reducer.ts
|
|
18516
|
+
import { safeParseJSON as safeParseJSON7 } from "@ai-sdk/provider-utils";
|
|
18517
|
+
function createInitialRealtimeState() {
|
|
18518
|
+
return {
|
|
18519
|
+
status: "disconnected",
|
|
18520
|
+
messages: [],
|
|
18521
|
+
events: [],
|
|
18522
|
+
isCapturing: false,
|
|
18523
|
+
isPlaying: false
|
|
18524
|
+
};
|
|
18525
|
+
}
|
|
18526
|
+
var RealtimeEventReducer = class {
|
|
18527
|
+
constructor(maxEvents = 500) {
|
|
18528
|
+
this.maxEvents = maxEvents;
|
|
18529
|
+
this.currentAssistantMessageId = null;
|
|
18530
|
+
this.textAccumulators = /* @__PURE__ */ new Map();
|
|
18531
|
+
this.toolArgAccumulators = /* @__PURE__ */ new Map();
|
|
18532
|
+
this.toolCallIdToMessageId = /* @__PURE__ */ new Map();
|
|
18533
|
+
this.toolCallIdToName = /* @__PURE__ */ new Map();
|
|
18534
|
+
this.inputAudioMessageInsertIndex = /* @__PURE__ */ new Map();
|
|
18535
|
+
this.itemIdToPartLocation = /* @__PURE__ */ new Map();
|
|
18536
|
+
}
|
|
18537
|
+
setStatus(state, status) {
|
|
18538
|
+
return { ...state, status };
|
|
18539
|
+
}
|
|
18540
|
+
setCapturing(state, isCapturing) {
|
|
18541
|
+
return { ...state, isCapturing };
|
|
18542
|
+
}
|
|
18543
|
+
setPlaying(state, isPlaying) {
|
|
18544
|
+
return { ...state, isPlaying };
|
|
18545
|
+
}
|
|
18546
|
+
addUserTextMessage(state, text2) {
|
|
18547
|
+
return {
|
|
18548
|
+
...state,
|
|
18549
|
+
messages: [
|
|
18550
|
+
...state.messages,
|
|
18551
|
+
{
|
|
18552
|
+
id: `user-${Date.now()}`,
|
|
18553
|
+
role: "user",
|
|
18554
|
+
parts: [{ type: "text", text: text2, state: "done" }]
|
|
18555
|
+
}
|
|
18556
|
+
]
|
|
18557
|
+
};
|
|
18558
|
+
}
|
|
18559
|
+
addToolOutput(state, callId, result) {
|
|
18560
|
+
return {
|
|
18561
|
+
state: this.updateToolPartState(state, callId, result),
|
|
18562
|
+
output: {
|
|
18563
|
+
callId,
|
|
18564
|
+
name: this.toolCallIdToName.get(callId),
|
|
18565
|
+
output: JSON.stringify(result)
|
|
18566
|
+
}
|
|
18567
|
+
};
|
|
18568
|
+
}
|
|
18569
|
+
async reduceServerEvent(state, event) {
|
|
18570
|
+
var _a25;
|
|
18571
|
+
let nextState = this.pushEvent(state, event);
|
|
18572
|
+
const effects = [];
|
|
18573
|
+
switch (event.type) {
|
|
18574
|
+
case "session-created":
|
|
18575
|
+
case "session-updated": {
|
|
18576
|
+
if (nextState.status === "connecting") {
|
|
18577
|
+
nextState = this.setStatus(nextState, "connected");
|
|
18578
|
+
}
|
|
18579
|
+
break;
|
|
18580
|
+
}
|
|
18581
|
+
case "audio-delta": {
|
|
18582
|
+
effects.push({
|
|
18583
|
+
type: "play-audio",
|
|
18584
|
+
itemId: event.itemId,
|
|
18585
|
+
delta: event.delta
|
|
17327
18586
|
});
|
|
17328
18587
|
break;
|
|
17329
18588
|
}
|
|
@@ -17383,7 +18642,7 @@ var RealtimeEventReducer = class {
|
|
|
17383
18642
|
case "function-call-arguments-done": {
|
|
17384
18643
|
this.toolArgAccumulators.delete(event.callId);
|
|
17385
18644
|
this.toolCallIdToName.set(event.callId, event.name);
|
|
17386
|
-
const parseResult = await
|
|
18645
|
+
const parseResult = await safeParseJSON7({ text: event.arguments });
|
|
17387
18646
|
const parsedInput = parseResult.success ? parseResult.value : {};
|
|
17388
18647
|
const messageId = this.toolCallIdToMessageId.get(event.callId);
|
|
17389
18648
|
if (messageId != null) {
|
|
@@ -17623,98 +18882,375 @@ var RealtimeEventReducer = class {
|
|
|
17623
18882
|
// src/realtime/realtime-session.ts
|
|
17624
18883
|
var AbstractRealtimeSession = class {
|
|
17625
18884
|
constructor(options) {
|
|
18885
|
+
this.options = options;
|
|
17626
18886
|
this.state = createInitialRealtimeState();
|
|
18887
|
+
this.publication = 0;
|
|
18888
|
+
this.captureGeneration = 0;
|
|
18889
|
+
this.captureRequested = false;
|
|
17627
18890
|
this.currentResponseItemId = null;
|
|
17628
|
-
// Tool calls requested by the current (tool-bearing) response, the outputs
|
|
17629
|
-
// that have been submitted for them, and whether that response has finished
|
|
17630
|
-
// delivering its tool calls. Used to request a single response only once
|
|
17631
|
-
// every tool output for the turn has been submitted.
|
|
17632
18891
|
this.toolCallsInResponse = /* @__PURE__ */ new Set();
|
|
17633
18892
|
this.submittedToolOutputs = /* @__PURE__ */ new Set();
|
|
17634
18893
|
this.responseToolCallsClosed = false;
|
|
17635
|
-
var _a25, _b25, _c, _d, _e
|
|
17636
|
-
|
|
17637
|
-
this.
|
|
17638
|
-
this.
|
|
18894
|
+
var _a25, _b25, _c, _d, _e;
|
|
18895
|
+
const capabilities = options.model.capabilities;
|
|
18896
|
+
this.sessionLifecycle = (capabilities == null ? void 0 : capabilities.startup) === "session-start" || (capabilities == null ? void 0 : capabilities.finalization) === "session-close";
|
|
18897
|
+
this.continuous = (capabilities == null ? void 0 : capabilities.conversation) === "continuous";
|
|
17639
18898
|
this.maxEvents = (_a25 = options.maxEvents) != null ? _a25 : 500;
|
|
18899
|
+
if (!Number.isSafeInteger(this.maxEvents) || this.maxEvents < 1)
|
|
18900
|
+
throw new Error("maxEvents must be a positive integer");
|
|
18901
|
+
for (const timeout of [
|
|
18902
|
+
(_b25 = options.startupTimeoutMs) != null ? _b25 : 3e4,
|
|
18903
|
+
(_c = options.closeTimeoutMs) != null ? _c : 15e3,
|
|
18904
|
+
(_d = options.rtcDisconnectTimeoutMs) != null ? _d : 5e3
|
|
18905
|
+
]) {
|
|
18906
|
+
if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647)
|
|
18907
|
+
throw new Error(
|
|
18908
|
+
"Realtime timeouts must be positive finite timer durations"
|
|
18909
|
+
);
|
|
18910
|
+
}
|
|
18911
|
+
const budget = (_e = options.maxPlaybackBufferSeconds) != null ? _e : 2;
|
|
18912
|
+
if (!Number.isFinite(budget) || budget <= 0)
|
|
18913
|
+
throw new Error("maxPlaybackBufferSeconds must be positive and finite");
|
|
17640
18914
|
this.reducer = new RealtimeEventReducer(this.maxEvents);
|
|
17641
18915
|
this.onToolCall = options.onToolCall;
|
|
17642
18916
|
this.onEvent = options.onEvent;
|
|
17643
18917
|
this.onError = options.onError;
|
|
17644
|
-
const sampleRate = (_b25 = options.sampleRate) != null ? _b25 : 24e3;
|
|
17645
|
-
const captureSampleRate = (_e = (_d = (_c = options.sessionConfig) == null ? void 0 : _c.inputAudioFormat) == null ? void 0 : _d.rate) != null ? _e : sampleRate;
|
|
17646
|
-
const playbackSampleRate = (_h = (_g = (_f = options.sessionConfig) == null ? void 0 : _f.outputAudioFormat) == null ? void 0 : _g.rate) != null ? _h : sampleRate;
|
|
17647
|
-
this.transport = new BrowserRealtimeTransport({
|
|
17648
|
-
model: this.model,
|
|
17649
|
-
onServerEvent: (event) => this.handleServerEvent(event),
|
|
17650
|
-
onError: (error) => {
|
|
17651
|
-
var _a26;
|
|
17652
|
-
this.applyState(this.reducer.setStatus(this.state, "error"));
|
|
17653
|
-
(_a26 = this.onError) == null ? void 0 : _a26.call(this, error);
|
|
17654
|
-
},
|
|
17655
|
-
onClose: () => {
|
|
17656
|
-
this.applyState(this.reducer.setStatus(this.state, "disconnected"));
|
|
17657
|
-
}
|
|
17658
|
-
});
|
|
17659
|
-
this.audio = new BrowserRealtimeAudio({
|
|
17660
|
-
captureSampleRate,
|
|
17661
|
-
playbackSampleRate,
|
|
17662
|
-
onAudio: (audio) => this.sendAudio(audio),
|
|
17663
|
-
onCapturingChange: (isCapturing) => {
|
|
17664
|
-
this.applyState(this.reducer.setCapturing(this.state, isCapturing));
|
|
17665
|
-
},
|
|
17666
|
-
onPlayingChange: (isPlaying) => {
|
|
17667
|
-
this.applyState(this.reducer.setPlaying(this.state, isPlaying));
|
|
17668
|
-
}
|
|
17669
|
-
});
|
|
17670
18918
|
}
|
|
17671
|
-
|
|
17672
|
-
|
|
17673
|
-
|
|
17674
|
-
|
|
18919
|
+
validateConnection() {
|
|
18920
|
+
var _a25, _b25, _c;
|
|
18921
|
+
const { model, api } = this.options;
|
|
18922
|
+
if ((!this.continuous || api.session != null) && this.options.maxPlaybackBufferSeconds != null)
|
|
18923
|
+
throw new Error(
|
|
18924
|
+
"maxPlaybackBufferSeconds is supported only for continuous PCM sessions"
|
|
18925
|
+
);
|
|
18926
|
+
const connection = api.session != null ? "webrtc" : api.token != null ? "client-secret-websocket" : "server-websocket";
|
|
18927
|
+
if (!((_b25 = (_a25 = model.capabilities) == null ? void 0 : _a25.connections) != null ? _b25 : ["client-secret-websocket"]).includes(connection) || ((_c = model.capabilities) == null ? void 0 : _c.transports) != null && !model.capabilities.transports.includes(
|
|
18928
|
+
api.session != null ? "webrtc" : "websocket"
|
|
18929
|
+
))
|
|
18930
|
+
throw new Error(`Realtime model does not support ${connection}`);
|
|
18931
|
+
if (api.session != null) {
|
|
18932
|
+
if (model.getWebRTCConfig == null)
|
|
18933
|
+
throw new Error("Realtime model does not support WebRTC configuration");
|
|
18934
|
+
return;
|
|
18935
|
+
}
|
|
18936
|
+
if (this.continuous && api.websocket == null)
|
|
18937
|
+
throw new Error(
|
|
18938
|
+
"Continuous PCM sessions require an application WebSocket relay"
|
|
18939
|
+
);
|
|
18940
|
+
if (api.token != null && model.getWebSocketConfig == null)
|
|
18941
|
+
throw new Error(
|
|
18942
|
+
"Realtime model does not support client-secret WebSocket configuration"
|
|
18943
|
+
);
|
|
18944
|
+
}
|
|
18945
|
+
async connect(connectOptions) {
|
|
18946
|
+
var _a25, _b25, _c, _d;
|
|
18947
|
+
if ((_a25 = this.attempt) == null ? void 0 : _a25.active)
|
|
18948
|
+
throw new Error("Realtime session is already active");
|
|
18949
|
+
const attempt = new RealtimeAttempt();
|
|
18950
|
+
this.attempt = attempt;
|
|
18951
|
+
const current = () => this.attempt === attempt && attempt.active;
|
|
17675
18952
|
try {
|
|
17676
|
-
|
|
17677
|
-
|
|
17678
|
-
|
|
17679
|
-
|
|
18953
|
+
this.applyState({ ...this.state, status: "connecting" });
|
|
18954
|
+
if (!current()) return;
|
|
18955
|
+
this.validateConnection();
|
|
18956
|
+
const { model, api, sessionConfig } = this.options;
|
|
18957
|
+
if ((connectOptions == null ? void 0 : connectOptions.capture) === false) this.stopAudioCapture();
|
|
18958
|
+
if (!current()) return;
|
|
18959
|
+
if ((connectOptions == null ? void 0 : connectOptions.stream) != null && connectOptions.stream !== this.suppliedStream) {
|
|
18960
|
+
if (this.captureRequested && !this.sessionLifecycle && !this.continuous && api.session == null)
|
|
18961
|
+
this.startAudioCapture(connectOptions.stream);
|
|
18962
|
+
else this.suppliedStream = connectOptions.stream;
|
|
18963
|
+
}
|
|
18964
|
+
if (!current()) return;
|
|
18965
|
+
this.reducer = new RealtimeEventReducer(this.maxEvents);
|
|
18966
|
+
this.currentResponseItemId = null;
|
|
18967
|
+
this.toolCallsInResponse.clear();
|
|
18968
|
+
this.submittedToolOutputs.clear();
|
|
18969
|
+
this.responseToolCallsClosed = false;
|
|
18970
|
+
if (this.sessionLifecycle)
|
|
18971
|
+
this.applyState({ ...this.state, session: createSessionState() });
|
|
18972
|
+
if (!current()) return;
|
|
18973
|
+
this.commands = new RealtimeCommandTracker((event) => {
|
|
18974
|
+
const writable = () => current() && !attempt.transportClosing && attempt.cause == null && (!attempt.closing || event.type === "session-close");
|
|
18975
|
+
if (!writable()) throw new Error("Realtime connection is closed");
|
|
18976
|
+
return this.sendTransport(event, writable);
|
|
18977
|
+
});
|
|
18978
|
+
attempt.timer("startup", (_b25 = this.options.startupTimeoutMs) != null ? _b25 : 3e4, () => {
|
|
18979
|
+
if (current())
|
|
18980
|
+
this.fail(new Error("Realtime session startup timed out"));
|
|
17680
18981
|
});
|
|
17681
|
-
|
|
17682
|
-
|
|
17683
|
-
|
|
17684
|
-
|
|
17685
|
-
|
|
17686
|
-
|
|
17687
|
-
|
|
17688
|
-
|
|
18982
|
+
const callbacks = {
|
|
18983
|
+
model,
|
|
18984
|
+
onEvent: async (event) => {
|
|
18985
|
+
var _a26, _b26, _c2;
|
|
18986
|
+
if (!current()) return;
|
|
18987
|
+
try {
|
|
18988
|
+
await this.handleServerEvent(event, attempt);
|
|
18989
|
+
} catch (error) {
|
|
18990
|
+
if (current())
|
|
18991
|
+
this.fail(
|
|
18992
|
+
error,
|
|
18993
|
+
(_c2 = (_b26 = (_a26 = this.rtc) != null ? _a26 : this.pcm) != null ? _b26 : this.transport) == null ? void 0 : _c2.finish()
|
|
18994
|
+
);
|
|
18995
|
+
}
|
|
18996
|
+
},
|
|
18997
|
+
onError: (error) => {
|
|
18998
|
+
if (current()) void this.reportError(error, attempt);
|
|
18999
|
+
},
|
|
19000
|
+
onFatalError: (error, drain) => {
|
|
19001
|
+
if (current()) this.fail(error, drain);
|
|
19002
|
+
},
|
|
19003
|
+
onClosing: () => {
|
|
19004
|
+
if (!current()) return;
|
|
19005
|
+
attempt.beginClose();
|
|
19006
|
+
attempt.transportClosing = true;
|
|
19007
|
+
attempt.clearTimer("startup");
|
|
19008
|
+
attempt.clearTimer("close");
|
|
19009
|
+
this.stopAudioCapture();
|
|
19010
|
+
if (!current()) return;
|
|
19011
|
+
this.applyState({
|
|
19012
|
+
...this.state,
|
|
19013
|
+
status: attempt.cause == null ? "closing" : "error"
|
|
19014
|
+
});
|
|
19015
|
+
},
|
|
19016
|
+
onClose: (error) => {
|
|
19017
|
+
var _a26, _b26;
|
|
19018
|
+
if (!current()) return;
|
|
19019
|
+
const finalizationConfirmed = ((_a26 = model.capabilities) == null ? void 0 : _a26.finalization) === "session-close" && ((_b26 = this.state.session) == null ? void 0 : _b26.finalization) === "confirmed";
|
|
19020
|
+
if (error != null && !finalizationConfirmed) this.fail(error);
|
|
19021
|
+
else if (!attempt.ready && !finalizationConfirmed)
|
|
19022
|
+
this.fail(
|
|
19023
|
+
new Error("Realtime connection closed before becoming ready")
|
|
19024
|
+
);
|
|
19025
|
+
else this.disconnect();
|
|
19026
|
+
},
|
|
19027
|
+
onCapturing: (isCapturing) => {
|
|
19028
|
+
if (current()) this.applyState({ ...this.state, isCapturing });
|
|
19029
|
+
},
|
|
19030
|
+
onPlaying: (isPlaying) => {
|
|
19031
|
+
if (current()) this.applyState({ ...this.state, isPlaying });
|
|
19032
|
+
}
|
|
17689
19033
|
};
|
|
17690
|
-
|
|
17691
|
-
|
|
17692
|
-
|
|
17693
|
-
|
|
17694
|
-
|
|
17695
|
-
|
|
17696
|
-
|
|
17697
|
-
|
|
19034
|
+
if (api.session != null) {
|
|
19035
|
+
this.rtc = new BrowserRealtimeWebRTC({
|
|
19036
|
+
...callbacks,
|
|
19037
|
+
disconnectTimeoutMs: this.options.rtcDisconnectTimeoutMs
|
|
19038
|
+
});
|
|
19039
|
+
await this.rtc.connect({
|
|
19040
|
+
api: api.session,
|
|
19041
|
+
sessionConfig,
|
|
19042
|
+
stream: connectOptions == null ? void 0 : connectOptions.stream,
|
|
19043
|
+
capture: connectOptions == null ? void 0 : connectOptions.capture,
|
|
19044
|
+
timeoutMs: (_c = this.options.startupTimeoutMs) != null ? _c : 3e4
|
|
19045
|
+
});
|
|
19046
|
+
} else if (api.websocket != null && ((_d = model.capabilities) == null ? void 0 : _d.conversation) === "continuous") {
|
|
19047
|
+
this.pcm = new BrowserRealtimeLiveWebSocket({
|
|
19048
|
+
...callbacks,
|
|
19049
|
+
sessionConfig,
|
|
19050
|
+
sampleRate: this.options.sampleRate,
|
|
19051
|
+
maxPlaybackBufferSeconds: this.options.maxPlaybackBufferSeconds
|
|
19052
|
+
});
|
|
19053
|
+
this.pcm.connect({
|
|
19054
|
+
url: api.websocket,
|
|
19055
|
+
protocols: api.protocols,
|
|
19056
|
+
stream: connectOptions == null ? void 0 : connectOptions.stream,
|
|
19057
|
+
capture: connectOptions == null ? void 0 : connectOptions.capture
|
|
19058
|
+
});
|
|
19059
|
+
} else {
|
|
19060
|
+
this.ensureAudio();
|
|
19061
|
+
let config = sessionConfig != null ? sessionConfig : {};
|
|
19062
|
+
let token;
|
|
19063
|
+
let url = api.websocket;
|
|
19064
|
+
if (api.token != null) {
|
|
19065
|
+
const response = await fetch(api.token, {
|
|
19066
|
+
method: "POST",
|
|
19067
|
+
headers: { "Content-Type": "application/json" },
|
|
19068
|
+
body: JSON.stringify({ sessionConfig }),
|
|
19069
|
+
signal: attempt.abort.signal
|
|
17698
19070
|
});
|
|
19071
|
+
if (!current()) return;
|
|
19072
|
+
if (!response.ok)
|
|
19073
|
+
throw new Error(
|
|
19074
|
+
`Failed to fetch realtime setup: ${response.status}`
|
|
19075
|
+
);
|
|
19076
|
+
const payload = await response.json().catch(() => {
|
|
19077
|
+
throw new Error("Invalid realtime setup response");
|
|
19078
|
+
});
|
|
19079
|
+
if (!current() || attempt.closing) return;
|
|
19080
|
+
const setup = validateRealtimeSetup(payload);
|
|
19081
|
+
token = setup.token;
|
|
19082
|
+
url = setup.url;
|
|
19083
|
+
config = {
|
|
19084
|
+
...sessionConfig,
|
|
19085
|
+
...setup.tools == null ? {} : { tools: setup.tools }
|
|
19086
|
+
};
|
|
17699
19087
|
}
|
|
17700
|
-
|
|
19088
|
+
if (url == null) throw new Error("Realtime WebSocket URL is missing");
|
|
19089
|
+
if (api.token != null && token == null)
|
|
19090
|
+
throw new Error("Realtime client-secret connection requires a token");
|
|
19091
|
+
this.ensureAudio().ensurePlaybackContext();
|
|
19092
|
+
if (!current()) return;
|
|
19093
|
+
this.transport = new BrowserRealtimeTransport({
|
|
19094
|
+
...callbacks,
|
|
19095
|
+
onServerEvent: callbacks.onEvent
|
|
19096
|
+
});
|
|
19097
|
+
this.transport.connect({
|
|
19098
|
+
...token != null ? { mode: "client-secret", token } : { mode: "relay", protocols: api.protocols },
|
|
19099
|
+
url,
|
|
19100
|
+
onOpen: () => {
|
|
19101
|
+
var _a26, _b26;
|
|
19102
|
+
const writable = () => current() && !attempt.closing && attempt.cause == null;
|
|
19103
|
+
if (!writable()) return;
|
|
19104
|
+
return this.sendTransport(
|
|
19105
|
+
{
|
|
19106
|
+
type: (_b26 = (_a26 = model.capabilities) == null ? void 0 : _a26.startup) != null ? _b26 : "session-update",
|
|
19107
|
+
config
|
|
19108
|
+
},
|
|
19109
|
+
writable
|
|
19110
|
+
);
|
|
19111
|
+
}
|
|
19112
|
+
});
|
|
19113
|
+
}
|
|
17701
19114
|
} catch (error) {
|
|
17702
|
-
|
|
17703
|
-
|
|
17704
|
-
|
|
17705
|
-
|
|
17706
|
-
|
|
19115
|
+
if (current()) this.fail(error);
|
|
19116
|
+
}
|
|
19117
|
+
}
|
|
19118
|
+
fail(error, drain) {
|
|
19119
|
+
const attempt = this.attempt;
|
|
19120
|
+
if (attempt == null || !attempt.active || attempt.cause != null) return;
|
|
19121
|
+
attempt.cause = error instanceof Error ? error : new Error(String(error));
|
|
19122
|
+
if (drain != null) attempt.beginClose();
|
|
19123
|
+
this.applyState({ ...this.state, status: "error" });
|
|
19124
|
+
if (this.attempt !== attempt || !attempt.active) return;
|
|
19125
|
+
this.stopAudioCapture();
|
|
19126
|
+
if (this.attempt !== attempt || !attempt.active) return;
|
|
19127
|
+
if (drain == null) this.disconnect();
|
|
19128
|
+
else void this.drainAttempt(attempt, drain);
|
|
19129
|
+
void this.reportError(attempt.cause, attempt);
|
|
19130
|
+
}
|
|
19131
|
+
async drainAttempt(attempt, drain) {
|
|
19132
|
+
if (this.attempt === attempt && attempt.active)
|
|
19133
|
+
attempt.timer("drain", 1e3, () => this.finishAttempt(attempt));
|
|
19134
|
+
try {
|
|
19135
|
+
await drain;
|
|
19136
|
+
} catch (e) {
|
|
19137
|
+
}
|
|
19138
|
+
this.finishAttempt(attempt);
|
|
19139
|
+
}
|
|
19140
|
+
finishAttempt(attempt) {
|
|
19141
|
+
if (this.attempt !== attempt || !attempt.active) return;
|
|
19142
|
+
try {
|
|
19143
|
+
this.disconnect();
|
|
19144
|
+
} catch (error) {
|
|
19145
|
+
void this.reportError(error, attempt);
|
|
17707
19146
|
}
|
|
17708
19147
|
}
|
|
19148
|
+
closeFailed(attempt, error) {
|
|
19149
|
+
var _a25, _b25;
|
|
19150
|
+
if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
|
|
19151
|
+
return;
|
|
19152
|
+
attempt.clearTimer("close");
|
|
19153
|
+
const transport = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.transport;
|
|
19154
|
+
void this.drainAttempt(attempt, transport == null ? void 0 : transport.finish());
|
|
19155
|
+
void this.reportError(error, attempt);
|
|
19156
|
+
}
|
|
17709
19157
|
disconnect() {
|
|
17710
|
-
this.
|
|
17711
|
-
this.
|
|
19158
|
+
const attempt = this.attempt;
|
|
19159
|
+
this.captureGeneration++;
|
|
19160
|
+
this.captureRequested = false;
|
|
19161
|
+
this.suppliedStream = void 0;
|
|
19162
|
+
const transport = this.transport;
|
|
19163
|
+
const audio = this.audio;
|
|
19164
|
+
const pcm = this.pcm;
|
|
19165
|
+
const rtc = this.rtc;
|
|
19166
|
+
this.transport = void 0;
|
|
19167
|
+
this.audio = void 0;
|
|
19168
|
+
this.pcm = void 0;
|
|
19169
|
+
this.rtc = void 0;
|
|
19170
|
+
this.commands = void 0;
|
|
19171
|
+
attempt == null ? void 0 : attempt.retire();
|
|
19172
|
+
transport == null ? void 0 : transport.dispose();
|
|
19173
|
+
audio == null ? void 0 : audio.dispose();
|
|
19174
|
+
pcm == null ? void 0 : pcm.dispose();
|
|
19175
|
+
rtc == null ? void 0 : rtc.dispose();
|
|
19176
|
+
if (this.attempt !== attempt) return;
|
|
19177
|
+
const session = this.state.session;
|
|
19178
|
+
this.applyState({
|
|
19179
|
+
...this.state,
|
|
19180
|
+
status: (attempt == null ? void 0 : attempt.cause) != null ? "error" : "disconnected",
|
|
19181
|
+
isCapturing: false,
|
|
19182
|
+
isPlaying: false,
|
|
19183
|
+
...session != null ? {
|
|
19184
|
+
session: session.finalization === "confirmed" ? session : { ...session, finalization: "unconfirmed" }
|
|
19185
|
+
} : {}
|
|
19186
|
+
});
|
|
19187
|
+
}
|
|
19188
|
+
/** Wait for final usage when the model supports a session-close acknowledgement. */
|
|
19189
|
+
close(options) {
|
|
19190
|
+
var _a25, _b25, _c, _d;
|
|
19191
|
+
const attempt = this.attempt;
|
|
19192
|
+
if ((attempt == null ? void 0 : attempt.closePromise) != null) return attempt.closePromise;
|
|
19193
|
+
if (this.state.status !== "connected" || ((_a25 = this.options.model.capabilities) == null ? void 0 : _a25.finalization) !== "session-close" || attempt == null) {
|
|
19194
|
+
this.disconnect();
|
|
19195
|
+
return Promise.resolve();
|
|
19196
|
+
}
|
|
19197
|
+
if ((options == null ? void 0 : options.eventId) != null) (_b25 = this.commands) == null ? void 0 : _b25.validateId(options.eventId);
|
|
19198
|
+
const promise = attempt.beginClose();
|
|
19199
|
+
this.applyState({ ...this.state, status: "closing" });
|
|
19200
|
+
if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
|
|
19201
|
+
return promise;
|
|
19202
|
+
this.stopAudioCapture();
|
|
19203
|
+
if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
|
|
19204
|
+
return promise;
|
|
19205
|
+
attempt.timer(
|
|
19206
|
+
"close",
|
|
19207
|
+
(_c = this.options.closeTimeoutMs) != null ? _c : 15e3,
|
|
19208
|
+
() => this.finishAttempt(attempt)
|
|
19209
|
+
);
|
|
19210
|
+
const failed = (error) => this.closeFailed(attempt, error);
|
|
19211
|
+
try {
|
|
19212
|
+
void ((_d = this.commands) == null ? void 0 : _d.send({ type: "session-close", eventId: options == null ? void 0 : options.eventId }).catch(failed));
|
|
19213
|
+
} catch (error) {
|
|
19214
|
+
failed(error);
|
|
19215
|
+
}
|
|
19216
|
+
return promise;
|
|
19217
|
+
}
|
|
19218
|
+
sendTransport(event, guard, automaticAudio = false) {
|
|
19219
|
+
var _a25, _b25;
|
|
19220
|
+
const transport = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.transport;
|
|
19221
|
+
if (transport == null) throw new Error("Realtime connection is not open");
|
|
19222
|
+
if (transport === this.transport)
|
|
19223
|
+
return this.transport.sendEvent(event, guard, automaticAudio);
|
|
19224
|
+
return transport.sendEvent(event, guard);
|
|
17712
19225
|
}
|
|
17713
|
-
// ── Sending events ─────────────────────────────────────────────────
|
|
17714
19226
|
sendEvent(event) {
|
|
17715
|
-
|
|
19227
|
+
var _a25, _b25, _c, _d;
|
|
19228
|
+
if (this.state.status === "error" || this.state.status === "closing" || ((_a25 = this.attempt) == null ? void 0 : _a25.closing) || !((_b25 = this.attempt) == null ? void 0 : _b25.active))
|
|
19229
|
+
throw new Error("Realtime session is not accepting submissions");
|
|
19230
|
+
if (this.sessionLifecycle && this.state.status !== "connected")
|
|
19231
|
+
throw new Error("Realtime session is not accepting submissions");
|
|
19232
|
+
if (this.rtc != null && event.type === "input-audio-append" || (this.continuous || this.rtc != null) && (event.type === "input-audio-commit" || event.type === "input-audio-clear"))
|
|
19233
|
+
throw new UnsupportedFunctionalityError6({
|
|
19234
|
+
functionality: "JSON audio commands unsupported for this session transport"
|
|
19235
|
+
});
|
|
19236
|
+
if (event.type === "session-close")
|
|
19237
|
+
return this.close({ eventId: event.eventId });
|
|
19238
|
+
if (event.type === "session-start" && this.sessionLifecycle)
|
|
19239
|
+
throw new Error("Realtime session has already started");
|
|
19240
|
+
if (!this.sessionLifecycle && this.state.session == null) {
|
|
19241
|
+
const attempt = this.attempt;
|
|
19242
|
+
return this.sendTransport(
|
|
19243
|
+
event,
|
|
19244
|
+
() => this.attempt === attempt && attempt.active && !attempt.closing && attempt.cause == null
|
|
19245
|
+
);
|
|
19246
|
+
}
|
|
19247
|
+
return (_d = (_c = this.commands) == null ? void 0 : _c.send(event)) != null ? _d : this.sendTransport(event);
|
|
17716
19248
|
}
|
|
17717
19249
|
sendTextMessage(text2) {
|
|
19250
|
+
if (this.continuous)
|
|
19251
|
+
throw new UnsupportedFunctionalityError6({
|
|
19252
|
+
functionality: "sendTextMessage for continuous sessions; client-delegation text is application-owned, use context-append for context"
|
|
19253
|
+
});
|
|
17718
19254
|
this.sendEvent({
|
|
17719
19255
|
type: "conversation-item-create",
|
|
17720
19256
|
item: { type: "text-message", role: "user", text: text2 }
|
|
@@ -17722,8 +19258,12 @@ var AbstractRealtimeSession = class {
|
|
|
17722
19258
|
this.sendEvent({ type: "response-create" });
|
|
17723
19259
|
this.applyState(this.reducer.addUserTextMessage(this.state, text2));
|
|
17724
19260
|
}
|
|
17725
|
-
sendAudio(
|
|
17726
|
-
|
|
19261
|
+
sendAudio(audio) {
|
|
19262
|
+
const attempt = this.attempt;
|
|
19263
|
+
void this.sendEvent({ type: "input-audio-append", audio }).catch((error) => {
|
|
19264
|
+
if ((attempt == null ? void 0 : attempt.active) && this.attempt === attempt)
|
|
19265
|
+
void this.reportError(error, attempt);
|
|
19266
|
+
});
|
|
17727
19267
|
}
|
|
17728
19268
|
commitAudio() {
|
|
17729
19269
|
this.sendEvent({ type: "input-audio-commit" });
|
|
@@ -17740,161 +19280,301 @@ var AbstractRealtimeSession = class {
|
|
|
17740
19280
|
cancelResponse() {
|
|
17741
19281
|
this.sendEvent({ type: "response-cancel" });
|
|
17742
19282
|
}
|
|
17743
|
-
|
|
19283
|
+
sendAutomaticAudio(audio) {
|
|
19284
|
+
var _a25;
|
|
19285
|
+
if (this.state.status !== "connected" || !((_a25 = this.attempt) == null ? void 0 : _a25.active) || this.attempt.closing)
|
|
19286
|
+
return;
|
|
19287
|
+
const attempt = this.attempt;
|
|
19288
|
+
const failed = (error) => {
|
|
19289
|
+
if ((attempt == null ? void 0 : attempt.active) && this.attempt === attempt && !attempt.closing)
|
|
19290
|
+
this.fail(error);
|
|
19291
|
+
};
|
|
19292
|
+
try {
|
|
19293
|
+
void this.sendTransport(
|
|
19294
|
+
{ type: "input-audio-append", audio },
|
|
19295
|
+
() => this.attempt === attempt && attempt.active && !attempt.closing,
|
|
19296
|
+
true
|
|
19297
|
+
).catch(failed);
|
|
19298
|
+
} catch (error) {
|
|
19299
|
+
failed(error);
|
|
19300
|
+
}
|
|
19301
|
+
}
|
|
17744
19302
|
addToolOutput(callId, result) {
|
|
19303
|
+
if (this.continuous)
|
|
19304
|
+
throw new UnsupportedFunctionalityError6({
|
|
19305
|
+
functionality: "addToolOutput for continuous sessions; client delegation is application-owned"
|
|
19306
|
+
});
|
|
19307
|
+
const attempt = this.attempt;
|
|
19308
|
+
if (!(attempt == null ? void 0 : attempt.active) || attempt.closing || attempt.cause != null)
|
|
19309
|
+
throw new Error("Realtime session is not accepting submissions");
|
|
17745
19310
|
const { state, output } = this.reducer.addToolOutput(
|
|
17746
19311
|
this.state,
|
|
17747
19312
|
callId,
|
|
17748
19313
|
result
|
|
17749
19314
|
);
|
|
17750
19315
|
this.applyState(state);
|
|
19316
|
+
if (this.attempt !== attempt || !(attempt == null ? void 0 : attempt.active)) return;
|
|
17751
19317
|
this.sendEvent({
|
|
17752
19318
|
type: "conversation-item-create",
|
|
17753
|
-
item: {
|
|
17754
|
-
type: "function-call-output",
|
|
17755
|
-
callId: output.callId,
|
|
17756
|
-
name: output.name,
|
|
17757
|
-
output: output.output
|
|
17758
|
-
}
|
|
19319
|
+
item: { type: "function-call-output", ...output }
|
|
17759
19320
|
});
|
|
17760
19321
|
this.submittedToolOutputs.add(callId);
|
|
17761
19322
|
this.maybeRequestToolResponse();
|
|
17762
19323
|
}
|
|
17763
|
-
/**
|
|
17764
|
-
* Requests a single response once the tool-bearing response has finished
|
|
17765
|
-
* delivering its tool calls and every one of them has an output. Requesting a
|
|
17766
|
-
* response after each individual output can cause the model to continue
|
|
17767
|
-
* without the full tool context on multi-tool turns.
|
|
17768
|
-
*/
|
|
17769
19324
|
maybeRequestToolResponse() {
|
|
17770
|
-
|
|
17771
|
-
if (this.toolCallsInResponse.size === 0
|
|
17772
|
-
|
|
17773
|
-
|
|
17774
|
-
|
|
19325
|
+
var _a25;
|
|
19326
|
+
if (this.state.status !== "connected" || ((_a25 = this.attempt) == null ? void 0 : _a25.closing) || !this.responseToolCallsClosed || this.toolCallsInResponse.size === 0 || [...this.toolCallsInResponse].some(
|
|
19327
|
+
(id) => !this.submittedToolOutputs.has(id)
|
|
19328
|
+
))
|
|
19329
|
+
return;
|
|
17775
19330
|
this.sendEvent({ type: "response-create" });
|
|
17776
19331
|
this.toolCallsInResponse.clear();
|
|
17777
19332
|
this.submittedToolOutputs.clear();
|
|
17778
19333
|
this.responseToolCallsClosed = false;
|
|
17779
19334
|
}
|
|
17780
|
-
// ── Audio capture ──────────────────────────────────────────────────
|
|
17781
19335
|
startAudioCapture(stream) {
|
|
17782
|
-
|
|
19336
|
+
var _a25;
|
|
19337
|
+
const legacy = !this.sessionLifecycle && !this.continuous && this.options.api.session == null;
|
|
19338
|
+
if (!legacy && this.state.status !== "connected")
|
|
19339
|
+
throw new Error("Realtime session is not accepting capture");
|
|
19340
|
+
if (((_a25 = this.attempt) == null ? void 0 : _a25.active) && (this.attempt.closing || this.attempt.cause != null))
|
|
19341
|
+
throw new Error("Realtime session is not accepting capture");
|
|
19342
|
+
this.suppliedStream = stream;
|
|
19343
|
+
if (legacy) {
|
|
19344
|
+
this.captureGeneration++;
|
|
19345
|
+
this.captureRequested = true;
|
|
19346
|
+
this.ensureAudio().startCapture(stream);
|
|
19347
|
+
return;
|
|
19348
|
+
}
|
|
19349
|
+
const attempt = this.attempt;
|
|
19350
|
+
void this.resumeAudioCapture().catch((error) => {
|
|
19351
|
+
if (attempt == null ? void 0 : attempt.active) void this.reportError(error, attempt);
|
|
19352
|
+
});
|
|
19353
|
+
}
|
|
19354
|
+
async resumeAudioCapture() {
|
|
19355
|
+
const accepting = () => {
|
|
19356
|
+
var _a25, _b25;
|
|
19357
|
+
return !((_a25 = this.attempt) == null ? void 0 : _a25.closing) && ((_b25 = this.attempt) == null ? void 0 : _b25.cause) == null && (this.state.status === "connected" || !this.sessionLifecycle && this.state.status === "connecting");
|
|
19358
|
+
};
|
|
19359
|
+
if (!accepting())
|
|
19360
|
+
throw new Error("Realtime session is not accepting capture");
|
|
19361
|
+
this.captureRequested = true;
|
|
19362
|
+
if (this.rtc != null) return this.rtc.startCapture(this.suppliedStream);
|
|
19363
|
+
if (this.pcm != null) return this.pcm.resumeCapture(this.suppliedStream);
|
|
19364
|
+
const audio = this.audio;
|
|
19365
|
+
if (audio == null)
|
|
19366
|
+
throw new Error("Realtime capture transport is not ready");
|
|
19367
|
+
const captureGeneration = ++this.captureGeneration;
|
|
19368
|
+
const attempt = this.attempt;
|
|
19369
|
+
const supplied = this.suppliedStream;
|
|
19370
|
+
const stream = supplied != null ? supplied : await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
19371
|
+
if (!(attempt == null ? void 0 : attempt.active) || this.attempt !== attempt || this.audio !== audio || !accepting() || captureGeneration !== this.captureGeneration) {
|
|
19372
|
+
if (supplied == null) stream.getTracks().forEach((track) => track.stop());
|
|
19373
|
+
return;
|
|
19374
|
+
}
|
|
19375
|
+
audio.startCapture(stream, {
|
|
19376
|
+
ownsStream: supplied == null || !this.sessionLifecycle || this.options.api.token != null
|
|
19377
|
+
});
|
|
17783
19378
|
}
|
|
17784
19379
|
stopAudioCapture() {
|
|
17785
|
-
this.
|
|
19380
|
+
this.captureGeneration++;
|
|
19381
|
+
this.captureRequested = false;
|
|
19382
|
+
if (this.audio != null && (!this.sessionLifecycle || this.options.api.token != null))
|
|
19383
|
+
this.suppliedStream = void 0;
|
|
19384
|
+
const pcm = this.pcm;
|
|
19385
|
+
const audio = this.audio;
|
|
19386
|
+
const rtc = this.rtc;
|
|
19387
|
+
const attempt = this.attempt;
|
|
19388
|
+
if (rtc != null)
|
|
19389
|
+
void rtc.stopCapture().catch((error) => {
|
|
19390
|
+
if ((attempt == null ? void 0 : attempt.active) && this.attempt === attempt)
|
|
19391
|
+
void this.reportError(error, attempt);
|
|
19392
|
+
});
|
|
19393
|
+
pcm == null ? void 0 : pcm.stopCapture();
|
|
19394
|
+
audio == null ? void 0 : audio.stopCapture();
|
|
17786
19395
|
}
|
|
17787
|
-
// ── Playback ───────────────────────────────────────────────────────
|
|
17788
19396
|
stopPlayback() {
|
|
17789
|
-
|
|
19397
|
+
var _a25, _b25, _c;
|
|
19398
|
+
(_c = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.audio) == null ? void 0 : _c.stopPlayback();
|
|
19399
|
+
}
|
|
19400
|
+
async resumePlayback() {
|
|
19401
|
+
var _a25, _b25, _c;
|
|
19402
|
+
await ((_c = (_b25 = (_a25 = this.rtc) != null ? _a25 : this.pcm) != null ? _b25 : this.audio) == null ? void 0 : _c.resumePlayback());
|
|
17790
19403
|
}
|
|
17791
|
-
// ── Cleanup ────────────────────────────────────────────────────────
|
|
17792
19404
|
dispose() {
|
|
17793
|
-
this.
|
|
17794
|
-
|
|
17795
|
-
|
|
17796
|
-
|
|
17797
|
-
|
|
17798
|
-
|
|
17799
|
-
|
|
17800
|
-
|
|
17801
|
-
|
|
17802
|
-
)
|
|
17803
|
-
|
|
19405
|
+
this.disconnect();
|
|
19406
|
+
}
|
|
19407
|
+
ensureAudio() {
|
|
19408
|
+
var _a25, _b25, _c, _d, _e, _f;
|
|
19409
|
+
if (this.audio != null) return this.audio;
|
|
19410
|
+
const { sessionConfig, sampleRate } = this.options;
|
|
19411
|
+
const audio = new BrowserRealtimeAudio({
|
|
19412
|
+
captureSampleRate: (_c = (_b25 = (_a25 = sessionConfig == null ? void 0 : sessionConfig.inputAudioFormat) == null ? void 0 : _a25.rate) != null ? _b25 : sampleRate) != null ? _c : 24e3,
|
|
19413
|
+
playbackSampleRate: (_f = (_e = (_d = sessionConfig == null ? void 0 : sessionConfig.outputAudioFormat) == null ? void 0 : _d.rate) != null ? _e : sampleRate) != null ? _f : 24e3,
|
|
19414
|
+
onAudio: (value) => {
|
|
19415
|
+
if (this.audio === audio) this.sendAutomaticAudio(value);
|
|
19416
|
+
},
|
|
19417
|
+
onError: (error) => {
|
|
19418
|
+
if (this.audio !== audio) return;
|
|
19419
|
+
void this.reportError(error, this.attempt);
|
|
19420
|
+
},
|
|
19421
|
+
onCapturingChange: (isCapturing) => {
|
|
19422
|
+
if (this.audio === audio)
|
|
19423
|
+
this.applyState({ ...this.state, isCapturing });
|
|
19424
|
+
},
|
|
19425
|
+
onPlayingChange: (isPlaying) => {
|
|
19426
|
+
if (this.audio === audio) this.applyState({ ...this.state, isPlaying });
|
|
19427
|
+
}
|
|
19428
|
+
});
|
|
19429
|
+
this.audio = audio;
|
|
19430
|
+
return audio;
|
|
17804
19431
|
}
|
|
17805
|
-
// ── Private helpers ────────────────────────────────────────────────
|
|
17806
19432
|
applyState(nextState) {
|
|
17807
|
-
const
|
|
19433
|
+
const publication = ++this.publication;
|
|
19434
|
+
const previous = this.state;
|
|
17808
19435
|
this.state = nextState;
|
|
17809
|
-
|
|
17810
|
-
this.
|
|
17811
|
-
|
|
17812
|
-
|
|
17813
|
-
|
|
17814
|
-
|
|
17815
|
-
|
|
17816
|
-
|
|
17817
|
-
|
|
17818
|
-
|
|
17819
|
-
|
|
17820
|
-
|
|
17821
|
-
if (
|
|
17822
|
-
this.setState("isPlaying", nextState.isPlaying);
|
|
17823
|
-
}
|
|
17824
|
-
}
|
|
17825
|
-
async executeToolCall({
|
|
17826
|
-
name: name25,
|
|
17827
|
-
args,
|
|
17828
|
-
callId
|
|
17829
|
-
}) {
|
|
17830
|
-
var _a25, _b25;
|
|
17831
|
-
if (this.onToolCall == null) {
|
|
17832
|
-
(_a25 = this.onError) == null ? void 0 : _a25.call(this, new Error(`No handler provided for tool "${name25}"`));
|
|
19436
|
+
const update = (key) => {
|
|
19437
|
+
if (publication === this.publication && previous[key] !== nextState[key])
|
|
19438
|
+
this.setState(key, nextState[key]);
|
|
19439
|
+
};
|
|
19440
|
+
update("status");
|
|
19441
|
+
update("messages");
|
|
19442
|
+
update("events");
|
|
19443
|
+
update("isCapturing");
|
|
19444
|
+
update("isPlaying");
|
|
19445
|
+
update("session");
|
|
19446
|
+
}
|
|
19447
|
+
async executeTool(callId, name25, args, attempt) {
|
|
19448
|
+
if (this.continuous || this.attempt !== attempt || !attempt.active || attempt.closing || attempt.cause != null)
|
|
17833
19449
|
return;
|
|
17834
|
-
}
|
|
17835
19450
|
try {
|
|
19451
|
+
if (this.onToolCall == null) {
|
|
19452
|
+
void this.reportError(
|
|
19453
|
+
new Error(`No handler provided for tool "${name25}"`),
|
|
19454
|
+
attempt
|
|
19455
|
+
);
|
|
19456
|
+
return;
|
|
19457
|
+
}
|
|
17836
19458
|
const result = await this.onToolCall({
|
|
17837
19459
|
toolCall: { toolCallId: callId, toolName: name25, args }
|
|
17838
19460
|
});
|
|
17839
|
-
if (result !== void 0)
|
|
19461
|
+
if (result !== void 0 && this.attempt === attempt && (attempt == null ? void 0 : attempt.active) && this.state.status === "connected")
|
|
17840
19462
|
this.addToolOutput(callId, result);
|
|
17841
|
-
}
|
|
17842
19463
|
} catch (error) {
|
|
17843
|
-
(
|
|
17844
|
-
this,
|
|
17845
|
-
error instanceof Error ? error : new Error(`Client tool execution failed: ${String(error)}`)
|
|
17846
|
-
);
|
|
19464
|
+
if (attempt.active) void this.reportError(error, attempt);
|
|
17847
19465
|
}
|
|
17848
19466
|
}
|
|
17849
|
-
async handleServerEvent(event) {
|
|
17850
|
-
var _a25;
|
|
19467
|
+
async handleServerEvent(event, attempt) {
|
|
19468
|
+
var _a25, _b25, _c, _d, _e, _f;
|
|
19469
|
+
const current = () => this.attempt === attempt && attempt.active;
|
|
19470
|
+
if (!current()) return;
|
|
19471
|
+
if (event.type === "session-started" && event.delegationMode === "provider")
|
|
19472
|
+
throw new UnsupportedFunctionalityError6({
|
|
19473
|
+
functionality: "Provider delegation mode; this realtime runtime supports client delegation only"
|
|
19474
|
+
});
|
|
19475
|
+
if (this.continuous && event.type === "audio-delta")
|
|
19476
|
+
throw new UnsupportedFunctionalityError6({
|
|
19477
|
+
functionality: "Turn-based audio-delta in a continuous PCM session; use audio-chunk"
|
|
19478
|
+
});
|
|
19479
|
+
const command = (_a25 = this.commands) == null ? void 0 : _a25.receive(event);
|
|
17851
19480
|
const result = await this.reducer.reduceServerEvent(this.state, event);
|
|
17852
|
-
|
|
17853
|
-
(
|
|
19481
|
+
if (!current()) return;
|
|
19482
|
+
const session = (_b25 = this.state.session) != null ? _b25 : event.type === "session-started" ? createSessionState() : void 0;
|
|
19483
|
+
this.applyState({
|
|
19484
|
+
...result.state,
|
|
19485
|
+
status: this.state.status,
|
|
19486
|
+
...session == null ? {} : {
|
|
19487
|
+
session: reduceSessionState(
|
|
19488
|
+
session,
|
|
19489
|
+
event,
|
|
19490
|
+
this.maxEvents,
|
|
19491
|
+
command == null ? void 0 : command.muted
|
|
19492
|
+
)
|
|
19493
|
+
}
|
|
19494
|
+
});
|
|
19495
|
+
if (!current()) return;
|
|
17854
19496
|
for (const effect of result.effects) {
|
|
17855
|
-
this.handleReducerEffect(effect);
|
|
19497
|
+
this.handleReducerEffect(effect, attempt);
|
|
19498
|
+
if (!current()) return;
|
|
17856
19499
|
}
|
|
19500
|
+
if (event.type === "audio-chunk")
|
|
19501
|
+
(_d = (_c = this.pcm) != null ? _c : this.audio) == null ? void 0 : _d.playAudio(event.delta);
|
|
19502
|
+
if (!current()) return;
|
|
17857
19503
|
if (event.type === "response-done" && this.toolCallsInResponse.size > 0) {
|
|
17858
19504
|
this.responseToolCallsClosed = true;
|
|
17859
19505
|
this.maybeRequestToolResponse();
|
|
17860
19506
|
}
|
|
19507
|
+
if (!current()) return;
|
|
19508
|
+
if (event.type === "session-closed" && session != null) {
|
|
19509
|
+
this.disconnect();
|
|
19510
|
+
this.notifyEvent(event, attempt, true);
|
|
19511
|
+
return;
|
|
19512
|
+
}
|
|
19513
|
+
if ((event.type === "session-started" || event.type === "session-created" || event.type === "session-updated") && attempt.active && !attempt.closing && attempt.cause == null) {
|
|
19514
|
+
const ready = ((_e = this.options.model.capabilities) == null ? void 0 : _e.startup) === "session-start" ? event.type === "session-started" : event.type !== "session-started";
|
|
19515
|
+
if (ready) {
|
|
19516
|
+
attempt.ready = true;
|
|
19517
|
+
attempt.clearTimer("startup");
|
|
19518
|
+
this.applyState({ ...this.state, status: "connected" });
|
|
19519
|
+
if (!current() || attempt.closing) return;
|
|
19520
|
+
(_f = this.pcm) == null ? void 0 : _f.startCapture();
|
|
19521
|
+
}
|
|
19522
|
+
}
|
|
19523
|
+
if (current()) this.notifyEvent(event, attempt);
|
|
17861
19524
|
}
|
|
17862
|
-
handleReducerEffect(effect) {
|
|
17863
|
-
var _a25;
|
|
19525
|
+
handleReducerEffect(effect, attempt) {
|
|
19526
|
+
var _a25, _b25, _c, _d;
|
|
17864
19527
|
switch (effect.type) {
|
|
17865
|
-
case "play-audio":
|
|
19528
|
+
case "play-audio":
|
|
17866
19529
|
this.currentResponseItemId = effect.itemId;
|
|
17867
|
-
this.audio.playAudio(effect.delta);
|
|
19530
|
+
(_a25 = this.audio) == null ? void 0 : _a25.playAudio(effect.delta);
|
|
17868
19531
|
break;
|
|
17869
|
-
|
|
17870
|
-
|
|
17871
|
-
|
|
17872
|
-
const
|
|
17873
|
-
this.audio.stopPlayback();
|
|
17874
|
-
if (this.
|
|
19532
|
+
case "speech-started":
|
|
19533
|
+
if (!this.continuous && this.state.isPlaying) {
|
|
19534
|
+
const playedMs = (_c = (_b25 = this.audio) == null ? void 0 : _b25.getPlaybackOffsetMs()) != null ? _c : 0;
|
|
19535
|
+
const itemId = this.currentResponseItemId;
|
|
19536
|
+
(_d = this.audio) == null ? void 0 : _d.stopPlayback();
|
|
19537
|
+
if (this.attempt !== attempt || !attempt.active) return;
|
|
19538
|
+
if (itemId != null && !attempt.closing && attempt.cause == null)
|
|
17875
19539
|
this.sendEvent({
|
|
17876
19540
|
type: "conversation-item-truncate",
|
|
17877
|
-
itemId
|
|
19541
|
+
itemId,
|
|
17878
19542
|
contentIndex: 0,
|
|
17879
19543
|
audioEndMs: Math.round(playedMs)
|
|
17880
19544
|
});
|
|
17881
|
-
}
|
|
17882
19545
|
}
|
|
17883
19546
|
break;
|
|
17884
|
-
|
|
17885
|
-
|
|
19547
|
+
case "tool-call":
|
|
19548
|
+
if (this.continuous) break;
|
|
17886
19549
|
this.toolCallsInResponse.add(effect.callId);
|
|
17887
|
-
void this.
|
|
17888
|
-
name: effect.name,
|
|
17889
|
-
args: effect.args,
|
|
17890
|
-
callId: effect.callId
|
|
17891
|
-
});
|
|
19550
|
+
void this.executeTool(effect.callId, effect.name, effect.args, attempt);
|
|
17892
19551
|
break;
|
|
17893
|
-
|
|
17894
|
-
|
|
17895
|
-
(_a25 = this.onError) == null ? void 0 : _a25.call(this, effect.error);
|
|
19552
|
+
case "error":
|
|
19553
|
+
void this.reportError(effect.error, attempt);
|
|
17896
19554
|
break;
|
|
17897
|
-
|
|
19555
|
+
}
|
|
19556
|
+
}
|
|
19557
|
+
async reportError(error, attempt) {
|
|
19558
|
+
var _a25;
|
|
19559
|
+
if (this.attempt !== attempt) return;
|
|
19560
|
+
try {
|
|
19561
|
+
await ((_a25 = this.onError) == null ? void 0 : _a25.call(
|
|
19562
|
+
this,
|
|
19563
|
+
error instanceof Error ? error : new Error(String(error))
|
|
19564
|
+
));
|
|
19565
|
+
} catch (e) {
|
|
19566
|
+
}
|
|
19567
|
+
}
|
|
19568
|
+
notifyEvent(event, attempt, terminal = false) {
|
|
19569
|
+
var _a25;
|
|
19570
|
+
if (this.attempt !== attempt || !attempt.active && !terminal) return;
|
|
19571
|
+
const report = (error) => {
|
|
19572
|
+
if (attempt.active || terminal) void this.reportError(error, attempt);
|
|
19573
|
+
};
|
|
19574
|
+
try {
|
|
19575
|
+
void Promise.resolve((_a25 = this.onEvent) == null ? void 0 : _a25.call(this, event)).catch(report);
|
|
19576
|
+
} catch (error) {
|
|
19577
|
+
report(error);
|
|
17898
19578
|
}
|
|
17899
19579
|
}
|
|
17900
19580
|
};
|
|
@@ -18551,7 +20231,7 @@ var DefaultTranscriptionResult = class {
|
|
|
18551
20231
|
|
|
18552
20232
|
// src/transcribe/stream-transcribe.ts
|
|
18553
20233
|
import {
|
|
18554
|
-
UnsupportedFunctionalityError as
|
|
20234
|
+
UnsupportedFunctionalityError as UnsupportedFunctionalityError7
|
|
18555
20235
|
} from "@ai-sdk/provider";
|
|
18556
20236
|
import {
|
|
18557
20237
|
DelayedPromise as DelayedPromise3,
|
|
@@ -18574,7 +20254,7 @@ function streamTranscribe({
|
|
|
18574
20254
|
}
|
|
18575
20255
|
const doStream = (_a25 = resolvedModel.doStream) == null ? void 0 : _a25.bind(resolvedModel);
|
|
18576
20256
|
if (doStream == null) {
|
|
18577
|
-
throw new
|
|
20257
|
+
throw new UnsupportedFunctionalityError7({
|
|
18578
20258
|
functionality: "streaming transcription",
|
|
18579
20259
|
message: `The ${resolvedModel.provider} model "${resolvedModel.modelId}" does not support streaming transcription.` + (typeof model === "string" ? " String model IDs resolve through the global provider (AI Gateway by default). If that provider does not support streaming transcription, pass a provider model instance instead (e.g. openai.transcription('gpt-realtime-whisper')) or upgrade @ai-sdk/gateway to a version with streaming transcription support." : "")
|
|
18580
20260
|
});
|
|
@@ -19150,13 +20830,13 @@ import {
|
|
|
19150
20830
|
} from "@ai-sdk/provider-utils";
|
|
19151
20831
|
|
|
19152
20832
|
// src/ui/convert-file-list-to-file-ui-parts.ts
|
|
19153
|
-
import { UnsupportedFunctionalityError as
|
|
20833
|
+
import { UnsupportedFunctionalityError as UnsupportedFunctionalityError8 } from "@ai-sdk/provider";
|
|
19154
20834
|
async function convertFileListToFileUIParts(files) {
|
|
19155
20835
|
if (files == null) {
|
|
19156
20836
|
return [];
|
|
19157
20837
|
}
|
|
19158
20838
|
if (!globalThis.FileList || !(files instanceof globalThis.FileList)) {
|
|
19159
|
-
throw new
|
|
20839
|
+
throw new UnsupportedFunctionalityError8({
|
|
19160
20840
|
functionality: "FileList",
|
|
19161
20841
|
message: "FileList is not supported in the current environment"
|
|
19162
20842
|
});
|
|
@@ -19340,8 +21020,8 @@ var DefaultChatTransport = class extends HttpChatTransport {
|
|
|
19340
21020
|
// src/ui/chat.ts
|
|
19341
21021
|
var AbstractChat = class {
|
|
19342
21022
|
constructor({
|
|
19343
|
-
generateId:
|
|
19344
|
-
id =
|
|
21023
|
+
generateId: generateId5 = generateIdFunc2,
|
|
21024
|
+
id = generateId5(),
|
|
19345
21025
|
transport = new DefaultChatTransport(),
|
|
19346
21026
|
messageMetadataSchema,
|
|
19347
21027
|
dataPartSchemas,
|
|
@@ -19553,7 +21233,7 @@ var AbstractChat = class {
|
|
|
19553
21233
|
};
|
|
19554
21234
|
this.id = id;
|
|
19555
21235
|
this.transport = transport;
|
|
19556
|
-
this.generateId =
|
|
21236
|
+
this.generateId = generateId5;
|
|
19557
21237
|
this.messageMetadataSchema = messageMetadataSchema;
|
|
19558
21238
|
this.dataPartSchemas = dataPartSchemas;
|
|
19559
21239
|
this.state = state;
|
|
@@ -20139,7 +21819,7 @@ export {
|
|
|
20139
21819
|
extractReasoningMiddleware,
|
|
20140
21820
|
fingerprintTools,
|
|
20141
21821
|
gateway3 as gateway,
|
|
20142
|
-
|
|
21822
|
+
generateId4 as generateId,
|
|
20143
21823
|
generateImage,
|
|
20144
21824
|
generateObject,
|
|
20145
21825
|
generateSpeech,
|